feature: API-ключи клиентов и журнал использования в Postgres (#4, #17)

- Postgres в compose; пакет db (SQLAlchemy async): clients, windows, requests.
- Все эндпоинты, кроме /health, — только с ключом в X-API-Key, иначе 401; без
  выпущенных ключей доступа нет. В базе — только sha256 ключа.
- Задача и окно сессии привязаны к клиенту: чужой ключ их не видит.
- Воркер пишет журнал: окно (открыто, последнее обращение, закрыто) и задачу
  (в очереди, начало, конец, итог, данные без секретов). Сбой записи не роняет задачу.
- python -m db.cli create|list|revoke|stats; make key-create/key-list/key-revoke/stats.
- smoke и loadtest с ключом.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/harness.py

@@ -1,5 +1,6 @@
 """api и worker в одном процессе без RabbitMQ и Redis: Redis — fakeredis, отправка в очередь подменена
 доставкой сообщения прямо в WindowPool воркера (tests/unit, tests/browser)."""
+import asyncio
 import json
 from contextlib import contextmanager
 
@@ -7,6 +8,9 @@ import fakeredis
 from fastapi.testclient import TestClient
 
 import main
+from db import clients
+from db.engine import connect as connect_db
+from db.journal import Journal
 from tasks import store as task_store
 from tasks.broker import RUN_TASK
 from worker import windows
@@ -49,17 +53,34 @@ class Harness:
 		return self.client.portal.call(fn, *args)
 
 
+def make_client_key(db, name="test") -> str:
+	async def create():
+		await db.init()
+		return (await clients.create_client(db, name))[1]
+	key = asyncio.run(create())
+	asyncio.run(db.close())  # соединения sqlite привязаны к циклу asyncio.run — закрываем, приложение откроет свои
+	return key
+
+
 @contextmanager
 def run_env(monkeypatch, tmp_path, slots=2):
 	server = fakeredis.FakeServer()
 	store = task_store.TaskStore(fakeredis.aioredis.FakeRedis(server=server, decode_responses=True))
-	pool = windows.WindowPool(store, WORKER_ID, slots=slots)
+	db = connect_db(f"sqlite+aiosqlite:///{tmp_path / 'robotex.db'}")
+	key = make_client_key(db)
+	pool = windows.WindowPool(store, WORKER_ID, slots=slots, journal=Journal(db))
 	harness = Harness(store, pool, fakeredis.FakeRedis(server=server, decode_responses=True))
-	monkeypatch.setattr(windows, "USER_DATA_ROOT", tmp_path)
+	harness.db, harness.api_key = db, key
+	# профили — в отдельном каталоге: в tmp_path ещё и файл базы
+	profiles = tmp_path / "profiles"
+	profiles.mkdir()
+	harness.profiles = profiles
+	monkeypatch.setattr(windows, "USER_DATA_ROOT", profiles)
 	monkeypatch.setattr(main, "store", store)
+	monkeypatch.setattr(main, "database", db)
 	monkeypatch.setattr(main, "dispatch", harness.dispatch)
 	harness.heartbeat()
-	with TestClient(main.app) as client:
+	with TestClient(main.app, headers={"X-API-Key": key}) as client:
 		harness.client = client
 		yield harness
 		client.portal.call(pool.close_all)

tests/loadtest.py

@@ -10,6 +10,7 @@
 import argparse
 import asyncio
 import json
+import os
 import statistics
 import subprocess
 import threading
@@ -94,7 +95,8 @@ async def run_level(n: int, args) -> None:
 	sampler = StatsSampler(args.container)
 	sampler.start()
 	t0 = time.monotonic()
-	async with httpx.AsyncClient(timeout=args.http_timeout, trust_env=False) as client:
+	headers = {"X-API-Key": args.api_key} if args.api_key else {}
+	async with httpx.AsyncClient(timeout=args.http_timeout, trust_env=False, headers=headers) as client:
 		results = await asyncio.gather(*(one(client, args.api, build_payload(args)) for _ in range(n)))
 	wall = time.monotonic() - t0
 	sampler.stop_event.set()
@@ -120,7 +122,9 @@ async def main():
 	p.add_argument("--selector", default='a[href*="/project/"]')
 	p.add_argument("--proxy", default="http://proxy:8080")
 	p.add_argument("--captcha-type", default="cloudflare")
-	p.add_argument("--container", default="webroboapi-api-1")
+	p.add_argument("--api-key", default=os.getenv("ROBOTEX_API_KEY"), help="X-API-Key (или ROBOTEX_API_KEY)")
+	# Chrome работает в воркере — его docker stats и снимаем
+	p.add_argument("--container", default="robotex-worker-1")
 	p.add_argument("--levels", type=int, nargs="+", default=[1, 5, 10])
 	p.add_argument("--pause", type=float, default=10, help="пауза между уровнями, с")
 	p.add_argument("--http-timeout", type=float, default=300)

tests/smoke.py

@@ -1,7 +1,9 @@
 """Проверка запущенного стенда (make up): api, очередь, воркер с Chrome, удержание окна.
 
-	make smoke                                   # http://localhost:8000
-	make smoke API_URL=https://app.robotex.sokolab.xyz
+	make smoke API_KEY=rbx_...                                   # http://localhost:8000
+	make smoke API_KEY=rbx_... API_URL=https://app.robotex.sokolab.xyz
+
+Ключ: make key-create name=smoke.
 """
 import sys
 import time
@@ -32,8 +34,13 @@ def windows(client: httpx.Client) -> int:
 	return sum(w["windows"] for w in client.get("/health").json()["workers"])
 
 
-def main(base_url: str) -> None:
-	client = httpx.Client(base_url=base_url, timeout=300)
+def main(base_url: str, api_key: str) -> None:
+	headers = {"X-API-Key": api_key}
+	client = httpx.Client(base_url=base_url, timeout=300, headers=headers)
+
+	print("auth")
+	resp = httpx.post(f"{base_url}/v1/solve", json={"url": URL, "actions": []}, timeout=30)
+	check(resp.status_code == 401, f"no key -> {resp.status_code}")
 
 	print("health")
 	health = client.get("/health").json()
@@ -46,7 +53,7 @@ def main(base_url: str) -> None:
 	check(data["actions"][0]["result"] == "Example Domain", "title = Example Domain")
 
 	print("v2 window held between tasks")
-	v2 = httpx.Client(base_url=base_url, timeout=60)  # своя сессия
+	v2 = httpx.Client(base_url=base_url, timeout=60, headers=headers)  # своя сессия
 	started = time.monotonic()
 	created = v2.post("/v2/tasks", json={"url": URL, "actions": READ_TITLE, "browser_idle_timeout": 2})
 	check(created.status_code == 202, f"POST /v2/tasks -> {created.status_code}")
@@ -73,4 +80,6 @@ def main(base_url: str) -> None:
 
 
 if __name__ == "__main__":
-	main(sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8000")
+	if len(sys.argv) < 3 or not sys.argv[2]:
+		sys.exit("usage: smoke.py <api url> <api key>   (make smoke API_KEY=...)")
+	main(sys.argv[1], sys.argv[2])

tests/unit/test_actors.py

@@ -22,7 +22,9 @@ def worker(monkeypatch, tmp_path):
 	monkeypatch.setattr(actions, "execute", fake_execute())
 	FakeSession.instances, FakeSession.error = [], None
 
+	from db.engine import connect as connect_db
 	from worker import actors
+	monkeypatch.setattr(actors, "connect_db", lambda: connect_db(f"sqlite+aiosqlite:///{tmp_path / 'robotex.db'}"))
 	w = dramatiq.Worker(actors.broker, worker_timeout=100, worker_threads=2)
 	w.start()
 	try:

tests/unit/test_auth_journal.py

@@ -0,0 +1,183 @@
+"""API-ключи (#4) и журнал окон и запросов (#17)."""
+import argparse
+import asyncio
+import uuid
+
+import pytest
+from sqlalchemy import select
+
+from db import cli, clients
+from db.engine import connect as connect_db
+from db.models import Client, Request, Window
+
+from .conftest import FakeSession
+
+URL = "https://example.com"
+
+
+def rows(env, model):
+	async def query():
+		async with env.db.session() as session:
+			return list(await session.scalars(select(model).order_by(model.id)))
+	return env.call(query)
+
+
+def new_key(env, name="other"):
+	return env.call(clients.create_client, env.db, name)[1]
+
+
+@pytest.mark.parametrize("request_", [
+	("post", "/v1/solve", {"url": URL, "actions": []}),
+	("post", "/v2/tasks", {"url": URL, "actions": []}),
+	("get", f"/v2/tasks/{uuid.uuid4()}", None),
+	("delete", "/v2/sessions/current", None),
+])
+@pytest.mark.parametrize("key", [None, "", "rbx_wrong"])
+def test_requests_without_valid_key_rejected(env, request_, key):
+	method, path, body = request_
+	headers = {"X-API-Key": key} if key is not None else {}
+	env.client.headers.pop("X-API-Key")
+	resp = env.client.request(method, path, json=body, headers=headers)
+	assert resp.status_code == 401
+	assert env.messages == []
+
+
+def test_health_is_open(env):
+	env.client.headers.pop("X-API-Key")
+	assert env.client.get("/health").status_code == 200
+
+
+def test_revoked_key_rejected(env):
+	assert env.client.post("/v1/solve", json={"url": URL, "actions": []}).status_code == 200
+	assert env.call(clients.revoke, env.db, 1)
+	assert env.client.post("/v1/solve", json={"url": URL, "actions": []}).status_code == 401
+
+
+def test_no_clients_means_no_access(env):
+	env.call(clients.revoke, env.db, 1)
+	assert [c.revoked_at is not None for c in rows(env, Client)] == [True]
+	assert env.client.post("/v2/tasks", json={"url": URL, "actions": []}).status_code == 401
+
+
+def test_last_access_tracked(env):
+	assert rows(env, Client)[0].last_access_at is None
+	env.client.post("/v2/tasks", json={"url": URL, "actions": []})
+	assert rows(env, Client)[0].last_access_at is not None
+
+
+def test_task_and_window_visible_only_to_their_client(env):
+	task_id = env.client.post("/v2/tasks", json={"url": URL, "actions": [], "browser_idle_timeout": 5}).json()["task_id"]
+	other = {"X-API-Key": new_key(env)}
+
+	# та же cookie сессии, но другой ключ: задача и окно — как чужие
+	assert env.client.get(f"/v2/tasks/{task_id}", headers=other).status_code == 404
+	assert env.client.post("/v2/tasks", json={"actions": []}, headers=other).status_code == 422
+	assert env.client.delete("/v2/sessions/current", headers=other).json() == {"closed": False}
+	assert env.pool.used == 1
+
+	assert env.client.get(f"/v2/tasks/{task_id}").json()["status"] == "done"
+	assert env.client.delete("/v2/sessions/current").json() == {"closed": True}
+
+
+def test_journal_request_and_window(env):
+	env.client.post("/v2/tasks", json={
+		"url": URL, "proxy": "http://user:TopSecret1@host:3128", "captcha": "KEYSECRET2",
+		"actions": [{"type": "typing", "selector": "#p", "value": "PassSecret3"}],
+	})
+	env.client.post("/v1/solve", json={"url": URL, "actions": []})
+
+	requests = rows(env, Request)
+	assert [(r.api, r.status, r.client_id) for r in requests] == [("v2", "ok", 1), ("v1", "ok", 1)]
+	for r in requests:
+		assert r.created_at <= r.started_at <= r.finished_at
+		assert r.duration_ms is not None and r.window_id is not None
+	data = str(requests[0].task_data)
+	assert requests[0].task_data["url"] == URL
+	for secret in ("user", "TopSecret1", "KEYSECRET2", "PassSecret3"):
+		assert secret not in data
+
+	windows = rows(env, Window)
+	assert len(windows) == 2
+	assert all(w.closed_at is not None and w.started_at <= w.closed_at for w in windows)
+	assert [w.task_id for w in windows] == [r.task_id for r in requests]
+
+
+def test_journal_held_window(env):
+	for body in ({"url": URL, "browser_idle_timeout": 5}, {"browser_idle_timeout": 5}):
+		env.client.post("/v2/tasks", json={"actions": [], **body})
+	window, = rows(env, Window)
+	assert window.closed_at is None and window.last_access_at > window.started_at
+	assert {r.window_id for r in rows(env, Request)} == {window.id}
+
+	env.client.delete("/v2/sessions/current")
+	assert rows(env, Window)[0].closed_at is not None
+
+
+def test_journal_failed_request(env):
+	FakeSession.error = RuntimeError("net::ERR_PROXY_CONNECTION_FAILED")
+	env.client.post("/v2/tasks", json={"url": URL, "actions": []})
+	request, = rows(env, Request)
+	assert (request.status, request.code) == ("err", "request_failed")
+
+
+def test_journal_failure_does_not_break_task(env, monkeypatch):
+	class DownDatabase:
+		def session(self):
+			raise RuntimeError("db is down")
+	monkeypatch.setattr(env.pool.journal, "db", DownDatabase())
+	task_id = env.client.post("/v2/tasks", json={"url": URL, "actions": []}).json()["task_id"]
+	assert env.client.get(f"/v2/tasks/{task_id}").json()["result"]["status"] == "ok"
+
+
+def test_cli_create_list_revoke(monkeypatch, tmp_path, capsys):
+	monkeypatch.setenv("DATABASE_URL", f"sqlite+aiosqlite:///{tmp_path / 'cli.db'}")
+	asyncio.run(cli.run(argparse.Namespace(command="create", name="acme")))
+	key = capsys.readouterr().out.strip().splitlines()[-1]
+	assert key.startswith(clients.KEY_PREFIX)
+
+	asyncio.run(cli.run(argparse.Namespace(command="list")))
+	listing = capsys.readouterr().out
+	assert "acme" in listing and "active" in listing and key not in listing
+
+	asyncio.run(cli.run(argparse.Namespace(command="revoke", id=1)))
+	assert capsys.readouterr().out.strip() == "revoked"
+
+	asyncio.run(cli.run(argparse.Namespace(command="stats", days=7)))
+	assert "запросов не было" in capsys.readouterr().out
+
+	async def check():
+		db = connect_db()
+		try:
+			return await clients.authenticate(db, key)
+		finally:
+			await db.close()
+	assert asyncio.run(check()) is None
+
+
+def test_usage_stats(env, monkeypatch):
+	from api import actions as api_actions
+	from db import stats
+
+	from .conftest import fake_execute
+
+	env.client.post("/v2/tasks", json={"url": URL, "actions": [], "browser_idle_timeout": 5})
+	env.client.post("/v2/tasks", json={"actions": []})  # в открытом окне, после неё окно закрывается
+	monkeypatch.setattr(api_actions, "execute", fake_execute(exc=api_actions.ChallengeFailed("cf")))
+	env.client.post("/v1/solve", json={"url": URL, "actions": []})
+
+	other = {"X-API-Key": new_key(env, "acme")}
+	env.client.cookies.clear()
+	monkeypatch.setattr(api_actions, "execute", fake_execute())
+	env.client.post("/v1/solve", json={"url": URL, "actions": []}, headers=other)
+
+	usage = {u.name: u for u in env.call(stats.usage, env.db, stats.since_days(1))}
+	test, acme = usage["test"], usage["acme"]
+	assert (test.requests, test.ok, test.errors, test.windows) == (3, 2, {"challenge_failed": 1}, 2)
+	assert (acme.requests, acme.ok, acme.windows) == (1, 1, 1)
+	assert test.avg_ms is not None and test.p95_ms is not None
+	assert test.browser_seconds > 0
+	# все окна закрыты: время браузера больше не растёт
+	again = {u.name: u.browser_seconds for u in env.call(stats.usage, env.db, stats.since_days(1))}
+	assert again == {name: u.browser_seconds for name, u in usage.items()}
+	# период фильтрует запросы и окна
+	assert env.call(stats.usage, env.db, stats.since_days(-1)) == []

tests/unit/test_store.py

@@ -67,8 +67,8 @@ async def test_finish_sets_result_and_ttl(store):
 
 
 async def test_live_window_requires_live_worker(store):
-	await store.set_session("s", "w1", 5, 300)
+	await store.set_session("s", "w1", 5, 300, client_id=7)
 	assert await store.live_window("s") is None
 	await store.heartbeat("w1", {"browser_slots": 1, "free_slots": 0})
-	assert await store.live_window("s") == {"worker_id": "w1", "idle_timeout": "5"}
+	assert await store.live_window("s") == {"worker_id": "w1", "idle_timeout": "5", "client_id": "7"}
 	assert await store.workers() == [{"worker_id": "w1", "browser_slots": 1, "free_slots": 0}]

tests/unit/test_v1.py

@@ -109,21 +109,21 @@ def test_invalid_request_is_422_without_task(env):
 	assert env.messages == []
 
 
-def test_session_cookie_reused(env, tmp_path):
+def test_session_cookie_reused(env):
 	sid = str(uuid.uuid4())
 	env.client.cookies.set("session_id", sid)
 	resp = post(env)
 	assert resp.cookies["session_id"] == sid
-	assert FakeSession.instances[0].kwargs["user_data_dir"] == str(tmp_path / sid)
+	assert FakeSession.instances[0].kwargs["user_data_dir"] == str(env.profiles / sid)
 
 
 @pytest.mark.parametrize("bad", ["../../etc", "not-a-uuid", "/tmp/x"])
-def test_bad_session_cookie_gets_new_session(env, tmp_path, bad):
+def test_bad_session_cookie_gets_new_session(env, bad):
 	env.client.cookies.set("session_id", bad)
 	resp = post(env)
 	sid = resp.cookies["session_id"]
 	assert sid != bad and uuid.UUID(sid)
-	assert [p.name for p in tmp_path.iterdir()] == [sid]
+	assert [p.name for p in env.profiles.iterdir()] == [sid]
 
 
 def test_user_agent_param_rejected(env):

tests/unit/test_v2.py

@@ -90,7 +90,8 @@ def test_window_held_and_reused(env):
 	sid = env.client.cookies["session_id"]
 	engine = FakeSession.instances[0]
 	assert not engine.closed and env.pool.used == 1
-	assert env.redis.hgetall(session_key(sid)) == {"worker_id": WORKER_ID, "idle_timeout": "5"}
+	window = env.redis.hgetall(session_key(sid))
+	assert (window["worker_id"], window["idle_timeout"], window["client_id"]) == (WORKER_ID, "5", "1")
 	assert 290 < env.redis.ttl(session_key(sid)) <= 300
 
 	# следующая задача сессии — в персональную очередь воркера с окном, без url — на текущей странице
@@ -180,7 +181,7 @@ def test_dead_worker_window_goes_to_common_queue(env):
 	assert env.messages[-1].queue_name == COMMON_QUEUE
 
 
-def test_old_profiles_cleaned_up(env, tmp_path):
+def test_old_profiles_cleaned_up(env):
 	"""#20: профили старше PROFILE_TTL_DAYS удаляются, кроме тех, где открыто окно."""
 	import os
 	import time
@@ -194,18 +195,18 @@ def test_old_profiles_cleaned_up(env, tmp_path):
 
 	long_ago = time.time() - 8 * 86400
 	for sid in (held, elsewhere, old, fresh, "not-a-session"):
-		(tmp_path / sid).mkdir(exist_ok=True)
-		marker = tmp_path / sid / LAST_USED_FILE
+		(env.profiles / sid).mkdir(exist_ok=True)
+		marker = env.profiles / sid / LAST_USED_FILE
 		marker.touch()
 		if sid != fresh:
 			os.utime(marker, (long_ago, long_ago))
 
 	assert env.call(env.pool.cleanup_profiles) == [old]
-	assert sorted(p.name for p in tmp_path.iterdir()) == sorted([held, elsewhere, fresh, "not-a-session"])
+	assert sorted(p.name for p in env.profiles.iterdir()) == sorted([held, elsewhere, fresh, "not-a-session"])
 
 
-def test_profile_marked_as_used(env, tmp_path):
+def test_profile_marked_as_used(env):
 	from worker.windows import LAST_USED_FILE
 
 	create(env, url="https://example.com")
-	assert (tmp_path / env.client.cookies["session_id"] / LAST_USED_FILE).exists()
+	assert (env.profiles / env.client.cookies["session_id"] / LAST_USED_FILE).exists()