- Postgres в compose; пакет db (SQLAlchemy async): clients, windows, requests. - Все эндпоинты, кроме /health, — только с ключом в X-API-Key, иначе 401; без выпущенных ключей доступа нет. В базе — только sha256 ключа. - Задача и окно сессии привязаны к клиенту: чужой ключ их не видит. - Воркер пишет журнал: окно (открыто, последнее обращение, закрыто) и задачу (в очереди, начало, конец, итог, данные без секретов). Сбой записи не роняет задачу. - python -m db.cli create|list|revoke|stats; make key-create/key-list/key-revoke/stats. - smoke и loadtest с ключом. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
protokeyandClaude Opus 5.5 committed
1 parent
9dc2c0b619
commit
1a4bd8d672
34 files changed
+857
-81
No files matched your search
+37
-14
@@ -29,8 +29,9 @@ from patchright.async_api import Page, ProxySettings
|
||||
|
||||
from api import actions, profile
|
||||
from api.geoip import resolve_geo
|
||||
from api.logs import safe_request_dump, safe_response_dump
|
||||
from api.logs import safe_request_data, safe_request_dump, safe_response_dump
|
||||
from api.schemas import TaskRequest
|
||||
from db.journal import Journal
|
||||
from engine import AsyncStealthySession
|
||||
from tasks.sessions import parse_session_id
|
||||
from tasks.store import RUNNING_TTL, Status, TaskStore
|
||||
@@ -64,6 +65,9 @@ class Window:
|
||||
proxy: str | None = None
|
||||
# окно закрыть, как только закончится текущая задача (DELETE /v2/sessions/current)
|
||||
closing: bool = False
|
||||
client_id: int | None = None
|
||||
# строка окна в журнале (#17): открыто — закрыто
|
||||
journal_id: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -158,8 +162,9 @@ def build_result(payload: TaskRequest, result: dict) -> dict:
|
||||
|
||||
|
||||
class WindowPool:
|
||||
def __init__(self, store: TaskStore, worker_id: str, slots: int):
|
||||
def __init__(self, store: TaskStore, worker_id: str, slots: int, journal: Journal | None = None):
|
||||
self.store = store
|
||||
self.journal = journal or Journal(None)
|
||||
self.worker_id = worker_id
|
||||
self.slots = slots
|
||||
self.used = 0
|
||||
@@ -198,8 +203,13 @@ class WindowPool:
|
||||
entry = self._locks.get(session_id)
|
||||
return bool(entry and entry.lock.locked())
|
||||
|
||||
async def handle(self, task_id: str, session_id: str, payload: dict, idle_timeout: int) -> str:
|
||||
"""Выполнить задачу. Результат — в Redis; возврат — что делать с сообщением очереди."""
|
||||
async def handle(
|
||||
self, task_id: str, session_id: str, payload: dict, idle_timeout: int, meta: dict | None = None,
|
||||
) -> str:
|
||||
"""Выполнить задачу. Результат — в Redis; возврат — что делать с сообщением очереди.
|
||||
meta: client_id и api (v1/v2) — для журнала."""
|
||||
meta = meta or {}
|
||||
client_id = meta.get("client_id")
|
||||
async with self._session_lock(session_id):
|
||||
window = self.windows.get(session_id)
|
||||
new_window = window is None
|
||||
@@ -210,7 +220,7 @@ class WindowPool:
|
||||
return Outcome.NO_SLOT if task and task["status"] == Status.QUEUED else Outcome.SKIPPED
|
||||
# слот занимается до первого await, иначе его может перехватить другая задача
|
||||
self.used += 1
|
||||
window = self.windows[session_id] = Window(session_id)
|
||||
window = self.windows[session_id] = Window(session_id, client_id=client_id)
|
||||
await self.report()
|
||||
|
||||
if not await self.store.claim(task_id, self.worker_id, running_here=task_id in self.running):
|
||||
@@ -219,9 +229,16 @@ class WindowPool:
|
||||
return Outcome.SKIPPED
|
||||
|
||||
self.running.add(task_id)
|
||||
task = TaskRequest.model_validate(payload)
|
||||
result = None
|
||||
queued = await self.store.get(task_id) or {}
|
||||
await self.journal.request_started(
|
||||
task_id, client_id, session_id, meta.get("api", "v2"),
|
||||
float(queued["created_at"]) if queued.get("created_at") else None, safe_request_data(task),
|
||||
)
|
||||
try:
|
||||
await self.store.set_session(session_id, self.worker_id, idle_timeout, RUNNING_TTL)
|
||||
result = await self._execute(window, TaskRequest.model_validate(payload))
|
||||
await self.store.set_session(session_id, self.worker_id, idle_timeout, RUNNING_TTL, client_id)
|
||||
result = await self._execute(window, task, task_id)
|
||||
await self.store.finish(task_id, result)
|
||||
except Exception as e:
|
||||
log.exception("task %s failed", task_id)
|
||||
@@ -234,15 +251,16 @@ class WindowPool:
|
||||
raise
|
||||
finally:
|
||||
self.running.discard(task_id)
|
||||
await self.journal.request_finished(task_id, window.journal_id, result)
|
||||
if idle_timeout and window.engine and not window.closing:
|
||||
await self.store.set_session(session_id, self.worker_id, idle_timeout, idle_timeout * 60)
|
||||
await self.store.set_session(session_id, self.worker_id, idle_timeout, idle_timeout * 60, client_id)
|
||||
else:
|
||||
await self._close(window)
|
||||
with suppress(Exception):
|
||||
await self.store.delete_session(session_id)
|
||||
return Outcome.DONE
|
||||
|
||||
async def _execute(self, window: Window, payload: TaskRequest) -> dict:
|
||||
async def _execute(self, window: Window, payload: TaskRequest, task_id: str) -> dict:
|
||||
started = time.monotonic()
|
||||
session_id = window.session_id
|
||||
log.info("Incoming request [%s]: %s", session_id, payload.url or "<current page>")
|
||||
@@ -264,11 +282,14 @@ class WindowPool:
|
||||
raise NoWindow("url is required: the session has no open browser window")
|
||||
window.engine = await self._open(session_id, payload, nav_kwargs)
|
||||
window.proxy = payload.proxy
|
||||
elif payload.cookies is not None:
|
||||
# переданные cookies заменяют cookies профиля, как и при запуске окна
|
||||
await window.engine.context.clear_cookies()
|
||||
if payload.cookies:
|
||||
await window.engine.context.add_cookies(payload.cookies)
|
||||
window.journal_id = await self.journal.window_opened(window.client_id, session_id, task_id, self.worker_id)
|
||||
else:
|
||||
await self.journal.window_used(window.journal_id)
|
||||
if payload.cookies is not None:
|
||||
# переданные cookies заменяют cookies профиля, как и при запуске окна
|
||||
await window.engine.context.clear_cookies()
|
||||
if payload.cookies:
|
||||
await window.engine.context.add_cookies(payload.cookies)
|
||||
|
||||
if payload.url:
|
||||
raw = await window.engine.fetch(
|
||||
@@ -324,6 +345,8 @@ class WindowPool:
|
||||
|
||||
async def _close_engine(self, window: Window) -> None:
|
||||
engine, window.engine = window.engine, None
|
||||
journal_id, window.journal_id = window.journal_id, None
|
||||
await self.journal.window_closed(journal_id)
|
||||
if engine is not None:
|
||||
try:
|
||||
await engine.close()
|
||||
|
||||
Reference in new issue
Block a user