feature: архитектура v2 — api + очередь + worker, удержание окна браузера (#78)

- Сервис api (FastAPI, без Chrome): POST /v2/tasks, GET /v2/tasks/{id} (доступ по cookie
  сессии), DELETE /v2/sessions/current; /v1/solve — прежний контракт поверх очереди.
- Сервис worker (dramatiq + RabbitMQ): слоты браузера, окна сессий между задачами
  (browser_idle_timeout 0–120 мин, простой — TTL ключа в Redis), heartbeat.
- Маршрутизация задач сессии в персональную очередь воркера с открытым окном.
- Redis: задачи, результаты (RESULT_TTL_HOURS), окна, воркеры. Прокси и ключ 2captcha
  в Redis не сохраняются.
- Движок: AsyncStealthySession.run() — действия на текущей странице без навигации.
- Два образа (robotex-api, robotex-worker), compose с rabbitmq и redis,
  make up/down/logs/smoke, релиз собирает все образы.
- Тесты: api + worker в одном процессе (fakeredis), настоящий воркер dramatiq на
  StubBroker, браузерные тесты на новой схеме, smoke для запущенного стенда.
- Документация: README, API (v2), Deployment, Usage.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
protokeyandClaude Opus 5.5 committed 2026-10-13 19:13:52 +04:00
1 parent fe509c25c3
commit 7b511d5d38
38 files changed
+2142 -545

No files matched your search

+45 -16
View File
@@ -1,16 +1,19 @@
"""Сквозные тесты с настоящим Chrome и сетью. Запускаются в Docker (нужны Chrome и Xvfb):
"""Сквозные тесты с настоящим Chrome и сетью: api и worker в одном процессе (tests/harness.py), без
RabbitMQ и Redis. Запускаются в Docker (нужны Chrome и Xvfb):
make test-browser
Весь стенд с очередью проверяет make smoke.
"""
import json
import re
import uuid
import pytest
from fastapi.testclient import TestClient
import main
from api import profile
from tests.harness import run_env
from worker import windows
pytestmark = pytest.mark.browser
@@ -34,9 +37,14 @@ FP_JS = """async () => {
@pytest.fixture
def client(monkeypatch, tmp_path):
monkeypatch.setattr(main, "USER_DATA_ROOT", tmp_path)
return TestClient(main.app)
def env(monkeypatch, tmp_path):
with run_env(monkeypatch, tmp_path) as harness:
yield harness
@pytest.fixture
def client(env):
return env.client
def solve(client, session_id=None, **body):
@@ -50,11 +58,11 @@ def solve(client, session_id=None, **body):
def test_basic_page(client):
data = solve(client, url="https://example.com", screenshot=True, actions=[
{"type": "wait_for", "selector": "a", "timeout": 10},
{"type": "inner_html", "selector": "a"},
{"type": "wait_for", "selector": "p", "timeout": 10},
{"type": "inner_html", "selector": "title"},
])
assert data["status"] == "ok", data
assert data["actions"][1]["result"] == "Learn more"
assert data["actions"][1]["result"] == "Example Domain"
assert len(data["screenshot"]) > 1000
@@ -88,7 +96,7 @@ def test_fingerprint_applied_and_stable(client, monkeypatch):
return json.loads(await page.inner_html("#fp"))
results = {}
original = main.make_page_action
original = windows.make_page_action
def capture(payload):
action = original(payload)
@@ -99,22 +107,22 @@ def test_fingerprint_applied_and_stable(client, monkeypatch):
return out
return wrapped
monkeypatch.setattr(main, "make_page_action", capture)
monkeypatch.setattr(windows, "make_page_action", capture)
class Spy(main.AsyncStealthySession):
async def fetch(self, url, page_action):
out = await super().fetch(url, page_action=page_action)
class Spy(windows.AsyncStealthySession):
async def fetch(self, url, page_action, **kwargs):
out = await super().fetch(url, page_action=page_action, **kwargs)
results.setdefault(self._user_data_dir, []).append(out["fp"])
return out
monkeypatch.setattr(main, "AsyncStealthySession", Spy)
monkeypatch.setattr(windows, "AsyncStealthySession", Spy)
sids = [solve(client, url="https://example.com")["session_id"] for _ in range(4)]
solve(client, sids[0], url="https://example.com")
fps = {}
for d, measured in results.items():
saved = profile.load(main.USER_DATA_ROOT / d.rsplit("/", 1)[-1])
saved = profile.load(windows.USER_DATA_ROOT / d.rsplit("/", 1)[-1])
fp = saved["fingerprint"]
for m in measured:
assert m["screen"] == [fp["screen"]["width"], fp["screen"]["height"], fp["device_scale_factor"]]
@@ -141,3 +149,24 @@ def test_cloudflare(client):
assert data["status"] == "ok", data.get("error")
assert "cf_clearance" in {c["name"] for c in data["cookies"]}
assert uuid.UUID(data["session_id"])
def test_window_held_between_tasks(env):
"""browser_idle_timeout: вторая задача без url выполняется в том же окне, на уже открытой странице."""
read = [{"type": "inner_html", "selector": "title"}]
client = env.client
first = client.post("/v2/tasks", json={"url": "https://example.com", "actions": read, "browser_idle_timeout": 1})
task = client.get(first.json()["url"]).json()
assert task["result"]["status"] == "ok", task
assert env.pool.used == 1
second = client.post("/v2/tasks", json={"actions": read, "browser_idle_timeout": 1}).json()
result = client.get(second["url"]).json()["result"]
assert result["status"] == "ok", result
assert result["actions"][0]["result"] == "Example Domain"
# окно не перезапускалось: страница не открывалась заново
assert result["duration_ms"] < task["result"]["duration_ms"]
assert client.delete("/v2/sessions/current").json() == {"closed": True}
assert env.pool.used == 0