feature: архитектура v2 — api + очередь + worker, удержание окна браузера (#78)
- Сервис api (FastAPI, без Chrome): POST /v2/tasks, GET /v2/tasks/{id} (доступ по cookie
сессии), DELETE /v2/sessions/current; /v1/solve — прежний контракт поверх очереди.
- Сервис worker (dramatiq + RabbitMQ): слоты браузера, окна сессий между задачами
(browser_idle_timeout 0–120 мин, простой — TTL ключа в Redis), heartbeat.
- Маршрутизация задач сессии в персональную очередь воркера с открытым окном.
- Redis: задачи, результаты (RESULT_TTL_HOURS), окна, воркеры. Прокси и ключ 2captcha
в Redis не сохраняются.
- Движок: AsyncStealthySession.run() — действия на текущей странице без навигации.
- Два образа (robotex-api, robotex-worker), compose с rabbitmq и redis,
make up/down/logs/smoke, релиз собирает все образы.
- Тесты: api + worker в одном процессе (fakeredis), настоящий воркер dramatiq на
StubBroker, браузерные тесты на новой схеме, smoke для запущенного стенда.
- Документация: README, API (v2), Deployment, Usage.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fe509c25c3
commit
7b511d5d38
38 files changed
+2142
-545
No files matched your search
+45
-16
@@ -1,16 +1,19 @@
|
||||
"""Сквозные тесты с настоящим Chrome и сетью. Запускаются в Docker (нужны Chrome и Xvfb):
|
||||
"""Сквозные тесты с настоящим Chrome и сетью: api и worker в одном процессе (tests/harness.py), без
|
||||
RabbitMQ и Redis. Запускаются в Docker (нужны Chrome и Xvfb):
|
||||
|
||||
make test-browser
|
||||
|
||||
Весь стенд с очередью проверяет make smoke.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import main
|
||||
from api import profile
|
||||
from tests.harness import run_env
|
||||
from worker import windows
|
||||
|
||||
pytestmark = pytest.mark.browser
|
||||
|
||||
@@ -34,9 +37,14 @@ FP_JS = """async () => {
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(main, "USER_DATA_ROOT", tmp_path)
|
||||
return TestClient(main.app)
|
||||
def env(monkeypatch, tmp_path):
|
||||
with run_env(monkeypatch, tmp_path) as harness:
|
||||
yield harness
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(env):
|
||||
return env.client
|
||||
|
||||
|
||||
def solve(client, session_id=None, **body):
|
||||
@@ -50,11 +58,11 @@ def solve(client, session_id=None, **body):
|
||||
|
||||
def test_basic_page(client):
|
||||
data = solve(client, url="https://example.com", screenshot=True, actions=[
|
||||
{"type": "wait_for", "selector": "a", "timeout": 10},
|
||||
{"type": "inner_html", "selector": "a"},
|
||||
{"type": "wait_for", "selector": "p", "timeout": 10},
|
||||
{"type": "inner_html", "selector": "title"},
|
||||
])
|
||||
assert data["status"] == "ok", data
|
||||
assert data["actions"][1]["result"] == "Learn more"
|
||||
assert data["actions"][1]["result"] == "Example Domain"
|
||||
assert len(data["screenshot"]) > 1000
|
||||
|
||||
|
||||
@@ -88,7 +96,7 @@ def test_fingerprint_applied_and_stable(client, monkeypatch):
|
||||
return json.loads(await page.inner_html("#fp"))
|
||||
|
||||
results = {}
|
||||
original = main.make_page_action
|
||||
original = windows.make_page_action
|
||||
|
||||
def capture(payload):
|
||||
action = original(payload)
|
||||
@@ -99,22 +107,22 @@ def test_fingerprint_applied_and_stable(client, monkeypatch):
|
||||
return out
|
||||
return wrapped
|
||||
|
||||
monkeypatch.setattr(main, "make_page_action", capture)
|
||||
monkeypatch.setattr(windows, "make_page_action", capture)
|
||||
|
||||
class Spy(main.AsyncStealthySession):
|
||||
async def fetch(self, url, page_action):
|
||||
out = await super().fetch(url, page_action=page_action)
|
||||
class Spy(windows.AsyncStealthySession):
|
||||
async def fetch(self, url, page_action, **kwargs):
|
||||
out = await super().fetch(url, page_action=page_action, **kwargs)
|
||||
results.setdefault(self._user_data_dir, []).append(out["fp"])
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(main, "AsyncStealthySession", Spy)
|
||||
monkeypatch.setattr(windows, "AsyncStealthySession", Spy)
|
||||
|
||||
sids = [solve(client, url="https://example.com")["session_id"] for _ in range(4)]
|
||||
solve(client, sids[0], url="https://example.com")
|
||||
|
||||
fps = {}
|
||||
for d, measured in results.items():
|
||||
saved = profile.load(main.USER_DATA_ROOT / d.rsplit("/", 1)[-1])
|
||||
saved = profile.load(windows.USER_DATA_ROOT / d.rsplit("/", 1)[-1])
|
||||
fp = saved["fingerprint"]
|
||||
for m in measured:
|
||||
assert m["screen"] == [fp["screen"]["width"], fp["screen"]["height"], fp["device_scale_factor"]]
|
||||
@@ -141,3 +149,24 @@ def test_cloudflare(client):
|
||||
assert data["status"] == "ok", data.get("error")
|
||||
assert "cf_clearance" in {c["name"] for c in data["cookies"]}
|
||||
assert uuid.UUID(data["session_id"])
|
||||
|
||||
|
||||
def test_window_held_between_tasks(env):
|
||||
"""browser_idle_timeout: вторая задача без url выполняется в том же окне, на уже открытой странице."""
|
||||
read = [{"type": "inner_html", "selector": "title"}]
|
||||
client = env.client
|
||||
|
||||
first = client.post("/v2/tasks", json={"url": "https://example.com", "actions": read, "browser_idle_timeout": 1})
|
||||
task = client.get(first.json()["url"]).json()
|
||||
assert task["result"]["status"] == "ok", task
|
||||
assert env.pool.used == 1
|
||||
|
||||
second = client.post("/v2/tasks", json={"actions": read, "browser_idle_timeout": 1}).json()
|
||||
result = client.get(second["url"]).json()["result"]
|
||||
assert result["status"] == "ok", result
|
||||
assert result["actions"][0]["result"] == "Example Domain"
|
||||
# окно не перезапускалось: страница не открывалась заново
|
||||
assert result["duration_ms"] < task["result"]["duration_ms"]
|
||||
|
||||
assert client.delete("/v2/sessions/current").json() == {"closed": True}
|
||||
assert env.pool.used == 0
|
||||
Reference in new issue
Block a user