feature: архитектура v2 — api + очередь + worker, удержание окна браузера (#78)
- Сервис api (FastAPI, без Chrome): POST /v2/tasks, GET /v2/tasks/{id} (доступ по cookie
сессии), DELETE /v2/sessions/current; /v1/solve — прежний контракт поверх очереди.
- Сервис worker (dramatiq + RabbitMQ): слоты браузера, окна сессий между задачами
(browser_idle_timeout 0–120 мин, простой — TTL ключа в Redis), heartbeat.
- Маршрутизация задач сессии в персональную очередь воркера с открытым окном.
- Redis: задачи, результаты (RESULT_TTL_HOURS), окна, воркеры. Прокси и ключ 2captcha
в Redis не сохраняются.
- Движок: AsyncStealthySession.run() — действия на текущей странице без навигации.
- Два образа (robotex-api, robotex-worker), compose с rabbitmq и redis,
make up/down/logs/smoke, релиз собирает все образы.
- Тесты: api + worker в одном процессе (fakeredis), настоящий воркер dramatiq на
StubBroker, браузерные тесты на новой схеме, smoke для запущенного стенда.
- Документация: README, API (v2), Deployment, Usage.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fe509c25c3
commit
7b511d5d38
38 files changed
+2142
-545
No files matched your search
@@ -0,0 +1,76 @@
|
||||
"""Проверка запущенного стенда (make up): api, очередь, воркер с Chrome, удержание окна.
|
||||
|
||||
make smoke # http://localhost:8000
|
||||
make smoke API_URL=https://app.robotex.sokolab.xyz
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
|
||||
import httpx
|
||||
|
||||
URL = "https://example.com"
|
||||
READ_TITLE = [{"type": "inner_html", "selector": "title"}]
|
||||
|
||||
|
||||
def check(cond: bool, what: str) -> None:
|
||||
print((" ok " if cond else " FAIL ") + what)
|
||||
if not cond:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def wait_task(client: httpx.Client, task_id: str, timeout: float = 180) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
task = client.get(f"/v2/tasks/{task_id}").json()
|
||||
if task["status"] not in ("queued", "running"):
|
||||
return task
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError(f"task {task_id} not finished in {timeout}s")
|
||||
|
||||
|
||||
def windows(client: httpx.Client) -> int:
|
||||
return sum(w["windows"] for w in client.get("/health").json()["workers"])
|
||||
|
||||
|
||||
def main(base_url: str) -> None:
|
||||
client = httpx.Client(base_url=base_url, timeout=300)
|
||||
|
||||
print("health")
|
||||
health = client.get("/health").json()
|
||||
check(len(health["workers"]) > 0, f"workers: {[w['worker_id'] for w in health['workers']]}, free slots {health['free_slots']}")
|
||||
|
||||
print("v1 /v1/solve")
|
||||
started = time.monotonic()
|
||||
data = client.post("/v1/solve", json={"url": URL, "actions": READ_TITLE}).json()
|
||||
check(data["status"] == "ok", f"status {data['status']} {data.get('error', '')} ({time.monotonic() - started:.1f}s)")
|
||||
check(data["actions"][0]["result"] == "Example Domain", "title = Example Domain")
|
||||
|
||||
print("v2 window held between tasks")
|
||||
v2 = httpx.Client(base_url=base_url, timeout=60) # своя сессия
|
||||
started = time.monotonic()
|
||||
created = v2.post("/v2/tasks", json={"url": URL, "actions": READ_TITLE, "browser_idle_timeout": 2})
|
||||
check(created.status_code == 202, f"POST /v2/tasks -> {created.status_code}")
|
||||
task = wait_task(v2, created.json()["task_id"])
|
||||
check(task["status"] == "done" and task["result"]["status"] == "ok", f"first task done ({time.monotonic() - started:.1f}s)")
|
||||
check(windows(client) >= 1, "window stays open")
|
||||
|
||||
started = time.monotonic()
|
||||
created = v2.post("/v2/tasks", json={"actions": READ_TITLE, "browser_idle_timeout": 2})
|
||||
task = wait_task(v2, created.json()["task_id"])
|
||||
check(
|
||||
task["status"] == "done" and task["result"]["actions"][0]["result"] == "Example Domain",
|
||||
f"second task without url ran in the open window ({time.monotonic() - started:.1f}s)",
|
||||
)
|
||||
|
||||
check(client.get(f"/v2/tasks/{created.json()['task_id']}").status_code == 404, "task hidden from other sessions")
|
||||
|
||||
check(v2.delete("/v2/sessions/current").json() == {"closed": True}, "DELETE /v2/sessions/current")
|
||||
deadline = time.monotonic() + 15
|
||||
while windows(client) and time.monotonic() < deadline:
|
||||
time.sleep(0.5)
|
||||
check(windows(client) == 0, "window closed, slot released")
|
||||
print("all good")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8000")
|
||||
Reference in new issue
Block a user