From 437ea27be7b38244355a5d9e047d54b1aba0ac52 Mon Sep 17 00:00:00 2001 From: protokey Date: Tue, 13 Oct 2026 14:52:12 +0400 Subject: [PATCH] =?UTF-8?q?fix:=20=D1=81=D0=BB=D0=BE=D1=82=D1=8B,=20=D0=B1?= =?UTF-8?q?=D0=B5=D0=B7=D0=BE=D0=BF=D0=B0=D1=81=D0=BD=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D1=8C,=20=D0=B5=D0=B4=D0=B8=D0=BD=D1=8B=D0=B9=20=D1=84=D0=BE?= =?UTF-8?q?=D1=80=D0=BC=D0=B0=D1=82=20=D0=BE=D1=82=D0=B2=D0=B5=D1=82=D0=B0?= =?UTF-8?q?=20=D0=B8=20=D0=B2=D0=B0=D0=BB=D0=B8=D0=B4=D0=B0=D1=86=D0=B8?= =?UTF-8?q?=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - слоты браузеров: asyncio.Semaphore, размер из BROWSER_SLOTS, при нехватке — 503 + Retry-After; устранена утечка счётчика при отказе (#12, #49) - session_id принимается только как UUID, путь профиля проверяется (#5) - секреты клиента в логах маскируются, полный дамп — только на DEBUG (#6) - ответ: status всегда "ok"/"err", поле code (no_slots / request_failed / challenge_failed / action_failed), response_model (#7) - непройденный антибот → challenge_failed, действия не выполняются (#8) - таймауты провайдеров в мс вынесены в константы; detect переживает навигацию посреди проверки; пауза между неудачными solve вместо холостого цикла (#9) - обязательные поля действий и формат прокси проверяются схемой → 422 до запуска браузера (#13) - прокси необязателен (#15) - user_agent сохраняется в профиле сессии и реально передаётся в браузер (#16) - duration_ms у действий и у запроса (#18), опция screenshot (#19) - общий конфиг логирования (LOG_LEVEL), print → логгер, CloudflareProvider, удалён cached_persistent, GET /health (#22) Co-Authored-By: Claude Opus 5.5 --- dot_env | 2 +- env.example | 8 + src/antibot/humanize.py | 5 - src/antibot/known_providers/cloudflare.py | 17 +- src/antibot/known_providers/recaptcha.py | 7 +- src/antibot/orchestrator.py | 21 +- src/api/actions.py | 53 ++-- src/api/schemas.py | 98 +++++++- src/captcha/base.py | 39 --- src/captcha/hcaptcha.py | 2 - src/captcha/recaptcha.py | 2 - src/captcha/turnstile.py | 2 - src/main.py | 292 ++++++++++++++-------- tests/loadtest.py | 13 +- 14 files changed, 360 insertions(+), 201 deletions(-) diff --git a/dot_env b/dot_env index b853412..8713326 100644 --- a/dot_env +++ b/dot_env @@ -1,4 +1,4 @@ API_PORT= API_CPUS=1 API_MEM_LIMIT=1g -API_MEM_LIMIT=1g +BROWSER_SLOTS=1 diff --git a/env.example b/env.example index 6a329b3..78de7d9 100644 --- a/env.example +++ b/env.example @@ -1 +1,9 @@ # Project Env +# порт, CPU и память контейнера (docker-compose) +API_PORT=8000 +API_CPUS=4 +API_MEM_LIMIT=8g +# сколько Chrome параллельно, ≈ числу ядер (docs/Deployment.md) +BROWSER_SLOTS=4 +NO_SLOTS_RETRY_AFTER=5 +LOG_LEVEL=INFO diff --git a/src/antibot/humanize.py b/src/antibot/humanize.py index 24b0ffb..db06c5d 100644 --- a/src/antibot/humanize.py +++ b/src/antibot/humanize.py @@ -85,7 +85,6 @@ async def move_mouse_human( if i < steps: px += random.uniform(-1.5, 1.5) py += random.uniform(-1.5, 1.5) - print("Move:", px, py) await page.mouse.move(px, py) await page.wait_for_timeout(step_delay_ms * random.uniform(0.7, 1.3)) @@ -103,13 +102,9 @@ async def click_human( """move_mouse_human + клик с человеческой задержкой между наведением и down/up, вместо мгновенного page.mouse.click(x, y).""" await move_mouse_human(page, x, y, **move_kwargs) - print('wait for timeout') await page.wait_for_timeout(random.uniform(50, 180)) - print('key down') await page.mouse.down(button=button) - print('sleep') await page.wait_for_timeout(random.uniform(40, 99)) - print('key up') await page.mouse.up(button=button) diff --git a/src/antibot/known_providers/cloudflare.py b/src/antibot/known_providers/cloudflare.py index c7f5401..e0d0119 100644 --- a/src/antibot/known_providers/cloudflare.py +++ b/src/antibot/known_providers/cloudflare.py @@ -72,8 +72,6 @@ from ..registry import register from ..humanize import wait_for_page_stability, wait_for_networkidle, click_human log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -log.addHandler(logging.StreamHandler()) class CloudflareDetector(Detector): @@ -185,9 +183,12 @@ __CF_PATTERN__ = re.compile(r"^https://challenges\.cloudflare\.com/cdn-cgi/chall class CloudflareSolver(Solver): + # опрос исчезновения маркера челленджа после клика / на interstitial + POLL_INTERVAL_MS = 250 + WAIT_CLEARED_S = 10 async def solve(self, page: Page, info: ChallengeInfo): - print("start solving") + log.debug("start solving") iframe = page.frame(url=__CF_PATTERN__) if iframe: await page.wait_for_timeout(randint(500, 1000)) @@ -212,11 +213,11 @@ class CloudflareSolver(Solver): finally: return else: - deadline = time.time() + 10 + deadline = time.time() + self.WAIT_CLEARED_S while time.time() < deadline: if not await page.query_selector(info.detect_locator): return - await page.wait_for_timeout(1) + await page.wait_for_timeout(self.POLL_INTERVAL_MS) return elif info.stage == Stage.INTERSTITIAL: if info.success_locator: @@ -227,11 +228,11 @@ class CloudflareSolver(Solver): else: return else: - deadline = time.time() + 10 + deadline = time.time() + self.WAIT_CLEARED_S while time.time() < deadline: if not await page.query_selector(info.detect_locator): return - await page.wait_for_timeout(1) + await page.wait_for_timeout(self.POLL_INTERVAL_MS) return else: @@ -257,7 +258,7 @@ class CloudflareSolver(Solver): @register -class ReCaptchaProvider(AntiBotProvider): +class CloudflareProvider(AntiBotProvider): provider = 'cloudflare' _solver_cls = CloudflareSolver _detector_cls = CloudflareDetector diff --git a/src/antibot/known_providers/recaptcha.py b/src/antibot/known_providers/recaptcha.py index d0967dd..2a03f31 100644 --- a/src/antibot/known_providers/recaptcha.py +++ b/src/antibot/known_providers/recaptcha.py @@ -11,8 +11,6 @@ from ..registry import register from ..exceptions import IframeLoadError log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -log.addHandler(logging.StreamHandler()) JS_CHALLENGE_SUBMIT = """(token) => { @@ -36,6 +34,8 @@ JS_CHALLENGE_SUBMIT = """(token) => { class ReCaptchaSolver(Solver): page: Page + # пауза на стадии INTERSTITIAL перед повторным detect + INTERSTITIAL_WAIT_MS = 15_000 def __init__(self, captcha_solver: Captcha, **kwargs) -> None: super().__init__(captcha_solver=captcha_solver, **kwargs) @@ -49,8 +49,7 @@ class ReCaptchaSolver(Solver): self.page = page log.debug("Начинаю решение recaptcha антибот преграды") if info.stage == Stage.INTERSTITIAL: - # TODO: вытащить - await page.wait_for_timeout(15) + await page.wait_for_timeout(self.INTERSTITIAL_WAIT_MS) return elif info.stage == Stage.CHECKBOX: log.debug("Нужно тыкнуть чекбокс") diff --git a/src/antibot/orchestrator.py b/src/antibot/orchestrator.py index 6007a94..d4353b8 100644 --- a/src/antibot/orchestrator.py +++ b/src/antibot/orchestrator.py @@ -3,9 +3,10 @@ """ import logging import time -import traceback from dataclasses import dataclass +from patchright.async_api import Error as PlaywrightError + from captcha import load_captcha from .base import Stage @@ -13,8 +14,8 @@ from .registry import select_one from .exceptions import ProviderNotDetected log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) -log.addHandler(logging.StreamHandler()) + +SOLVE_RETRY_DELAY_MS = 1000 @dataclass @@ -53,7 +54,14 @@ async def pass_challenges( info = None while time.monotonic() < deadline: - info = await provider.detect(page) # noqa + try: + info = await provider.detect(page) # noqa + except PlaywrightError as e: + # челлендж сам переходит на целевую страницу посреди detect() + # («Execution context was destroyed») — даём навигации закончиться и смотрим заново + log.debug("%s: detect interrupted (%s), retrying", provider.provider, e.message.splitlines()[0]) + await page.wait_for_load_state("load") + continue # 3. проверяем что решение успешно if info.success: # noqa return ChallengeOutcome(True, reason=(info.meta or {}).get("reason")) @@ -61,7 +69,10 @@ async def pass_challenges( try: await provider.solve(page, info) except Exception as e: - traceback.print_exc() + log.warning("%s: solve failed (%s), retrying", provider.provider, str(e).splitlines()[0] if str(e) else type(e).__name__) + log.debug("solve traceback", exc_info=True) + # пауза, иначе при стабильно падающем solve цикл крутится вхолостую до дедлайна + await page.wait_for_timeout(SOLVE_RETRY_DELAY_MS) continue else: return ChallengeOutcome( diff --git a/src/api/actions.py b/src/api/actions.py index f66c177..0b30b2c 100644 --- a/src/api/actions.py +++ b/src/api/actions.py @@ -1,8 +1,8 @@ import asyncio +import time from patchright.async_api import Page -from antibot.base import click_and_race from antibot.orchestrator import pass_challenges from captcha import load_captcha from api.schemas import ActionRequest @@ -10,50 +10,52 @@ from api.schemas import ActionRequest _actions: dict = {} +class ChallengeFailed(Exception): + pass + + def register(fun) -> None: _actions[fun.__name__] = fun return fun +async def _query(page: Page, selector: str): + element = await page.query_selector(selector) + if element is None: + raise LookupError(f"selector not found: {selector}") + return element + + @register async def fill(page: Page, act: ActionRequest, **kwargs) -> str | None: - assert act.selector - assert act.value await page.fill(act.selector, act.value) @register async def captcha(page: Page, act: ActionRequest, captcha_key: str, **kwargs) -> str | None: - assert act.selector - assert act.value - assert captcha_key captcha_solver = load_captcha(captcha_key, act.value) await captcha_solver.solve(act.selector, page) @register async def delay(page: Page, act: ActionRequest, **kwargs) -> str | None: - assert act.value await asyncio.sleep(int(act.value)) @register async def mouse_move(page: Page, act: ActionRequest, **kwargs) -> str | None: - assert act.selector - element = await page.query_selector(act.selector) + element = await _query(page, act.selector) await element.hover(scroll="auto") @register async def click(page: Page, act: ActionRequest, **kwargs) -> str | None: - assert act.selector element = page.locator(act.selector) await element.click() @register async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None: - assert act.selector element = page.locator(act.selector) async with page.expect_event( @@ -65,15 +67,12 @@ async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None: @register async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None: - assert act.selector - element = await page.query_selector(act.selector) + element = await _query(page, act.selector) return await element.inner_html() @register async def wait_for(page: Page, act: ActionRequest, **kwargs) -> str | None: - assert act.selector - assert 1 <= act.timeout <= 7200 await page.wait_for_selector(act.selector, timeout=act.timeout * 1000) @@ -99,7 +98,10 @@ async def execute( captcha_key: str | None = None, antibot_name: str | None = None, success_locator: str | None = None, detect_locator: str | None = None) -> list[dict]: - """Выполнение очереди автоматизации""" + """Выполнение очереди автоматизации. + + Если антибот не пройден — бросает ChallengeFailed, действия не выполняются: на странице-заглушке + они бы упали с невнятной ошибкой селектора.""" actions_result = [] ctx = { "captcha_key": captcha_key, @@ -107,27 +109,32 @@ async def execute( "success_locator": success_locator, "detect_locator": detect_locator, } - await pass_challenges( + outcome = await pass_challenges( antibot_name, page, captcha=captcha_key, success_locator=success_locator, detect_locator=detect_locator, ) + if not outcome.cleared: + stage = outcome.last_stage.value if outcome.last_stage else None + raise ChallengeFailed(f"{antibot_name} not passed: {outcome.reason} (stage: {stage})") + for act in actions: new_line = { "type": act.type, } + started = time.monotonic() try: - # TODO: добавить время выполнения result = await _actions[act.type](page, act, **ctx) except Exception as e: new_line["ok"] = False - new_line["result"] = str(e) - actions_result.append(new_line) - break + new_line["result"] = str(e) or type(e).__name__ else: new_line["ok"] = True if result: new_line["result"] = result - actions_result.append(new_line) - return actions_result if len(actions_result) else [{"status": "pass"}] + new_line["duration_ms"] = int((time.monotonic() - started) * 1000) + actions_result.append(new_line) + if not new_line["ok"]: + break + return actions_result diff --git a/src/api/schemas.py b/src/api/schemas.py index dc0d0f5..e5da3c6 100644 --- a/src/api/schemas.py +++ b/src/api/schemas.py @@ -1,25 +1,70 @@ -from typing import Any +from typing import Any, Literal +from urllib.parse import urlsplit -from pydantic import BaseModel +from pydantic import BaseModel, field_validator, model_validator + +CAPTCHA_TYPES = ("cloudflare", "recaptcha", "hcaptcha") +PROXY_SCHEMES = ("http", "https", "socks5") + +# тип действия -> обязательные поля (кроме type) +_REQUIRED_FIELDS: dict[str, tuple[str, ...]] = { + "fill": ("selector", "value"), + "captcha": ("selector", "value"), + "delay": ("value",), + "mouse_move": ("selector",), + "click": ("selector",), + "submit": ("selector",), + "inner_html": ("selector",), + "wait_for": ("selector",), + "pass_challenge": (), +} +WAIT_FOR_DEFAULT_TIMEOUT = 30 class ActionRequest(BaseModel): """ Действие автоматизации. Характеризуется типом и производится над селектором. Набор параметров зависит от типа, этот класс перечисляет все параметры. + Обязательные поля для каждого типа проверяются здесь же — неверное действие + отклоняется с 422 до запуска браузера. """ type: str selector: str | None = None value: str | None = None timeout: int | None = None + @model_validator(mode="after") + def check_fields(self): + if self.type not in _REQUIRED_FIELDS: + raise ValueError(f"unknown action type {self.type!r}, expected one of: {', '.join(_REQUIRED_FIELDS)}") + + missing = [f for f in _REQUIRED_FIELDS[self.type] if getattr(self, f) in (None, "")] + if missing: + raise ValueError(f"action {self.type!r} requires: {', '.join(missing)}") + + if self.type == "wait_for": + if self.timeout is None: + self.timeout = WAIT_FOR_DEFAULT_TIMEOUT + if not 1 <= self.timeout <= 7200: + raise ValueError("wait_for timeout must be between 1 and 7200 seconds") + elif self.type == "delay": + try: + delay = int(self.value) + except ValueError: + raise ValueError("delay value must be an integer number of seconds") from None + if not 0 <= delay <= 600: + raise ValueError("delay value must be between 0 and 600 seconds") + elif self.type == "captcha" and self.value not in CAPTCHA_TYPES: + raise ValueError(f"captcha value must be one of: {', '.join(CAPTCHA_TYPES)}") + return self + class SolveRequest(BaseModel): """ Объект запроса для автоматизации на сайте. """ url: str - proxy: str + proxy: str | None = None timeout: int | None = None user_agent: str | None = None cookies: list[dict[str, Any]] | None = None @@ -28,7 +73,7 @@ class SolveRequest(BaseModel): actions: list[ActionRequest] html: bool = False - # TODO: screenshot: bool = False + screenshot: bool = False captcha: str | None = None captcha_detect_locator: str | None = None @@ -43,3 +88,48 @@ class SolveRequest(BaseModel): retry_delay: float = 1.0 wait: int = 0 google_search: bool = False + + @field_validator("proxy") + @classmethod + def check_proxy(cls, proxy: str | None) -> str | None: + if not proxy: + return None + try: + parts = urlsplit(proxy) + port = parts.port + except ValueError as e: + raise ValueError(f"invalid proxy url: {e}") from None + if parts.scheme not in PROXY_SCHEMES: + raise ValueError(f"proxy scheme must be one of: {', '.join(PROXY_SCHEMES)}") + if not parts.hostname or not port: + raise ValueError("proxy must be in form scheme://[user:pass@]host:port") + return proxy + + @model_validator(mode="after") + def check_captcha_key(self): + if not self.captcha and any(a.type == "captcha" for a in self.actions): + raise ValueError("action 'captcha' requires request field 'captcha' (2captcha key)") + return self + + +class ActionResult(BaseModel): + type: str + ok: bool + result: str | None = None + duration_ms: int + + +class SolveResponse(BaseModel): + """ + Ответ /v1/solve. status — "ok" (антибот пройден, все действия выполнены) или "err". + code — машиночитаемая причина ошибки, error — текст для человека. + """ + status: Literal["ok", "err"] + code: Literal["no_slots", "request_failed", "challenge_failed", "action_failed"] | None = None + error: str | None = None + user_agent: str | None = None + cookies: list[dict[str, Any]] | None = None + actions: list[ActionResult] = [] + html: str | None = None + screenshot: str | None = None + duration_ms: int | None = None diff --git a/src/captcha/base.py b/src/captcha/base.py index 52c5683..3b19ed0 100644 --- a/src/captcha/base.py +++ b/src/captcha/base.py @@ -1,48 +1,9 @@ import asyncio -import json -import hashlib -import time -from pathlib import Path from patchright.async_api import Page from twocaptcha import TwoCaptcha -def cached_persistent(file_name=None): - - def key(*args, **kwargs): - return hashlib.sha256(f'{str(args)}-{str(kwargs)}'.encode()).hexdigest() - - def wrapper(fun): - - def clear_cache(): - cache_file.write_text(json.dumps({})) - - cache_file = Path(file_name) - if not cache_file.exists(): - cache_file.write_text(json.dumps({})) - - async def wrapped(*args, **kwargs): - data = json.loads(cache_file.read_text()) - _key = key(fun.__name__, *args, **kwargs) - - if cached_value := data.get(_key): - cached_value, cache_time = cached_value - if time.time() - cache_time > 300: - cached_value = None - - value = cached_value or await fun(*args, **kwargs) - - data[_key] = [value, time.time()] - cache_file.write_text(json.dumps(data)) - return value - - setattr(wrapped, '_clear_cache', clear_cache) - return wrapped - - return wrapper - - class Captcha: ctype = "captcha" diff --git a/src/captcha/hcaptcha.py b/src/captcha/hcaptcha.py index 9d3f933..aacb5ba 100644 --- a/src/captcha/hcaptcha.py +++ b/src/captcha/hcaptcha.py @@ -16,8 +16,6 @@ import logging from .base import Captcha, Page logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -logger.addHandler(logging.StreamHandler()) class SolverHCaptcha(Captcha): diff --git a/src/captcha/recaptcha.py b/src/captcha/recaptcha.py index 993118b..269992f 100644 --- a/src/captcha/recaptcha.py +++ b/src/captcha/recaptcha.py @@ -7,8 +7,6 @@ from urllib.parse import urlparse, parse_qs from .base import Captcha, Page logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -logger.addHandler(logging.StreamHandler()) class SolverReCaptcha(Captcha): diff --git a/src/captcha/turnstile.py b/src/captcha/turnstile.py index e087c26..cb50a77 100644 --- a/src/captcha/turnstile.py +++ b/src/captcha/turnstile.py @@ -16,8 +16,6 @@ import logging from .base import Captcha, Page logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -logger.addHandler(logging.StreamHandler()) class SolverTurnstile(Captcha): diff --git a/src/main.py b/src/main.py index 42dfebb..b953eea 100644 --- a/src/main.py +++ b/src/main.py @@ -1,6 +1,9 @@ +import asyncio +import base64 import json import logging -import traceback +import os +import time import uuid from pathlib import Path from typing import Annotated @@ -8,19 +11,29 @@ from urllib.parse import urlsplit from fastapi import Cookie from fastapi import FastAPI -from fastapi import Request from fastapi import Response +from fastapi.responses import JSONResponse from patchright.async_api import Page, ProxySettings -from api import actions -from api.geoip import resolve_geo -from api.schemas import SolveRequest -from engine import AsyncStealthySession +logging.basicConfig( + level=os.getenv("LOG_LEVEL", "INFO").upper(), + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) + +from api import actions # noqa: E402 — модули ниже логируют при импорте, конфиг логов должен быть раньше +from api.geoip import resolve_geo # noqa: E402 +from api.schemas import SolveRequest, SolveResponse # noqa: E402 +from engine import AsyncStealthySession # noqa: E402 logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) -logger.addHandler(logging.StreamHandler()) -BROWSER_SLOTS = 10 + +# сколько Chrome запускается параллельно; подбирать ≈ числу ядер (см. docs/Deployment.md) +BROWSER_SLOTS = int(os.getenv("BROWSER_SLOTS", "10")) +RETRY_AFTER = os.getenv("NO_SLOTS_RETRY_AFTER", "5") +_slots = asyncio.Semaphore(BROWSER_SLOTS) + +USER_DATA_ROOT = (Path(__file__).parent / "../extra/user_data_dir").resolve() +SESSION_SETTINGS = "robotex_session.json" def parse_proxy(proxy_url: str) -> ProxySettings: @@ -34,23 +47,97 @@ def parse_proxy(proxy_url: str) -> ProxySettings: return settings +def mask_proxy(proxy_url: str | None) -> str | None: + if not proxy_url: + return proxy_url + parts = urlsplit(proxy_url) + return f"{parts.scheme}://{'***@' if parts.username else ''}{parts.hostname}:{parts.port}" + + +def safe_request_dump(payload: SolveRequest) -> str: + """Запрос для лога без секретов клиента: креды прокси, ключ 2captcha, вводимые значения, cookies.""" + data = payload.model_dump(exclude_none=True) + data["proxy"] = mask_proxy(payload.proxy) + if payload.captcha: + data["captcha"] = "***" + if payload.cookies: + data["cookies"] = [c.get("name") for c in payload.cookies] + for act in data["actions"]: + if act["type"] == "fill" and "value" in act: + act["value"] = "***" + return json.dumps(data, indent=2, ensure_ascii=False) + + +def safe_response_dump(result: dict) -> str: + """Ответ для лога: без значений cookies, html и скриншота.""" + data = dict(result) + if data.get("cookies"): + data["cookies"] = [c.get("name") for c in data["cookies"]] + for key in ("html", "screenshot"): + if data.get(key): + data[key] = f"<{len(data[key])} chars>" + return json.dumps(data, indent=2, ensure_ascii=False) + + +def session_dir(session_id: str | None) -> tuple[str, Path]: + """Каталог профиля по session_id из cookie. Принимается только UUID — иначе (в т.ч. попытка + подсунуть путь вида ../..) выдаётся новая сессия.""" + try: + session_id = str(uuid.UUID(session_id)) if session_id else None + except ValueError: + logger.warning("invalid session_id cookie, issuing a new session") + session_id = None + session_id = session_id or str(uuid.uuid4()) + + user_data_dir = (USER_DATA_ROOT / session_id).resolve() + if user_data_dir.parent != USER_DATA_ROOT: + raise ValueError("invalid session path") + return session_id, user_data_dir + + +def load_session_settings(user_data_dir: Path) -> dict: + try: + return json.loads((user_data_dir / SESSION_SETTINGS).read_text()) + except (OSError, json.JSONDecodeError): + return {} + + +def save_session_settings(user_data_dir: Path, settings: dict) -> None: + (user_data_dir / SESSION_SETTINGS).write_text(json.dumps(settings, ensure_ascii=False)) + + def make_page_action(payload: SolveRequest): """Замыкание для AsyncStealthySession.fetch(page_action=...): выполняет очередь действий (включая прохождение капчи/антибота через api.actions.execute) и достаёт из уже навигированной страницы всё, что нужно для ответа API. Возврат этой функции — то, что - вернёт fetch() (см. engine/stealthy.py).""" + вернёт fetch() (см. engine/stealthy.py). + + Непройденный антибот не бросается наружу, а возвращается в challenge_error — чтобы в ответ + всё равно попали cookies, html и скриншот страницы-заглушки.""" async def page_action(page: Page) -> dict: - actions_result = await actions.execute( - page, payload.actions, payload.captcha, - antibot_name=payload.captcha_type, - success_locator=payload.captcha_success_locator, - detect_locator=payload.captcha_detect_locator, - ) + challenge_error = None + try: + actions_result = await actions.execute( + page, payload.actions, payload.captcha, + antibot_name=payload.captcha_type, + success_locator=payload.captcha_success_locator, + detect_locator=payload.captcha_detect_locator, + ) + except actions.ChallengeFailed as e: + challenge_error = str(e) + actions_result = [] + + screenshot = None + if payload.screenshot: + screenshot = base64.b64encode(await page.screenshot(type="png")).decode() + return { + "challenge_error": challenge_error, "actions": actions_result, "cookies": await page.context.cookies(), "html": await page.inner_html("html"), + "screenshot": screenshot, "user_agent": await page.evaluate("() => navigator.userAgent"), } @@ -61,100 +148,105 @@ app = FastAPI() @app.get("/") -async def root(): - return {"message": "Hello World"} +@app.get("/health") +async def health(): + return {"status": "ok", "browser_slots": BROWSER_SLOTS, "free_slots": _slots._value} # noqa -@app.post("/v1/solve") +@app.post("/v1/solve", response_model=SolveResponse, response_model_exclude_none=True) async def solve( - request: Request, payload: SolveRequest, response: Response, session_id: Annotated[str | None, Cookie()] = None ): - # TODO: добавить поддержку api ключей - api_key = request.headers.get('X-API-Key') + # TODO: добавить поддержку api ключей (#4) + started = time.monotonic() - logger.info("Incoming payload: [%s]\n%s", session_id, payload.model_dump_json(indent=2, ensure_ascii=True)) - - global BROWSER_SLOTS - - try: - assert BROWSER_SLOTS > 0 and (BROWSER_SLOTS := BROWSER_SLOTS.__sub__(1)) >= 0, "No slots available" - - proxy = parse_proxy(payload.proxy) - - if not session_id: - session_id = str(uuid.uuid4()) - response.set_cookie("session_id", session_id) - logger.info("Setup new session with session_id: %s", session_id) - - # профиль (cookies/local storage/fingerprint.json) хранится на диске по - # session_id — если он уже приходил в куках, ниже подгружаем существующий - # fingerprint из этой папки; если нет (новый session_id) — генерируем новый - # и сохраняем туда же (см. api/fingerprint.py:load_or_create_fingerprint) - user_data_dir = Path(__file__).parent / f"../extra/user_data_dir/{session_id}" - if not user_data_dir.exists(): - user_data_dir.mkdir(parents=True) - - # geo — до создания fingerprint'а, чтобы производная от прокси locale - # (см. api/geoip.py) попала в сам fingerprint, а не подменяла его - # задним числом - geo = await resolve_geo(proxy) - locale = geo.get("locale") if geo else None - - fp_kwargs = {} - if payload.user_agent: - fp_kwargs["user_agent"] = payload.user_agent - # TODO: если есть session_id, то user_agent нужно взять из настроек пользователя - if locale: - fp_kwargs["locale"] = locale - - nav_timeout_ms = payload.timeout * 1000 if payload.timeout else 60_000 - blocked_domains = set(payload.blocked_domains) if payload.blocked_domains else None - - kwargs = dict( - headless=False, - proxy=proxy, - user_data_dir=str(user_data_dir), - real_chrome=True, - locale=locale, - timezone_id=geo.get("timezone_id") if geo else None, - cookies=payload.cookies, - disable_resources=payload.disable_resources, - blocked_domains=blocked_domains, - timeout=nav_timeout_ms, - retries=max(1, payload.retries), - retry_delay=payload.retry_delay, - wait=payload.wait, - google_search=payload.google_search + if _slots.locked(): + logger.warning("no free browser slots (%d), rejecting %s", BROWSER_SLOTS, payload.url) + return JSONResponse( + status_code=503, + headers={"Retry-After": RETRY_AFTER}, + content={"status": "err", "code": "no_slots", "error": "No slots available"}, ) - async with AsyncStealthySession(**kwargs) as engine: - result = await engine.fetch(payload.url, page_action=make_page_action(payload)) - except Exception as e: - traceback.print_exc() - result_payload = { - "error": str(e) or type(e).__name__, - "status": False - } - else: - failed = [a for a in result["actions"] if a.get("ok") is False] - status = not failed - error = None if status else f"{failed[0]['type']}: {failed[0].get('result')}" + async with _slots: + session_id, user_data_dir = session_dir(session_id) + response.set_cookie("session_id", session_id) + logger.info("Incoming request [%s]: %s", session_id, payload.url) + logger.debug("Incoming payload [%s]:\n%s", session_id, safe_request_dump(payload)) - result_payload = { - "status": ["err", "ok"][status], - "user_agent": result["user_agent"], - "cookies": result["cookies"], - "actions": result["actions"], - } - if payload.html: - result_payload["html"] = result["html"] - if error: - result_payload["error"] = error - finally: - BROWSER_SLOTS += 1 + try: + user_data_dir.mkdir(parents=True, exist_ok=True) - logger.info("Outgoing payload:\n%s", json.dumps(result_payload, indent=2, sort_keys=True, ensure_ascii=True)) + # user_agent сессии сохраняется рядом с профилем: при повторном запросе с тем же + # session_id без user_agent отпечаток не должен меняться + settings = load_session_settings(user_data_dir) + if payload.user_agent: + settings["user_agent"] = payload.user_agent + save_session_settings(user_data_dir, settings) + user_agent = settings.get("user_agent") + + proxy = parse_proxy(payload.proxy) if payload.proxy else None + + # geo — по exit-IP прокси (см. api/geoip.py); без прокси оставляем настройки браузера + geo = await resolve_geo(proxy) if proxy else None + locale = geo.get("locale") if geo else None + + nav_timeout_ms = payload.timeout * 1000 if payload.timeout else 60_000 + blocked_domains = set(payload.blocked_domains) if payload.blocked_domains else None + + kwargs = dict( + headless=False, + proxy=proxy, + user_data_dir=str(user_data_dir), + real_chrome=True, + useragent=user_agent, + locale=locale, + timezone_id=geo.get("timezone_id") if geo else None, + cookies=payload.cookies, + disable_resources=payload.disable_resources, + blocked_domains=blocked_domains, + timeout=nav_timeout_ms, + retries=max(1, payload.retries), + retry_delay=payload.retry_delay, + wait=payload.wait, + google_search=payload.google_search + ) + + async with AsyncStealthySession(**kwargs) as engine: + result = await engine.fetch(payload.url, page_action=make_page_action(payload)) + except Exception as e: + logger.exception("request failed [%s]", session_id) + result_payload = { + "status": "err", + "code": "request_failed", + "error": str(e) or type(e).__name__, + } + else: + failed = [a for a in result["actions"] if a.get("ok") is False] + result_payload = { + "status": "ok", + "user_agent": result["user_agent"], + "cookies": result["cookies"], + "actions": result["actions"], + } + if result["challenge_error"]: + result_payload.update(status="err", code="challenge_failed", error=result["challenge_error"]) + elif failed: + result_payload.update( + status="err", code="action_failed", + error=f"{failed[0]['type']}: {failed[0].get('result')}", + ) + if payload.html: + result_payload["html"] = result["html"] + if payload.screenshot: + result_payload["screenshot"] = result["screenshot"] + + result_payload["duration_ms"] = int((time.monotonic() - started) * 1000) + logger.info( + "Outgoing [%s]: %s %s %dms", session_id, result_payload["status"], + result_payload.get("error", ""), result_payload["duration_ms"], + ) + logger.debug("Outgoing payload:\n%s", safe_response_dump(result_payload)) return result_payload diff --git a/tests/loadtest.py b/tests/loadtest.py index 90fe47e..53e305b 100644 --- a/tests/loadtest.py +++ b/tests/loadtest.py @@ -35,15 +35,16 @@ def classify(resp: httpx.Response | Exception) -> str: if isinstance(resp, Exception): return f"exc:{type(resp).__name__}" data = resp.json() - if "No slots" in str(data.get("error", "")): + code = data.get("code") + if code == "no_slots": return "no_slots" - actions = data.get("actions") or [] - if not actions: - return f"error:{str(data.get('error'))[:60]}" - last = actions[-1] - if not last.get("ok") or last.get("result"): + if code == "challenge_failed": + return "cf_blocked" + if code == "action_failed": html = data.get("html") or "" return "cf_blocked" if any(m in html for m in CF_MARKERS) else "action_failed" + if data.get("status") != "ok": + return f"error:{str(data.get('error'))[:60]}" return "ok"