fix: слоты, безопасность, единый формат ответа и валидация

- слоты браузеров: asyncio.Semaphore, размер из BROWSER_SLOTS, при нехватке — 503 + Retry-After;
  устранена утечка счётчика при отказе (#12, #49)
- session_id принимается только как UUID, путь профиля проверяется (#5)
- секреты клиента в логах маскируются, полный дамп — только на DEBUG (#6)
- ответ: status всегда "ok"/"err", поле code (no_slots / request_failed / challenge_failed /
  action_failed), response_model (#7)
- непройденный антибот → challenge_failed, действия не выполняются (#8)
- таймауты провайдеров в мс вынесены в константы; detect переживает навигацию посреди
  проверки; пауза между неудачными solve вместо холостого цикла (#9)
- обязательные поля действий и формат прокси проверяются схемой → 422 до запуска браузера (#13)
- прокси необязателен (#15)
- user_agent сохраняется в профиле сессии и реально передаётся в браузер (#16)
- duration_ms у действий и у запроса (#18), опция screenshot (#19)
- общий конфиг логирования (LOG_LEVEL), print → логгер, CloudflareProvider,
  удалён cached_persistent, GET /health (#22)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
protokeyandClaude Opus 5.5 committed 2026-10-13 14:52:12 +04:00
1 parent f58306cbb4
commit 437ea27be7
14 files changed
+360 -201

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
API_PORT= API_PORT=
API_CPUS=1 API_CPUS=1
API_MEM_LIMIT=1g API_MEM_LIMIT=1g
API_MEM_LIMIT=1g BROWSER_SLOTS=1
+8
View File
@@ -1 +1,9 @@
# Project Env # Project Env
# порт, CPU и память контейнера (docker-compose)
API_PORT=8000
API_CPUS=4
API_MEM_LIMIT=8g
# сколько Chrome параллельно, ≈ числу ядер (docs/Deployment.md)
BROWSER_SLOTS=4
NO_SLOTS_RETRY_AFTER=5
LOG_LEVEL=INFO
-5
View File
@@ -85,7 +85,6 @@ async def move_mouse_human(
if i < steps: if i < steps:
px += random.uniform(-1.5, 1.5) px += random.uniform(-1.5, 1.5)
py += random.uniform(-1.5, 1.5) py += random.uniform(-1.5, 1.5)
print("Move:", px, py)
await page.mouse.move(px, py) await page.mouse.move(px, py)
await page.wait_for_timeout(step_delay_ms * random.uniform(0.7, 1.3)) await page.wait_for_timeout(step_delay_ms * random.uniform(0.7, 1.3))
@@ -103,13 +102,9 @@ async def click_human(
"""move_mouse_human + клик с человеческой задержкой между наведением и down/up, вместо """move_mouse_human + клик с человеческой задержкой между наведением и down/up, вместо
мгновенного page.mouse.click(x, y).""" мгновенного page.mouse.click(x, y)."""
await move_mouse_human(page, x, y, **move_kwargs) await move_mouse_human(page, x, y, **move_kwargs)
print('wait for timeout')
await page.wait_for_timeout(random.uniform(50, 180)) await page.wait_for_timeout(random.uniform(50, 180))
print('key down')
await page.mouse.down(button=button) await page.mouse.down(button=button)
print('sleep')
await page.wait_for_timeout(random.uniform(40, 99)) await page.wait_for_timeout(random.uniform(40, 99))
print('key up')
await page.mouse.up(button=button) await page.mouse.up(button=button)
+9 -8
View File
@@ -72,8 +72,6 @@ from ..registry import register
from ..humanize import wait_for_page_stability, wait_for_networkidle, click_human from ..humanize import wait_for_page_stability, wait_for_networkidle, click_human
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler())
class CloudflareDetector(Detector): class CloudflareDetector(Detector):
@@ -185,9 +183,12 @@ __CF_PATTERN__ = re.compile(r"^https://challenges\.cloudflare\.com/cdn-cgi/chall
class CloudflareSolver(Solver): class CloudflareSolver(Solver):
# опрос исчезновения маркера челленджа после клика / на interstitial
POLL_INTERVAL_MS = 250
WAIT_CLEARED_S = 10
async def solve(self, page: Page, info: ChallengeInfo): async def solve(self, page: Page, info: ChallengeInfo):
print("start solving") log.debug("start solving")
iframe = page.frame(url=__CF_PATTERN__) iframe = page.frame(url=__CF_PATTERN__)
if iframe: if iframe:
await page.wait_for_timeout(randint(500, 1000)) await page.wait_for_timeout(randint(500, 1000))
@@ -212,11 +213,11 @@ class CloudflareSolver(Solver):
finally: finally:
return return
else: else:
deadline = time.time() + 10 deadline = time.time() + self.WAIT_CLEARED_S
while time.time() < deadline: while time.time() < deadline:
if not await page.query_selector(info.detect_locator): if not await page.query_selector(info.detect_locator):
return return
await page.wait_for_timeout(1) await page.wait_for_timeout(self.POLL_INTERVAL_MS)
return return
elif info.stage == Stage.INTERSTITIAL: elif info.stage == Stage.INTERSTITIAL:
if info.success_locator: if info.success_locator:
@@ -227,11 +228,11 @@ class CloudflareSolver(Solver):
else: else:
return return
else: else:
deadline = time.time() + 10 deadline = time.time() + self.WAIT_CLEARED_S
while time.time() < deadline: while time.time() < deadline:
if not await page.query_selector(info.detect_locator): if not await page.query_selector(info.detect_locator):
return return
await page.wait_for_timeout(1) await page.wait_for_timeout(self.POLL_INTERVAL_MS)
return return
else: else:
@@ -257,7 +258,7 @@ class CloudflareSolver(Solver):
@register @register
class ReCaptchaProvider(AntiBotProvider): class CloudflareProvider(AntiBotProvider):
provider = 'cloudflare' provider = 'cloudflare'
_solver_cls = CloudflareSolver _solver_cls = CloudflareSolver
_detector_cls = CloudflareDetector _detector_cls = CloudflareDetector
+3 -4
View File
@@ -11,8 +11,6 @@ from ..registry import register
from ..exceptions import IframeLoadError from ..exceptions import IframeLoadError
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler())
JS_CHALLENGE_SUBMIT = """(token) => { JS_CHALLENGE_SUBMIT = """(token) => {
@@ -36,6 +34,8 @@ JS_CHALLENGE_SUBMIT = """(token) => {
class ReCaptchaSolver(Solver): class ReCaptchaSolver(Solver):
page: Page page: Page
# пауза на стадии INTERSTITIAL перед повторным detect
INTERSTITIAL_WAIT_MS = 15_000
def __init__(self, captcha_solver: Captcha, **kwargs) -> None: def __init__(self, captcha_solver: Captcha, **kwargs) -> None:
super().__init__(captcha_solver=captcha_solver, **kwargs) super().__init__(captcha_solver=captcha_solver, **kwargs)
@@ -49,8 +49,7 @@ class ReCaptchaSolver(Solver):
self.page = page self.page = page
log.debug("Начинаю решение recaptcha антибот преграды") log.debug("Начинаю решение recaptcha антибот преграды")
if info.stage == Stage.INTERSTITIAL: if info.stage == Stage.INTERSTITIAL:
# TODO: вытащить await page.wait_for_timeout(self.INTERSTITIAL_WAIT_MS)
await page.wait_for_timeout(15)
return return
elif info.stage == Stage.CHECKBOX: elif info.stage == Stage.CHECKBOX:
log.debug("Нужно тыкнуть чекбокс") log.debug("Нужно тыкнуть чекбокс")
+16 -5
View File
@@ -3,9 +3,10 @@
""" """
import logging import logging
import time import time
import traceback
from dataclasses import dataclass from dataclasses import dataclass
from patchright.async_api import Error as PlaywrightError
from captcha import load_captcha from captcha import load_captcha
from .base import Stage from .base import Stage
@@ -13,8 +14,8 @@ from .registry import select_one
from .exceptions import ProviderNotDetected from .exceptions import ProviderNotDetected
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
log.addHandler(logging.StreamHandler()) SOLVE_RETRY_DELAY_MS = 1000
@dataclass @dataclass
@@ -53,7 +54,14 @@ async def pass_challenges(
info = None info = None
while time.monotonic() < deadline: while time.monotonic() < deadline:
info = await provider.detect(page) # noqa try:
info = await provider.detect(page) # noqa
except PlaywrightError as e:
# челлендж сам переходит на целевую страницу посреди detect()
# («Execution context was destroyed») — даём навигации закончиться и смотрим заново
log.debug("%s: detect interrupted (%s), retrying", provider.provider, e.message.splitlines()[0])
await page.wait_for_load_state("load")
continue
# 3. проверяем что решение успешно # 3. проверяем что решение успешно
if info.success: # noqa if info.success: # noqa
return ChallengeOutcome(True, reason=(info.meta or {}).get("reason")) return ChallengeOutcome(True, reason=(info.meta or {}).get("reason"))
@@ -61,7 +69,10 @@ async def pass_challenges(
try: try:
await provider.solve(page, info) await provider.solve(page, info)
except Exception as e: except Exception as e:
traceback.print_exc() log.warning("%s: solve failed (%s), retrying", provider.provider, str(e).splitlines()[0] if str(e) else type(e).__name__)
log.debug("solve traceback", exc_info=True)
# пауза, иначе при стабильно падающем solve цикл крутится вхолостую до дедлайна
await page.wait_for_timeout(SOLVE_RETRY_DELAY_MS)
continue continue
else: else:
return ChallengeOutcome( return ChallengeOutcome(
+30 -23
View File
@@ -1,8 +1,8 @@
import asyncio import asyncio
import time
from patchright.async_api import Page from patchright.async_api import Page
from antibot.base import click_and_race
from antibot.orchestrator import pass_challenges from antibot.orchestrator import pass_challenges
from captcha import load_captcha from captcha import load_captcha
from api.schemas import ActionRequest from api.schemas import ActionRequest
@@ -10,50 +10,52 @@ from api.schemas import ActionRequest
_actions: dict = {} _actions: dict = {}
class ChallengeFailed(Exception):
pass
def register(fun) -> None: def register(fun) -> None:
_actions[fun.__name__] = fun _actions[fun.__name__] = fun
return fun return fun
async def _query(page: Page, selector: str):
element = await page.query_selector(selector)
if element is None:
raise LookupError(f"selector not found: {selector}")
return element
@register @register
async def fill(page: Page, act: ActionRequest, **kwargs) -> str | None: async def fill(page: Page, act: ActionRequest, **kwargs) -> str | None:
assert act.selector
assert act.value
await page.fill(act.selector, act.value) await page.fill(act.selector, act.value)
@register @register
async def captcha(page: Page, act: ActionRequest, captcha_key: str, **kwargs) -> str | None: async def captcha(page: Page, act: ActionRequest, captcha_key: str, **kwargs) -> str | None:
assert act.selector
assert act.value
assert captcha_key
captcha_solver = load_captcha(captcha_key, act.value) captcha_solver = load_captcha(captcha_key, act.value)
await captcha_solver.solve(act.selector, page) await captcha_solver.solve(act.selector, page)
@register @register
async def delay(page: Page, act: ActionRequest, **kwargs) -> str | None: async def delay(page: Page, act: ActionRequest, **kwargs) -> str | None:
assert act.value
await asyncio.sleep(int(act.value)) await asyncio.sleep(int(act.value))
@register @register
async def mouse_move(page: Page, act: ActionRequest, **kwargs) -> str | None: async def mouse_move(page: Page, act: ActionRequest, **kwargs) -> str | None:
assert act.selector element = await _query(page, act.selector)
element = await page.query_selector(act.selector)
await element.hover(scroll="auto") await element.hover(scroll="auto")
@register @register
async def click(page: Page, act: ActionRequest, **kwargs) -> str | None: async def click(page: Page, act: ActionRequest, **kwargs) -> str | None:
assert act.selector
element = page.locator(act.selector) element = page.locator(act.selector)
await element.click() await element.click()
@register @register
async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None: async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None:
assert act.selector
element = page.locator(act.selector) element = page.locator(act.selector)
async with page.expect_event( async with page.expect_event(
@@ -65,15 +67,12 @@ async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None:
@register @register
async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None: async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None:
assert act.selector element = await _query(page, act.selector)
element = await page.query_selector(act.selector)
return await element.inner_html() return await element.inner_html()
@register @register
async def wait_for(page: Page, act: ActionRequest, **kwargs) -> str | None: async def wait_for(page: Page, act: ActionRequest, **kwargs) -> str | None:
assert act.selector
assert 1 <= act.timeout <= 7200
await page.wait_for_selector(act.selector, timeout=act.timeout * 1000) await page.wait_for_selector(act.selector, timeout=act.timeout * 1000)
@@ -99,7 +98,10 @@ async def execute(
captcha_key: str | None = None, antibot_name: str | None = None, captcha_key: str | None = None, antibot_name: str | None = None,
success_locator: str | None = None, success_locator: str | None = None,
detect_locator: str | None = None) -> list[dict]: detect_locator: str | None = None) -> list[dict]:
"""Выполнение очереди автоматизации""" """Выполнение очереди автоматизации.
Если антибот не пройден — бросает ChallengeFailed, действия не выполняются: на странице-заглушке
они бы упали с невнятной ошибкой селектора."""
actions_result = [] actions_result = []
ctx = { ctx = {
"captcha_key": captcha_key, "captcha_key": captcha_key,
@@ -107,27 +109,32 @@ async def execute(
"success_locator": success_locator, "success_locator": success_locator,
"detect_locator": detect_locator, "detect_locator": detect_locator,
} }
await pass_challenges( outcome = await pass_challenges(
antibot_name, page, antibot_name, page,
captcha=captcha_key, captcha=captcha_key,
success_locator=success_locator, success_locator=success_locator,
detect_locator=detect_locator, detect_locator=detect_locator,
) )
if not outcome.cleared:
stage = outcome.last_stage.value if outcome.last_stage else None
raise ChallengeFailed(f"{antibot_name} not passed: {outcome.reason} (stage: {stage})")
for act in actions: for act in actions:
new_line = { new_line = {
"type": act.type, "type": act.type,
} }
started = time.monotonic()
try: try:
# TODO: добавить время выполнения
result = await _actions[act.type](page, act, **ctx) result = await _actions[act.type](page, act, **ctx)
except Exception as e: except Exception as e:
new_line["ok"] = False new_line["ok"] = False
new_line["result"] = str(e) new_line["result"] = str(e) or type(e).__name__
actions_result.append(new_line)
break
else: else:
new_line["ok"] = True new_line["ok"] = True
if result: if result:
new_line["result"] = result new_line["result"] = result
actions_result.append(new_line) new_line["duration_ms"] = int((time.monotonic() - started) * 1000)
return actions_result if len(actions_result) else [{"status": "pass"}] actions_result.append(new_line)
if not new_line["ok"]:
break
return actions_result
+94 -4
View File
@@ -1,25 +1,70 @@
from typing import Any from typing import Any, Literal
from urllib.parse import urlsplit
from pydantic import BaseModel from pydantic import BaseModel, field_validator, model_validator
CAPTCHA_TYPES = ("cloudflare", "recaptcha", "hcaptcha")
PROXY_SCHEMES = ("http", "https", "socks5")
# тип действия -> обязательные поля (кроме type)
_REQUIRED_FIELDS: dict[str, tuple[str, ...]] = {
"fill": ("selector", "value"),
"captcha": ("selector", "value"),
"delay": ("value",),
"mouse_move": ("selector",),
"click": ("selector",),
"submit": ("selector",),
"inner_html": ("selector",),
"wait_for": ("selector",),
"pass_challenge": (),
}
WAIT_FOR_DEFAULT_TIMEOUT = 30
class ActionRequest(BaseModel): class ActionRequest(BaseModel):
""" """
Действие автоматизации. Характеризуется типом и производится над селектором. Действие автоматизации. Характеризуется типом и производится над селектором.
Набор параметров зависит от типа, этот класс перечисляет все параметры. Набор параметров зависит от типа, этот класс перечисляет все параметры.
Обязательные поля для каждого типа проверяются здесь же — неверное действие
отклоняется с 422 до запуска браузера.
""" """
type: str type: str
selector: str | None = None selector: str | None = None
value: str | None = None value: str | None = None
timeout: int | None = None timeout: int | None = None
@model_validator(mode="after")
def check_fields(self):
if self.type not in _REQUIRED_FIELDS:
raise ValueError(f"unknown action type {self.type!r}, expected one of: {', '.join(_REQUIRED_FIELDS)}")
missing = [f for f in _REQUIRED_FIELDS[self.type] if getattr(self, f) in (None, "")]
if missing:
raise ValueError(f"action {self.type!r} requires: {', '.join(missing)}")
if self.type == "wait_for":
if self.timeout is None:
self.timeout = WAIT_FOR_DEFAULT_TIMEOUT
if not 1 <= self.timeout <= 7200:
raise ValueError("wait_for timeout must be between 1 and 7200 seconds")
elif self.type == "delay":
try:
delay = int(self.value)
except ValueError:
raise ValueError("delay value must be an integer number of seconds") from None
if not 0 <= delay <= 600:
raise ValueError("delay value must be between 0 and 600 seconds")
elif self.type == "captcha" and self.value not in CAPTCHA_TYPES:
raise ValueError(f"captcha value must be one of: {', '.join(CAPTCHA_TYPES)}")
return self
class SolveRequest(BaseModel): class SolveRequest(BaseModel):
""" """
Объект запроса для автоматизации на сайте. Объект запроса для автоматизации на сайте.
""" """
url: str url: str
proxy: str proxy: str | None = None
timeout: int | None = None timeout: int | None = None
user_agent: str | None = None user_agent: str | None = None
cookies: list[dict[str, Any]] | None = None cookies: list[dict[str, Any]] | None = None
@@ -28,7 +73,7 @@ class SolveRequest(BaseModel):
actions: list[ActionRequest] actions: list[ActionRequest]
html: bool = False html: bool = False
# TODO: screenshot: bool = False screenshot: bool = False
captcha: str | None = None captcha: str | None = None
captcha_detect_locator: str | None = None captcha_detect_locator: str | None = None
@@ -43,3 +88,48 @@ class SolveRequest(BaseModel):
retry_delay: float = 1.0 retry_delay: float = 1.0
wait: int = 0 wait: int = 0
google_search: bool = False google_search: bool = False
@field_validator("proxy")
@classmethod
def check_proxy(cls, proxy: str | None) -> str | None:
if not proxy:
return None
try:
parts = urlsplit(proxy)
port = parts.port
except ValueError as e:
raise ValueError(f"invalid proxy url: {e}") from None
if parts.scheme not in PROXY_SCHEMES:
raise ValueError(f"proxy scheme must be one of: {', '.join(PROXY_SCHEMES)}")
if not parts.hostname or not port:
raise ValueError("proxy must be in form scheme://[user:pass@]host:port")
return proxy
@model_validator(mode="after")
def check_captcha_key(self):
if not self.captcha and any(a.type == "captcha" for a in self.actions):
raise ValueError("action 'captcha' requires request field 'captcha' (2captcha key)")
return self
class ActionResult(BaseModel):
type: str
ok: bool
result: str | None = None
duration_ms: int
class SolveResponse(BaseModel):
"""
Ответ /v1/solve. status — "ok" (антибот пройден, все действия выполнены) или "err".
code — машиночитаемая причина ошибки, error — текст для человека.
"""
status: Literal["ok", "err"]
code: Literal["no_slots", "request_failed", "challenge_failed", "action_failed"] | None = None
error: str | None = None
user_agent: str | None = None
cookies: list[dict[str, Any]] | None = None
actions: list[ActionResult] = []
html: str | None = None
screenshot: str | None = None
duration_ms: int | None = None
-39
View File
@@ -1,48 +1,9 @@
import asyncio import asyncio
import json
import hashlib
import time
from pathlib import Path
from patchright.async_api import Page from patchright.async_api import Page
from twocaptcha import TwoCaptcha from twocaptcha import TwoCaptcha
def cached_persistent(file_name=None):
def key(*args, **kwargs):
return hashlib.sha256(f'{str(args)}-{str(kwargs)}'.encode()).hexdigest()
def wrapper(fun):
def clear_cache():
cache_file.write_text(json.dumps({}))
cache_file = Path(file_name)
if not cache_file.exists():
cache_file.write_text(json.dumps({}))
async def wrapped(*args, **kwargs):
data = json.loads(cache_file.read_text())
_key = key(fun.__name__, *args, **kwargs)
if cached_value := data.get(_key):
cached_value, cache_time = cached_value
if time.time() - cache_time > 300:
cached_value = None
value = cached_value or await fun(*args, **kwargs)
data[_key] = [value, time.time()]
cache_file.write_text(json.dumps(data))
return value
setattr(wrapped, '_clear_cache', clear_cache)
return wrapped
return wrapper
class Captcha: class Captcha:
ctype = "captcha" ctype = "captcha"
-2
View File
@@ -16,8 +16,6 @@ import logging
from .base import Captcha, Page from .base import Captcha, Page
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler())
class SolverHCaptcha(Captcha): class SolverHCaptcha(Captcha):
-2
View File
@@ -7,8 +7,6 @@ from urllib.parse import urlparse, parse_qs
from .base import Captcha, Page from .base import Captcha, Page
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler())
class SolverReCaptcha(Captcha): class SolverReCaptcha(Captcha):
-2
View File
@@ -16,8 +16,6 @@ import logging
from .base import Captcha, Page from .base import Captcha, Page
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler())
class SolverTurnstile(Captcha): class SolverTurnstile(Captcha):
+192 -100
View File
@@ -1,6 +1,9 @@
import asyncio
import base64
import json import json
import logging import logging
import traceback import os
import time
import uuid import uuid
from pathlib import Path from pathlib import Path
from typing import Annotated from typing import Annotated
@@ -8,19 +11,29 @@ from urllib.parse import urlsplit
from fastapi import Cookie from fastapi import Cookie
from fastapi import FastAPI from fastapi import FastAPI
from fastapi import Request
from fastapi import Response from fastapi import Response
from fastapi.responses import JSONResponse
from patchright.async_api import Page, ProxySettings from patchright.async_api import Page, ProxySettings
from api import actions logging.basicConfig(
from api.geoip import resolve_geo level=os.getenv("LOG_LEVEL", "INFO").upper(),
from api.schemas import SolveRequest format="%(asctime)s %(levelname)s %(name)s: %(message)s",
from engine import AsyncStealthySession )
from api import actions # noqa: E402 — модули ниже логируют при импорте, конфиг логов должен быть раньше
from api.geoip import resolve_geo # noqa: E402
from api.schemas import SolveRequest, SolveResponse # noqa: E402
from engine import AsyncStealthySession # noqa: E402
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(logging.StreamHandler()) # сколько Chrome запускается параллельно; подбирать ≈ числу ядер (см. docs/Deployment.md)
BROWSER_SLOTS = 10 BROWSER_SLOTS = int(os.getenv("BROWSER_SLOTS", "10"))
RETRY_AFTER = os.getenv("NO_SLOTS_RETRY_AFTER", "5")
_slots = asyncio.Semaphore(BROWSER_SLOTS)
USER_DATA_ROOT = (Path(__file__).parent / "../extra/user_data_dir").resolve()
SESSION_SETTINGS = "robotex_session.json"
def parse_proxy(proxy_url: str) -> ProxySettings: def parse_proxy(proxy_url: str) -> ProxySettings:
@@ -34,23 +47,97 @@ def parse_proxy(proxy_url: str) -> ProxySettings:
return settings return settings
def mask_proxy(proxy_url: str | None) -> str | None:
if not proxy_url:
return proxy_url
parts = urlsplit(proxy_url)
return f"{parts.scheme}://{'***@' if parts.username else ''}{parts.hostname}:{parts.port}"
def safe_request_dump(payload: SolveRequest) -> str:
"""Запрос для лога без секретов клиента: креды прокси, ключ 2captcha, вводимые значения, cookies."""
data = payload.model_dump(exclude_none=True)
data["proxy"] = mask_proxy(payload.proxy)
if payload.captcha:
data["captcha"] = "***"
if payload.cookies:
data["cookies"] = [c.get("name") for c in payload.cookies]
for act in data["actions"]:
if act["type"] == "fill" and "value" in act:
act["value"] = "***"
return json.dumps(data, indent=2, ensure_ascii=False)
def safe_response_dump(result: dict) -> str:
"""Ответ для лога: без значений cookies, html и скриншота."""
data = dict(result)
if data.get("cookies"):
data["cookies"] = [c.get("name") for c in data["cookies"]]
for key in ("html", "screenshot"):
if data.get(key):
data[key] = f"<{len(data[key])} chars>"
return json.dumps(data, indent=2, ensure_ascii=False)
def session_dir(session_id: str | None) -> tuple[str, Path]:
"""Каталог профиля по session_id из cookie. Принимается только UUID — иначе (в т.ч. попытка
подсунуть путь вида ../..) выдаётся новая сессия."""
try:
session_id = str(uuid.UUID(session_id)) if session_id else None
except ValueError:
logger.warning("invalid session_id cookie, issuing a new session")
session_id = None
session_id = session_id or str(uuid.uuid4())
user_data_dir = (USER_DATA_ROOT / session_id).resolve()
if user_data_dir.parent != USER_DATA_ROOT:
raise ValueError("invalid session path")
return session_id, user_data_dir
def load_session_settings(user_data_dir: Path) -> dict:
try:
return json.loads((user_data_dir / SESSION_SETTINGS).read_text())
except (OSError, json.JSONDecodeError):
return {}
def save_session_settings(user_data_dir: Path, settings: dict) -> None:
(user_data_dir / SESSION_SETTINGS).write_text(json.dumps(settings, ensure_ascii=False))
def make_page_action(payload: SolveRequest): def make_page_action(payload: SolveRequest):
"""Замыкание для AsyncStealthySession.fetch(page_action=...): выполняет очередь действий """Замыкание для AsyncStealthySession.fetch(page_action=...): выполняет очередь действий
(включая прохождение капчи/антибота через api.actions.execute) и достаёт из уже (включая прохождение капчи/антибота через api.actions.execute) и достаёт из уже
навигированной страницы всё, что нужно для ответа API. Возврат этой функции — то, что навигированной страницы всё, что нужно для ответа API. Возврат этой функции — то, что
вернёт fetch() (см. engine/stealthy.py).""" вернёт fetch() (см. engine/stealthy.py).
Непройденный антибот не бросается наружу, а возвращается в challenge_error — чтобы в ответ
всё равно попали cookies, html и скриншот страницы-заглушки."""
async def page_action(page: Page) -> dict: async def page_action(page: Page) -> dict:
actions_result = await actions.execute( challenge_error = None
page, payload.actions, payload.captcha, try:
antibot_name=payload.captcha_type, actions_result = await actions.execute(
success_locator=payload.captcha_success_locator, page, payload.actions, payload.captcha,
detect_locator=payload.captcha_detect_locator, antibot_name=payload.captcha_type,
) success_locator=payload.captcha_success_locator,
detect_locator=payload.captcha_detect_locator,
)
except actions.ChallengeFailed as e:
challenge_error = str(e)
actions_result = []
screenshot = None
if payload.screenshot:
screenshot = base64.b64encode(await page.screenshot(type="png")).decode()
return { return {
"challenge_error": challenge_error,
"actions": actions_result, "actions": actions_result,
"cookies": await page.context.cookies(), "cookies": await page.context.cookies(),
"html": await page.inner_html("html"), "html": await page.inner_html("html"),
"screenshot": screenshot,
"user_agent": await page.evaluate("() => navigator.userAgent"), "user_agent": await page.evaluate("() => navigator.userAgent"),
} }
@@ -61,100 +148,105 @@ app = FastAPI()
@app.get("/") @app.get("/")
async def root(): @app.get("/health")
return {"message": "Hello World"} async def health():
return {"status": "ok", "browser_slots": BROWSER_SLOTS, "free_slots": _slots._value} # noqa
@app.post("/v1/solve") @app.post("/v1/solve", response_model=SolveResponse, response_model_exclude_none=True)
async def solve( async def solve(
request: Request,
payload: SolveRequest, payload: SolveRequest,
response: Response, response: Response,
session_id: Annotated[str | None, Cookie()] = None session_id: Annotated[str | None, Cookie()] = None
): ):
# TODO: добавить поддержку api ключей # TODO: добавить поддержку api ключей (#4)
api_key = request.headers.get('X-API-Key') started = time.monotonic()
logger.info("Incoming payload: [%s]\n%s", session_id, payload.model_dump_json(indent=2, ensure_ascii=True)) if _slots.locked():
logger.warning("no free browser slots (%d), rejecting %s", BROWSER_SLOTS, payload.url)
global BROWSER_SLOTS return JSONResponse(
status_code=503,
try: headers={"Retry-After": RETRY_AFTER},
assert BROWSER_SLOTS > 0 and (BROWSER_SLOTS := BROWSER_SLOTS.__sub__(1)) >= 0, "No slots available" content={"status": "err", "code": "no_slots", "error": "No slots available"},
proxy = parse_proxy(payload.proxy)
if not session_id:
session_id = str(uuid.uuid4())
response.set_cookie("session_id", session_id)
logger.info("Setup new session with session_id: %s", session_id)
# профиль (cookies/local storage/fingerprint.json) хранится на диске по
# session_id — если он уже приходил в куках, ниже подгружаем существующий
# fingerprint из этой папки; если нет (новый session_id) — генерируем новый
# и сохраняем туда же (см. api/fingerprint.py:load_or_create_fingerprint)
user_data_dir = Path(__file__).parent / f"../extra/user_data_dir/{session_id}"
if not user_data_dir.exists():
user_data_dir.mkdir(parents=True)
# geo — до создания fingerprint'а, чтобы производная от прокси locale
# (см. api/geoip.py) попала в сам fingerprint, а не подменяла его
# задним числом
geo = await resolve_geo(proxy)
locale = geo.get("locale") if geo else None
fp_kwargs = {}
if payload.user_agent:
fp_kwargs["user_agent"] = payload.user_agent
# TODO: если есть session_id, то user_agent нужно взять из настроек пользователя
if locale:
fp_kwargs["locale"] = locale
nav_timeout_ms = payload.timeout * 1000 if payload.timeout else 60_000
blocked_domains = set(payload.blocked_domains) if payload.blocked_domains else None
kwargs = dict(
headless=False,
proxy=proxy,
user_data_dir=str(user_data_dir),
real_chrome=True,
locale=locale,
timezone_id=geo.get("timezone_id") if geo else None,
cookies=payload.cookies,
disable_resources=payload.disable_resources,
blocked_domains=blocked_domains,
timeout=nav_timeout_ms,
retries=max(1, payload.retries),
retry_delay=payload.retry_delay,
wait=payload.wait,
google_search=payload.google_search
) )
async with AsyncStealthySession(**kwargs) as engine: async with _slots:
result = await engine.fetch(payload.url, page_action=make_page_action(payload)) session_id, user_data_dir = session_dir(session_id)
except Exception as e: response.set_cookie("session_id", session_id)
traceback.print_exc() logger.info("Incoming request [%s]: %s", session_id, payload.url)
result_payload = { logger.debug("Incoming payload [%s]:\n%s", session_id, safe_request_dump(payload))
"error": str(e) or type(e).__name__,
"status": False
}
else:
failed = [a for a in result["actions"] if a.get("ok") is False]
status = not failed
error = None if status else f"{failed[0]['type']}: {failed[0].get('result')}"
result_payload = { try:
"status": ["err", "ok"][status], user_data_dir.mkdir(parents=True, exist_ok=True)
"user_agent": result["user_agent"],
"cookies": result["cookies"],
"actions": result["actions"],
}
if payload.html:
result_payload["html"] = result["html"]
if error:
result_payload["error"] = error
finally:
BROWSER_SLOTS += 1
logger.info("Outgoing payload:\n%s", json.dumps(result_payload, indent=2, sort_keys=True, ensure_ascii=True)) # user_agent сессии сохраняется рядом с профилем: при повторном запросе с тем же
# session_id без user_agent отпечаток не должен меняться
settings = load_session_settings(user_data_dir)
if payload.user_agent:
settings["user_agent"] = payload.user_agent
save_session_settings(user_data_dir, settings)
user_agent = settings.get("user_agent")
proxy = parse_proxy(payload.proxy) if payload.proxy else None
# geo — по exit-IP прокси (см. api/geoip.py); без прокси оставляем настройки браузера
geo = await resolve_geo(proxy) if proxy else None
locale = geo.get("locale") if geo else None
nav_timeout_ms = payload.timeout * 1000 if payload.timeout else 60_000
blocked_domains = set(payload.blocked_domains) if payload.blocked_domains else None
kwargs = dict(
headless=False,
proxy=proxy,
user_data_dir=str(user_data_dir),
real_chrome=True,
useragent=user_agent,
locale=locale,
timezone_id=geo.get("timezone_id") if geo else None,
cookies=payload.cookies,
disable_resources=payload.disable_resources,
blocked_domains=blocked_domains,
timeout=nav_timeout_ms,
retries=max(1, payload.retries),
retry_delay=payload.retry_delay,
wait=payload.wait,
google_search=payload.google_search
)
async with AsyncStealthySession(**kwargs) as engine:
result = await engine.fetch(payload.url, page_action=make_page_action(payload))
except Exception as e:
logger.exception("request failed [%s]", session_id)
result_payload = {
"status": "err",
"code": "request_failed",
"error": str(e) or type(e).__name__,
}
else:
failed = [a for a in result["actions"] if a.get("ok") is False]
result_payload = {
"status": "ok",
"user_agent": result["user_agent"],
"cookies": result["cookies"],
"actions": result["actions"],
}
if result["challenge_error"]:
result_payload.update(status="err", code="challenge_failed", error=result["challenge_error"])
elif failed:
result_payload.update(
status="err", code="action_failed",
error=f"{failed[0]['type']}: {failed[0].get('result')}",
)
if payload.html:
result_payload["html"] = result["html"]
if payload.screenshot:
result_payload["screenshot"] = result["screenshot"]
result_payload["duration_ms"] = int((time.monotonic() - started) * 1000)
logger.info(
"Outgoing [%s]: %s %s %dms", session_id, result_payload["status"],
result_payload.get("error", ""), result_payload["duration_ms"],
)
logger.debug("Outgoing payload:\n%s", safe_response_dump(result_payload))
return result_payload return result_payload
+7 -6
View File
@@ -35,15 +35,16 @@ def classify(resp: httpx.Response | Exception) -> str:
if isinstance(resp, Exception): if isinstance(resp, Exception):
return f"exc:{type(resp).__name__}" return f"exc:{type(resp).__name__}"
data = resp.json() data = resp.json()
if "No slots" in str(data.get("error", "")): code = data.get("code")
if code == "no_slots":
return "no_slots" return "no_slots"
actions = data.get("actions") or [] if code == "challenge_failed":
if not actions: return "cf_blocked"
return f"error:{str(data.get('error'))[:60]}" if code == "action_failed":
last = actions[-1]
if not last.get("ok") or last.get("result"):
html = data.get("html") or "" html = data.get("html") or ""
return "cf_blocked" if any(m in html for m in CF_MARKERS) else "action_failed" return "cf_blocked" if any(m in html for m in CF_MARKERS) else "action_failed"
if data.get("status") != "ok":
return f"error:{str(data.get('error'))[:60]}"
return "ok" return "ok"