fix: слоты, безопасность, единый формат ответа и валидация
- слоты браузеров: asyncio.Semaphore, размер из BROWSER_SLOTS, при нехватке — 503 + Retry-After; устранена утечка счётчика при отказе (#12, #49) - session_id принимается только как UUID, путь профиля проверяется (#5) - секреты клиента в логах маскируются, полный дамп — только на DEBUG (#6) - ответ: status всегда "ok"/"err", поле code (no_slots / request_failed / challenge_failed / action_failed), response_model (#7) - непройденный антибот → challenge_failed, действия не выполняются (#8) - таймауты провайдеров в мс вынесены в константы; detect переживает навигацию посреди проверки; пауза между неудачными solve вместо холостого цикла (#9) - обязательные поля действий и формат прокси проверяются схемой → 422 до запуска браузера (#13) - прокси необязателен (#15) - user_agent сохраняется в профиле сессии и реально передаётся в браузер (#16) - duration_ms у действий и у запроса (#18), опция screenshot (#19) - общий конфиг логирования (LOG_LEVEL), print → логгер, CloudflareProvider, удалён cached_persistent, GET /health (#22) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f58306cbb4
commit
437ea27be7
14 files changed
+360
-201
No files matched your search
@@ -1,4 +1,4 @@
|
|||||||
API_PORT=
|
API_PORT=
|
||||||
API_CPUS=1
|
API_CPUS=1
|
||||||
API_MEM_LIMIT=1g
|
API_MEM_LIMIT=1g
|
||||||
API_MEM_LIMIT=1g
|
BROWSER_SLOTS=1
|
||||||
@@ -1 +1,9 @@
|
|||||||
# Project Env
|
# Project Env
|
||||||
|
# порт, CPU и память контейнера (docker-compose)
|
||||||
|
API_PORT=8000
|
||||||
|
API_CPUS=4
|
||||||
|
API_MEM_LIMIT=8g
|
||||||
|
# сколько Chrome параллельно, ≈ числу ядер (docs/Deployment.md)
|
||||||
|
BROWSER_SLOTS=4
|
||||||
|
NO_SLOTS_RETRY_AFTER=5
|
||||||
|
LOG_LEVEL=INFO
|
||||||
@@ -85,7 +85,6 @@ async def move_mouse_human(
|
|||||||
if i < steps:
|
if i < steps:
|
||||||
px += random.uniform(-1.5, 1.5)
|
px += random.uniform(-1.5, 1.5)
|
||||||
py += random.uniform(-1.5, 1.5)
|
py += random.uniform(-1.5, 1.5)
|
||||||
print("Move:", px, py)
|
|
||||||
await page.mouse.move(px, py)
|
await page.mouse.move(px, py)
|
||||||
await page.wait_for_timeout(step_delay_ms * random.uniform(0.7, 1.3))
|
await page.wait_for_timeout(step_delay_ms * random.uniform(0.7, 1.3))
|
||||||
|
|
||||||
@@ -103,13 +102,9 @@ async def click_human(
|
|||||||
"""move_mouse_human + клик с человеческой задержкой между наведением и down/up, вместо
|
"""move_mouse_human + клик с человеческой задержкой между наведением и down/up, вместо
|
||||||
мгновенного page.mouse.click(x, y)."""
|
мгновенного page.mouse.click(x, y)."""
|
||||||
await move_mouse_human(page, x, y, **move_kwargs)
|
await move_mouse_human(page, x, y, **move_kwargs)
|
||||||
print('wait for timeout')
|
|
||||||
await page.wait_for_timeout(random.uniform(50, 180))
|
await page.wait_for_timeout(random.uniform(50, 180))
|
||||||
print('key down')
|
|
||||||
await page.mouse.down(button=button)
|
await page.mouse.down(button=button)
|
||||||
print('sleep')
|
|
||||||
await page.wait_for_timeout(random.uniform(40, 99))
|
await page.wait_for_timeout(random.uniform(40, 99))
|
||||||
print('key up')
|
|
||||||
await page.mouse.up(button=button)
|
await page.mouse.up(button=button)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -72,8 +72,6 @@ from ..registry import register
|
|||||||
from ..humanize import wait_for_page_stability, wait_for_networkidle, click_human
|
from ..humanize import wait_for_page_stability, wait_for_networkidle, click_human
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
log.setLevel(logging.DEBUG)
|
|
||||||
log.addHandler(logging.StreamHandler())
|
|
||||||
|
|
||||||
|
|
||||||
class CloudflareDetector(Detector):
|
class CloudflareDetector(Detector):
|
||||||
@@ -185,9 +183,12 @@ __CF_PATTERN__ = re.compile(r"^https://challenges\.cloudflare\.com/cdn-cgi/chall
|
|||||||
|
|
||||||
|
|
||||||
class CloudflareSolver(Solver):
|
class CloudflareSolver(Solver):
|
||||||
|
# опрос исчезновения маркера челленджа после клика / на interstitial
|
||||||
|
POLL_INTERVAL_MS = 250
|
||||||
|
WAIT_CLEARED_S = 10
|
||||||
|
|
||||||
async def solve(self, page: Page, info: ChallengeInfo):
|
async def solve(self, page: Page, info: ChallengeInfo):
|
||||||
print("start solving")
|
log.debug("start solving")
|
||||||
iframe = page.frame(url=__CF_PATTERN__)
|
iframe = page.frame(url=__CF_PATTERN__)
|
||||||
if iframe:
|
if iframe:
|
||||||
await page.wait_for_timeout(randint(500, 1000))
|
await page.wait_for_timeout(randint(500, 1000))
|
||||||
@@ -212,11 +213,11 @@ class CloudflareSolver(Solver):
|
|||||||
finally:
|
finally:
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
deadline = time.time() + 10
|
deadline = time.time() + self.WAIT_CLEARED_S
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
if not await page.query_selector(info.detect_locator):
|
if not await page.query_selector(info.detect_locator):
|
||||||
return
|
return
|
||||||
await page.wait_for_timeout(1)
|
await page.wait_for_timeout(self.POLL_INTERVAL_MS)
|
||||||
return
|
return
|
||||||
elif info.stage == Stage.INTERSTITIAL:
|
elif info.stage == Stage.INTERSTITIAL:
|
||||||
if info.success_locator:
|
if info.success_locator:
|
||||||
@@ -227,11 +228,11 @@ class CloudflareSolver(Solver):
|
|||||||
else:
|
else:
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
deadline = time.time() + 10
|
deadline = time.time() + self.WAIT_CLEARED_S
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
if not await page.query_selector(info.detect_locator):
|
if not await page.query_selector(info.detect_locator):
|
||||||
return
|
return
|
||||||
await page.wait_for_timeout(1)
|
await page.wait_for_timeout(self.POLL_INTERVAL_MS)
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
|
|
||||||
@@ -257,7 +258,7 @@ class CloudflareSolver(Solver):
|
|||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
class ReCaptchaProvider(AntiBotProvider):
|
class CloudflareProvider(AntiBotProvider):
|
||||||
provider = 'cloudflare'
|
provider = 'cloudflare'
|
||||||
_solver_cls = CloudflareSolver
|
_solver_cls = CloudflareSolver
|
||||||
_detector_cls = CloudflareDetector
|
_detector_cls = CloudflareDetector
|
||||||
@@ -11,8 +11,6 @@ from ..registry import register
|
|||||||
from ..exceptions import IframeLoadError
|
from ..exceptions import IframeLoadError
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
log.setLevel(logging.DEBUG)
|
|
||||||
log.addHandler(logging.StreamHandler())
|
|
||||||
|
|
||||||
|
|
||||||
JS_CHALLENGE_SUBMIT = """(token) => {
|
JS_CHALLENGE_SUBMIT = """(token) => {
|
||||||
@@ -36,6 +34,8 @@ JS_CHALLENGE_SUBMIT = """(token) => {
|
|||||||
|
|
||||||
class ReCaptchaSolver(Solver):
|
class ReCaptchaSolver(Solver):
|
||||||
page: Page
|
page: Page
|
||||||
|
# пауза на стадии INTERSTITIAL перед повторным detect
|
||||||
|
INTERSTITIAL_WAIT_MS = 15_000
|
||||||
|
|
||||||
def __init__(self, captcha_solver: Captcha, **kwargs) -> None:
|
def __init__(self, captcha_solver: Captcha, **kwargs) -> None:
|
||||||
super().__init__(captcha_solver=captcha_solver, **kwargs)
|
super().__init__(captcha_solver=captcha_solver, **kwargs)
|
||||||
@@ -49,8 +49,7 @@ class ReCaptchaSolver(Solver):
|
|||||||
self.page = page
|
self.page = page
|
||||||
log.debug("Начинаю решение recaptcha антибот преграды")
|
log.debug("Начинаю решение recaptcha антибот преграды")
|
||||||
if info.stage == Stage.INTERSTITIAL:
|
if info.stage == Stage.INTERSTITIAL:
|
||||||
# TODO: вытащить
|
await page.wait_for_timeout(self.INTERSTITIAL_WAIT_MS)
|
||||||
await page.wait_for_timeout(15)
|
|
||||||
return
|
return
|
||||||
elif info.stage == Stage.CHECKBOX:
|
elif info.stage == Stage.CHECKBOX:
|
||||||
log.debug("Нужно тыкнуть чекбокс")
|
log.debug("Нужно тыкнуть чекбокс")
|
||||||
|
|||||||
@@ -3,9 +3,10 @@
|
|||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
import traceback
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from patchright.async_api import Error as PlaywrightError
|
||||||
|
|
||||||
from captcha import load_captcha
|
from captcha import load_captcha
|
||||||
|
|
||||||
from .base import Stage
|
from .base import Stage
|
||||||
@@ -13,8 +14,8 @@ from .registry import select_one
|
|||||||
from .exceptions import ProviderNotDetected
|
from .exceptions import ProviderNotDetected
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
log.setLevel(logging.DEBUG)
|
|
||||||
log.addHandler(logging.StreamHandler())
|
SOLVE_RETRY_DELAY_MS = 1000
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -53,7 +54,14 @@ async def pass_challenges(
|
|||||||
|
|
||||||
info = None
|
info = None
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
info = await provider.detect(page) # noqa
|
try:
|
||||||
|
info = await provider.detect(page) # noqa
|
||||||
|
except PlaywrightError as e:
|
||||||
|
# челлендж сам переходит на целевую страницу посреди detect()
|
||||||
|
# («Execution context was destroyed») — даём навигации закончиться и смотрим заново
|
||||||
|
log.debug("%s: detect interrupted (%s), retrying", provider.provider, e.message.splitlines()[0])
|
||||||
|
await page.wait_for_load_state("load")
|
||||||
|
continue
|
||||||
# 3. проверяем что решение успешно
|
# 3. проверяем что решение успешно
|
||||||
if info.success: # noqa
|
if info.success: # noqa
|
||||||
return ChallengeOutcome(True, reason=(info.meta or {}).get("reason"))
|
return ChallengeOutcome(True, reason=(info.meta or {}).get("reason"))
|
||||||
@@ -61,7 +69,10 @@ async def pass_challenges(
|
|||||||
try:
|
try:
|
||||||
await provider.solve(page, info)
|
await provider.solve(page, info)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
traceback.print_exc()
|
log.warning("%s: solve failed (%s), retrying", provider.provider, str(e).splitlines()[0] if str(e) else type(e).__name__)
|
||||||
|
log.debug("solve traceback", exc_info=True)
|
||||||
|
# пауза, иначе при стабильно падающем solve цикл крутится вхолостую до дедлайна
|
||||||
|
await page.wait_for_timeout(SOLVE_RETRY_DELAY_MS)
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
return ChallengeOutcome(
|
return ChallengeOutcome(
|
||||||
|
|||||||
+30
-23
@@ -1,8 +1,8 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
from patchright.async_api import Page
|
from patchright.async_api import Page
|
||||||
|
|
||||||
from antibot.base import click_and_race
|
|
||||||
from antibot.orchestrator import pass_challenges
|
from antibot.orchestrator import pass_challenges
|
||||||
from captcha import load_captcha
|
from captcha import load_captcha
|
||||||
from api.schemas import ActionRequest
|
from api.schemas import ActionRequest
|
||||||
@@ -10,50 +10,52 @@ from api.schemas import ActionRequest
|
|||||||
_actions: dict = {}
|
_actions: dict = {}
|
||||||
|
|
||||||
|
|
||||||
|
class ChallengeFailed(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def register(fun) -> None:
|
def register(fun) -> None:
|
||||||
_actions[fun.__name__] = fun
|
_actions[fun.__name__] = fun
|
||||||
return fun
|
return fun
|
||||||
|
|
||||||
|
|
||||||
|
async def _query(page: Page, selector: str):
|
||||||
|
element = await page.query_selector(selector)
|
||||||
|
if element is None:
|
||||||
|
raise LookupError(f"selector not found: {selector}")
|
||||||
|
return element
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
async def fill(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
async def fill(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
||||||
assert act.selector
|
|
||||||
assert act.value
|
|
||||||
await page.fill(act.selector, act.value)
|
await page.fill(act.selector, act.value)
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
async def captcha(page: Page, act: ActionRequest, captcha_key: str, **kwargs) -> str | None:
|
async def captcha(page: Page, act: ActionRequest, captcha_key: str, **kwargs) -> str | None:
|
||||||
assert act.selector
|
|
||||||
assert act.value
|
|
||||||
assert captcha_key
|
|
||||||
captcha_solver = load_captcha(captcha_key, act.value)
|
captcha_solver = load_captcha(captcha_key, act.value)
|
||||||
await captcha_solver.solve(act.selector, page)
|
await captcha_solver.solve(act.selector, page)
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
async def delay(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
async def delay(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
||||||
assert act.value
|
|
||||||
await asyncio.sleep(int(act.value))
|
await asyncio.sleep(int(act.value))
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
async def mouse_move(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
async def mouse_move(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
||||||
assert act.selector
|
element = await _query(page, act.selector)
|
||||||
element = await page.query_selector(act.selector)
|
|
||||||
await element.hover(scroll="auto")
|
await element.hover(scroll="auto")
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
async def click(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
async def click(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
||||||
assert act.selector
|
|
||||||
element = page.locator(act.selector)
|
element = page.locator(act.selector)
|
||||||
await element.click()
|
await element.click()
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
||||||
assert act.selector
|
|
||||||
element = page.locator(act.selector)
|
element = page.locator(act.selector)
|
||||||
|
|
||||||
async with page.expect_event(
|
async with page.expect_event(
|
||||||
@@ -65,15 +67,12 @@ async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
|||||||
|
|
||||||
@register
|
@register
|
||||||
async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
||||||
assert act.selector
|
element = await _query(page, act.selector)
|
||||||
element = await page.query_selector(act.selector)
|
|
||||||
return await element.inner_html()
|
return await element.inner_html()
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
async def wait_for(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
async def wait_for(page: Page, act: ActionRequest, **kwargs) -> str | None:
|
||||||
assert act.selector
|
|
||||||
assert 1 <= act.timeout <= 7200
|
|
||||||
await page.wait_for_selector(act.selector, timeout=act.timeout * 1000)
|
await page.wait_for_selector(act.selector, timeout=act.timeout * 1000)
|
||||||
|
|
||||||
|
|
||||||
@@ -99,7 +98,10 @@ async def execute(
|
|||||||
captcha_key: str | None = None, antibot_name: str | None = None,
|
captcha_key: str | None = None, antibot_name: str | None = None,
|
||||||
success_locator: str | None = None,
|
success_locator: str | None = None,
|
||||||
detect_locator: str | None = None) -> list[dict]:
|
detect_locator: str | None = None) -> list[dict]:
|
||||||
"""Выполнение очереди автоматизации"""
|
"""Выполнение очереди автоматизации.
|
||||||
|
|
||||||
|
Если антибот не пройден — бросает ChallengeFailed, действия не выполняются: на странице-заглушке
|
||||||
|
они бы упали с невнятной ошибкой селектора."""
|
||||||
actions_result = []
|
actions_result = []
|
||||||
ctx = {
|
ctx = {
|
||||||
"captcha_key": captcha_key,
|
"captcha_key": captcha_key,
|
||||||
@@ -107,27 +109,32 @@ async def execute(
|
|||||||
"success_locator": success_locator,
|
"success_locator": success_locator,
|
||||||
"detect_locator": detect_locator,
|
"detect_locator": detect_locator,
|
||||||
}
|
}
|
||||||
await pass_challenges(
|
outcome = await pass_challenges(
|
||||||
antibot_name, page,
|
antibot_name, page,
|
||||||
captcha=captcha_key,
|
captcha=captcha_key,
|
||||||
success_locator=success_locator,
|
success_locator=success_locator,
|
||||||
detect_locator=detect_locator,
|
detect_locator=detect_locator,
|
||||||
)
|
)
|
||||||
|
if not outcome.cleared:
|
||||||
|
stage = outcome.last_stage.value if outcome.last_stage else None
|
||||||
|
raise ChallengeFailed(f"{antibot_name} not passed: {outcome.reason} (stage: {stage})")
|
||||||
|
|
||||||
for act in actions:
|
for act in actions:
|
||||||
new_line = {
|
new_line = {
|
||||||
"type": act.type,
|
"type": act.type,
|
||||||
}
|
}
|
||||||
|
started = time.monotonic()
|
||||||
try:
|
try:
|
||||||
# TODO: добавить время выполнения
|
|
||||||
result = await _actions[act.type](page, act, **ctx)
|
result = await _actions[act.type](page, act, **ctx)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
new_line["ok"] = False
|
new_line["ok"] = False
|
||||||
new_line["result"] = str(e)
|
new_line["result"] = str(e) or type(e).__name__
|
||||||
actions_result.append(new_line)
|
|
||||||
break
|
|
||||||
else:
|
else:
|
||||||
new_line["ok"] = True
|
new_line["ok"] = True
|
||||||
if result:
|
if result:
|
||||||
new_line["result"] = result
|
new_line["result"] = result
|
||||||
actions_result.append(new_line)
|
new_line["duration_ms"] = int((time.monotonic() - started) * 1000)
|
||||||
return actions_result if len(actions_result) else [{"status": "pass"}]
|
actions_result.append(new_line)
|
||||||
|
if not new_line["ok"]:
|
||||||
|
break
|
||||||
|
return actions_result
|
||||||
+94
-4
@@ -1,25 +1,70 @@
|
|||||||
from typing import Any
|
from typing import Any, Literal
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel, field_validator, model_validator
|
||||||
|
|
||||||
|
CAPTCHA_TYPES = ("cloudflare", "recaptcha", "hcaptcha")
|
||||||
|
PROXY_SCHEMES = ("http", "https", "socks5")
|
||||||
|
|
||||||
|
# тип действия -> обязательные поля (кроме type)
|
||||||
|
_REQUIRED_FIELDS: dict[str, tuple[str, ...]] = {
|
||||||
|
"fill": ("selector", "value"),
|
||||||
|
"captcha": ("selector", "value"),
|
||||||
|
"delay": ("value",),
|
||||||
|
"mouse_move": ("selector",),
|
||||||
|
"click": ("selector",),
|
||||||
|
"submit": ("selector",),
|
||||||
|
"inner_html": ("selector",),
|
||||||
|
"wait_for": ("selector",),
|
||||||
|
"pass_challenge": (),
|
||||||
|
}
|
||||||
|
WAIT_FOR_DEFAULT_TIMEOUT = 30
|
||||||
|
|
||||||
|
|
||||||
class ActionRequest(BaseModel):
|
class ActionRequest(BaseModel):
|
||||||
"""
|
"""
|
||||||
Действие автоматизации. Характеризуется типом и производится над селектором.
|
Действие автоматизации. Характеризуется типом и производится над селектором.
|
||||||
Набор параметров зависит от типа, этот класс перечисляет все параметры.
|
Набор параметров зависит от типа, этот класс перечисляет все параметры.
|
||||||
|
Обязательные поля для каждого типа проверяются здесь же — неверное действие
|
||||||
|
отклоняется с 422 до запуска браузера.
|
||||||
"""
|
"""
|
||||||
type: str
|
type: str
|
||||||
selector: str | None = None
|
selector: str | None = None
|
||||||
value: str | None = None
|
value: str | None = None
|
||||||
timeout: int | None = None
|
timeout: int | None = None
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def check_fields(self):
|
||||||
|
if self.type not in _REQUIRED_FIELDS:
|
||||||
|
raise ValueError(f"unknown action type {self.type!r}, expected one of: {', '.join(_REQUIRED_FIELDS)}")
|
||||||
|
|
||||||
|
missing = [f for f in _REQUIRED_FIELDS[self.type] if getattr(self, f) in (None, "")]
|
||||||
|
if missing:
|
||||||
|
raise ValueError(f"action {self.type!r} requires: {', '.join(missing)}")
|
||||||
|
|
||||||
|
if self.type == "wait_for":
|
||||||
|
if self.timeout is None:
|
||||||
|
self.timeout = WAIT_FOR_DEFAULT_TIMEOUT
|
||||||
|
if not 1 <= self.timeout <= 7200:
|
||||||
|
raise ValueError("wait_for timeout must be between 1 and 7200 seconds")
|
||||||
|
elif self.type == "delay":
|
||||||
|
try:
|
||||||
|
delay = int(self.value)
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError("delay value must be an integer number of seconds") from None
|
||||||
|
if not 0 <= delay <= 600:
|
||||||
|
raise ValueError("delay value must be between 0 and 600 seconds")
|
||||||
|
elif self.type == "captcha" and self.value not in CAPTCHA_TYPES:
|
||||||
|
raise ValueError(f"captcha value must be one of: {', '.join(CAPTCHA_TYPES)}")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
class SolveRequest(BaseModel):
|
class SolveRequest(BaseModel):
|
||||||
"""
|
"""
|
||||||
Объект запроса для автоматизации на сайте.
|
Объект запроса для автоматизации на сайте.
|
||||||
"""
|
"""
|
||||||
url: str
|
url: str
|
||||||
proxy: str
|
proxy: str | None = None
|
||||||
timeout: int | None = None
|
timeout: int | None = None
|
||||||
user_agent: str | None = None
|
user_agent: str | None = None
|
||||||
cookies: list[dict[str, Any]] | None = None
|
cookies: list[dict[str, Any]] | None = None
|
||||||
@@ -28,7 +73,7 @@ class SolveRequest(BaseModel):
|
|||||||
actions: list[ActionRequest]
|
actions: list[ActionRequest]
|
||||||
|
|
||||||
html: bool = False
|
html: bool = False
|
||||||
# TODO: screenshot: bool = False
|
screenshot: bool = False
|
||||||
|
|
||||||
captcha: str | None = None
|
captcha: str | None = None
|
||||||
captcha_detect_locator: str | None = None
|
captcha_detect_locator: str | None = None
|
||||||
@@ -43,3 +88,48 @@ class SolveRequest(BaseModel):
|
|||||||
retry_delay: float = 1.0
|
retry_delay: float = 1.0
|
||||||
wait: int = 0
|
wait: int = 0
|
||||||
google_search: bool = False
|
google_search: bool = False
|
||||||
|
|
||||||
|
@field_validator("proxy")
|
||||||
|
@classmethod
|
||||||
|
def check_proxy(cls, proxy: str | None) -> str | None:
|
||||||
|
if not proxy:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parts = urlsplit(proxy)
|
||||||
|
port = parts.port
|
||||||
|
except ValueError as e:
|
||||||
|
raise ValueError(f"invalid proxy url: {e}") from None
|
||||||
|
if parts.scheme not in PROXY_SCHEMES:
|
||||||
|
raise ValueError(f"proxy scheme must be one of: {', '.join(PROXY_SCHEMES)}")
|
||||||
|
if not parts.hostname or not port:
|
||||||
|
raise ValueError("proxy must be in form scheme://[user:pass@]host:port")
|
||||||
|
return proxy
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def check_captcha_key(self):
|
||||||
|
if not self.captcha and any(a.type == "captcha" for a in self.actions):
|
||||||
|
raise ValueError("action 'captcha' requires request field 'captcha' (2captcha key)")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ActionResult(BaseModel):
|
||||||
|
type: str
|
||||||
|
ok: bool
|
||||||
|
result: str | None = None
|
||||||
|
duration_ms: int
|
||||||
|
|
||||||
|
|
||||||
|
class SolveResponse(BaseModel):
|
||||||
|
"""
|
||||||
|
Ответ /v1/solve. status — "ok" (антибот пройден, все действия выполнены) или "err".
|
||||||
|
code — машиночитаемая причина ошибки, error — текст для человека.
|
||||||
|
"""
|
||||||
|
status: Literal["ok", "err"]
|
||||||
|
code: Literal["no_slots", "request_failed", "challenge_failed", "action_failed"] | None = None
|
||||||
|
error: str | None = None
|
||||||
|
user_agent: str | None = None
|
||||||
|
cookies: list[dict[str, Any]] | None = None
|
||||||
|
actions: list[ActionResult] = []
|
||||||
|
html: str | None = None
|
||||||
|
screenshot: str | None = None
|
||||||
|
duration_ms: int | None = None
|
||||||
@@ -1,48 +1,9 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
|
||||||
import hashlib
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from patchright.async_api import Page
|
from patchright.async_api import Page
|
||||||
from twocaptcha import TwoCaptcha
|
from twocaptcha import TwoCaptcha
|
||||||
|
|
||||||
|
|
||||||
def cached_persistent(file_name=None):
|
|
||||||
|
|
||||||
def key(*args, **kwargs):
|
|
||||||
return hashlib.sha256(f'{str(args)}-{str(kwargs)}'.encode()).hexdigest()
|
|
||||||
|
|
||||||
def wrapper(fun):
|
|
||||||
|
|
||||||
def clear_cache():
|
|
||||||
cache_file.write_text(json.dumps({}))
|
|
||||||
|
|
||||||
cache_file = Path(file_name)
|
|
||||||
if not cache_file.exists():
|
|
||||||
cache_file.write_text(json.dumps({}))
|
|
||||||
|
|
||||||
async def wrapped(*args, **kwargs):
|
|
||||||
data = json.loads(cache_file.read_text())
|
|
||||||
_key = key(fun.__name__, *args, **kwargs)
|
|
||||||
|
|
||||||
if cached_value := data.get(_key):
|
|
||||||
cached_value, cache_time = cached_value
|
|
||||||
if time.time() - cache_time > 300:
|
|
||||||
cached_value = None
|
|
||||||
|
|
||||||
value = cached_value or await fun(*args, **kwargs)
|
|
||||||
|
|
||||||
data[_key] = [value, time.time()]
|
|
||||||
cache_file.write_text(json.dumps(data))
|
|
||||||
return value
|
|
||||||
|
|
||||||
setattr(wrapped, '_clear_cache', clear_cache)
|
|
||||||
return wrapped
|
|
||||||
|
|
||||||
return wrapper
|
|
||||||
|
|
||||||
|
|
||||||
class Captcha:
|
class Captcha:
|
||||||
ctype = "captcha"
|
ctype = "captcha"
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,6 @@ import logging
|
|||||||
from .base import Captcha, Page
|
from .base import Captcha, Page
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
logger.setLevel(logging.DEBUG)
|
|
||||||
logger.addHandler(logging.StreamHandler())
|
|
||||||
|
|
||||||
|
|
||||||
class SolverHCaptcha(Captcha):
|
class SolverHCaptcha(Captcha):
|
||||||
|
|||||||
@@ -7,8 +7,6 @@ from urllib.parse import urlparse, parse_qs
|
|||||||
from .base import Captcha, Page
|
from .base import Captcha, Page
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
logger.setLevel(logging.DEBUG)
|
|
||||||
logger.addHandler(logging.StreamHandler())
|
|
||||||
|
|
||||||
|
|
||||||
class SolverReCaptcha(Captcha):
|
class SolverReCaptcha(Captcha):
|
||||||
|
|||||||
@@ -16,8 +16,6 @@ import logging
|
|||||||
from .base import Captcha, Page
|
from .base import Captcha, Page
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
logger.setLevel(logging.DEBUG)
|
|
||||||
logger.addHandler(logging.StreamHandler())
|
|
||||||
|
|
||||||
|
|
||||||
class SolverTurnstile(Captcha):
|
class SolverTurnstile(Captcha):
|
||||||
|
|||||||
+192
-100
@@ -1,6 +1,9 @@
|
|||||||
|
import asyncio
|
||||||
|
import base64
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import traceback
|
import os
|
||||||
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Annotated
|
from typing import Annotated
|
||||||
@@ -8,19 +11,29 @@ from urllib.parse import urlsplit
|
|||||||
|
|
||||||
from fastapi import Cookie
|
from fastapi import Cookie
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi import Request
|
|
||||||
from fastapi import Response
|
from fastapi import Response
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
from patchright.async_api import Page, ProxySettings
|
from patchright.async_api import Page, ProxySettings
|
||||||
|
|
||||||
from api import actions
|
logging.basicConfig(
|
||||||
from api.geoip import resolve_geo
|
level=os.getenv("LOG_LEVEL", "INFO").upper(),
|
||||||
from api.schemas import SolveRequest
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
||||||
from engine import AsyncStealthySession
|
)
|
||||||
|
|
||||||
|
from api import actions # noqa: E402 — модули ниже логируют при импорте, конфиг логов должен быть раньше
|
||||||
|
from api.geoip import resolve_geo # noqa: E402
|
||||||
|
from api.schemas import SolveRequest, SolveResponse # noqa: E402
|
||||||
|
from engine import AsyncStealthySession # noqa: E402
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
logger.setLevel(logging.DEBUG)
|
|
||||||
logger.addHandler(logging.StreamHandler())
|
# сколько Chrome запускается параллельно; подбирать ≈ числу ядер (см. docs/Deployment.md)
|
||||||
BROWSER_SLOTS = 10
|
BROWSER_SLOTS = int(os.getenv("BROWSER_SLOTS", "10"))
|
||||||
|
RETRY_AFTER = os.getenv("NO_SLOTS_RETRY_AFTER", "5")
|
||||||
|
_slots = asyncio.Semaphore(BROWSER_SLOTS)
|
||||||
|
|
||||||
|
USER_DATA_ROOT = (Path(__file__).parent / "../extra/user_data_dir").resolve()
|
||||||
|
SESSION_SETTINGS = "robotex_session.json"
|
||||||
|
|
||||||
|
|
||||||
def parse_proxy(proxy_url: str) -> ProxySettings:
|
def parse_proxy(proxy_url: str) -> ProxySettings:
|
||||||
@@ -34,23 +47,97 @@ def parse_proxy(proxy_url: str) -> ProxySettings:
|
|||||||
return settings
|
return settings
|
||||||
|
|
||||||
|
|
||||||
|
def mask_proxy(proxy_url: str | None) -> str | None:
|
||||||
|
if not proxy_url:
|
||||||
|
return proxy_url
|
||||||
|
parts = urlsplit(proxy_url)
|
||||||
|
return f"{parts.scheme}://{'***@' if parts.username else ''}{parts.hostname}:{parts.port}"
|
||||||
|
|
||||||
|
|
||||||
|
def safe_request_dump(payload: SolveRequest) -> str:
|
||||||
|
"""Запрос для лога без секретов клиента: креды прокси, ключ 2captcha, вводимые значения, cookies."""
|
||||||
|
data = payload.model_dump(exclude_none=True)
|
||||||
|
data["proxy"] = mask_proxy(payload.proxy)
|
||||||
|
if payload.captcha:
|
||||||
|
data["captcha"] = "***"
|
||||||
|
if payload.cookies:
|
||||||
|
data["cookies"] = [c.get("name") for c in payload.cookies]
|
||||||
|
for act in data["actions"]:
|
||||||
|
if act["type"] == "fill" and "value" in act:
|
||||||
|
act["value"] = "***"
|
||||||
|
return json.dumps(data, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def safe_response_dump(result: dict) -> str:
|
||||||
|
"""Ответ для лога: без значений cookies, html и скриншота."""
|
||||||
|
data = dict(result)
|
||||||
|
if data.get("cookies"):
|
||||||
|
data["cookies"] = [c.get("name") for c in data["cookies"]]
|
||||||
|
for key in ("html", "screenshot"):
|
||||||
|
if data.get(key):
|
||||||
|
data[key] = f"<{len(data[key])} chars>"
|
||||||
|
return json.dumps(data, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def session_dir(session_id: str | None) -> tuple[str, Path]:
|
||||||
|
"""Каталог профиля по session_id из cookie. Принимается только UUID — иначе (в т.ч. попытка
|
||||||
|
подсунуть путь вида ../..) выдаётся новая сессия."""
|
||||||
|
try:
|
||||||
|
session_id = str(uuid.UUID(session_id)) if session_id else None
|
||||||
|
except ValueError:
|
||||||
|
logger.warning("invalid session_id cookie, issuing a new session")
|
||||||
|
session_id = None
|
||||||
|
session_id = session_id or str(uuid.uuid4())
|
||||||
|
|
||||||
|
user_data_dir = (USER_DATA_ROOT / session_id).resolve()
|
||||||
|
if user_data_dir.parent != USER_DATA_ROOT:
|
||||||
|
raise ValueError("invalid session path")
|
||||||
|
return session_id, user_data_dir
|
||||||
|
|
||||||
|
|
||||||
|
def load_session_settings(user_data_dir: Path) -> dict:
|
||||||
|
try:
|
||||||
|
return json.loads((user_data_dir / SESSION_SETTINGS).read_text())
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def save_session_settings(user_data_dir: Path, settings: dict) -> None:
|
||||||
|
(user_data_dir / SESSION_SETTINGS).write_text(json.dumps(settings, ensure_ascii=False))
|
||||||
|
|
||||||
|
|
||||||
def make_page_action(payload: SolveRequest):
|
def make_page_action(payload: SolveRequest):
|
||||||
"""Замыкание для AsyncStealthySession.fetch(page_action=...): выполняет очередь действий
|
"""Замыкание для AsyncStealthySession.fetch(page_action=...): выполняет очередь действий
|
||||||
(включая прохождение капчи/антибота через api.actions.execute) и достаёт из уже
|
(включая прохождение капчи/антибота через api.actions.execute) и достаёт из уже
|
||||||
навигированной страницы всё, что нужно для ответа API. Возврат этой функции — то, что
|
навигированной страницы всё, что нужно для ответа API. Возврат этой функции — то, что
|
||||||
вернёт fetch() (см. engine/stealthy.py)."""
|
вернёт fetch() (см. engine/stealthy.py).
|
||||||
|
|
||||||
|
Непройденный антибот не бросается наружу, а возвращается в challenge_error — чтобы в ответ
|
||||||
|
всё равно попали cookies, html и скриншот страницы-заглушки."""
|
||||||
|
|
||||||
async def page_action(page: Page) -> dict:
|
async def page_action(page: Page) -> dict:
|
||||||
actions_result = await actions.execute(
|
challenge_error = None
|
||||||
page, payload.actions, payload.captcha,
|
try:
|
||||||
antibot_name=payload.captcha_type,
|
actions_result = await actions.execute(
|
||||||
success_locator=payload.captcha_success_locator,
|
page, payload.actions, payload.captcha,
|
||||||
detect_locator=payload.captcha_detect_locator,
|
antibot_name=payload.captcha_type,
|
||||||
)
|
success_locator=payload.captcha_success_locator,
|
||||||
|
detect_locator=payload.captcha_detect_locator,
|
||||||
|
)
|
||||||
|
except actions.ChallengeFailed as e:
|
||||||
|
challenge_error = str(e)
|
||||||
|
actions_result = []
|
||||||
|
|
||||||
|
screenshot = None
|
||||||
|
if payload.screenshot:
|
||||||
|
screenshot = base64.b64encode(await page.screenshot(type="png")).decode()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
"challenge_error": challenge_error,
|
||||||
"actions": actions_result,
|
"actions": actions_result,
|
||||||
"cookies": await page.context.cookies(),
|
"cookies": await page.context.cookies(),
|
||||||
"html": await page.inner_html("html"),
|
"html": await page.inner_html("html"),
|
||||||
|
"screenshot": screenshot,
|
||||||
"user_agent": await page.evaluate("() => navigator.userAgent"),
|
"user_agent": await page.evaluate("() => navigator.userAgent"),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,100 +148,105 @@ app = FastAPI()
|
|||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
async def root():
|
@app.get("/health")
|
||||||
return {"message": "Hello World"}
|
async def health():
|
||||||
|
return {"status": "ok", "browser_slots": BROWSER_SLOTS, "free_slots": _slots._value} # noqa
|
||||||
|
|
||||||
|
|
||||||
@app.post("/v1/solve")
|
@app.post("/v1/solve", response_model=SolveResponse, response_model_exclude_none=True)
|
||||||
async def solve(
|
async def solve(
|
||||||
request: Request,
|
|
||||||
payload: SolveRequest,
|
payload: SolveRequest,
|
||||||
response: Response,
|
response: Response,
|
||||||
session_id: Annotated[str | None, Cookie()] = None
|
session_id: Annotated[str | None, Cookie()] = None
|
||||||
):
|
):
|
||||||
# TODO: добавить поддержку api ключей
|
# TODO: добавить поддержку api ключей (#4)
|
||||||
api_key = request.headers.get('X-API-Key')
|
started = time.monotonic()
|
||||||
|
|
||||||
logger.info("Incoming payload: [%s]\n%s", session_id, payload.model_dump_json(indent=2, ensure_ascii=True))
|
if _slots.locked():
|
||||||
|
logger.warning("no free browser slots (%d), rejecting %s", BROWSER_SLOTS, payload.url)
|
||||||
global BROWSER_SLOTS
|
return JSONResponse(
|
||||||
|
status_code=503,
|
||||||
try:
|
headers={"Retry-After": RETRY_AFTER},
|
||||||
assert BROWSER_SLOTS > 0 and (BROWSER_SLOTS := BROWSER_SLOTS.__sub__(1)) >= 0, "No slots available"
|
content={"status": "err", "code": "no_slots", "error": "No slots available"},
|
||||||
|
|
||||||
proxy = parse_proxy(payload.proxy)
|
|
||||||
|
|
||||||
if not session_id:
|
|
||||||
session_id = str(uuid.uuid4())
|
|
||||||
response.set_cookie("session_id", session_id)
|
|
||||||
logger.info("Setup new session with session_id: %s", session_id)
|
|
||||||
|
|
||||||
# профиль (cookies/local storage/fingerprint.json) хранится на диске по
|
|
||||||
# session_id — если он уже приходил в куках, ниже подгружаем существующий
|
|
||||||
# fingerprint из этой папки; если нет (новый session_id) — генерируем новый
|
|
||||||
# и сохраняем туда же (см. api/fingerprint.py:load_or_create_fingerprint)
|
|
||||||
user_data_dir = Path(__file__).parent / f"../extra/user_data_dir/{session_id}"
|
|
||||||
if not user_data_dir.exists():
|
|
||||||
user_data_dir.mkdir(parents=True)
|
|
||||||
|
|
||||||
# geo — до создания fingerprint'а, чтобы производная от прокси locale
|
|
||||||
# (см. api/geoip.py) попала в сам fingerprint, а не подменяла его
|
|
||||||
# задним числом
|
|
||||||
geo = await resolve_geo(proxy)
|
|
||||||
locale = geo.get("locale") if geo else None
|
|
||||||
|
|
||||||
fp_kwargs = {}
|
|
||||||
if payload.user_agent:
|
|
||||||
fp_kwargs["user_agent"] = payload.user_agent
|
|
||||||
# TODO: если есть session_id, то user_agent нужно взять из настроек пользователя
|
|
||||||
if locale:
|
|
||||||
fp_kwargs["locale"] = locale
|
|
||||||
|
|
||||||
nav_timeout_ms = payload.timeout * 1000 if payload.timeout else 60_000
|
|
||||||
blocked_domains = set(payload.blocked_domains) if payload.blocked_domains else None
|
|
||||||
|
|
||||||
kwargs = dict(
|
|
||||||
headless=False,
|
|
||||||
proxy=proxy,
|
|
||||||
user_data_dir=str(user_data_dir),
|
|
||||||
real_chrome=True,
|
|
||||||
locale=locale,
|
|
||||||
timezone_id=geo.get("timezone_id") if geo else None,
|
|
||||||
cookies=payload.cookies,
|
|
||||||
disable_resources=payload.disable_resources,
|
|
||||||
blocked_domains=blocked_domains,
|
|
||||||
timeout=nav_timeout_ms,
|
|
||||||
retries=max(1, payload.retries),
|
|
||||||
retry_delay=payload.retry_delay,
|
|
||||||
wait=payload.wait,
|
|
||||||
google_search=payload.google_search
|
|
||||||
)
|
)
|
||||||
|
|
||||||
async with AsyncStealthySession(**kwargs) as engine:
|
async with _slots:
|
||||||
result = await engine.fetch(payload.url, page_action=make_page_action(payload))
|
session_id, user_data_dir = session_dir(session_id)
|
||||||
except Exception as e:
|
response.set_cookie("session_id", session_id)
|
||||||
traceback.print_exc()
|
logger.info("Incoming request [%s]: %s", session_id, payload.url)
|
||||||
result_payload = {
|
logger.debug("Incoming payload [%s]:\n%s", session_id, safe_request_dump(payload))
|
||||||
"error": str(e) or type(e).__name__,
|
|
||||||
"status": False
|
|
||||||
}
|
|
||||||
else:
|
|
||||||
failed = [a for a in result["actions"] if a.get("ok") is False]
|
|
||||||
status = not failed
|
|
||||||
error = None if status else f"{failed[0]['type']}: {failed[0].get('result')}"
|
|
||||||
|
|
||||||
result_payload = {
|
try:
|
||||||
"status": ["err", "ok"][status],
|
user_data_dir.mkdir(parents=True, exist_ok=True)
|
||||||
"user_agent": result["user_agent"],
|
|
||||||
"cookies": result["cookies"],
|
|
||||||
"actions": result["actions"],
|
|
||||||
}
|
|
||||||
if payload.html:
|
|
||||||
result_payload["html"] = result["html"]
|
|
||||||
if error:
|
|
||||||
result_payload["error"] = error
|
|
||||||
finally:
|
|
||||||
BROWSER_SLOTS += 1
|
|
||||||
|
|
||||||
logger.info("Outgoing payload:\n%s", json.dumps(result_payload, indent=2, sort_keys=True, ensure_ascii=True))
|
# user_agent сессии сохраняется рядом с профилем: при повторном запросе с тем же
|
||||||
|
# session_id без user_agent отпечаток не должен меняться
|
||||||
|
settings = load_session_settings(user_data_dir)
|
||||||
|
if payload.user_agent:
|
||||||
|
settings["user_agent"] = payload.user_agent
|
||||||
|
save_session_settings(user_data_dir, settings)
|
||||||
|
user_agent = settings.get("user_agent")
|
||||||
|
|
||||||
|
proxy = parse_proxy(payload.proxy) if payload.proxy else None
|
||||||
|
|
||||||
|
# geo — по exit-IP прокси (см. api/geoip.py); без прокси оставляем настройки браузера
|
||||||
|
geo = await resolve_geo(proxy) if proxy else None
|
||||||
|
locale = geo.get("locale") if geo else None
|
||||||
|
|
||||||
|
nav_timeout_ms = payload.timeout * 1000 if payload.timeout else 60_000
|
||||||
|
blocked_domains = set(payload.blocked_domains) if payload.blocked_domains else None
|
||||||
|
|
||||||
|
kwargs = dict(
|
||||||
|
headless=False,
|
||||||
|
proxy=proxy,
|
||||||
|
user_data_dir=str(user_data_dir),
|
||||||
|
real_chrome=True,
|
||||||
|
useragent=user_agent,
|
||||||
|
locale=locale,
|
||||||
|
timezone_id=geo.get("timezone_id") if geo else None,
|
||||||
|
cookies=payload.cookies,
|
||||||
|
disable_resources=payload.disable_resources,
|
||||||
|
blocked_domains=blocked_domains,
|
||||||
|
timeout=nav_timeout_ms,
|
||||||
|
retries=max(1, payload.retries),
|
||||||
|
retry_delay=payload.retry_delay,
|
||||||
|
wait=payload.wait,
|
||||||
|
google_search=payload.google_search
|
||||||
|
)
|
||||||
|
|
||||||
|
async with AsyncStealthySession(**kwargs) as engine:
|
||||||
|
result = await engine.fetch(payload.url, page_action=make_page_action(payload))
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("request failed [%s]", session_id)
|
||||||
|
result_payload = {
|
||||||
|
"status": "err",
|
||||||
|
"code": "request_failed",
|
||||||
|
"error": str(e) or type(e).__name__,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
failed = [a for a in result["actions"] if a.get("ok") is False]
|
||||||
|
result_payload = {
|
||||||
|
"status": "ok",
|
||||||
|
"user_agent": result["user_agent"],
|
||||||
|
"cookies": result["cookies"],
|
||||||
|
"actions": result["actions"],
|
||||||
|
}
|
||||||
|
if result["challenge_error"]:
|
||||||
|
result_payload.update(status="err", code="challenge_failed", error=result["challenge_error"])
|
||||||
|
elif failed:
|
||||||
|
result_payload.update(
|
||||||
|
status="err", code="action_failed",
|
||||||
|
error=f"{failed[0]['type']}: {failed[0].get('result')}",
|
||||||
|
)
|
||||||
|
if payload.html:
|
||||||
|
result_payload["html"] = result["html"]
|
||||||
|
if payload.screenshot:
|
||||||
|
result_payload["screenshot"] = result["screenshot"]
|
||||||
|
|
||||||
|
result_payload["duration_ms"] = int((time.monotonic() - started) * 1000)
|
||||||
|
logger.info(
|
||||||
|
"Outgoing [%s]: %s %s %dms", session_id, result_payload["status"],
|
||||||
|
result_payload.get("error", ""), result_payload["duration_ms"],
|
||||||
|
)
|
||||||
|
logger.debug("Outgoing payload:\n%s", safe_response_dump(result_payload))
|
||||||
return result_payload
|
return result_payload
|
||||||
+7
-6
@@ -35,15 +35,16 @@ def classify(resp: httpx.Response | Exception) -> str:
|
|||||||
if isinstance(resp, Exception):
|
if isinstance(resp, Exception):
|
||||||
return f"exc:{type(resp).__name__}"
|
return f"exc:{type(resp).__name__}"
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
if "No slots" in str(data.get("error", "")):
|
code = data.get("code")
|
||||||
|
if code == "no_slots":
|
||||||
return "no_slots"
|
return "no_slots"
|
||||||
actions = data.get("actions") or []
|
if code == "challenge_failed":
|
||||||
if not actions:
|
return "cf_blocked"
|
||||||
return f"error:{str(data.get('error'))[:60]}"
|
if code == "action_failed":
|
||||||
last = actions[-1]
|
|
||||||
if not last.get("ok") or last.get("result"):
|
|
||||||
html = data.get("html") or ""
|
html = data.get("html") or ""
|
||||||
return "cf_blocked" if any(m in html for m in CF_MARKERS) else "action_failed"
|
return "cf_blocked" if any(m in html for m in CF_MARKERS) else "action_failed"
|
||||||
|
if data.get("status") != "ok":
|
||||||
|
return f"error:{str(data.get('error'))[:60]}"
|
||||||
return "ok"
|
return "ok"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user