diff --git a/src/antibot/humanize.py b/src/antibot/humanize.py index d7ddd4c..8b2591b 100644 --- a/src/antibot/humanize.py +++ b/src/antibot/humanize.py @@ -31,10 +31,28 @@ page.click()) телепортирует курсор мгновенно и та """ import random import string +from typing import Optional +from playwright.async_api import TimeoutError from playwright.async_api import Page +async def wait_for_networkidle(page: Page, timeout: Optional[int] = None): + """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" + try: + await page.wait_for_load_state("networkidle", timeout=timeout) + except TimeoutError: + pass + + +async def wait_for_page_stability(page: Page, load_dom: bool, network_idle: bool): + await page.wait_for_load_state(state="load") + if load_dom: + await page.wait_for_load_state(state="domcontentloaded") + if network_idle: + await wait_for_networkidle(page) + + async def typing(page: Page, locator: str, text: str): element = page.locator(locator) diff --git a/src/antibot/known_providers/cloudflare.py b/src/antibot/known_providers/cloudflare.py index 60d6a44..fdc8553 100644 --- a/src/antibot/known_providers/cloudflare.py +++ b/src/antibot/known_providers/cloudflare.py @@ -59,13 +59,16 @@ iFrame with checkbox """ import logging +import re import time +from random import randint from playwright.async_api import Page from ..base import Detector, ChallengeInfo, AntiBotProvider, Solver from ..base import Stage from ..registry import register +from ..humanize import wait_for_page_stability, wait_for_networkidle log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -74,15 +77,23 @@ log.addHandler(logging.StreamHandler()) class CloudflareDetector(Detector): + # степени риска, которые Cloudflare сам проставляет в конфиге своего + # челлендж-скрипта на странице (см. `cType: '...'` в html/js) — + # по нему и определяем стадию, а не по факту наличия виджета на глаз + _CTYPES = ("non-interactive", "managed", "interactive") + async def detect(self, page: Page, info: ChallengeInfo): log.debug("CloudflareDetector run") + await wait_for_page_stability(page, True, True) + info.meta = {} - checkbox_selector = "иногда надо тыкнуть, сейчас этого нет" + widget_selector = "div.cf-turnstile[data-sitekey], div[data-sitekey].cf-turnstile" marker_selector = info.detect_locator or "//input[@name='cf-turnstile-response']" info.detect_locator = marker_selector log.debug(f"ищем {marker_selector} успех если есть {info.success_locator}") - if await page.locator(info.detect_locator).count() == 0: + + if not await page.query_selector(marker_selector): log.debug("на странице нет маркера преграды") if info.success_locator: info.success = bool(await page.query_selector(info.success_locator)) @@ -96,19 +107,67 @@ class CloudflareDetector(Detector): log.debug("Решение не требуется") return - info.stage = Stage.INTERSTITIAL - return - if await page.query_selector(checkbox_selector): + content = await page.content() + cf_type = next((t for t in self._CTYPES if f"cType: '{t}'" in content), None) + info.meta["cf_type"] = cf_type + log.debug(f"cType со страницы: {cf_type!r}") + + widget = await page.query_selector(widget_selector) + + if cf_type == "non-interactive": + # фулл-пейдж интерстишл без чекбокса — просто ждать, пока пропадёт + # или эскалирует в managed/interactive (тогда cf_type сменится на + # следующем detect()) + info.stage = Stage.INTERSTITIAL + elif cf_type in ("managed", "interactive"): + # CF уже решил, что нужен клик по виджету внутри своего iframe + # (challenges.cloudflare.com) — сам клик этот проект не решает info.stage = Stage.CHECKBOX + info.meta["requires_click"] = True + elif widget: + # обычный встроенный виджет на странице (без cType, просто + #
) — решаем через + # captcha_solver по токену, см. CloudflareSolver + info.stage = Stage.CHECKBOX + info.meta["container_selector"] = widget_selector else: info.stage = Stage.INTERSTITIAL +__CF_PATTERN__ = re.compile(r"^https://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*") + + class CloudflareSolver(Solver): async def solve(self, page: Page, info: ChallengeInfo): if info.stage == Stage.CHECKBOX: - raise NotImplementedError + iframe = page.frame(url=__CF_PATTERN__) + if iframe: + frame_el = await iframe.frame_element() + box = await frame_el.bounding_box() + else: + box = await page.locator("//input[@name='cf-turnstile-response]").bounding_box() + # Calculate the Captcha coordinates for any viewport + captcha_x, captcha_y = box["x"] + randint(26, 28), box["y"] + randint(25, 27) + # Move the mouse to the center of the window, then press and hold the left mouse button + + # TODO: не кликает! + await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left") + await wait_for_networkidle(page) + + if info.success_locator: + try: + await page.wait_for_selector(info.success_locator) + except: + pass + finally: + return + else: + deadline = time.time() + 10 + while time.time() < deadline: + if not await page.query_selector(info.detect_locator): + return + return elif info.stage == Stage.INTERSTITIAL: if info.success_locator: try: @@ -118,13 +177,17 @@ class CloudflareSolver(Solver): else: return else: - deadline = time.time() + 30 + deadline = time.time() + 10 while time.time() < deadline: if not await page.query_selector(info.detect_locator): return return else: - raise NotImplementedError + captcha_solver = self._opt.get("captcha_solver") + if not captcha_solver: + raise RuntimeError("Нужен captcha_solver (2captcha) чтобы решить Turnstile") + await captcha_solver.solve(info.meta["container_selector"], page) + # TODO: возбудить js @register diff --git a/src/antibot/orchestrator.py b/src/antibot/orchestrator.py index 51388bd..d3ccad8 100644 --- a/src/antibot/orchestrator.py +++ b/src/antibot/orchestrator.py @@ -34,19 +34,17 @@ async def pass_challenges( deadline = time.monotonic() + timeout_ms / 1000 # 1. определить кто блокирует - try: - if captcha: - captcha_solver = load_captcha(captcha, antibot_name) - else: - captcha_solver = None - provider = await select_one( - page, antibot_name, - captcha_solver=captcha_solver, - success_locator=success_locator, - detect_locator=detect_locator, - ) - except ProviderNotDetected: - return ChallengeOutcome(False, None, None, "Provider not detected") + if captcha: + captcha_solver = load_captcha(captcha, antibot_name) + else: + captcha_solver = None + + provider = await select_one( + page, antibot_name, + captcha_solver=captcha_solver, + success_locator=success_locator, + detect_locator=detect_locator, + ) info = None while time.monotonic() < deadline: diff --git a/src/antibot/stages/__init__.py b/src/antibot/stages/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/antibot/stages/checkbox.py b/src/antibot/stages/checkbox.py deleted file mode 100644 index ebf3ddd..0000000 --- a/src/antibot/stages/checkbox.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -стадия B: универсальный click-solver + сигнатуры провайдеров -""" - -from ..registry import detect_any -from ..base import Stage - - -async def solve_checkbox(page, info, max_attempts=3): - for attempt in range(max_attempts): - frame = page.locator(page, info.frame_locator) - - - - # после клика — либо чисто, либо эскалация в картинки (стадия C) - if await detect_any(page, Stage.CHECKBOX) is None: - return True - if await detect_any(page, Stage.INTERACTIVE): - return False # передаём выше, пусть оркестратор переключит стадию - return False diff --git a/src/antibot/stages/interactive.py b/src/antibot/stages/interactive.py deleted file mode 100644 index 6f6418f..0000000 --- a/src/antibot/stages/interactive.py +++ /dev/null @@ -1,4 +0,0 @@ -# стадия C: мост к src/captcha (hcaptcha/recaptcha/...) -async def solve_interactive(page, info, captcha): - await captcha.solve(info.frame_locator or info.meta["container_selector"], page) - return True diff --git a/src/antibot/stages/interstitial.py b/src/antibot/stages/interstitial.py deleted file mode 100644 index 01d1766..0000000 --- a/src/antibot/stages/interstitial.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -стадия A: универсальный wait-solver + сигнатуры провайдеров -""" -from ..base import Stage -from ..registry import detect_any - - -async def solve_interstitial(page, info, timeout_ms=15000, poll_ms=500): - elapsed = 0 - while elapsed < timeout_ms: - if await detect_any(page, Stage.INTERSTITIAL) is None: - return True - await page.wait_for_timeout(poll_ms) - elapsed += poll_ms - return False diff --git a/src/captcha/__init__.py b/src/captcha/__init__.py index 03766fb..5b890c5 100644 --- a/src/captcha/__init__.py +++ b/src/captcha/__init__.py @@ -1,3 +1,4 @@ from .hcaptcha import SolverHCaptcha from .recaptcha import SolverReCaptcha +from .turnstile import SolverTurnstile from .base import load_captcha diff --git a/src/captcha/base.py b/src/captcha/base.py index cef68aa..73bee1c 100644 --- a/src/captcha/base.py +++ b/src/captcha/base.py @@ -68,6 +68,10 @@ class Captcha: resp = await self._run(self._solver.recaptcha, site_key, url) return resp + async def turnstile(self, url, site_key): + resp = await self._run(self._solver.turnstile, site_key, url) + return resp + def load_captcha(key, captcha_type) -> Captcha: for cls in Captcha.__subclasses__(): diff --git a/src/captcha/turnstile.py b/src/captcha/turnstile.py new file mode 100644 index 0000000..e087c26 --- /dev/null +++ b/src/captcha/turnstile.py @@ -0,0 +1,44 @@ +""" + +
+
+
+ +
+
+ +Кликабельный чекбокс Cloudflare рендерит в закрытом shadow DOM — туда лезть +не нужно и бессмысленно. sitekey и поле для ответа лежат в обычном DOM +контейнера, поэтому решаем через API 2captcha и просто подставляем токен. +""" +import logging + +from .base import Captcha, Page + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.StreamHandler()) + + +class SolverTurnstile(Captcha): + ctype = "cloudflare" + + async def solve(self, selector, page: Page): + container = page.locator(selector) + await container.wait_for() + + site_key = await container.get_attribute("data-sitekey") + + response_input = container.locator('input[name="cf-turnstile-response"]') + await response_input.wait_for(state="attached") + + code = await self.turnstile(page.url, site_key) + + await response_input.evaluate(( + "(el, value) => {" + "el.value = value;" + "el.dispatchEvent(new Event('input', {bubbles: true}));" + "el.dispatchEvent(new Event('change', {bubbles: true}));" + "}" + ), code) + return code diff --git a/src/engine/actions.py b/src/engine/actions.py index e9bfcbd..cefe961 100644 --- a/src/engine/actions.py +++ b/src/engine/actions.py @@ -2,6 +2,7 @@ import asyncio from playwright.async_api import Page +from antibot.base import click_and_race from antibot.orchestrator import pass_challenges from captcha import load_captcha from engine.schemas import ActionRequest @@ -50,6 +51,18 @@ async def click(page: Page, act: ActionRequest, **kwargs) -> str | None: await element.click() +@register +async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None: + assert act.selector + element = page.locator(act.selector) + + async with page.expect_event( + "framenavigated", + predicate=lambda frame: frame == page.main_frame, + ): + await element.click() + + @register async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None: assert act.selector @@ -61,7 +74,10 @@ async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None: async def wait_for(page: Page, act: ActionRequest, **kwargs) -> str | None: assert act.selector assert 1 <= act.timeout <= 7200 - await page.wait_for_selector(act.selector, timeout=act.timeout * 1000) + try: + await page.wait_for_selector(act.selector, timeout=act.timeout * 1000) + except Exception as e: + return str(e) @register @@ -70,12 +86,13 @@ async def pass_challenge( antibot_name, success_locator=None, captcha_key=None, detect_locator=None, **kwargs ) -> str | None: - await pass_challenges( + result = await pass_challenges( antibot_name, page, captcha=captcha_key, success_locator=act.selector or success_locator, detect_locator=detect_locator ) + return result.reason async def execute( @@ -102,6 +119,7 @@ async def execute( "type": act.type, } try: + # TODO: добавить время выполнения result = await _actions[act.type](page, act, **ctx) except Exception as e: new_line["ok"] = False diff --git a/src/main.py b/src/main.py index 0847576..257e451 100644 --- a/src/main.py +++ b/src/main.py @@ -82,7 +82,7 @@ async def solve( ): api_key = request.headers.get('X-API-Key') - logger.info("Incoming payload: [%s] %s", session_id, payload) + logger.info("Incoming payload: [%s]\n%s", session_id, payload.model_dump_json(indent=2, ensure_ascii=True)) global BROWSER_SLOTS if BROWSER_SLOTS <= 0: @@ -159,5 +159,5 @@ async def solve( result["html"] = html if error: result["error"] = error - logger.info("Outgoing payload: %s", result) + logger.info("Outgoing payload:\n%s", json.dumps(result, indent=2, sort_keys=True, ensure_ascii=True)) return result