feature!: добавлен обход claudflare

не кликает!

src/antibot/humanize.py

@@ -31,10 +31,28 @@ page.click()) телепортирует курсор мгновенно и та
 """
 import random
 import string
+from typing import Optional
 
+from playwright.async_api import TimeoutError
 from playwright.async_api import Page
 
 
+async def wait_for_networkidle(page: Page, timeout: Optional[int] = None):
+	"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""
+	try:
+		await page.wait_for_load_state("networkidle", timeout=timeout)
+	except TimeoutError:
+		pass
+
+
+async def wait_for_page_stability(page: Page, load_dom: bool, network_idle: bool):
+	await page.wait_for_load_state(state="load")
+	if load_dom:
+		await page.wait_for_load_state(state="domcontentloaded")
+	if network_idle:
+		await wait_for_networkidle(page)
+
+
 async def typing(page: Page, locator: str, text: str):
 	element = page.locator(locator)
 

src/antibot/known_providers/cloudflare.py

@@ -59,13 +59,16 @@ iFrame with checkbox
 </div>
 """
 import logging
+import re
 import time
+from random import randint
 
 from playwright.async_api import Page
 
 from ..base import Detector, ChallengeInfo, AntiBotProvider, Solver
 from ..base import Stage
 from ..registry import register
+from ..humanize import wait_for_page_stability, wait_for_networkidle
 
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
@@ -74,15 +77,23 @@ log.addHandler(logging.StreamHandler())
 
 class CloudflareDetector(Detector):
 
+	# степени риска, которые Cloudflare сам проставляет в конфиге своего
+	# челлендж-скрипта на странице (см. `cType: '...'` в html/js) —
+	# по нему и определяем стадию, а не по факту наличия виджета на глаз
+	_CTYPES = ("non-interactive", "managed", "interactive")
+
 	async def detect(self, page: Page, info: ChallengeInfo):
 		log.debug("CloudflareDetector run")
+		await wait_for_page_stability(page, True, True)
+
 		info.meta = {}
-		checkbox_selector = "иногда надо тыкнуть, сейчас этого нет"
+		widget_selector = "div.cf-turnstile[data-sitekey], div[data-sitekey].cf-turnstile"
 		marker_selector = info.detect_locator or "//input[@name='cf-turnstile-response']"
 		info.detect_locator = marker_selector
 
 		log.debug(f"ищем {marker_selector} успех если есть {info.success_locator}")
-		if await page.locator(info.detect_locator).count() == 0:
+
+		if not await page.query_selector(marker_selector):
 			log.debug("на странице нет маркера преграды")
 			if info.success_locator:
 				info.success = bool(await page.query_selector(info.success_locator))
@@ -96,19 +107,67 @@ class CloudflareDetector(Detector):
 			log.debug("Решение не требуется")
 			return
 
-		info.stage = Stage.INTERSTITIAL
+		content = await page.content()
-		return
+		cf_type = next((t for t in self._CTYPES if f"cType: '{t}'" in content), None)
-		if await page.query_selector(checkbox_selector):
+		info.meta["cf_type"] = cf_type
+		log.debug(f"cType со страницы: {cf_type!r}")
+
+		widget = await page.query_selector(widget_selector)
+
+		if cf_type == "non-interactive":
+			# фулл-пейдж интерстишл без чекбокса — просто ждать, пока пропадёт
+			# или эскалирует в managed/interactive (тогда cf_type сменится на
+			# следующем detect())
+			info.stage = Stage.INTERSTITIAL
+		elif cf_type in ("managed", "interactive"):
+			# CF уже решил, что нужен клик по виджету внутри своего iframe
+			# (challenges.cloudflare.com) — сам клик этот проект не решает
 			info.stage = Stage.CHECKBOX
+			info.meta["requires_click"] = True
+		elif widget:
+			# обычный встроенный виджет на странице (без cType, просто
+			# <div class="cf-turnstile" data-sitekey="...">) — решаем через
+			# captcha_solver по токену, см. CloudflareSolver
+			info.stage = Stage.CHECKBOX
+			info.meta["container_selector"] = widget_selector
 		else:
 			info.stage = Stage.INTERSTITIAL
 
 
+__CF_PATTERN__ = re.compile(r"^https://challenges\.cloudflare\.com/cdn-cgi/challenge-platform/.*")
+
+
 class CloudflareSolver(Solver):
 
 	async def solve(self, page: Page, info: ChallengeInfo):
 		if info.stage == Stage.CHECKBOX:
-			raise NotImplementedError
+			iframe = page.frame(url=__CF_PATTERN__)
+			if iframe:
+				frame_el = await iframe.frame_element()
+				box = await frame_el.bounding_box()
+			else:
+				box = await page.locator("//input[@name='cf-turnstile-response]").bounding_box()
+			# Calculate the Captcha coordinates for any viewport
+			captcha_x, captcha_y = box["x"] + randint(26, 28), box["y"] + randint(25, 27)
+			# Move the mouse to the center of the window, then press and hold the left mouse button
+
+			# TODO: не кликает!
+			await page.mouse.click(captcha_x, captcha_y, delay=randint(100, 200), button="left")
+			await wait_for_networkidle(page)
+
+			if info.success_locator:
+				try:
+					await page.wait_for_selector(info.success_locator)
+				except:
+					pass
+				finally:
+					return
+			else:
+				deadline = time.time() + 10
+				while time.time() < deadline:
+					if not await page.query_selector(info.detect_locator):
+						return
+			return
 		elif info.stage == Stage.INTERSTITIAL:
 			if info.success_locator:
 				try:
@@ -118,13 +177,17 @@ class CloudflareSolver(Solver):
 				else:
 					return
 			else:
-				deadline = time.time() + 30
+				deadline = time.time() + 10
 				while time.time() < deadline:
 					if not await page.query_selector(info.detect_locator):
 						return
 				return
 		else:
-			raise NotImplementedError
+			captcha_solver = self._opt.get("captcha_solver")
+			if not captcha_solver:
+				raise RuntimeError("Нужен captcha_solver (2captcha) чтобы решить Turnstile")
+			await captcha_solver.solve(info.meta["container_selector"], page)
+			# TODO: возбудить js
 
 
 @register

src/antibot/orchestrator.py

@@ -34,19 +34,17 @@ async def pass_challenges(
 	deadline = time.monotonic() + timeout_ms / 1000
 
 	# 1. определить кто блокирует
-	try:
+	if captcha:
-		if captcha:
+		captcha_solver = load_captcha(captcha, antibot_name)
-			captcha_solver = load_captcha(captcha, antibot_name)
+	else:
-		else:
+		captcha_solver = None
-			captcha_solver = None
+
-		provider = await select_one(
+	provider = await select_one(
-			page, antibot_name,
+		page, antibot_name,
-			captcha_solver=captcha_solver,
+		captcha_solver=captcha_solver,
-			success_locator=success_locator,
+		success_locator=success_locator,
-			detect_locator=detect_locator,
+		detect_locator=detect_locator,
-		)
+	)
-	except ProviderNotDetected:
-		return ChallengeOutcome(False, None, None, "Provider not detected")
 
 	info = None
 	while time.monotonic() < deadline:

src/antibot/stages/checkbox.py

@@ -1,20 +0,0 @@
-"""
-стадия B: универсальный click-solver + сигнатуры провайдеров
-"""
-
-from ..registry import detect_any
-from ..base import Stage
-
-
-async def solve_checkbox(page, info, max_attempts=3):
-	for attempt in range(max_attempts):
-		frame = page.locator(page, info.frame_locator)
-
-
-
-		# после клика — либо чисто, либо эскалация в картинки (стадия C)
-		if await detect_any(page, Stage.CHECKBOX) is None:
-			return True
-		if await detect_any(page, Stage.INTERACTIVE):
-			return False  # передаём выше, пусть оркестратор переключит стадию
-	return False

src/antibot/stages/interactive.py

@@ -1,4 +0,0 @@
-# стадия C: мост к src/captcha (hcaptcha/recaptcha/...)
-async def solve_interactive(page, info, captcha):
-	await captcha.solve(info.frame_locator or info.meta["container_selector"], page)
-	return True

src/antibot/stages/interstitial.py

@@ -1,15 +0,0 @@
-"""
-стадия A: универсальный wait-solver + сигнатуры провайдеров
-"""
-from ..base import Stage
-from ..registry import detect_any
-
-
-async def solve_interstitial(page, info, timeout_ms=15000, poll_ms=500):
-	elapsed = 0
-	while elapsed < timeout_ms:
-		if await detect_any(page, Stage.INTERSTITIAL) is None:
-			return True
-		await page.wait_for_timeout(poll_ms)
-		elapsed += poll_ms
-	return False

src/captcha/__init__.py

@@ -1,3 +1,4 @@
 from .hcaptcha import SolverHCaptcha
 from .recaptcha import SolverReCaptcha
+from .turnstile import SolverTurnstile
 from .base import load_captcha

src/captcha/base.py

@@ -68,6 +68,10 @@ class Captcha:
 		resp = await self._run(self._solver.recaptcha, site_key, url)
 		return resp
 
+	async def turnstile(self, url, site_key):
+		resp = await self._run(self._solver.turnstile, site_key, url)
+		return resp
+
 
 def load_captcha(key, captcha_type) -> Captcha:
 	for cls in Captcha.__subclasses__():

src/captcha/turnstile.py

@@ -0,0 +1,44 @@
+"""
+
+<div id="cf-turnstile" class="cf-turnstile" data-sitekey="3x00000000000000000000FF">
+  <div>
+    <div></div>
+    <input type="hidden" name="cf-turnstile-response" id="cf-chl-widget-a3ptc_response">
+  </div>
+</div>
+
+Кликабельный чекбокс Cloudflare рендерит в закрытом shadow DOM — туда лезть
+не нужно и бессмысленно. sitekey и поле для ответа лежат в обычном DOM
+контейнера, поэтому решаем через API 2captcha и просто подставляем токен.
+"""
+import logging
+
+from .base import Captcha, Page
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+logger.addHandler(logging.StreamHandler())
+
+
+class SolverTurnstile(Captcha):
+	ctype = "cloudflare"
+
+	async def solve(self, selector, page: Page):
+		container = page.locator(selector)
+		await container.wait_for()
+
+		site_key = await container.get_attribute("data-sitekey")
+
+		response_input = container.locator('input[name="cf-turnstile-response"]')
+		await response_input.wait_for(state="attached")
+
+		code = await self.turnstile(page.url, site_key)
+
+		await response_input.evaluate((
+			"(el, value) => {"
+			"el.value = value;"
+			"el.dispatchEvent(new Event('input', {bubbles: true}));"
+			"el.dispatchEvent(new Event('change', {bubbles: true}));"
+			"}"
+		), code)
+		return code

src/engine/actions.py

@@ -2,6 +2,7 @@ import asyncio
 
 from playwright.async_api import Page
 
+from antibot.base import click_and_race
 from antibot.orchestrator import pass_challenges
 from captcha import load_captcha
 from engine.schemas import ActionRequest
@@ -50,6 +51,18 @@ async def click(page: Page, act: ActionRequest, **kwargs) -> str | None:
 	await element.click()
 
 
+@register
+async def submit(page: Page, act: ActionRequest, **kwargs) -> str | None:
+	assert act.selector
+	element = page.locator(act.selector)
+
+	async with page.expect_event(
+		"framenavigated",
+		predicate=lambda frame: frame == page.main_frame,
+	):
+		await element.click()
+
+
 @register
 async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None:
 	assert act.selector
@@ -61,7 +74,10 @@ async def inner_html(page: Page, act: ActionRequest, **kwargs) -> str | None:
 async def wait_for(page: Page, act: ActionRequest, **kwargs) -> str | None:
 	assert act.selector
 	assert 1 <=  act.timeout <= 7200
-	await page.wait_for_selector(act.selector, timeout=act.timeout * 1000)
+	try:
+		await page.wait_for_selector(act.selector, timeout=act.timeout * 1000)
+	except Exception as e:
+		return str(e)
 
 
 @register
@@ -70,12 +86,13 @@ async def pass_challenge(
 	antibot_name, success_locator=None, captcha_key=None, detect_locator=None,
 	**kwargs
 ) -> str | None:
-	await pass_challenges(
+	result = await pass_challenges(
 		antibot_name, page,
 		captcha=captcha_key,
 		success_locator=act.selector or success_locator,
 		detect_locator=detect_locator
 	)
+	return result.reason
 
 
 async def execute(
@@ -102,6 +119,7 @@ async def execute(
 			"type": act.type,
 		}
 		try:
+			# TODO: добавить время выполнения
 			result = await _actions[act.type](page, act, **ctx)
 		except Exception as e:
 			new_line["ok"] = False

src/main.py

@@ -82,7 +82,7 @@ async def solve(
 ):
 	api_key = request.headers.get('X-API-Key')
 
-	logger.info("Incoming payload: [%s] %s", session_id, payload)
+	logger.info("Incoming payload: [%s]\n%s", session_id, payload.model_dump_json(indent=2, ensure_ascii=True))
 	global BROWSER_SLOTS
 
 	if BROWSER_SLOTS <= 0:
@@ -159,5 +159,5 @@ async def solve(
 		result["html"] = html
 	if error:
 		result["error"] = error
-	logger.info("Outgoing payload: %s", result)
+	logger.info("Outgoing payload:\n%s", json.dumps(result, indent=2, sort_keys=True, ensure_ascii=True))
 	return result