chore: настроен обход антибот защиты

src/antibot/base.py

@@ -16,36 +16,36 @@ from playwright.async_api import TimeoutError as PWTimeoutError
 
 
 async def click_and_race(page, element, challenge_locator, timeout=30_000):
-    nav_task = asyncio.create_task(
+	nav_task = asyncio.create_task(
-        page.wait_for_event(
+		page.wait_for_event(
-            "framenavigated",
+			"framenavigated",
-            predicate=lambda frame: frame == page.main_frame,
+			predicate=lambda frame: frame == page.main_frame,
-            timeout=timeout,
+			timeout=timeout,
-        )
+		)
-    )
+	)
-    challenge_task = asyncio.create_task(
+	challenge_task = asyncio.create_task(
-        challenge_locator.wait_for(state="attached", timeout=timeout)
+		challenge_locator.wait_for(state="attached", timeout=timeout)
-    )
+	)
 
-    await element.click()
+	await element.click()
 
-    done, pending = await asyncio.wait(
+	done, pending = await asyncio.wait(
-        {nav_task, challenge_task}, return_when=asyncio.FIRST_COMPLETED
+		{nav_task, challenge_task}, return_when=asyncio.FIRST_COMPLETED
-    )
+	)
-    for t in pending:
+	for t in pending:
-        t.cancel()
+		t.cancel()
 
-    if nav_task in done and nav_task.exception() is None:
+	if nav_task in done and nav_task.exception() is None:
-        return "navigation"
+		return "navigation"
-    if challenge_task in done and challenge_task.exception() is None:
+	if challenge_task in done and challenge_task.exception() is None:
-        return "challenge"
+		return "challenge"
 
-    # оба протухли по таймауту / или упали с другой ошибкой
+	# оба протухли по таймауту / или упали с другой ошибкой
-    for t in done:
+	for t in done:
-        exc = t.exception()
+		exc = t.exception()
-        if exc and not isinstance(exc, PWTimeoutError):
+		if exc and not isinstance(exc, PWTimeoutError):
-            raise exc
+			raise exc
-    raise PWTimeoutError("Ни навигация, ни challenge-iframe не появились вовремя")
+	raise PWTimeoutError("Ни навигация, ни challenge-iframe не появились вовремя")
 
 
 class Stage(str, Enum):
@@ -97,10 +97,10 @@ class AntiBotProvider:
 
 	async def detect(self, page: Page) -> ChallengeInfo:
 		info = ChallengeInfo(
-			stage = Stage.INTERSTITIAL,
+			stage=Stage.INTERSTITIAL,
-			provider = self.provider,
+			provider=self.provider,
-			success_locator = self._success_locator,
+			success_locator=self._success_locator,
-			detect_locator = self._detect_locator,
+			detect_locator=self._detect_locator,
 		)
 		# обновление info
 		await self.detector.detect(page, info)

src/antibot/known_providers/recaptcha.py

@@ -1,3 +1,4 @@
+import json
 import logging
 
 from playwright.async_api import Page
@@ -23,13 +24,14 @@ class ReCaptchaSolver(Solver):
 	async def solve(self, page: Page, info: ChallengeInfo):
 		log.debug("Начинаю решение recaptcha антибот преграды")
 		if info.stage == Stage.INTERSTITIAL:
+			# TODO: вытащить
 			await page.wait_for_timeout(15)
 			return
 		elif info.stage == Stage.CHECKBOX:
 			log.debug("Нужно тыкнуть чекбокс")
 			checkbox_selector = "#recaptcha-anchor"
 			element = page.frame_locator("iframe[title='reCAPTCHA']").locator(checkbox_selector)
-			interactive_marker_selector = "xpath=//div[@class='g-recaptcha-bubble-arrow'][1]"
+			interactive_marker_selector = "//div[@class='g-recaptcha-bubble-arrow']/..//iframe"
 			challenge_locator = page.locator(interactive_marker_selector)
 			result = await click_and_race(page, element, challenge_locator)
 			log.debug(f"Ожидание завершено: {result}")
@@ -48,24 +50,29 @@ class ReCaptchaSolver(Solver):
 			log.debug("Капча решена. Возбуждаем JS")
 			try:
 				async with page.expect_navigation(wait_until='load'):
-					await page.evaluate("""
+					js = """(token) => {
-	try {
+					    const clients = window.___grecaptcha_cfg && window.___grecaptcha_cfg.clients;
-	const clients = window.___grecaptcha_cfg && window.___grecaptcha_cfg.clients;
+					    if (!clients) return false;
-	for (const cid in clients) {
+					    for (const cid in clients) {
-		for (const key in clients[cid]) {
+					        for (const key in clients[cid]) {
-			const obj = clients[cid][key];
+					            const obj = clients[cid][key];
-			for (const prop in obj) {
+					            for (const prop in obj) {
-				const cb = obj[prop] && obj[prop].callback;
+					                const cb = obj[prop] && obj[prop].callback;
-				if (typeof cb === 'function') { cb(token); return; }
+					                if (typeof cb === 'function') { cb(token); return true; }
-				if (typeof cb === 'string' && typeof window[cb] === 'function') {
+					                if (typeof cb === 'string' && typeof window[cb] === 'function') {
-					window[cb](token); return;
+					                    window[cb](token); return true;
-				}
+					                }
-			}
+					            }
-		}
+					        }
-	}
+					    }
-	} catch (e) {}""", token)
+					    return false;
+					}"""
+					found = await page.evaluate(f"mw:({js})({json.dumps(token)})")
+					assert found, "Recaptcha callback function not found"
 			except Exception as e:
 				log.warning("JS navigation error", exc_info=True)
+			else:
+				self.captcha_solver.recaptcha._clear_cache()
 			log.debug("Все готово. Проверяем еще раз.")
 			return
 

src/captcha/base.py

@@ -1,7 +1,48 @@
+import asyncio
+import json
+import hashlib
+import time
+from pathlib import Path
+
 from playwright.async_api import Page
 from twocaptcha import TwoCaptcha
 
 
+def cached_persistent(file_name=None):
+
+	def key(*args, **kwargs):
+		return hashlib.sha256(f'{str(args)}-{str(kwargs)}'.encode()).hexdigest()
+
+	def wrapper(fun):
+
+		def clear_cache():
+			cache_file.write_text(json.dumps({}))
+
+		cache_file = Path(file_name)
+		if not cache_file.exists():
+			cache_file.write_text(json.dumps({}))
+
+		async def wrapped(*args, **kwargs):
+			data = json.loads(cache_file.read_text())
+			_key = key(fun.__name__, *args, **kwargs)
+
+			if cached_value := data.get(_key):
+				cached_value, cache_time = cached_value
+				if time.time() - cache_time > 300:
+					cached_value = None
+
+			value = cached_value or await fun(*args, **kwargs)
+
+			data[_key] = [value, time.time()]
+			cache_file.write_text(json.dumps(data))
+			return value
+
+		setattr(wrapped, '_clear_cache', clear_cache)
+		return wrapped
+
+	return wrapper
+
+
 class Captcha:
 	ctype = "captcha"
 
@@ -12,9 +53,22 @@ class Captcha:
 	def get_balance(self) -> float:
 		return self._solver.balance()
 
-	async def solve(self, selector: str, page: Page):
+	async def solve(self, selector: str, page: Page) -> str:
 		raise NotImplementedError()
 
+	async def _run(self, fun, /, *args, **kwargs):
+		resp = await asyncio.to_thread(fun, *args, **kwargs)
+		return resp["code"]
+
+	async def hcaptcha(self, url, site_key, is_invisible):
+		resp = await self._run(self._solver.hcaptcha, site_key, url, invisible=is_invisible)
+		return resp
+
+	@cached_persistent('recaptcha')
+	async def recaptcha(self, url, site_key):
+		resp = await self._run(self._solver.recaptcha, site_key, url)
+		return resp
+
 
 def load_captcha(key, captcha_type) -> Captcha:
 	for cls in Captcha.__subclasses__():

src/captcha/hcaptcha.py

@@ -10,6 +10,7 @@ textarea[id="h-captcha-response-0dm47zf0gn8t"]
 
 
 """
+import asyncio
 import logging
 
 from .base import Captcha, Page
@@ -23,12 +24,11 @@ class SolverHCaptcha(Captcha):
 	ctype = "hcaptcha"
 
 	async def solve(self, selector, page: Page):
+		# проверка что элемент с капчей есть
 		div_container = page.locator(selector)
 		await div_container.wait_for()
 
-		if not div_container:
+		# поиск данных и элементов
-			return None
-
 		site_key = await div_container.get_attribute("data-sitekey")
 		is_invisible = bool(await div_container.get_attribute("data-invisible"))
 
@@ -39,7 +39,10 @@ class SolverHCaptcha(Captcha):
 		textarea_g = await page.query_selector(f"#g-recaptcha-response-{textarea_id_postfix}")
 		textarea_h = await page.query_selector(f"#h-captcha-response-{textarea_id_postfix}")
 
+		# решение
 		code = await self.hcaptcha(page.url, site_key, is_invisible)
+
+		# подстановка решения
 		await textarea_g.evaluate((
 			"(el, value) => {"
 			"el.value = value;"
@@ -52,9 +55,4 @@ class SolverHCaptcha(Captcha):
 			"el.dispatchEvent(new Event('input', {bubbles: true}));"
 			"}"
 		), code)
-		return None
+		return code
-
-	async def hcaptcha(self, url, site_key, is_invisible):
-		resp = self._solver.hcaptcha(site_key, url, invisible=is_invisible)
-		logger.debug(f"hcaptcha response: {resp}")
-		return resp["code"]

src/captcha/recaptcha.py

@@ -30,9 +30,6 @@ class SolverReCaptcha(Captcha):
 		div_container = page.locator(selector)
 		await div_container.wait_for()
 
-		if not div_container:
-			return None
-
 		site_key_input_element = page.locator("iframe[title='reCAPTCHA']")
 		src_value = await site_key_input_element.get_attribute("src")
 		url_splitted = urlparse(src_value)
@@ -49,9 +46,4 @@ class SolverReCaptcha(Captcha):
 			"el.dispatchEvent(new Event('input', {bubbles: true}));"
 			"}"
 		), code)
-		return None
+		return code
-
-	async def recaptcha(self, url, site_key):
-		resp = self._solver.recaptcha(site_key, url)
-		logger.debug(f"recaptcha response: {resp}")
-		return resp["code"]

src/main.py

@@ -117,6 +117,7 @@ async def solve(
 		i_know_what_im_doing=True,
 		geoip=True,
 		humanize=True,
+		main_world_eval=True,
 		**ctx_opts,
 	) as camoufo:
 		await camoufo.add_init_script(config["init_script"])