protokeyandClaude Opus 5.5
7b511d5d38
- Сервис api (FastAPI, без Chrome): POST /v2/tasks, GET /v2/tasks/{id} (доступ по cookie
сессии), DELETE /v2/sessions/current; /v1/solve — прежний контракт поверх очереди.
- Сервис worker (dramatiq + RabbitMQ): слоты браузера, окна сессий между задачами
(browser_idle_timeout 0–120 мин, простой — TTL ключа в Redis), heartbeat.
- Маршрутизация задач сессии в персональную очередь воркера с открытым окном.
- Redis: задачи, результаты (RESULT_TTL_HOURS), окна, воркеры. Прокси и ключ 2captcha
в Redis не сохраняются.
- Движок: AsyncStealthySession.run() — действия на текущей странице без навигации.
- Два образа (robotex-api, robotex-worker), compose с rabbitmq и redis,
make up/down/logs/smoke, релиз собирает все образы.
- Тесты: api + worker в одном процессе (fakeredis), настоящий воркер dramatiq на
StubBroker, браузерные тесты на новой схеме, smoke для запущенного стенда.
- Документация: README, API (v2), Deployment, Usage.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
219 lines
12 KiB
Python
219 lines
12 KiB
Python
"""
|
||
Stealthy-браузерная сессия (persistent context + пул страниц) — портировано и урезано из
|
||
scrapling.engines._browsers._stealth.AsyncStealthySession.
|
||
|
||
Отличия от апстрима:
|
||
- Только async-версия (проект целиком async).
|
||
- `fetch()` не строит `scrapling.Response` (это тянет `scrapling.parser.Selector` — тяжёлый
|
||
HTML-парсер, который в проекте не используется: html/cookies/user_agent достаются вручную
|
||
через сырой Playwright API). Вместо этого, если передан `page_action`, `fetch()` возвращает
|
||
РЕЗУЛЬТАТ `page_action(page)` напрямую (в апстриме он отбрасывается) — так `page_action`
|
||
становится единственной точкой, где происходит выполнение action-очереди
|
||
(`api.actions.execute`) и извлечение html/cookies/user_agent для ответа API. Если
|
||
`page_action` не передан, возвращается сам `page`.
|
||
- Без решения Cloudflare-челленджей (`_cloudflare_solver`/`solve_cloudflare`) — в проекте это
|
||
делает `antibot.orchestrator.pass_challenges`, вызываемый из `page_action`.
|
||
"""
|
||
import logging
|
||
from asyncio import sleep as asyncio_sleep
|
||
|
||
from playwright.async_api import Page, Locator
|
||
from patchright.async_api import async_playwright
|
||
from typing_extensions import Unpack
|
||
|
||
from engine.page_pool import PageInfo
|
||
from engine.proxy_rotation import is_proxy_error
|
||
from engine.session import AsyncSession, StealthySessionMixin
|
||
from engine.session_types import StealthSession, StealthFetchParams
|
||
from engine.validators import validate_fetch as _validate, StealthConfig
|
||
from engine._types import Any, List, Optional, ProxyType
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
|
||
class AsyncStealthySession(AsyncSession, StealthySessionMixin):
|
||
"""An async Stealthy Browser session manager with page pooling."""
|
||
|
||
__slots__ = (
|
||
"_config",
|
||
"_context_options",
|
||
"_browser_options",
|
||
"_user_data_dir",
|
||
"_headers_keys",
|
||
)
|
||
|
||
def __init__(self, **kwargs: Unpack[StealthSession]):
|
||
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
|
||
|
||
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched.
|
||
:param useragent: Pass a useragent string to be used. Otherwise the browser's own default is used.
|
||
:param cookies: Set cookies for the next request.
|
||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
||
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning.
|
||
:param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need
|
||
(in this project: `api.actions.execute`). Its return value is what `fetch()` returns.
|
||
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
||
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc.
|
||
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
||
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
||
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
||
:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
|
||
:param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely.
|
||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
||
:param google_search: Enabled by default, Scrapling will set a Google referer header.
|
||
:param extra_headers: A dictionary of extra headers to add to the request.
|
||
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
||
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
|
||
:param extra_flags: A list of additional browser flags to pass to the browser on launch.
|
||
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than the settings above.
|
||
"""
|
||
self.__validate__(**kwargs)
|
||
super().__init__(max_pages=self._config.max_pages)
|
||
|
||
async def start(self) -> None:
|
||
"""Create a browser for this instance and context."""
|
||
if not self.playwright:
|
||
self.playwright = await async_playwright().start()
|
||
try:
|
||
if self._config.cdp_url:
|
||
self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
|
||
if not self._config.proxy_rotator:
|
||
assert self.browser is not None
|
||
self.context = await self.browser.new_context(**self._context_options)
|
||
elif self._config.proxy_rotator:
|
||
self.browser = await self.playwright.chromium.launch(**self._browser_options)
|
||
else:
|
||
persistent_options = (
|
||
self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}
|
||
)
|
||
self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)
|
||
|
||
if self.context:
|
||
self.context = await self._initialize_context(self._config, self.context)
|
||
|
||
self._is_alive = True
|
||
except Exception:
|
||
# Clean up playwright if browser setup fails
|
||
await self.playwright.stop()
|
||
self.playwright = None
|
||
raise
|
||
else:
|
||
raise RuntimeError("Session has been already started")
|
||
|
||
async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Any:
|
||
"""Opens up the browser and navigates to `url`.
|
||
|
||
If `page_action` is given, returns its return value (see class docstring). Otherwise returns
|
||
the Playwright `page` navigated to `url`, still open (belongs to the session's page pool/context).
|
||
|
||
:param url: The Target url.
|
||
:param google_search: Enabled by default, sets a Google referer header.
|
||
:param timeout: The timeout in milliseconds used in all operations and waits through the page.
|
||
:param wait: The time (milliseconds) to wait after everything finishes before returning.
|
||
:param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need. Its return value is propagated as `fetch()`'s return value.
|
||
:param page_setup: A function that takes the `page` object, runs before navigation.
|
||
:param extra_headers: A dictionary of extra headers to add to the request.
|
||
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
||
:param blocked_domains: A set of domain names to block requests to.
|
||
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
||
:param wait_selector_state: The state to wait for the selector given with `wait_selector`.
|
||
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
||
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
||
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
|
||
:return: `page_action`'s return value, or the navigated `page` if no `page_action` was given.
|
||
"""
|
||
static_proxy = kwargs.pop("proxy", None)
|
||
|
||
params = _validate(kwargs, self, StealthConfig)
|
||
if not self._is_alive: # pragma: no cover
|
||
raise RuntimeError("Context manager has been closed")
|
||
|
||
request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
|
||
referer = (
|
||
"https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
|
||
)
|
||
|
||
for attempt in range(self._config.retries):
|
||
proxy: Optional[ProxyType] = None
|
||
if self._config.proxy_rotator and static_proxy is None:
|
||
proxy = self._config.proxy_rotator.get_proxy()
|
||
else:
|
||
proxy = static_proxy
|
||
|
||
async with self._page_generator(
|
||
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
||
) as page_info:
|
||
page_info: PageInfo
|
||
page: Page = page_info.page
|
||
if params.page_setup:
|
||
try:
|
||
await params.page_setup(page)
|
||
except Exception as e: # pragma: no cover
|
||
log.error(f"Error executing page_setup: {e}")
|
||
|
||
try:
|
||
first_response = await page.goto(url, referer=referer)
|
||
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
||
|
||
if not first_response:
|
||
raise RuntimeError(f"Failed to get response for {url}")
|
||
|
||
result = await params.page_action(page) if params.page_action else None
|
||
|
||
if params.wait_selector:
|
||
try:
|
||
waiter: Locator = page.locator(params.wait_selector)
|
||
await waiter.first.wait_for(state=params.wait_selector_state)
|
||
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
||
except Exception as e: # pragma: no cover
|
||
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
|
||
|
||
await page.wait_for_timeout(params.wait)
|
||
|
||
return result if params.page_action else page
|
||
|
||
except Exception as e:
|
||
page_info.mark_error()
|
||
if attempt < self._config.retries - 1:
|
||
if is_proxy_error(e):
|
||
log.warning(
|
||
f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..."
|
||
)
|
||
else:
|
||
log.warning(
|
||
f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..."
|
||
)
|
||
await asyncio_sleep(self._config.retry_delay)
|
||
else:
|
||
log.error(f"Failed after {self._config.retries} attempts: {e}")
|
||
raise
|
||
|
||
raise RuntimeError("Request failed") # pragma: no cover
|
||
|
||
async def run(self, page_action, **kwargs: Unpack[StealthFetchParams]) -> Any:
|
||
"""Выполнить `page_action` на текущей странице сессии, без навигации — для окна, удерживаемого
|
||
между задачами (#82). Страница после `fetch()` остаётся в пуле открытой, её и берём.
|
||
|
||
:param page_action: функция от `page`; её результат возвращается.
|
||
:param timeout / disable_resources / blocked_domains / extra_headers: как у `fetch()`.
|
||
"""
|
||
params = _validate(kwargs, self, StealthConfig)
|
||
if not self._is_alive: # pragma: no cover
|
||
raise RuntimeError("Context manager has been closed")
|
||
|
||
async with self._page_generator(
|
||
params.timeout, params.extra_headers, params.disable_resources, None, params.blocked_domains
|
||
) as page_info:
|
||
try:
|
||
return await page_action(page_info.page)
|
||
except Exception:
|
||
page_info.mark_error()
|
||
raise
|