""" Stealthy-браузерная сессия (persistent context + пул страниц) — портировано и урезано из scrapling.engines._browsers._stealth.AsyncStealthySession. Отличия от апстрима: - Только async-версия (проект целиком async). - `fetch()` не строит `scrapling.Response` (это тянет `scrapling.parser.Selector` — тяжёлый HTML-парсер, который в проекте не используется: html/cookies/user_agent достаются вручную через сырой Playwright API). Вместо этого, если передан `page_action`, `fetch()` возвращает РЕЗУЛЬТАТ `page_action(page)` напрямую (в апстриме он отбрасывается) — так `page_action` становится единственной точкой, где происходит выполнение action-очереди (`api.actions.execute`) и извлечение html/cookies/user_agent для ответа API. Если `page_action` не передан, возвращается сам `page`. - Без решения Cloudflare-челленджей (`_cloudflare_solver`/`solve_cloudflare`) — в проекте это делает `antibot.orchestrator.pass_challenges`, вызываемый из `page_action`. """ import logging from asyncio import sleep as asyncio_sleep from playwright.async_api import Page, Locator from patchright.async_api import async_playwright from typing_extensions import Unpack from engine.page_pool import PageInfo from engine.proxy_rotation import is_proxy_error from engine.session import AsyncSession, StealthySessionMixin from engine.session_types import StealthSession, StealthFetchParams from engine.validators import validate_fetch as _validate, StealthConfig from engine._types import Any, List, Optional, ProxyType log = logging.getLogger(__name__) class AsyncStealthySession(AsyncSession, StealthySessionMixin): """An async Stealthy Browser session manager with page pooling.""" __slots__ = ( "_config", "_context_options", "_browser_options", "_user_data_dir", "_headers_keys", ) def __init__(self, **kwargs: Unpack[StealthSession]): """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. :param headless: Run the browser in headless/hidden (default), or headful/visible mode. :param disable_resources: Drop requests for unnecessary resources for a speed boost. :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched. :param useragent: Pass a useragent string to be used. Otherwise the browser's own default is used. :param cookies: Set cookies for the next request. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning. :param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need (in this project: `api.actions.execute`). Its return value is what `fetch()` returns. :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. :param google_search: Enabled by default, Scrapling will set a Google referer header. :param extra_headers: A dictionary of extra headers to add to the request. :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. :param extra_flags: A list of additional browser flags to pass to the browser on launch. :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than the settings above. """ self.__validate__(**kwargs) super().__init__(max_pages=self._config.max_pages) async def start(self) -> None: """Create a browser for this instance and context.""" if not self.playwright: self.playwright = await async_playwright().start() try: if self._config.cdp_url: self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) if not self._config.proxy_rotator: assert self.browser is not None self.context = await self.browser.new_context(**self._context_options) elif self._config.proxy_rotator: self.browser = await self.playwright.chromium.launch(**self._browser_options) else: persistent_options = ( self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir} ) self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options) if self.context: self.context = await self._initialize_context(self._config, self.context) self._is_alive = True except Exception: # Clean up playwright if browser setup fails await self.playwright.stop() self.playwright = None raise else: raise RuntimeError("Session has been already started") async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Any: """Opens up the browser and navigates to `url`. If `page_action` is given, returns its return value (see class docstring). Otherwise returns the Playwright `page` navigated to `url`, still open (belongs to the session's page pool/context). :param url: The Target url. :param google_search: Enabled by default, sets a Google referer header. :param timeout: The timeout in milliseconds used in all operations and waits through the page. :param wait: The time (milliseconds) to wait after everything finishes before returning. :param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need. Its return value is propagated as `fetch()`'s return value. :param page_setup: A function that takes the `page` object, runs before navigation. :param extra_headers: A dictionary of extra headers to add to the request. :param disable_resources: Drop requests for unnecessary resources for a speed boost. :param blocked_domains: A set of domain names to block requests to. :param wait_selector: Wait for a specific CSS selector to be in a specific state. :param wait_selector_state: The state to wait for the selector given with `wait_selector`. :param network_idle: Wait for the page until there are no network connections for at least 500 ms. :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it. :return: `page_action`'s return value, or the navigated `page` if no `page_action` was given. """ static_proxy = kwargs.pop("proxy", None) params = _validate(kwargs, self, StealthConfig) if not self._is_alive: # pragma: no cover raise RuntimeError("Context manager has been closed") request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() referer = ( "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None ) for attempt in range(self._config.retries): proxy: Optional[ProxyType] = None if self._config.proxy_rotator and static_proxy is None: proxy = self._config.proxy_rotator.get_proxy() else: proxy = static_proxy async with self._page_generator( params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains ) as page_info: page_info: PageInfo page: Page = page_info.page if params.page_setup: try: await params.page_setup(page) except Exception as e: # pragma: no cover log.error(f"Error executing page_setup: {e}") try: first_response = await page.goto(url, referer=referer) await self._wait_for_page_stability(page, params.load_dom, params.network_idle) if not first_response: raise RuntimeError(f"Failed to get response for {url}") result = await params.page_action(page) if params.page_action else None if params.wait_selector: try: waiter: Locator = page.locator(params.wait_selector) await waiter.first.wait_for(state=params.wait_selector_state) await self._wait_for_page_stability(page, params.load_dom, params.network_idle) except Exception as e: # pragma: no cover log.error(f"Error waiting for selector {params.wait_selector}: {e}") await page.wait_for_timeout(params.wait) return result if params.page_action else page except Exception as e: page_info.mark_error() if attempt < self._config.retries - 1: if is_proxy_error(e): log.warning( f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..." ) else: log.warning( f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..." ) await asyncio_sleep(self._config.retry_delay) else: log.error(f"Failed after {self._config.retries} attempts: {e}") raise raise RuntimeError("Request failed") # pragma: no cover