199 lines
11 KiB
Python
199 lines
11 KiB
Python
"""
|
|
Stealthy-браузерная сессия (persistent context + пул страниц) — портировано и урезано из
|
|
scrapling.engines._browsers._stealth.AsyncStealthySession.
|
|
|
|
Отличия от апстрима:
|
|
- Только async-версия (проект целиком async).
|
|
- `fetch()` не строит `scrapling.Response` (это тянет `scrapling.parser.Selector` — тяжёлый
|
|
HTML-парсер, который в проекте не используется: html/cookies/user_agent достаются вручную
|
|
через сырой Playwright API). Вместо этого, если передан `page_action`, `fetch()` возвращает
|
|
РЕЗУЛЬТАТ `page_action(page)` напрямую (в апстриме он отбрасывается) — так `page_action`
|
|
становится единственной точкой, где происходит выполнение action-очереди
|
|
(`api.actions.execute`) и извлечение html/cookies/user_agent для ответа API. Если
|
|
`page_action` не передан, возвращается сам `page`.
|
|
- Без решения Cloudflare-челленджей (`_cloudflare_solver`/`solve_cloudflare`) — в проекте это
|
|
делает `antibot.orchestrator.pass_challenges`, вызываемый из `page_action`.
|
|
"""
|
|
import logging
|
|
from asyncio import sleep as asyncio_sleep
|
|
|
|
from playwright.async_api import Page, Locator
|
|
from patchright.async_api import async_playwright
|
|
from typing_extensions import Unpack
|
|
|
|
from engine.page_pool import PageInfo
|
|
from engine.proxy_rotation import is_proxy_error
|
|
from engine.session import AsyncSession, StealthySessionMixin
|
|
from engine.session_types import StealthSession, StealthFetchParams
|
|
from engine.validators import validate_fetch as _validate, StealthConfig
|
|
from engine._types import Any, List, Optional, ProxyType
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class AsyncStealthySession(AsyncSession, StealthySessionMixin):
|
|
"""An async Stealthy Browser session manager with page pooling."""
|
|
|
|
__slots__ = (
|
|
"_config",
|
|
"_context_options",
|
|
"_browser_options",
|
|
"_user_data_dir",
|
|
"_headers_keys",
|
|
)
|
|
|
|
def __init__(self, **kwargs: Unpack[StealthSession]):
|
|
"""A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory.
|
|
|
|
:param headless: Run the browser in headless/hidden (default), or headful/visible mode.
|
|
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
|
:param blocked_domains: A set of domain names to block requests to. Subdomains are also matched.
|
|
:param useragent: Pass a useragent string to be used. Otherwise the browser's own default is used.
|
|
:param cookies: Set cookies for the next request.
|
|
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
|
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000
|
|
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning.
|
|
:param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need
|
|
(in this project: `api.actions.execute`). Its return value is what `fetch()` returns.
|
|
:param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads.
|
|
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
|
:param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session.
|
|
:param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc.
|
|
:param timezone_id: Changes the timezone of the browser. Defaults to the system timezone.
|
|
:param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
|
|
:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
|
|
:param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
|
|
:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
|
|
:param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely.
|
|
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
|
:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
|
|
:param google_search: Enabled by default, Scrapling will set a Google referer header.
|
|
:param extra_headers: A dictionary of extra headers to add to the request.
|
|
:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
|
|
:param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory.
|
|
:param extra_flags: A list of additional browser flags to pass to the browser on launch.
|
|
:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than the settings above.
|
|
"""
|
|
self.__validate__(**kwargs)
|
|
super().__init__(max_pages=self._config.max_pages)
|
|
|
|
async def start(self) -> None:
|
|
"""Create a browser for this instance and context."""
|
|
if not self.playwright:
|
|
self.playwright = await async_playwright().start()
|
|
try:
|
|
if self._config.cdp_url:
|
|
self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url)
|
|
if not self._config.proxy_rotator:
|
|
assert self.browser is not None
|
|
self.context = await self.browser.new_context(**self._context_options)
|
|
elif self._config.proxy_rotator:
|
|
self.browser = await self.playwright.chromium.launch(**self._browser_options)
|
|
else:
|
|
persistent_options = (
|
|
self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir}
|
|
)
|
|
self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options)
|
|
|
|
if self.context:
|
|
self.context = await self._initialize_context(self._config, self.context)
|
|
|
|
self._is_alive = True
|
|
except Exception:
|
|
# Clean up playwright if browser setup fails
|
|
await self.playwright.stop()
|
|
self.playwright = None
|
|
raise
|
|
else:
|
|
raise RuntimeError("Session has been already started")
|
|
|
|
async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Any:
|
|
"""Opens up the browser and navigates to `url`.
|
|
|
|
If `page_action` is given, returns its return value (see class docstring). Otherwise returns
|
|
the Playwright `page` navigated to `url`, still open (belongs to the session's page pool/context).
|
|
|
|
:param url: The Target url.
|
|
:param google_search: Enabled by default, sets a Google referer header.
|
|
:param timeout: The timeout in milliseconds used in all operations and waits through the page.
|
|
:param wait: The time (milliseconds) to wait after everything finishes before returning.
|
|
:param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need. Its return value is propagated as `fetch()`'s return value.
|
|
:param page_setup: A function that takes the `page` object, runs before navigation.
|
|
:param extra_headers: A dictionary of extra headers to add to the request.
|
|
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
|
|
:param blocked_domains: A set of domain names to block requests to.
|
|
:param wait_selector: Wait for a specific CSS selector to be in a specific state.
|
|
:param wait_selector_state: The state to wait for the selector given with `wait_selector`.
|
|
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
|
|
:param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute.
|
|
:param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it.
|
|
:return: `page_action`'s return value, or the navigated `page` if no `page_action` was given.
|
|
"""
|
|
static_proxy = kwargs.pop("proxy", None)
|
|
|
|
params = _validate(kwargs, self, StealthConfig)
|
|
if not self._is_alive: # pragma: no cover
|
|
raise RuntimeError("Context manager has been closed")
|
|
|
|
request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set()
|
|
referer = (
|
|
"https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None
|
|
)
|
|
|
|
for attempt in range(self._config.retries):
|
|
proxy: Optional[ProxyType] = None
|
|
if self._config.proxy_rotator and static_proxy is None:
|
|
proxy = self._config.proxy_rotator.get_proxy()
|
|
else:
|
|
proxy = static_proxy
|
|
|
|
async with self._page_generator(
|
|
params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains
|
|
) as page_info:
|
|
page_info: PageInfo
|
|
page: Page = page_info.page
|
|
if params.page_setup:
|
|
try:
|
|
await params.page_setup(page)
|
|
except Exception as e: # pragma: no cover
|
|
log.error(f"Error executing page_setup: {e}")
|
|
|
|
try:
|
|
first_response = await page.goto(url, referer=referer)
|
|
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
|
|
|
if not first_response:
|
|
raise RuntimeError(f"Failed to get response for {url}")
|
|
|
|
result = await params.page_action(page) if params.page_action else None
|
|
|
|
if params.wait_selector:
|
|
try:
|
|
waiter: Locator = page.locator(params.wait_selector)
|
|
await waiter.first.wait_for(state=params.wait_selector_state)
|
|
await self._wait_for_page_stability(page, params.load_dom, params.network_idle)
|
|
except Exception as e: # pragma: no cover
|
|
log.error(f"Error waiting for selector {params.wait_selector}: {e}")
|
|
|
|
await page.wait_for_timeout(params.wait)
|
|
|
|
return result if params.page_action else page
|
|
|
|
except Exception as e:
|
|
page_info.mark_error()
|
|
if attempt < self._config.retries - 1:
|
|
if is_proxy_error(e):
|
|
log.warning(
|
|
f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..."
|
|
)
|
|
else:
|
|
log.warning(
|
|
f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..."
|
|
)
|
|
await asyncio_sleep(self._config.retry_delay)
|
|
else:
|
|
log.error(f"Failed after {self._config.retries} attempts: {e}")
|
|
raise
|
|
|
|
raise RuntimeError("Request failed") # pragma: no cover
|