chore: перетащил обертку над браузером из scrapling

src/engine/session.py

@@ -0,0 +1,391 @@
+"""
+Управление контекстом/пулом страниц — портировано из scrapling.engines._browsers._base.
+
+Только async-сессия (проект целиком async, см. patchright.async_api в src/api/actions.py) и без
+методов детекта Cloudflare (`_detect_cloudflare`/`_challenge_cleared` в апстриме зависят от
+scrapling.parser.Selector и дублируют antibot.orchestrator.pass_challenges, который в проекте
+уже решает эту задачу — см. src/api/actions.py:execute).
+"""
+import logging
+from time import time
+from re import search as re_search
+from asyncio import sleep as asyncio_sleep, Lock
+from contextlib import asynccontextmanager, suppress
+
+from playwright.async_api._generated import Page as AsyncPage
+from playwright.async_api import (
+	Frame as AsyncFrame,
+	Response as AsyncPlaywrightResponse,
+	BrowserContext as AsyncBrowserContext,
+)
+from playwright._impl._errors import Error as PlaywrightError
+
+from engine.page_pool import PageInfo, PagePool
+from engine.validators import validate, PlaywrightConfig, StealthConfig
+from engine.navigation import construct_proxy_dict, create_async_intercept_handler
+from engine._types import (
+	Any,
+	Awaitable,
+	Dict,
+	List,
+	Set,
+	Optional,
+	Callable,
+	TYPE_CHECKING,
+	cast,
+	overload,
+	Tuple,
+	ProxyType,
+	AsyncGenerator,
+)
+from engine.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS
+
+log = logging.getLogger(__name__)
+
+
+class AsyncSession:
+	_config: "PlaywrightConfig | StealthConfig"
+	_context_options: Dict[str, Any]
+	if TYPE_CHECKING:
+		_build_context_with_proxy: Callable[..., Dict[str, Any]]
+
+	def __init__(self, max_pages: int = 1):
+		self.max_pages = max_pages
+		self.page_pool = PagePool(max_pages)
+		self._max_wait_for_page = 60
+		self.playwright: Any = None
+		self.context: Any = None
+		self.browser: Any = None
+		self._is_alive = False
+		self._lock = Lock()
+
+	async def start(self) -> None:
+		pass
+
+	async def close_pages(self) -> None:
+		"""Close every open tab in the session's pool. The next request opens a fresh tab."""
+		for page_info in self.page_pool.clear():
+			with suppress(Exception):
+				await cast(AsyncPage, page_info.page).close()
+
+	async def close(self):
+		"""Close all resources"""
+		if not self._is_alive:  # pragma: no cover
+			return
+
+		await self.close_pages()
+		if self.context:
+			await self.context.close()
+			self.context = None  # pyright: ignore
+
+		if self.browser:
+			await self.browser.close()
+			self.browser = None
+
+		if self.playwright:
+			await self.playwright.stop()
+			self.playwright = None  # pyright: ignore
+
+		self._is_alive = False
+
+	async def __aenter__(self):
+		await self.start()
+		return self
+
+	async def __aexit__(self, exc_type, exc_val, exc_tb):
+		await self.close()
+
+	async def _initialize_context(
+		self, config: PlaywrightConfig | StealthConfig, ctx: AsyncBrowserContext
+	) -> AsyncBrowserContext:
+		"""Initialize the browser context."""
+		if config.init_script:  # pragma: no cover
+			await ctx.add_init_script(path=config.init_script)
+
+		# Аналог `init_script`, но для инлайнового JS (например, InjectFunction(fingerprint) из
+		# browserforge — он отдаёт готовый скрипт строкой, а не путь к файлу), не апстримное
+		# поле scrapling.
+		if config.init_script_content:
+			await ctx.add_init_script(config.init_script_content)
+
+		if config.cookies:  # pragma: no cover
+			await ctx.add_cookies(config.cookies)
+
+		return ctx
+
+	async def _get_page(
+		self,
+		timeout: int | float,
+		extra_headers: Optional[Dict[str, str]],
+		disable_resources: bool,
+		blocked_domains: Optional[Set[str]] = None,
+		context: Optional[AsyncBrowserContext] = None,
+	) -> PageInfo:  # pragma: no cover
+		"""Get a ready page from the pool, or open a new one"""
+		ctx = context if context is not None else self.context
+		if TYPE_CHECKING:
+			assert ctx is not None, "Browser context not initialized"
+
+		async with self._lock:
+			page_info = self.page_pool.get_ready_page() if context is None else None
+			if page_info is None and context is None and self.page_pool.pages_count >= self.max_pages:
+				# At max capacity with the persistent context, so wait for a busy page to become ready
+				start_time = time()
+				while time() - start_time < self._max_wait_for_page:
+					await asyncio_sleep(0.05)
+					page_info = self.page_pool.get_ready_page()
+					if page_info is not None:
+						break
+				else:
+					raise TimeoutError(
+						f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period"
+					)
+
+			if page_info is None:
+				page_info = self.page_pool.add_page(await ctx.new_page())
+
+			page = cast(AsyncPage, page_info.page)
+			page.set_default_navigation_timeout(timeout)
+			page.set_default_timeout(timeout)
+			await page.set_extra_http_headers(extra_headers or {})
+			await page.unroute_all(behavior="ignoreErrors")
+			if disable_resources or blocked_domains:
+				await page.route("**/*", create_async_intercept_handler(disable_resources, blocked_domains))
+			return page_info
+
+	def get_pool_stats(self) -> Dict[str, int]:
+		"""Get statistics about the current page pool"""
+		return {
+			"total_pages": self.page_pool.pages_count,
+			"busy_pages": self.page_pool.busy_count,
+			"max_pages": self.max_pages,
+		}
+
+	@staticmethod
+	async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[int] = None):
+		"""Wait for the page to become idle (no network activity) even if there are never-ending requests."""
+		try:
+			await page.wait_for_load_state("networkidle", timeout=timeout)
+		except (PlaywrightError, Exception):
+			pass
+
+	async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool):
+		await page.wait_for_load_state(state="load")
+		if load_dom:
+			await page.wait_for_load_state(state="domcontentloaded")
+		if network_idle:
+			await self._wait_for_networkidle(page)
+
+	@staticmethod
+	def _create_response_handler(
+		page_info: PageInfo,
+		response_container: List,
+		xhr_pattern: Optional[str] = None,
+		xhr_container: Optional[List] = None,
+	) -> Callable[[AsyncPlaywrightResponse], Awaitable[None]]:
+		"""Create an async response handler that captures the final navigation response and optionally XHR/fetch responses.
+
+		:param page_info: The PageInfo object containing the page
+		:param response_container: A list to store the final response (mutable container)
+		:param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs
+		:param xhr_container: Optional list to store captured XHR/fetch responses
+		:return: A callback function for page.on("response", ...)
+		"""
+
+		async def handle_response(finished_response: AsyncPlaywrightResponse) -> None:
+			if (
+				finished_response.request.resource_type == "document"
+				and finished_response.request.is_navigation_request()
+				and finished_response.request.frame == page_info.page.main_frame
+			):
+				response_container[0] = finished_response
+			elif (
+				xhr_pattern
+				and xhr_container is not None
+				and finished_response.request.resource_type in ("xhr", "fetch")
+				and re_search(xhr_pattern, finished_response.url)
+			):
+				xhr_container.append(finished_response)
+
+		return handle_response
+
+	@asynccontextmanager
+	async def _page_generator(
+		self,
+		timeout: int | float,
+		extra_headers: Optional[Dict[str, str]],
+		disable_resources: bool,
+		proxy: Optional[ProxyType] = None,
+		blocked_domains: Optional[Set[str]] = None,
+	) -> AsyncGenerator["PageInfo", None]:
+		"""Acquire a page - either from persistent context or fresh context with proxy."""
+		if proxy:
+			# Rotation mode: create fresh context with the provided proxy
+			if not self.browser:  # pragma: no cover
+				raise RuntimeError("Browser not initialized for proxy rotation mode")
+			context_options = self._build_context_with_proxy(proxy)
+			context: AsyncBrowserContext = await self.browser.new_context(**context_options)
+
+			page_info = None
+			try:
+				context = await self._initialize_context(self._config, context)
+				page_info = await self._get_page(
+					timeout, extra_headers, disable_resources, blocked_domains, context=context
+				)
+				yield page_info
+			finally:
+				if page_info is not None:
+					self.page_pool.remove_page(page_info)
+				await context.close()
+		else:
+			# Standard mode: use PagePool with persistent context
+			page_info = await self._get_page(timeout, extra_headers, disable_resources, blocked_domains)
+			try:
+				yield page_info
+			finally:
+				if page_info.state == "error" or page_info.page.is_closed():
+					with suppress(Exception):
+						await page_info.page.close()
+					self.page_pool.remove_page(page_info)
+				else:
+					page_info.mark_ready()
+
+
+class BaseSessionMixin:
+	_config: "PlaywrightConfig | StealthConfig"
+
+	@overload
+	def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ...
+
+	@overload
+	def __validate_routine__(self, params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ...
+
+	def __validate_routine__(
+		self, params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]
+	) -> PlaywrightConfig | StealthConfig:
+		# Dark color scheme bypasses the 'prefersLightColor' check in creepjs
+		self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2}
+		self._browser_options: Dict[str, Any] = {
+			"args": DEFAULT_ARGS,
+			"ignore_default_args": HARMFUL_ARGS,
+		}
+		if "__max_pages" in params:
+			params["max_pages"] = params.pop("__max_pages")
+
+		config = validate(params, model=model)
+		self._headers_keys = (
+			{header.lower() for header in config.extra_headers.keys()} if config.extra_headers else set()
+		)
+
+		return config
+
+	def __generate_options__(self, extra_flags: Tuple | None = None) -> None:
+		config: PlaywrightConfig | StealthConfig = self._config
+		self._context_options.update(
+			{
+				"proxy": config.proxy,
+				"timezone_id": config.timezone_id,
+				"extra_http_headers": config.extra_headers,
+			}
+		)
+		if config.locale and config.cdp_url:
+			# Launch flags can't be set on remote browsers, so the detectable context option is the best effort left
+			self._context_options["locale"] = config.locale
+		if config.useragent:
+			self._context_options["user_agent"] = config.useragent
+
+		if not config.cdp_url:
+			flags = self._browser_options["args"]
+			if config.extra_flags or extra_flags:
+				flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ())))
+
+			if config.dns_over_https:
+				doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query"
+				if isinstance(flags, list):
+					flags.append(doh_flag)
+				else:
+					flags = list(flags) + [doh_flag]
+
+			if config.locale:
+				# The context `locale` option patches the main thread only, so Web Workers keep the browser's real
+				# language and WAFs like Cloudflare flag the mismatch. Launch flags set it browser-wide instead,
+				# so workers, `Intl`, and the `Accept-Language` header all follow natively.
+				base_lang = config.locale.split("-")[0].lower()
+				accept_lang = f"{config.locale},{base_lang}" if base_lang != config.locale.lower() else config.locale
+				flags = (flags if isinstance(flags, list) else list(flags)) + [
+					f"--lang={config.locale}",
+					f"--accept-lang={accept_lang}",
+				]
+
+			self._browser_options.update(
+				{
+					"args": flags,
+					"headless": config.headless,
+					"channel": "chrome" if config.real_chrome else "chromium",
+				}
+			)
+			if config.executable_path:
+				self._browser_options["executable_path"] = config.executable_path
+
+			self._user_data_dir = config.user_data_dir
+		else:
+			self._browser_options = {}
+
+		if config.additional_args:
+			self._context_options.update(config.additional_args)
+
+	def _build_context_with_proxy(self, proxy: Optional[ProxyType] = None) -> Dict[str, Any]:
+		"""
+		Build context options with a specific proxy for rotation mode.
+
+		:param proxy: Proxy URL string or Playwright-style proxy dict to use for this context.
+		:return: Dictionary of context options for browser.new_context().
+		"""
+
+		context_options = self._context_options.copy()
+
+		# Override proxy if provided
+		if proxy:
+			context_options["proxy"] = construct_proxy_dict(proxy)
+
+		return context_options
+
+
+class StealthySessionMixin(BaseSessionMixin):
+	def __validate__(self, **params):
+		self._config = self.__validate_routine__(params, model=StealthConfig)
+		self._context_options.update(
+			{
+				"is_mobile": False,
+				"has_touch": False,
+				# I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now
+				"service_workers": "allow",
+				"ignore_https_errors": True,
+				"screen": {"width": 1920, "height": 1080},
+				"viewport": {"width": 1920, "height": 1080},
+				"permissions": ["geolocation", "notifications"],
+			}
+		)
+		self.__generate_stealth_options()
+
+	def __generate_stealth_options(self) -> None:
+		config = cast(StealthConfig, self._config)
+		flags: Tuple[str, ...] = tuple()
+		if not config.cdp_url:
+			flags = tuple(DEFAULT_ARGS) + tuple(STEALTH_ARGS)
+
+			if config.block_webrtc:
+				flags += (
+					"--webrtc-ip-handling-policy=disable_non_proxied_udp",
+					"--force-webrtc-ip-handling-policy",  # Ensures the policy is enforced
+				)
+			if not config.allow_webgl:
+				flags += (
+					"--disable-webgl",
+					"--disable-webgl-image-chromium",
+					"--disable-webgl2",
+				)
+			if config.hide_canvas:
+				flags += ("--fingerprinting-canvas-image-data-noise",)
+
+		super(StealthySessionMixin, self).__generate_options__(flags)