feature: переезд на хром
This commit is contained in:
protokey committed
1 parent
714cf9f922
commit
f2d4d441f4
6 files changed
+355
-102
No files matched your search
+63
-70
@@ -6,17 +6,17 @@ from pathlib import Path
|
||||
from typing import Annotated
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from camoufox.async_api import AsyncCamoufox
|
||||
from camoufox.fingerprints import generate_context_fingerprint
|
||||
from camoufox.utils import determine_ua_os
|
||||
from browserforge.fingerprints import Screen
|
||||
from fastapi import Cookie
|
||||
from fastapi import FastAPI
|
||||
from fastapi import Request
|
||||
from fastapi import Response
|
||||
from playwright.async_api import ProxySettings
|
||||
from ua_parser import user_agent_parser
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from engine import actions
|
||||
from engine.fingerprint import apply_fingerprint, context_options_for, load_or_create_fingerprint
|
||||
from engine.geoip import resolve_geo
|
||||
from engine.schemas import SolveRequest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -24,8 +24,6 @@ logger.setLevel(logging.DEBUG)
|
||||
logger.addHandler(logging.StreamHandler())
|
||||
BROWSER_SLOTS = 10
|
||||
|
||||
IDENTITY_FILE = Path("identity.json")
|
||||
|
||||
|
||||
def parse_proxy(proxy_url: str) -> ProxySettings:
|
||||
parts = urlsplit(proxy_url)
|
||||
@@ -38,22 +36,6 @@ def parse_proxy(proxy_url: str) -> ProxySettings:
|
||||
return settings
|
||||
|
||||
|
||||
def load_or_create_identity(user_data_dir, os=None, ff_version=None, locale=None, time_zone=None, user_agent=None) -> dict:
|
||||
identity_file = user_data_dir / IDENTITY_FILE
|
||||
|
||||
if identity_file.exists():
|
||||
return json.loads(identity_file.read_text())
|
||||
|
||||
kwargs = {}
|
||||
if user_agent:
|
||||
os, ff_version = parse_os_and_ff(user_agent)
|
||||
kwargs["config_overrides"] = {"navigator.UserAgent": user_agent}
|
||||
|
||||
fp = generate_context_fingerprint(os=os, ff_version=ff_version, locale=locale, timezone=time_zone)
|
||||
identity_file.write_text(json.dumps(fp))
|
||||
return fp
|
||||
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
@@ -62,17 +44,6 @@ async def root():
|
||||
return {"message": "Hello World"}
|
||||
|
||||
|
||||
_OS_MAP = {'win': 'windows', 'mac': 'macos', 'lin': 'linux'}
|
||||
|
||||
|
||||
def parse_os_and_ff(user_agent) -> tuple[str, str]:
|
||||
os_name = _OS_MAP[determine_ua_os(user_agent)]
|
||||
parsed = user_agent_parser.ParseUserAgent(user_agent)
|
||||
ff_version = int(parsed["major"]) if (
|
||||
parsed.get("family") == "Firefox" and parsed.get("major")) else 150
|
||||
return os_name, str(ff_version)
|
||||
|
||||
|
||||
@app.post("/v1/solve")
|
||||
async def solve(
|
||||
request: Request,
|
||||
@@ -106,52 +77,74 @@ async def solve(
|
||||
if not user_data_dir.exists():
|
||||
user_data_dir.mkdir(parents=True)
|
||||
|
||||
config = load_or_create_identity(user_data_dir, user_agent=payload.user_agent)
|
||||
ctx_opts = {**config["context_options"], "config": config["config"]}
|
||||
fp_kwargs = {}
|
||||
if payload.user_agent:
|
||||
fp_kwargs["user_agent"] = payload.user_agent
|
||||
fingerprint = load_or_create_fingerprint(
|
||||
user_data_dir,
|
||||
# только нижняя граница: реалистичный монитор должен быть не меньше запрошенного окна,
|
||||
# а точный верхний потолок слишком часто не находится в датасете и тихо "ослабляется"
|
||||
# browserforge (актуальный размер окна мы всё равно берём из payload.screen ниже, не отсюда)
|
||||
screen=Screen(min_width=scr_size[0], min_height=scr_size[1]),
|
||||
**fp_kwargs,
|
||||
)
|
||||
|
||||
async with AsyncCamoufox(
|
||||
persistent_context=True,
|
||||
user_data_dir=user_data_dir,
|
||||
proxy=proxy,
|
||||
window=scr_size,
|
||||
i_know_what_im_doing=True,
|
||||
geoip=True,
|
||||
humanize=True,
|
||||
main_world_eval=True,
|
||||
**ctx_opts,
|
||||
) as camoufo:
|
||||
await camoufo.add_init_script(config["init_script"])
|
||||
page = camoufo.pages[0] if camoufo.pages else await camoufo.new_page()
|
||||
if payload.cookies:
|
||||
await camoufo.add_cookies(payload.cookies)
|
||||
page.set_default_timeout(30_000)
|
||||
page.set_default_navigation_timeout(60_000)
|
||||
ctx_kwargs = context_options_for(
|
||||
fingerprint,
|
||||
viewport={"width": scr_size[0], "height": scr_size[1]},
|
||||
)
|
||||
|
||||
await page.goto(url=payload.url)
|
||||
geo = await resolve_geo(proxy)
|
||||
if geo:
|
||||
if geo.get("timezone_id"):
|
||||
ctx_kwargs["timezone_id"] = geo["timezone_id"]
|
||||
if geo.get("geolocation"):
|
||||
ctx_kwargs["geolocation"] = geo["geolocation"]
|
||||
ctx_kwargs["permissions"] = ["geolocation"]
|
||||
|
||||
actions_result = await actions.execute(
|
||||
page, payload.actions, payload.captcha,
|
||||
antibot_name=payload.captcha_type,
|
||||
success_locator=payload.captcha_success_locator,
|
||||
detect_locator=payload.captcha_detect_locator,
|
||||
async with async_playwright() as p:
|
||||
context = await p.chromium.launch_persistent_context(
|
||||
user_data_dir=str(user_data_dir),
|
||||
proxy=proxy,
|
||||
headless=False,
|
||||
**ctx_kwargs,
|
||||
)
|
||||
try:
|
||||
await apply_fingerprint(context, fingerprint)
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
if payload.cookies:
|
||||
await context.add_cookies(payload.cookies)
|
||||
page.set_default_timeout(30_000)
|
||||
page.set_default_navigation_timeout(60_000)
|
||||
|
||||
status = not "error" in actions_result[-1]
|
||||
error = ["Something error", None][status]
|
||||
cookies = await camoufo.cookies()
|
||||
html = await page.inner_html('html')
|
||||
# TODO: добавить как опцию. передачу скрина в base64.
|
||||
# await page.screenshot(
|
||||
# path=(
|
||||
# f"/home/sokol/PycharmProjects/WebRoboApi/extra/screenshots/"
|
||||
# f"{datetime.timestamp(datetime.now())}.png"
|
||||
# )
|
||||
# )
|
||||
await page.goto(url=payload.url)
|
||||
|
||||
actions_result = await actions.execute(
|
||||
page, payload.actions, payload.captcha,
|
||||
antibot_name=payload.captcha_type,
|
||||
success_locator=payload.captcha_success_locator,
|
||||
detect_locator=payload.captcha_detect_locator,
|
||||
)
|
||||
|
||||
status = not "error" in actions_result[-1]
|
||||
error = ["Something error", None][status]
|
||||
cookies = await context.cookies()
|
||||
html = await page.inner_html('html')
|
||||
user_agent = await page.evaluate("() => navigator.userAgent")
|
||||
# TODO: добавить как опцию. передачу скрина в base64.
|
||||
# await page.screenshot(
|
||||
# path=(
|
||||
# f"/home/sokol/PycharmProjects/WebRoboApi/extra/screenshots/"
|
||||
# f"{datetime.timestamp(datetime.now())}.png"
|
||||
# )
|
||||
# )
|
||||
finally:
|
||||
await context.close()
|
||||
|
||||
BROWSER_SLOTS += 1
|
||||
result = {
|
||||
"status": ["err", "ok"][status],
|
||||
"user_agent": ctx_opts["user_agent"],
|
||||
"user_agent": user_agent,
|
||||
"cookies": cookies,
|
||||
"actions": actions_result,
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user