feature!: параметр user_agent убран, UA всегда родной; docs/Usage.md
- user_agent в запросе → 422 с подсказкой брать UA из ответа: UA, отличный от реального Chrome, расходится с userAgentData и движком, Cloudflare такой браузер не пропускает (замер: родной 3/3, Chrome 147 и Windows — 0) (#87) - профиль хранит только отпечаток (экран, DPR), UA не подменяется (#89) - docs/Usage.md: практики для клиентов API — тот же UA и прокси, что у браузера, сессии, cookies, ошибки и повторы, таймауты, пример на httpx - README, docs/API.md, docs/Introduction.md; тесты под новый контракт Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0ae5b0c83f
commit
fe509c25c3
11 files changed
+202
-125
No files matched your search
@@ -3,6 +3,7 @@
|
||||
make test-browser
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
@@ -63,19 +64,17 @@ def test_action_failed_on_real_page(client):
|
||||
assert data["error"] == "inner_html: selector not found: #missing"
|
||||
|
||||
|
||||
def test_user_agent_once_and_cookies_replace(client):
|
||||
def test_cookies_replace_profile_cookies(client):
|
||||
url = "https://httpbin.org/cookies"
|
||||
read = [{"type": "inner_html", "selector": "pre"}]
|
||||
cookie = lambda name: {"name": name, "value": "1", "url": "https://httpbin.org", "expires": 4102444800}
|
||||
|
||||
first = solve(client, url=url, user_agent="First/1", cookies=[cookie("a")], actions=read)
|
||||
first = solve(client, url=url, cookies=[cookie("a")], actions=read)
|
||||
sid = first["session_id"]
|
||||
assert first["user_agent"] == "First/1"
|
||||
assert json.loads(first["actions"][0]["result"])["cookies"] == {"a": "1"}
|
||||
|
||||
# повторный UA игнорируется; без cookies в запросе профильные сохраняются
|
||||
second = solve(client, sid, url=url, user_agent="Second/2", actions=read)
|
||||
assert second["user_agent"] == "First/1"
|
||||
# без cookies в запросе профильные сохраняются
|
||||
second = solve(client, sid, url=url, actions=read)
|
||||
assert json.loads(second["actions"][0]["result"])["cookies"] == {"a": "1"}
|
||||
|
||||
# переданные cookies заменяют профильные
|
||||
@@ -83,7 +82,7 @@ def test_user_agent_once_and_cookies_replace(client):
|
||||
assert json.loads(third["actions"][0]["result"])["cookies"] == {"b": "1"}
|
||||
|
||||
|
||||
def test_generated_fingerprint_applied_and_stable(client, monkeypatch):
|
||||
def test_fingerprint_applied_and_stable(client, monkeypatch):
|
||||
async def page_fp(page):
|
||||
await page.evaluate(FP_JS)
|
||||
return json.loads(await page.inner_html("#fp"))
|
||||
@@ -113,20 +112,28 @@ def test_generated_fingerprint_applied_and_stable(client, monkeypatch):
|
||||
sids = [solve(client, url="https://example.com")["session_id"] for _ in range(4)]
|
||||
solve(client, sids[0], url="https://example.com")
|
||||
|
||||
major = profile.chrome_major()
|
||||
fps = {}
|
||||
for d, measured in results.items():
|
||||
saved = profile.load(main.USER_DATA_ROOT / d.rsplit("/", 1)[-1])
|
||||
fp = saved["fingerprint"]
|
||||
for m in measured:
|
||||
assert m["screen"] == [fp["screen"]["width"], fp["screen"]["height"], fp["device_scale_factor"]]
|
||||
assert f"Chrome/{major}." in m["ua"] and f"Google Chrome/{major}" in m["ua_ch"]
|
||||
# UA родной: версия в строке UA совпадает с userAgentData
|
||||
major = re.search(r"Google Chrome/(\d+)", m["ua_ch"]).group(1)
|
||||
assert f"Chrome/{major}." in m["ua"]
|
||||
assert all(m == measured[0] for m in measured), "fingerprint changed within a profile"
|
||||
fps[d] = measured[0]
|
||||
assert len(fps) == 4
|
||||
print("distinct fingerprints:", len({json.dumps(f, sort_keys=True) for f in fps.values()}), "of", len(fps))
|
||||
|
||||
|
||||
def test_ua_in_response_is_native(client):
|
||||
"""user_agent в ответе — родной UA браузера, тот, с которым получен cf_clearance."""
|
||||
data = solve(client, url="https://example.com")
|
||||
assert re.fullmatch(r"Mozilla/5\.0 \(X11; Linux x86_64\) .* Chrome/\d+\.0\.0\.0 Safari/537\.36", data["user_agent"])
|
||||
assert "Headless" not in data["user_agent"]
|
||||
|
||||
|
||||
def test_cloudflare(client):
|
||||
data = solve(client, url=CF_URL, captcha_type="cloudflare", timeout=120, actions=[
|
||||
{"type": "wait_for", "selector": CF_SELECTOR, "timeout": 60},
|
||||
|
||||
+17
-38
@@ -1,36 +1,24 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from api import profile
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def chrome_155(monkeypatch):
|
||||
monkeypatch.setattr(profile, "chrome_major", lambda: "155")
|
||||
def test_fingerprint_generated_once(tmp_path):
|
||||
first = profile.load_or_create(tmp_path)
|
||||
assert profile.load_or_create(tmp_path) == first
|
||||
assert json.loads((tmp_path / profile.SETTINGS_FILE).read_text()) == first
|
||||
|
||||
|
||||
def test_passed_user_agent_is_saved_once(tmp_path):
|
||||
first = profile.load_or_create(tmp_path, "First/1")
|
||||
second = profile.load_or_create(tmp_path, "Second/2")
|
||||
|
||||
assert first == second == {"user_agent": "First/1"}
|
||||
assert json.loads((tmp_path / profile.SETTINGS_FILE).read_text()) == {"user_agent": "First/1"}
|
||||
|
||||
|
||||
def test_generated_fingerprint_without_user_agent(tmp_path):
|
||||
settings = profile.load_or_create(tmp_path, None)
|
||||
|
||||
assert settings["user_agent"] == profile.UA_TEMPLATE.format(major="155")
|
||||
screen = settings["fingerprint"]["screen"]
|
||||
screen = first["fingerprint"]["screen"]
|
||||
assert screen["width"] >= profile.MIN_SCREEN[0] and screen["height"] >= profile.MIN_SCREEN[1]
|
||||
assert settings["fingerprint"]["device_scale_factor"] >= 1
|
||||
assert first["fingerprint"]["device_scale_factor"] >= 1
|
||||
assert "user_agent" not in first
|
||||
|
||||
|
||||
def test_generated_fingerprint_is_stable_and_ignores_later_user_agent(tmp_path):
|
||||
first = profile.load_or_create(tmp_path, None)
|
||||
assert profile.load_or_create(tmp_path, None) == first
|
||||
assert profile.load_or_create(tmp_path, "Later/1") == first
|
||||
def test_old_profile_settings_get_fingerprint(tmp_path):
|
||||
"""Профиль из версии, где сохранялся user_agent клиента: UA отбрасывается, отпечаток создаётся."""
|
||||
(tmp_path / profile.SETTINGS_FILE).write_text(json.dumps({"user_agent": "First/1"}))
|
||||
settings = profile.load_or_create(tmp_path)
|
||||
assert set(settings) == {"fingerprint"}
|
||||
|
||||
|
||||
def test_profiles_get_independent_fingerprints(tmp_path):
|
||||
@@ -39,7 +27,7 @@ def test_profiles_get_independent_fingerprints(tmp_path):
|
||||
for i in range(30):
|
||||
d = tmp_path / str(i)
|
||||
d.mkdir()
|
||||
fp = profile.load_or_create(d, None)["fingerprint"]
|
||||
fp = profile.load_or_create(d)["fingerprint"]
|
||||
variants.add((fp["screen"]["width"], fp["screen"]["height"], fp["device_scale_factor"]))
|
||||
assert len(variants) > 1
|
||||
|
||||
@@ -50,18 +38,9 @@ def test_generate_falls_back_when_browserforge_fails(monkeypatch):
|
||||
raise ValueError("Cannot generate headers")
|
||||
|
||||
monkeypatch.setattr(profile, "FingerprintGenerator", Failing)
|
||||
fp = profile.generate_fingerprint(attempts=2)
|
||||
assert fp["screen"] == {"width": 1920, "height": 1080}
|
||||
assert fp["device_scale_factor"] == 1
|
||||
assert profile.generate_fingerprint(attempts=2) == {"screen": {"width": 1920, "height": 1080}, "device_scale_factor": 1}
|
||||
|
||||
|
||||
def test_browser_kwargs():
|
||||
assert profile.browser_kwargs({"user_agent": "UA/1"}) == {"useragent": "UA/1"}
|
||||
kwargs = profile.browser_kwargs({
|
||||
"user_agent": "UA/1",
|
||||
"fingerprint": {"screen": {"width": 2560, "height": 1440}, "device_scale_factor": 1.25},
|
||||
})
|
||||
assert kwargs == {
|
||||
"useragent": "UA/1",
|
||||
"additional_args": {"screen": {"width": 2560, "height": 1440}, "device_scale_factor": 1.25},
|
||||
}
|
||||
def test_browser_kwargs_do_not_touch_user_agent():
|
||||
kwargs = profile.browser_kwargs({"fingerprint": {"screen": {"width": 2560, "height": 1440}, "device_scale_factor": 1.25}})
|
||||
assert kwargs == {"additional_args": {"screen": {"width": 2560, "height": 1440}, "device_scale_factor": 1.25}}
|
||||
@@ -68,6 +68,11 @@ def test_proxy_optional():
|
||||
assert solve(proxy="").proxy is None
|
||||
|
||||
|
||||
def test_user_agent_rejected():
|
||||
with pytest.raises(ValidationError, match="user_agent is not supported"):
|
||||
solve(user_agent="Mozilla/5.0")
|
||||
|
||||
|
||||
def test_captcha_action_requires_key():
|
||||
action = {"type": "captcha", "selector": "#c", "value": "recaptcha"}
|
||||
with pytest.raises(ValidationError, match="requires request field 'captcha'"):
|
||||
|
||||
@@ -54,7 +54,6 @@ def client(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(main, "AsyncStealthySession", FakeSession)
|
||||
monkeypatch.setattr(main, "USER_DATA_ROOT", tmp_path)
|
||||
monkeypatch.setattr(main, "_slots", asyncio.Semaphore(2))
|
||||
monkeypatch.setattr(profile, "chrome_major", lambda: "155")
|
||||
return TestClient(main.app)
|
||||
|
||||
|
||||
@@ -168,18 +167,18 @@ def test_bad_session_cookie_gets_new_session(client, monkeypatch, tmp_path, bad)
|
||||
assert [p.name for p in tmp_path.iterdir()] == [sid]
|
||||
|
||||
|
||||
def test_user_agent_set_once_per_profile(client, monkeypatch):
|
||||
monkeypatch.setattr(actions, "execute", fake_execute([]))
|
||||
post(client, user_agent="First/1")
|
||||
post(client, user_agent="Second/2")
|
||||
assert [c["useragent"] for c in FakeSession.calls] == ["First/1", "First/1"]
|
||||
def test_user_agent_param_rejected(client):
|
||||
resp = post(client, user_agent="First/1")
|
||||
assert resp.status_code == 422
|
||||
assert "take user_agent from the response" in resp.text
|
||||
assert FakeSession.calls == []
|
||||
|
||||
|
||||
def test_generated_fingerprint_passed_to_browser(client, monkeypatch):
|
||||
def test_fingerprint_passed_to_browser_and_ua_untouched(client, monkeypatch):
|
||||
monkeypatch.setattr(actions, "execute", fake_execute([]))
|
||||
post(client)
|
||||
kwargs = FakeSession.calls[0]
|
||||
assert kwargs["useragent"] == profile.UA_TEMPLATE.format(major="155")
|
||||
assert "useragent" not in kwargs
|
||||
assert set(kwargs["additional_args"]) == {"screen", "device_scale_factor"}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user