build: лимиты ресурсов, прокси и нагрузочный тест; docs: API

- docker-compose: cpus/mem_limit через API_CPUS/API_MEM_LIMIT, прокси gost в профиле loadtest
- tests/loadtest.py: параллельные запросы к /v1/solve + пики docker stats
- docs/API.md: справочник по запросу, действиям, ответу и сессиям
- docs/Deployment.md: результаты замеров и правило подбора BROWSER_SLOTS

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
protokeyandClaude Opus 5.5 committed 2026-10-12 13:55:36 +04:00
1 parent df108dae3f
commit a319b81d8b
6 files changed
+330 -105

No files matched your search

+137
View File
@@ -0,0 +1,137 @@
"""Нагрузочный тест /v1/solve: N параллельных запросов на каждом уровне + пиковое потребление
контейнера по `docker stats`.
docker compose --profile loadtest up -d
python tests/loadtest.py --levels 1 5 10 15
Помимо status ответа, неуспех раскладывается по причинам: нет слотов, застряли на Cloudflare,
упало действие, исключение до действий.
"""
import argparse
import asyncio
import json
import statistics
import subprocess
import threading
import time
import httpx
CF_MARKERS = ("challenges.cloudflare.com", "Just a moment", "cf-turnstile")
def build_payload(args) -> dict:
return {
"url": args.url,
"proxy": args.proxy,
"timeout": 120,
"html": True,
"captcha_type": args.captcha_type,
"actions": [{"type": "wait_for", "selector": args.selector, "timeout": 60}],
}
def classify(resp: httpx.Response | Exception) -> str:
if isinstance(resp, Exception):
return f"exc:{type(resp).__name__}"
data = resp.json()
if "No slots" in str(data.get("error", "")):
return "no_slots"
actions = data.get("actions") or []
if not actions:
return f"error:{str(data.get('error'))[:60]}"
last = actions[-1]
if not last.get("ok") or last.get("result"):
html = data.get("html") or ""
return "cf_blocked" if any(m in html for m in CF_MARKERS) else "action_failed"
return "ok"
class StatsSampler(threading.Thread):
"""Раз в ~2с снимает docker stats контейнера, копит пики CPU/памяти и число процессов."""
def __init__(self, container: str):
super().__init__(daemon=True)
self.container = container
self.stop_event = threading.Event()
self.cpu = 0.0
self.mem_mb = 0.0
self.pids = 0
@staticmethod
def _to_mb(value: str) -> float:
units = {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024, "kB": 1 / 1000, "MB": 1, "GB": 1000, "B": 1 / 2**20}
for unit, k in units.items():
if value.endswith(unit):
return float(value[: -len(unit)]) * k
return 0.0
def run(self):
while not self.stop_event.is_set():
out = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{json .}}", self.container],
capture_output=True, text=True,
).stdout.strip()
if out:
s = json.loads(out)
self.cpu = max(self.cpu, float(s["CPUPerc"].rstrip("%")))
self.mem_mb = max(self.mem_mb, self._to_mb(s["MemUsage"].split("/")[0].strip()))
self.pids = max(self.pids, int(s["PIDs"]))
self.stop_event.wait(1)
async def one(client: httpx.AsyncClient, api: str, payload: dict) -> tuple[str, float]:
t0 = time.monotonic()
try:
resp = await client.post(f"{api}/v1/solve", json=payload)
except Exception as e:
resp = e
return classify(resp), time.monotonic() - t0
async def run_level(n: int, args) -> None:
sampler = StatsSampler(args.container)
sampler.start()
t0 = time.monotonic()
async with httpx.AsyncClient(timeout=args.http_timeout, trust_env=False) as client:
results = await asyncio.gather(*(one(client, args.api, build_payload(args)) for _ in range(n)))
wall = time.monotonic() - t0
sampler.stop_event.set()
sampler.join()
outcomes: dict[str, int] = {}
for outcome, _ in results:
outcomes[outcome] = outcomes.get(outcome, 0) + 1
ok_times = [d for o, d in results if o == "ok"]
median = f"{statistics.median(ok_times):.1f}" if ok_times else "-"
print(
f"| {n:>3} | {outcomes.get('ok', 0):>3}/{n:<3} | {median:>8} | {wall:>7.1f} "
f"| {sampler.cpu:>7.0f} | {sampler.mem_mb / 1024:>7.2f} | {sampler.pids:>5} | "
+ ", ".join(f"{k}={v}" for k, v in sorted(outcomes.items()) if k != "ok"),
flush=True,
)
async def main():
p = argparse.ArgumentParser()
p.add_argument("--api", default="http://127.0.0.1:8000")
p.add_argument("--url", default="https://freelancehunt.com/projects")
p.add_argument("--selector", default='a[href*="/project/"]')
p.add_argument("--proxy", default="http://proxy:8080")
p.add_argument("--captcha-type", default="cloudflare")
p.add_argument("--container", default="webroboapi-api-1")
p.add_argument("--levels", type=int, nargs="+", default=[1, 5, 10])
p.add_argument("--pause", type=float, default=10, help="пауза между уровнями, с")
p.add_argument("--http-timeout", type=float, default=300)
args = p.parse_args()
print("| par | ok | med ok,s | wall,s | peak CPU% | peak GiB | PIDs | прочее")
print("|-----|---------|----------|---------|-----------|----------|------|-------")
for i, n in enumerate(args.levels):
if i:
await asyncio.sleep(args.pause)
await run_level(n, args)
if __name__ == "__main__":
asyncio.run(main())