diff --git a/requirements.txt b/requirements.txt index c489c3b..eacd032 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,7 @@ mozrunner==8.4.0 mozsystemmonitor==1.0.1 mozterm==1.0.0 mozversion==2.4.0 +msgspec==0.21.1 multidict==6.7.1 numpy==2.5.3 orjson==3.12.0 diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/engine/actions.py b/src/api/actions.py similarity index 98% rename from src/engine/actions.py rename to src/api/actions.py index 250abcd..ffe9f06 100644 --- a/src/engine/actions.py +++ b/src/api/actions.py @@ -5,7 +5,7 @@ from patchright.async_api import Page from antibot.base import click_and_race from antibot.orchestrator import pass_challenges from captcha import load_captcha -from engine.schemas import ActionRequest +from api.schemas import ActionRequest _actions: dict = {} diff --git a/src/engine/fingerprint.py b/src/api/fingerprint.py similarity index 62% rename from src/engine/fingerprint.py rename to src/api/fingerprint.py index 395e0af..30d7e1f 100644 --- a/src/engine/fingerprint.py +++ b/src/api/fingerprint.py @@ -50,26 +50,31 @@ def load_or_create_fingerprint(user_data_dir: Path, **generator_kwargs) -> Finge return fingerprint -def context_options_for(fingerprint: Fingerprint, **overrides) -> dict: - """Опции контекста (launch_persistent_context/new_context), согласованные с fingerprint'ом. - Headers сюда намеренно не кладём — их полный набор выставляет apply_fingerprint() через - set_extra_http_headers() уже после создания контекста, а он не мёржит, а заменяет, - так что выставлять их дважды бессмысленно.""" +def browser_kwargs_for(fingerprint: Fingerprint, browser_name: str = "chromium") -> dict: + """kwargs для `AsyncStealthySession(**kwargs)`, полностью выставляющие браузер под fingerprint — + UA, заголовки, init-script (подделка navigator/screen/WebGL/codecs/battery) и размер окна + (viewport/screen/device_scale_factor). + + Всё уходит обычными kwargs сессии — тем же путём, что и остальная конфигурация + (proxy/cookies/locale и т.д.), как это делает сама scrapling — а не отдельным вызовом после + запуска контекста (add_init_script/set_extra_http_headers руками требуют context, который + есть только после входа в `async with AsyncStealthySession(...)`, то есть после того, как + решение о конфигурации уже "должно было" быть принято до запуска браузера). + + :param fingerprint: Сгенерированный/загруженный fingerprint (см. load_or_create_fingerprint). + :param browser_name: Имя браузера для фильтрации заголовков (only_injectable_headers) — + "chromium", раз движок у нас реальный Chromium (см. docstring модуля). + :return: dict с ключами `useragent`, `extra_headers`, `init_script_content`, `additional_args` + (viewport/screen/device_scale_factor) — распаковывается в kwargs AsyncStealthySession. + """ + screen = {"width": fingerprint.screen.width, "height": fingerprint.screen.height} return { - "user_agent": fingerprint.navigator.userAgent, - "viewport": { - "width": fingerprint.screen.width, - "height": fingerprint.screen.height, - **overrides.pop("viewport", {}), + "useragent": fingerprint.navigator.userAgent, + "extra_headers": only_injectable_headers(fingerprint.headers, browser_name), + "init_script_content": InjectFunction(fingerprint), + "additional_args": { + "viewport": dict(screen), + "screen": dict(screen), + "device_scale_factor": fingerprint.screen.devicePixelRatio, }, - "device_scale_factor": fingerprint.screen.devicePixelRatio, - **overrides, } - - -async def apply_fingerprint(context, fingerprint: Fingerprint, browser_name: str = "chromium"): - """Довешивает на уже созданный контекст заголовки и init-script с подделкой - navigator/screen/WebGL/codecs/battery. Вызывать сразу после launch_persistent_context/new_context, - до первого page.goto().""" - await context.set_extra_http_headers(only_injectable_headers(fingerprint.headers, browser_name)) - await context.add_init_script(InjectFunction(fingerprint)) diff --git a/src/engine/geoip.py b/src/api/geoip.py similarity index 100% rename from src/engine/geoip.py rename to src/api/geoip.py diff --git a/src/engine/schemas.py b/src/api/schemas.py similarity index 98% rename from src/engine/schemas.py rename to src/api/schemas.py index 5db6d48..dc0d0f5 100644 --- a/src/engine/schemas.py +++ b/src/api/schemas.py @@ -21,7 +21,6 @@ class SolveRequest(BaseModel): url: str proxy: str timeout: int | None = None - screen: str = "1280x720" user_agent: str | None = None cookies: list[dict[str, Any]] | None = None # List[{name: str, value: str, domain: str, path: str, expires: float, httpOnly: bool, secure: bool, sameSite: Union["Lax", "None", "Strict"], partitionKey: Union[str, None]}] diff --git a/src/engine/__init__.py b/src/engine/__init__.py index e69de29..eddb28c 100644 --- a/src/engine/__init__.py +++ b/src/engine/__init__.py @@ -0,0 +1,3 @@ +from engine.stealthy import AsyncStealthySession + +__all__ = ["AsyncStealthySession"] diff --git a/src/engine/_types.py b/src/engine/_types.py new file mode 100644 index 0000000..0ea901b --- /dev/null +++ b/src/engine/_types.py @@ -0,0 +1,48 @@ +""" +Локальные тайп-алиасы для движка браузера — вендорено из scrapling.core._types +(урезано до того, что реально используется StealthySession/toolbelt; без RequestsSession/ +GetRequestParams/DataRequestParams — они тянут curl_cffi, который в проекте не установлен +и не нужен, т.к. не используется HTTP-фетчер scrapling). +""" +from typing import ( + TYPE_CHECKING, + TypeAlias, + cast, + overload, + Any, + Callable, + Dict, + Generator, + AsyncGenerator, + Generic, + List, + Set, + Literal, + Optional, + Sequence, + Tuple, + TypeVar, + Union, + Mapping, + Awaitable, +) +from typing_extensions import TypedDict + +# Прокси — строка-URL или словарь в формате Playwright ({"server": ..., "username": ..., "password": ...}) +ProxyType = Union[str, Dict[str, str]] +SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"] +FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]] + + +# Скопировано из playwright._impl._api_structures.SetCookieParam +class SetCookieParam(TypedDict, total=False): + name: str + value: str + url: Optional[str] + domain: Optional[str] + path: Optional[str] + expires: Optional[float] + httpOnly: Optional[bool] + secure: Optional[bool] + sameSite: Optional[Literal["Lax", "None", "Strict"]] + partitionKey: Optional[str] diff --git a/src/engine/ad_domains.py b/src/engine/ad_domains.py new file mode 100644 index 0000000..026a936 --- /dev/null +++ b/src/engine/ad_domains.py @@ -0,0 +1,3537 @@ +""" +Built-in ad/tracker domain list for use with block_ads=True. + +Source: Peter Lowe's ad and tracking server list https://pgl.yoyo.org/adservers/ +Used config: https://pgl.yoyo.org/adservers/serverlist.php?hostformat=plain&showintro=0&startyear=2000&mimetype=plaintext +""" + +AD_DOMAINS: frozenset = frozenset( + { + "101com.com", + "180hits.de", + "180searchassistant.com", + "1rx.io", + "2020mustang.com", + "207.net", + "247media.com", + "24log.com", + "24pm-affiliation.com", + "2linkpath.com", + "2mdn.net", + "2o7.net", + "2znp09oa.com", + "30ads.com", + "3337723.com", + "33across.com", + "360yield.com", + "3lift.com", + "3o9s.short.gy", + "4d5.net", + "4info.com", + "4jnzhl0d0.com", + "50websads.com", + "518ad.com", + "6sc.co", + "777partner.com", + "77tracking.com", + "7bpeople.com", + "7cnq.net", + "82o9v830.com", + "a-ads.com", + "a.mktw.net", + "a.muloqot.uz", + "a.sakh.com", + "a.ucoz.net", + "a.ucoz.ru", + "a.vartoken.com", + "a.vdo.ai", + "a.vfghd.com", + "a.vfgtb.com", + "a.xanga.com", + "a11.click", + "a135.wftv.com", + "a5.overclockers.ua", + "aa-metrics.beauty.hotpepper.jp", + "aa-metrics.recruit-card.jp", + "aa-metrics.trip-ai.jp", + "aaddzz.com", + "aax-eu-dub.amazon.com", + "aaxads.com", + "abacho.net", + "abc-ads.com", + "ablink.comms.trainline.com", + "ablink.info.wise.com", + "ablink.news.emails-puregym.com", + "ablinks.mail.hinge.co", + "aboardlevel.com", + "absolutering.com", + "absorbingband.com", + "abstractedauthority.com", + "abtasty.com", + "ac.rnm.ca", + "accountsdoor.com", + "acebunny.com", + "acemlnb.com", + "acobt.tech", + "acridtwist.com", + "actionsplash.com", + "actonsoftware.com", + "actualdeals.com", + "actuallysheep.com", + "actuallysnake.com", + "acuityads.com", + "acuityplatform.com", + "acustomizedgift.com", + "ad-balancer.at", + "ad-balancer.net", + "ad-cupid.com", + "ad-delivery.net", + "ad-pay.de", + "ad-rotator.com", + "ad-score.com", + "ad-server.gulasidorna.se", + "ad-space.net", + "ad-up.com", + "ad.71i.de", + "ad.a8.net", + "ad.abcnews.com", + "ad.abctv.com", + "ad.aboutwebservices.com", + "ad.abum.com", + "ad.admitad.com", + "ad.allboxing.ru", + "ad.altervista.org", + "ad.amgdgt.com", + "ad.anuntis.com", + "ad.auditude.com", + "ad.bitmedia.io", + "ad.bizo.com", + "ad.bondage.com", + "ad.centrum.cz", + "ad.cgi.cz", + "ad.choiceradio.com", + "ad.cooks.com", + "ad.digitallook.com", + "ad.dnoticias.pt", + "ad.domainfactory.de", + "ad.exyws.org", + "ad.grafika.cz", + "ad.gt", + "ad.hbv.de", + "ad.hyena.cz", + "ad.iinfo.cz", + "ad.infoseek.com", + "ad.intl.xiaomi.com", + "ad.jetsoftware.com", + "ad.keenspace.com", + "ad.lgappstv.com", + "ad.liveinternet.ru", + "ad.lupa.cz", + "ad.mediastorm.hu", + "ad.mg", + "ad.missena.io", + "ad.musicmatch.com", + "ad.myapple.pl", + "ad.mynetreklam.com.streamprovider.net", + "ad.nachtagenten.de", + "ad.nettvservices.com", + "ad.nttnavi.co.jp", + "ad.nwt.cz", + "ad.period-calendar.com", + "ad.profiwin.de", + "ad.prv.pl", + "ad.reachlocal.com", + "ad.simgames.net", + "ad.style", + "ad.technoratimedia.com", + "ad.tv2.no", + "ad.universcine.com", + "ad.usatoday.com", + "ad.virtual-nights.com", + "ad.wavu.hu", + "ad.weatherbug.com", + "ad.wsod.com", + "ad.wz.cz", + "ad.xiaomi.com", + "ad.xmovies8.si", + "ad.xrea.com", + "ad.ztylez.com", + "ad0.bigmir.net", + "ad01.mediacorpsingapore.com", + "ad1.emule-project.org", + "ad1.kde.cz", + "ad2.iinfo.cz", + "ad2.lupa.cz", + "ad2.netriota.hu", + "ad2.nmm.de", + "ad2.xrea.com", + "ad3.iinfo.cz", + "ad3.xrea.com", + "ad4game.com", + "ad4mat.com", + "ad4mat.de", + "ad4mat.net", + "adabra.com", + "adaction.de", + "adadvisor.net", + "adalliance.io", + "adap.tv", + "adapt.tv", + "adbilty.me", + "adblade.com", + "adblade.org", + "adblockanalytics.com", + "adbooth.net", + "adbot.com", + "adbrite.com", + "adbroker.de", + "adbutler.com", + "adbuyer3.lycos.com", + "adcampo.com", + "adcannyads.com", + "adcash.com", + "adcast.deviantart.com", + "adcel.co", + "adcell.de", + "adcenter.net", + "adclick.com", + "adclient1.tucows.com", + "adclixx.net", + "adcolony.com", + "adcomplete.com", + "adconion.com", + "adcontent.gamespy.com", + "adcovery.com", + "adcycle.com", + "add.newmedia.cz", + "addfreestats.com", + "addme.com", + "addressfriend.com", + "ade.clmbtech.com", + "adecn.com", + "adeimptrck.com", + "ademails.com", + "adengage.com", + "adetracking.com", + "adexchangegate.com", + "adexchangeprediction.com", + "adexpose.com", + "adext.inkclub.com", + "adf.ly", + "adfeed.marchex.com", + "adflight.com", + "adforce.com", + "adform.com", + "adform.net", + "adformdsp.net", + "adhaven.com", + "adhese.be", + "adhese.com", + "adhigh.net", + "adhub.media", + "adhunter.media", + "adimage.guardian.co.uk", + "adimages.been.com", + "adimages.carsoup.com", + "adimages.go.com", + "adimages.homestore.com", + "adimages.omroepzeeland.nl", + "adimages.sanomawsoy.fi", + "adimg.com.com", + "adimg.uimserv.net", + "adimg1.chosun.com", + "adimgs.sapo.pt", + "adingo.jp", + "adinjector.net", + "adinterax.com", + "adisfy.com", + "adition.com", + "adition.de", + "adition.net", + "adizio.com", + "adjix.com", + "adjug.com", + "adjuggler.com", + "adjuggler.yourdictionary.com", + "adjust.com", + "adjustnetwork.com", + "adk2.com", + "adland.ru", + "adlegend.com", + "adlightning.com", + "adlog.com.com", + "adloox.com", + "adlooxtracking.com", + "adlure.net", + "adm.fwmrm.net", + "admailtiser.com", + "adman.gr", + "adman.otenet.gr", + "admanagement.ch", + "admanager.btopenworld.com", + "admanager.carsoup.com", + "admanmedia.com", + "admantx.com", + "admarketplace.net", + "admarvel.com", + "admaster.com.cn", + "admatchly.com", + "admedia.com", + "admeld.com", + "admeridianads.com", + "admex.com", + "admidadsp.com", + "adminder.com", + "adminshop.com", + "admixer.net", + "admized.com", + "admob.com", + "admonitor.com", + "adn.lrb.co.uk", + "adnami.io", + "adnet.asahi.com", + "adnet.biz", + "adnet.de", + "adnet.ru", + "adnetasia.com", + "adnetwork.net", + "adnetworkperformance.com", + "adnews.maddog2000.de", + "adnium.com", + "adnxs-simple.com", + "adnxs.com", + "adocean.pl", + "adonspot.com", + "adopsboost.com", + "adoptum.net", + "adoric-om.com", + "adorigin.com", + "adotmob.com", + "adpepper.dk", + "adpepper.nl", + "adperium.com", + "adpia.vn", + "adplxmd.com", + "adprofits.ru", + "adpushup.com", + "adrazzi.com", + "adreactor.com", + "adrecover.com", + "adrecreate.com", + "adremedy.com", + "adreporting.com", + "adrevolver.com", + "adriver.ru", + "adrolays.de", + "adrotate.de", + "adrotic.girlonthenet.com", + "adrta.com", + "ads-backend.chaincliq.com", + "ads-bilek.com", + "ads-dev.pinterest.com", + "ads-game-187f4.firebaseapp.com", + "ads-img.mozilla.org", + "ads-portal-cdn.vidaatv.net", + "ads-twitter.com", + "ads.365.mk", + "ads.5ci.lt", + "ads.73dpi.com", + "ads.aavv.com", + "ads.abovetopsecret.com", + "ads.aceweb.net", + "ads.acpc.cat", + "ads.acrosspf.com", + "ads.activestate.com", + "ads.adfox.ru", + "ads.administrator.de", + "ads.adred.de", + "ads.adstream.com.ro", + "ads.adultfriendfinder.com", + "ads.advance.net", + "ads.adverline.com", + "ads.alive.com", + "ads.alt.com", + "ads.amdmb.com", + "ads.amigos.com", + "ads.annabac.com", + "ads.apn.co.nz", + "ads.appsgeyser.com", + "ads.as4x.tmcs.net", + "ads.as4x.tmcs.ticketmaster.com", + "ads.asiafriendfinder.com", + "ads.avazu.net", + "ads.bb59.ru", + "ads.betfair.com", + "ads.bigchurch.com", + "ads.bigfoot.com", + "ads.bing.com", + "ads.bittorrent.com", + "ads.blog.com", + "ads.bluemountain.com", + "ads.boerding.com", + "ads.boylesports.com", + "ads.canalblog.com", + "ads.casinocity.com", + "ads.casumoaffiliates.com", + "ads.cbc.ca", + "ads.cc", + "ads.cc-dt.com", + "ads.centraliprom.com", + "ads.channel4.com", + "ads.cheabit.com", + "ads.citymagazine.si", + "ads.clasificadox.com", + "ads.co.com", + "ads.colombiaonline.com", + "ads.com.com", + "ads.comeon.com", + "ads.creative-serving.com", + "ads.cybersales.cz", + "ads.dada.it", + "ads.dailycamera.com", + "ads.deltha.hu", + "ads.dennisnet.co.uk", + "ads.desmoinesregister.com", + "ads.deviantart.com", + "ads.devmates.com", + "ads.digital-digest.com", + "ads.digitalmedianet.com", + "ads.digitalpoint.com", + "ads.doit.com.cn", + "ads.domeus.com", + "ads.eagletribune.com", + "ads.easy-forex.com", + "ads.economist.com", + "ads.elcarado.com", + "ads.electrocelt.com", + "ads.elitetrader.com", + "ads.emdee.ca", + "ads.emirates.net.ae", + "ads.epi.sk", + "ads.epltalk.com", + "ads.eu.msn.com", + "ads.fairfax.com.au", + "ads.fastcomgroup.it", + "ads.femmefab.nl", + "ads.ferianc.com", + "ads.filmup.com", + "ads.financialcontent.com", + "ads.flooble.com", + "ads.fool.com", + "ads.footymad.net", + "ads.forbes.net", + "ads.formit.cz", + "ads.fortunecity.com", + "ads.fotosidan.se", + "ads.friendfinder.com", + "ads.gamecity.net", + "ads.gamespyid.com", + "ads.gamigo.de", + "ads.gaming-universe.de", + "ads.gaming1.com", + "ads.getlucky.com", + "ads.gld.dk", + "ads.gmodules.com", + "ads.goyk.com", + "ads.gradfinder.com", + "ads.grindinggears.com", + "ads.gsm-exchange.com", + "ads.gsmexchange.com", + "ads.guardian.co.uk", + "ads.guardianunlimited.co.uk", + "ads.hbv.de", + "ads.hearstmags.com", + "ads.heartlight.org", + "ads.hollywood.com", + "ads.horsehero.com", + "ads.hsoub.com", + "ads.ibest.com.br", + "ads.ibryte.com", + "ads.icq.com", + "ads.ign.com", + "ads.imagistica.com", + "ads.imgur.com", + "ads.independent.com.mt", + "ads.infi.net", + "ads.internic.co.il", + "ads.ipowerweb.com", + "ads.itv.com", + "ads.jewishfriendfinder.com", + "ads.jobsite.co.uk", + "ads.justhungry.com", + "ads.kabooaffiliates.com", + "ads.kaktuz.net", + "ads.kelbymediagroup.com", + "ads.kinxxx.com", + "ads.kompass.com", + "ads.krawall.de", + "ads.leovegas.com", + "ads.lesbianpersonals.com", + "ads.liberte.pl", + "ads.linkedin.com", + "ads.livenation.com", + "ads.ma7.tv", + "ads.mail.bg", + "ads.massinfra.nl", + "ads.mcafee.com", + "ads.mediaodyssey.com", + "ads.mediasmart.es", + "ads.medienhaus.de", + "ads.meetcelebs.com", + "ads.mgnetwork.com", + "ads.miarroba.com", + "ads.mic.com", + "ads.mmania.com", + "ads.mobilebet.com", + "ads.mozilla.org", + "ads.msn.com", + "ads.multimania.lycos.fr", + "ads.muslimehelfen.org", + "ads.mvscoelho.com", + "ads.myadv.org", + "ads.ndtv1.com", + "ads.networksolutions.com", + "ads.newgrounds.com", + "ads.newmedia.cz", + "ads.newsint.co.uk", + "ads.newsquest.co.uk", + "ads.nj.com", + "ads.nola.com", + "ads.nordichardware.com", + "ads.nordichardware.se", + "ads.nyi.net", + "ads.nytimes.com", + "ads.nyx.cz", + "ads.nzcity.co.nz", + "ads.o2.pl", + "ads.oddschecker.com", + "ads.okcimg.com", + "ads.ole.com", + "ads.oneplace.com", + "ads.opensubtitles.org", + "ads.optusnet.com.au", + "ads.outpersonals.com", + "ads.oxyshop.cz", + "ads.passion.com", + "ads.paymonex.net", + "ads.pexi.nl", + "ads.pfl.ua", + "ads.phpclasses.org", + "ads.pinterest.com", + "ads.planet.nl", + "ads.pni.com", + "ads.pof.com", + "ads.powweb.com", + "ads.printscr.com", + "ads.prisacom.com", + "ads.prod.webservices.mozgcp.net", + "ads.program3.com", + "ads.psd2html.com", + "ads.pubmatic.com", + "ads.quoka.de", + "ads.radio1.lv", + "ads.recoletos.es", + "ads.rediff.com", + "ads.redlightcenter.com", + "ads.revjet.com", + "ads.reward-hunt.com", + "ads.samsung.com", + "ads.saymedia.com", + "ads.schmoozecom.net", + "ads.scifi.com", + "ads.seniorfriendfinder.com", + "ads.servebom.com", + "ads.shizmoo.com", + "ads.shopstyle.com", + "ads.sift.co.uk", + "ads.sjon.info", + "ads.smartclick.com", + "ads.socialtheater.com", + "ads.soft32.com", + "ads.soweb.gr", + "ads.space.com", + "ads.sun.com", + "ads.suomiautomaatti.com", + "ads.supplyframe.com", + "ads.syscdn.de", + "ads.themovienation.com", + "ads.thestar.com", + "ads.thrillsaffiliates.com", + "ads.tiktok.com", + "ads.tmcs.net", + "ads.todoti.com.br", + "ads.toplayaffiliates.com", + "ads.townhall.com", + "ads.travelaudience.com", + "ads.trinitymirror.co.uk", + "ads.tripod.com", + "ads.tripod.lycos.co.uk", + "ads.tripod.lycos.es", + "ads.tripod.lycos.it", + "ads.tripod.lycos.nl", + "ads.tso.dennisnet.co.uk", + "ads.twitter.com", + "ads.twojatv.info", + "ads.ultimate-guitar.com", + "ads.uncrate.com", + "ads.unison.bg", + "ads.usatoday.com", + "ads.uxs.at", + "ads.v-lazer.com", + "ads.verticalresponse.com", + "ads.vgchartz.com", + "ads.virtual-nights.com", + "ads.virtuopolitan.com", + "ads.vnumedia.com", + "ads.walkiberia.com", + "ads.watson.ch", + "ads.weather.ca", + "ads.web.de", + "ads.webinak.sk", + "ads.webmasterpoint.org", + "ads.whoishostingthis.com", + "ads.wiezoekje.nl", + "ads.wikia.nocookie.net", + "ads.wineenthusiast.com", + "ads.wwe.biz", + "ads.xhamster.com", + "ads.xtra.co.nz", + "ads.yahoo.com", + "ads.yimg.com", + "ads.yldmgrimg.net", + "ads.youtube.com", + "ads.yumenetworks.com", + "ads1-adnow.com", + "ads1.msn.com", + "ads1.virtual-nights.com", + "ads10.speedbit.com", + "ads180.com", + "ads2.brazzers.com", + "ads2.contentabc.com", + "ads2.femmefab.nl", + "ads2.gamecity.net", + "ads2.hsoub.com", + "ads2.virtual-nights.com", + "ads2.xnet.cz", + "ads2004.treiberupdate.de", + "ads24h.net", + "ads3-adnow.com", + "ads3.contentabc.com", + "ads3.gamecity.net", + "ads3.virtual-nights.com", + "ads4.gamecity.net", + "ads4.virtual-nights.com", + "ads4homes.com", + "ads5.virtual-nights.com", + "ads6.gamecity.net", + "ads7.gamecity.net", + "adsafeprotected.com", + "adsatt.abc.starwave.com", + "adsatt.abcnews.starwave.com", + "adsatt.espn.go.com", + "adsatt.espn.starwave.com", + "adsatt.go.starwave.com", + "adsby.bidtheatre.com", + "adsbydelema.com", + "adscale.de", + "adscholar.com", + "adscience.nl", + "adsco.re", + "adscpm.com", + "adsdaq.com", + "adsdk.yandex.ru", + "adsend.de", + "adsensecustomsearchads.com", + "adserve.ams.rhythmxchange.com", + "adserve.gkeurope.de", + "adserve.io", + "adserve.jbs.org", + "adserver.71i.de", + "adserver.adultfriendfinder.com", + "adserver.adverty.com", + "adserver.anawe.cz", + "adserver.ariase.org", + "adserver.bdoce.cl", + "adserver.betandwin.de", + "adserver.bing.com", + "adserver.bizedge.com", + "adserver.bizhat.com", + "adserver.break-even.it", + "adserver.cams.com", + "adserver.cdnstream.com", + "adserver.cherryfind.co.uk", + "adserver.com", + "adserver.diariodosertao.com.br", + "adserver.digitoday.com", + "adserver.echdk.pl", + "adserver.friendfinder.com", + "adserver.generationiron.com", + "adserver.hwupgrade.it", + "adserver.ilango.de", + "adserver.industryarena.com", + "adserver.info7.mx", + "adserver.irishwebmasterforum.com", + "adserver.janes.com", + "adserver.kontent.com", + "adserver.mobi", + "adserver.news.com.au", + "adserver.nydailynews.com", + "adserver.o2.pl", + "adserver.oddschecker.com", + "adserver.omroepzeeland.nl", + "adserver.otthonom.hu", + "adserver.pampa.com.br", + "adserver.piksel.mk", + "adserver.pl", + "adserver.portugalmail.net", + "adserver.pressboard.ca", + "adserver.sanomawsoy.fi", + "adserver.sciflicks.com", + "adserver.scr.sk", + "adserver.smgfiles.com", + "adserver.trojaner-info.de", + "adserver.tupolska.com", + "adserver.twitpic.com", + "adserver.virginmedia.com", + "adserver.waggonerguide.com", + "adserver01.de", + "adserverplus.com", + "adserverpub.com", + "adserversolutions.com", + "adserverxxl.de", + "adservice.google.com", + "adservice.google.com.mt", + "adserving.unibet.com", + "adservingfront.com", + "adservrs.com", + "adservrs.com.edgekey.net", + "adsfac.eu", + "adsfac.net", + "adsfac.us", + "adsflowprime.top", + "adshrink.it", + "adside.com", + "adsiduous.com", + "adskeeper.co.uk", + "adskeeper.com", + "adsklick.de", + "adsmart.net", + "adsmartracker.com", + "adsmetadata.startappservice.com", + "adsmogo.com", + "adsnative.com", + "adsoftware.com", + "adsolut.in", + "adspeed.net", + "adspirit.de", + "adsponse.de", + "adspredictiv.com", + "adspsp.com", + "adsroller.com", + "adsrv.deviantart.com", + "adsrv.eacdn.com", + "adsrv.iol.co.za", + "adsrv.kobi.tv", + "adsrv.moebelmarkt.tv", + "adsrv2.swidnica24.pl", + "adsrvr.org", + "adstacks.in", + "adstanding.com", + "adstat.4u.pl", + "adstest.weather.com", + "adstub.net", + "adsupply.com", + "adswizz.com", + "adsxyz.com", + "adsynergy.com", + "adsys.townnews.com", + "adsystem.simplemachines.org", + "adt598.com", + "adtech.com", + "adtech.de", + "adtechjp.com", + "adtechus.com", + "adtegrity.net", + "adthis.com", + "adthrive.com", + "adtiger.de", + "adtilt.com", + "adtng.com", + "adtology.com", + "adtoma.com", + "adtrace.org", + "adtrack.voicestar.com", + "adtraction.com", + "adtrade.net", + "adultadvertising.com", + "adv-adserver.com", + "adv.donejty.pl", + "adv.freeonline.it", + "adv.hwupgrade.it", + "adv.mpvc.it", + "adv.nexthardware.com", + "adv.webmd.com", + "adv.wp.pl", + "adv.yo.cz", + "advangelists.com", + "advariant.com", + "adventory.com", + "adventurousamount.com", + "advert.bayarea.com", + "advert.dyna.ultraweb.hu", + "adverticum.com", + "adverticum.net", + "advertipros.com", + "advertise.com", + "advertiserurl.com", + "advertising.com", + "advertisingbanners.com", + "advertisingbox.com", + "advertmarket.com", + "advertmedia.de", + "advertpro.ya.com", + "advertserve.com", + "advertwizard.com", + "advideo.uimserv.net", + "adview.com", + "advisormedia.cz", + "adviva.net", + "advnt.com", + "adwebone.com", + "adwhirl.com", + "adworldnetwork.com", + "adworx.at", + "adworx.nl", + "adx.gayboy.at", + "adxpansion.com", + "adxpose.com", + "adyoulike.com", + "adz.rashflash.com", + "adzbazar.com", + "adzerk.net", + "adzerk.s3.amazonaws.com", + "adzestocp.com", + "aerialapps.com", + "aerserv.com", + "af-ad.co.uk", + "affec.tv", + "affectionknit.com", + "affili.net", + "affiliate.1800flowers.com", + "affiliate.dtiserv.com", + "affiliate.travelnow.com", + "affiliate.treated.com", + "affiliatefuture.com", + "affiliates.allposters.com", + "affiliates.babylon.com", + "affiliates.globat.com", + "affiliates.streamray.com", + "affiliates.thinkhost.net", + "affiliates.thrixxx.com", + "affiliates.ultrahosting.com", + "affiliatetracking.com", + "affiliatetracking.net", + "affiliatewindow.com", + "afflnx.com", + "afftracking.justanswer.com", + "afraidlanguage.com", + "agencysignals.com", + "agkn.com", + "ah-ha.com", + "ahalogy.com", + "aheadday.com", + "aim4media.com", + "airplanecoffee.com", + "airpush.com", + "aistat.net", + "ak0gsh40.com", + "alchemist.go2cloud.org", + "alclick.com", + "alenty.com", + "alexa-sitestats.s3.amazonaws.com", + "algorix.co", + "aliasanvil.com", + "alikeaddition.com", + "alipromo.com", + "alluringbucket.com", + "aloofmetal.com", + "aloofvest.com", + "alphonso.tv", + "als-svc.nytimes.com", + "amazingcounters.com", + "amazon-adsystem.com", + "americash.com", + "amidsttruly.com", + "amung.us", + "analytics-production.hapyak.com", + "analytics.adpost.org", + "analytics.algoepico.net", + "analytics.bitrix.info", + "analytics.cloudron.io", + "analytics.ext.go-tellm.com", + "analytics.google.com", + "analytics.htmedia.in", + "analytics.icons8.com", + "analytics.inlinemanual.com", + "analytics.jst.ai", + "analytics.justuno.com", + "analytics.lucid.app", + "analytics.mailmunch.co", + "analytics.mobile.yandex.net", + "analytics.momentum-institut.at", + "analytics.myfinance.com", + "analytics.ostr.io", + "analytics.phando.com", + "analytics.picsart.com", + "analytics.pinterest.com", + "analytics.pointdrive.linkedin.com", + "analytics.poolshool.com", + "analytics.posttv.com", + "analytics.santander.co.uk", + "analytics.swiggy.com", + "analytics.tiktok.com", + "analytics.xelondigital.com", + "analytics.yahoo.com", + "analyticsapi.happypancake.net", + "ancientact.com", + "androiddownload.net", + "aniview.com", + "annonser.dagbladet.no", + "annoyedairport.com", + "annoyingacoustics.com", + "anrdoezrs.net", + "anstrex.com", + "anxiousapples.com", + "api.affiliations.site", + "api.amplitude.com", + "api.appmetrica.yandex.ru", + "api.eu.amplitude.com", + "api.intensifier.de", + "api.iterable.com", + "api.kameleoon.com", + "api.lab.amplitude.com", + "api.rudderlabs.com", + "api.swetrix.com", + "api2.amplitude.com", + "apolloprogram.io", + "app-analytics-v2.snapchat.com", + "app-analytics.snapchat.com", + "app-measurement.com", + "app.pendo.io", + "app2.salesmanago.pl", + "appboycdn.com", + "applovin.com", + "appsflyer.com", + "aps.hearstnp.com", + "apsalar.com", + "aptabase.com", + "apture.com", + "apu.samsungelectronics.com", + "aquaticowl.com", + "aralego.com", + "arc1.msn.com", + "archswimming.com", + "ard.xxxblackbook.com", + "aromamirror.com", + "as.webmd.com", + "as2.adserverhd.com", + "aserv.motorsgate.com", + "asewlfjqwlflkew.com", + "askdriver.com", + "aso1.net", + "assets1.exgfnetwork.com", + "assoc-amazon.com", + "aswpapius.com", + "aswpsdkus.com", + "async.gsyndication.com", + "at-adserver.alltop.com", + "at-o.net", + "atdmt.com", + "athena-ads.wikia.com", + "ato.mx", + "attractionbanana.com", + "attribution.report", + "atwola.com", + "auctionads.com", + "auctionads.net", + "aud.pubmatic.com", + "audience.media", + "audience2media.com", + "audienceinsights.com", + "audio.ad", + "audit.median.hu", + "audit.webinform.hu", + "augur.io", + "auto-bannertausch.de", + "avalonalbum.com", + "avazutracking.net", + "avenuea.com", + "avocet.io", + "awempire.com", + "awin1.com", + "awstrack.me", + "awzbijw.com", + "axonix.com", + "ay.delivery", + "ayads.co", + "b-s.tercept.com", + "b.videoamp.com", + "b3.videoamp.com", + "ba.afl.rakuten.co.jp", + "backbeatmedia.com", + "bagsurprise.com", + "banik.redigy.cz", + "banner.ambercoastcasino.com", + "banner.buempliz-online.ch", + "banner.cotedazurpalace.com", + "banner.easyspace.com", + "banner.elisa.net", + "banner.finzoom.ro", + "banner.goldenpalace.com", + "banner.inyourpocket.com", + "banner.linux.se", + "banner.media-system.de", + "banner.nixnet.cz", + "banner.noblepoker.com", + "banner.northsky.com", + "banner.reinstil.de", + "banner.tanto.de", + "banner.titan-dsl.de", + "banner10.zetasystem.dk", + "bannerads.de", + "bannerboxes.com", + "bannerconnect.com", + "bannerconnect.net", + "bannergrabber.internet.gr", + "bannerimage.com", + "bannermall.com", + "bannermanager.bnr.bg", + "bannerpower.com", + "banners.adultfriendfinder.com", + "banners.amigos.com", + "banners.asiafriendfinder.com", + "banners.babylon-x.com", + "banners.bol.com.br", + "banners.cams.com", + "banners.czi.cz", + "banners.dine.com", + "banners.direction-x.com", + "banners.freett.com", + "banners.friendfinder.com", + "banners.getiton.com", + "banners.iq.pl", + "banners.passion.com", + "banners.payserve.com", + "banners.resultonline.com", + "banners.sys-con.com", + "banners.thomsonlocal.com", + "banners.virtuagirlhd.com", + "bannerserver.com", + "bannershotlink.perfectgonzo.com", + "bannersng.yell.com", + "bannerspace.com", + "bannerswap.com", + "bannertrack.net", + "bannery.cz", + "bannieres.wdmedia.net", + "bans.bride.ru", + "baremetrics.com", + "barnesandnoble.bfast.com", + "basebanner.com", + "basketballbelieve.com", + "baskettexture.com", + "bastingart.com", + "bat.bing.com", + "bbelements.com", + "bbn.img.com.ua", + "beachfront.com", + "beacon.gu-web.net", + "beacon.netflix.com", + "beacons.gcp.gvt2.com", + "beacons.gvt2.com", + "bebi.com", + "beemray.com", + "begun.ru", + "behavioralengine.com", + "belstat.com", + "belstat.nl", + "benefits.sovendus.com", + "benfly.net", + "berp.com", + "bespoke.iln8.net", + "bestboundary.com", + "bestbuy.7tiv.net", + "bewilderedblade.com", + "bfmio.com", + "bhcumsc.com", + "bid.pubmatic.com", + "bidbarrel.cbsnews.com", + "bidclix.com", + "bidclix.net", + "bidpapers.com", + "bidr.io", + "bidsopt.com", + "bidswitch.net", + "bidtellect.com", + "bidvertiser.com", + "big-bang-ads.com", + "bigbangmedia.com", + "bigclicks.com", + "bigfishapparel.com", + "bigreal.org", + "bigworldfashion.com", + "billboard.cz", + "billtable.com", + "birthdaybelief.com", + "bitmedianetwork.com", + "bizible.com", + "bizographics.com", + "bizrate.com", + "bizzclick.com", + "bkrtx.com", + "bleachbubble.com", + "blingbucks.com", + "blis.com", + "blockadblock.com", + "blogads.com", + "blogcounter.de", + "blogherads.com", + "blogtoplist.se", + "blogtopsites.com", + "blueconic.com", + "blueconic.net", + "bluekai.com", + "bluelithium.com", + "bluenest.net", + "bluewhaleweb.com", + "blushingbeast.com", + "blushingbread.com", + "bm.annonce.cz", + "bn.bfast.com", + "bnrs.ilm.ee", + "boffoadsapi.com", + "boilingbeetle.com", + "boilingmadsoup.com", + "bongacash.com", + "boomads.com", + "boomtrain.com", + "boredcrown.com", + "boudja.com", + "bounceads.net", + "bounceexchange.com", + "bowie-cdn.fathomdns.com", + "box.anchorfree.net", + "bpath.com", + "bpu.samsungelectronics.com", + "braincash.com", + "brand-display.com", + "brandreachsys.com", + "brandybison.com", + "braze.eu", + "breadbalance.com", + "breakableinsurance.com", + "breaktime.com.tw", + "brealtime.com", + "bridgetrack.com", + "brightcom.com", + "brightinfo.com", + "brightmountainmedia.com", + "broadcastbed.com", + "broadstreetads.com", + "browser-http-intake.logs.datadoghq.com", + "browser-http-intake.logs.datadoghq.eu", + "bs.yandex.ru", + "btglss.net", + "btrll.com", + "bttrack.com", + "bubblyaction.com", + "buddycanvas.com", + "buysellads.com", + "buzzonclick.com", + "bwp.download.com", + "c.bigmir.net", + "c1exchange.com", + "c212.net", + "cakesdrum.com", + "calculatingcircle.com", + "calculatorstatement.com", + "call-ad-network-api.marchex.com", + "callousbrake.com", + "callrail.com", + "calmcactus.com", + "calypsocapsule.com", + "campaign.bharatmatrimony.com", + "caniamedia.com", + "capriciouscorn.com", + "captainbicycle.com", + "captainstick.com", + "carambo.la", + "carbonads.com", + "carbonads.net", + "cardmethod.com", + "caringcast.com", + "carsarace.com", + "cartstack.com", + "carvecakes.com", + "casalemedia.com", + "casalmedia.com", + "cash4members.com", + "cash4popup.de", + "cashcrate.com", + "cashengines.com", + "cashfiesta.com", + "cashpartner.com", + "cashstaging.me", + "casinopays.com", + "casinorewards.com", + "casinotraffic.com", + "cattlecommittee.com", + "causecherry.com", + "cautiouscredit.com", + "cbanners.virtuagirlhd.com", + "cbzxy.com", + "cdn.bannerflow.com", + "cdn.branch.io", + "cdn.freshmarketer.com", + "cdn.heapanalytics.com", + "cdn.keywee.co", + "cdn.mouseflow.com", + "cdn.onesignal.com", + "cdn.scarabresearch.com", + "cdn.segment.com", + "cdnondemand.org", + "cedato.com", + "celtra.com", + "centerpointmedia.com", + "cetrk.com", + "cgicounter.puretec.de", + "chairscrack.com", + "channelintelligence.com", + "chargecracker.com", + "chart.dk", + "chartbeat.com", + "chartbeat.net", + "chartboost.com", + "checkstat.nl", + "cherriescare.com", + "childlikecrowd.com", + "chubbycreature.com", + "citrusad.net", + "cityads.telus.net", + "cj.com", + "cjbmanagement.com", + "cjlog.com", + "cl.turkishairlines.com", + "cl0udh0st1ng.com", + "clck.ru", + "clevernt.com", + "click-1.pl", + "click.airmalta-mail.com", + "click.aliexpress.com", + "click.allkeyshop.com", + "click.bkdpt.com", + "click.cartsguru.io", + "click.ccg.nintendo.com", + "click.cision.com", + "click.classmates.com", + "click.comm.rcibank.co.uk", + "click.crm.ba.com", + "click.digital.metaquestmail.com", + "click.discord.com", + "click.e.bbcmail.co.uk", + "click.e.progressive.com", + "click.e.zoom.us", + "click.em.blizzard.com", + "click.email.bbc.com", + "click.email.lhh.com", + "click.email.microsoftemail.com", + "click.email.sonos.com", + "click.emails.argos.co.uk", + "click.emails.tuclothing.sainsburys.co.uk", + "click.fool.com", + "click.hookupinyourcity.com", + "click.hooligapps.com", + "click.i.southwesternrailway.com", + "click.infoblox.com", + "click.justwatch.com", + "click.kmindex.ru", + "click.liftoff.io", + "click.mail.hotels.com", + "click.mail.salesforce.com", + "click.mailing.ticketmaster.com", + "click.mkt.grab.com", + "click.news.vans.com", + "click.nl.npr.org", + "click.nvgaming.nvidia.com", + "click.redditmail.com", + "click.uber.com", + "click.v.visionlab.es", + "click2freemoney.com", + "click360v2-ingest.azurewebsites.net", + "click4.pro", + "clickadddilla.com", + "clickadz.com", + "clickagents.com", + "clickbank.com", + "clickbooth.com", + "clickboothlnk.com", + "clickbrokers.com", + "clickcease.com", + "clickcompare.co.uk", + "clickdensity.com", + "clickedyclick.com", + "clickfuse.com", + "clickhereforcellphones.com", + "clickngo.pro", + "clickngo.top", + "clickonometrics.pl", + "clicks.checkatrade.com", + "clicks.deliveroo.co.uk", + "clicks.equantum.com", + "clicks.eventbrite.com", + "clicks.monzo.com", + "clickserve.cc-dt.com", + "clicksinfo.thefork.co.uk", + "clicktag.de", + "clickthruserver.com", + "clickthrutraffic.com", + "clicktrack.pubmatic.com", + "clicktrack.ziyu.net", + "clicktracks.com", + "clicktrade.com", + "clickxchange.com", + "clickyab.com", + "clickz.com", + "clientgear.com", + "clientmetrics-pa.googleapis.com", + "clikerz.net", + "cliksolution.com", + "clixgalore.com", + "clk1005.com", + "clk1011.com", + "clk1015.com", + "clkrev.com", + "clksite.com", + "cloudflareinsights.com", + "cloudguppy.com", + "clrstm.com", + "cluster.adultworld.com", + "clustrmaps.com", + "cmp.dmgmediaprivacy.co.uk", + "cmvrclicks000.com", + "cnomy.com", + "cnt1.pocitadlo.cz", + "cnvlink.com", + "cny.yoyo.org", + "codeadnetwork.com", + "cognitiv.ai", + "cointraffic.io", + "coldbalance.com", + "collector-dev.cdp-dev.cnn.com", + "collector.cdp.cnn.com", + "colonize.com", + "colorfulpet.com", + "coloroptimizer.com", + "comfortablecheese.com", + "commissionmonster.com", + "communications.melitaltd.com", + "compactbanner.com", + "comparereaction.com", + "compilework.com", + "comprabanner.it", + "conditionchange.com", + "conductrics.com", + "confiant-integrations.net", + "confidencetalks.com", + "configv2.unityads.unity3d.com", + "connatix.com", + "connectad.io", + "connextra.com", + "consciouscheese.com", + "consensad.com", + "consensu.org", + "contadores.miarroba.com", + "content.acc-hd.de", + "content.ad", + "content22.online.citi.com", + "contextweb.com", + "contrack.link", + "converge-digital.com", + "conversantmedia.com", + "conversionbet.com", + "conversionruler.com", + "convertingtraffic.com", + "convrse.media", + "cookies.cmpnet.com", + "coolkidsdistrict.com", + "cootlogix.com", + "copperstills.net", + "copycarpenter.com", + "copyrightaccesscontrols.com", + "coremetrics.com", + "cosmosjackson.com", + "count.rin.ru", + "count.west263.com", + "counted.com", + "counter.bloke.com", + "counter.cnw.cz", + "counter.cz", + "counter.dreamhost.com", + "counter.mirohost.net", + "counter.mojgorod.ru", + "counter.rambler.ru", + "counter.search.bg", + "counter.snackly.co", + "counting.kmindex.ru", + "coupling-media.de", + "cowbelltime.com", + "coxmt.com", + "cozyhillside.com", + "cpalead.com", + "cpays.com", + "cpmstar.com", + "cpu.samsungelectronics.com", + "cpx.to", + "cpxinteractive.com", + "cqcounter.com", + "crabbychin.com", + "craktraffic.com", + "crashchance.com", + "crashlytics.com", + "crashlyticsreports-pa.googleapis.com", + "cratecamera.com", + "crawlability.com", + "crazyegg.com", + "crazypopups.com", + "creatives.livejasmin.com", + "creatopy.net", + "crimsonmeadow.com", + "criteo.com", + "criteo.net", + "critictruck.com", + "crowdedmass.com", + "crowdgravity.com", + "crsspxl.com", + "crta.dailymail.co.uk", + "crtv.mate1.com", + "crwdcntrl.net", + "crypto-loot.org", + "crystalboulevard.com", + "cs.co", + "curbminers.com", + "curiousmetal.com", + "curtaincows.com", + "curveshore.com", + "cushiondrum.com", + "customad.cnn.com", + "customads.co", + "customerevents.netflix.com", + "customers.kameleoon.com", + "cutechin.com", + "cxense.com", + "cyberbounty.com", + "cyclinghere.com", + "d-collect.jennifersoft.com", + "d-collector.jennifersoft.com", + "d.adroll.com", + "d1f0tbk1v3e25u.cloudfront.net", + "d2cmedia.ca", + "d81mfvml8p5ml.cloudfront.net", + "dabiaozhi.com", + "dacdn.visualwebsiteoptimizer.com", + "dacdn.vwo.com", + "dakic-ia-300.com", + "damageddistance.com", + "damdoor.com", + "dapper.net", + "data.namesakeoscilloscopemarquis.com", + "datapickles.com", + "datenow.link", + "daybreaklights.com", + "dc-storm.com", + "de17a.com", + "dealdotcom.com", + "decenterads.com", + "decisivebase.com", + "decisivedrawer.com", + "decknetwork.net", + "deepintent.com", + "delicatecascade.com", + "deloo.de", + "demandbase.com", + "demdex.net", + "deployads.com", + "desertgates.com", + "desiredirt.com", + "detailedgovernment.com", + "detectdiscovery.com", + "dev.visualwebsiteoptimizer.com", + "dewdroplagoon.com", + "dianomi.com", + "dicecandies.com", + "didtheyreadit.com", + "digestiondrawer.com", + "digital-ads.s3.amazonaws.com", + "direct-events-collector.spot.im", + "direct-promo.pro", + "directaclick.com", + "directorym.com", + "discountclick.com", + "discreetfield.com", + "displayvertising.com", + "disqusads.com", + "dist.belnk.com", + "distillery.wistia.com", + "districtm.ca", + "districtm.io", + "dmp.mall.tv", + "dmtracker.com", + "dmtracking.alibaba.com", + "dmtracking2.alibaba.com", + "dnsdelegation.io", + "do-global.com", + "dockdigestion.com", + "dogcollarfavourbluff.com", + "domaining.in", + "domdex.com", + "dotmetrics.net", + "dotomi.com", + "doubleclick.com", + "doubleclick.de", + "doubleclick.net", + "doublepimp.com", + "doubleverify.com", + "dpbolvw.net", + "dpu.samsungelectronics.com", + "dq95d35.com", + "dragonbats.com", + "drumcash.com", + "dsp.colpirio.com", + "dsp.io", + "dstillery.com", + "dustyhammer.com", + "dyntrk.com", + "e-eu.customeriomail.com", + "e-m.fr", + "e-planning.net", + "e.customeriomail.com", + "e.kde.cz", + "e37364.dscd.akamaiedge.net", + "eadexchange.com", + "eas.almamedia.fi", + "easyhits4u.com", + "ebayadvertising.com", + "ebuzzing.com", + "echoacloud.com", + "ecircle-ag.com", + "ecleneue.com", + "eclick.vn", + "eclicks.deliveroo.com", + "eclkspbn.com", + "ecoupons.com", + "edaa.eu", + "edgexads.com", + "eighthlayer.net", + "eiv.baidu.com", + "elasticchange.com", + "elderlytown.com", + "elitedollars.com", + "em1.yoursantander.co.uk", + "email-link.adtidy.info", + "email-link.adtidy.net", + "email-link.adtidy.org", + "email-links.crowdfireapp.com", + "email-open.adtidy.net", + "email-open.adtidy.org", + "email-trk.ihg-businessedge.com", + "email.mg1.substack.com", + "emailer.stockbit.com", + "emaillinks.soundiiz.com", + "emebo.io", + "emerse.com", + "emetriq.de", + "emjcd.com", + "eml.blackduck.com", + "emltrk.com", + "emodoinc.com", + "emptyescort.com", + "emxdigital.com", + "energeticladybug.com", + "engage.tines.com", + "engage.windows.com", + "engagebdr.com", + "engageya.com", + "engine.espace.netavenir.com", + "engineertrick.com", + "enginenetwork.com", + "enormousearth.com", + "enquisite.com", + "ensighten.com", + "entercasino.com", + "entrecard.s3.amazonaws.com", + "enviousthread.com", + "epom.com", + "epp.bih.net.ba", + "eqads.com", + "eqy.link", + "erne.co", + "ero-advertising.com", + "essaygiants.com", + "esty.com", + "et.educationdynamics.com", + "et.nytimes.com", + "etahub.com", + "etargetnet.com", + "etracker.com", + "etracker.de", + "eu-adcenter.net", + "eule1.pmu.fr", + "eulerian.net", + "eurekster.com", + "euros4click.de", + "eusta.de", + "evadav.com", + "evadavdsp.pro", + "evencoating.com", + "eventexistence.com", + "events-eu.freshsuccess.com", + "events-us.freshsuccess.com", + "everestads.net", + "everesttech.net", + "evergage.com", + "eversales.space", + "evs.sgmt.loom.com", + "evyy.net", + "exampleshake.com", + "exchange-it.com", + "exchangead.com", + "exchangeclicksonline.com", + "exelate.com", + "exelator.com", + "exhibitsneeze.com", + "exit76.com", + "exitexchange.com", + "exitfuel.com", + "exoclick.com", + "exosrv.com", + "experianmarketingservices.digital", + "explorads.com", + "exponea.com", + "exponential.com", + "exportdialog.com", + "express-submit.de", + "extractobservation.com", + "extreme-dm.com", + "extremetracking.com", + "eyeota.net", + "eyeviewads.com", + "eyewonder.com", + "ezula.com", + "f.pie.org", + "f7ds.liberation.fr", + "fabric.io", + "fadedsnow.com", + "fairfeeling.com", + "fakedisguise.com", + "fallaciousfifth.com", + "fallingshoals.com", + "fam-ad.com", + "farethief.com", + "farmergoldfish.com", + "fast-redirecting.com", + "fastclick.com", + "fastclick.com.edgesuite.net", + "fastclick.net", + "fastly-insights.com", + "faultycanvas.com", + "fave.co", + "fc.webmasterpro.de", + "feedbackresearch.com", + "feedjit.com", + "feedmob.com", + "femalecook.com", + "figsprotein.com", + "figurehunter.net", + "fillthemap.com", + "fimserve.com", + "findcommerce.com", + "findyourcasino.com", + "fireads.online", + "fireads.org", + "fireworkadservices.com", + "fireworkanalytics.com", + "fireworks-advertising.com", + "firstlightera.com", + "fishingtoolsbox.com", + "fixedfold.com", + "fjordsand.com", + "flairadscpc.com", + "flakyfeast.com", + "flashtalking.com", + "flashtexting.com", + "fleshlightcash.com", + "flexbanner.com", + "flimsycircle.com", + "flimsythought.com", + "floodprincipal.com", + "flourishinginnovation.com", + "floweryflavor.com", + "flowgo.com", + "flurry.com", + "fontserif.com", + "foo.cosmocode.de", + "foresee.com", + "forex-affiliate.net", + "forkcdn.com", + "fourlevelsgame.com", + "fpctraffic.com", + "fpjs.io", + "fqtag.com", + "fraysystems.com", + "free-counter.co.uk", + "freebanner.com", + "freecounterstat.com", + "freedomgrail.com", + "freelogs.com", + "freepay.com", + "freestats.com", + "freestats.tv", + "freewebcounter.com", + "freewheel.com", + "freewheel.tv", + "freezingbuilding.com", + "frequentflesh.com", + "freshrelevance.com", + "fronttoad.com", + "frtyj.com", + "frtyk.com", + "fudgegenie.com", + "fullrestore.net", + "fullstory.com", + "functionalcrown.com", + "functionalfeather.com", + "funklicks.com", + "funnelytics.io", + "funtoyplanet.com", + "furryfork.com", + "fusionads.net", + "fusionquest.com", + "futuristicfifth.com", + "futuristicframe.com", + "fuzzybasketball.com", + "fwcdn1.com", + "fwcdn2.com", + "fxstyle.net", + "g2.gumgum.com", + "ga.clearbit.com", + "gadsbee.com", + "galaxien.com", + "game-advertising-online.com", + "gamesites100.net", + "gamesites200.com", + "gammamaximum.com", + "gaug.es", + "gavvia.com", + "gearwom.de", + "geo.digitalpoint.com", + "geobanner.adultfriendfinder.com", + "georiot.com", + "geovisite.com", + "getclicky.com", + "getintent.com", + "getmyads.com", + "getxmlisi.com", + "giddycoat.com", + "glasscoyote.com", + "glisteningsign.com", + "globalismedia.com", + "gloriousbeef.com", + "gmads.net", + "gml.email", + "go-clicks.de", + "go-link.network", + "go-mpulse.net", + "go-rank.de", + "go.clickwww.com", + "go.dhs.gov", + "go.eu.sparkpostmail1.com", + "go.first.org", + "go.icann.org", + "go.scmagazine.com", + "go.usa.gov", + "go.xlirdr.com", + "go2affise.com", + "godseedband.com", + "goingplatinum.com", + "goldstats.com", + "gondolagnome.com", + "goodcontentservice.top", + "google-analytics.com", + "googleadservices.com", + "googleanalytics.com", + "googlesyndication.com", + "googletagmanager.com", + "googletagservices.com", + "goqon.com", + "gostats.com", + "gothamads.com", + "gotoyahoo.com", + "gotraffic.net", + "gp.dejanews.com", + "grandfatherguitar.com", + "granlite.com", + "grapeshot.co.uk", + "greystripe.com", + "groovespacing.com", + "grouchybrothers.com", + "groundtruth.com", + "growthrx.in", + "gscontxt.net", + "guardeddirection.com", + "guardedschool.com", + "gumyfui.com", + "gunggo.com", + "h-bid.com", + "h-trck.com", + "h0.t.hubspotemail.net", + "haikusoap.com", + "halcyoncanyon.com", + "halocolor.com", + "haltingbadge.com", + "hammerhearing.com", + "hamsterspot.com", + "handsomehose.com", + "handyfireman.com", + "handyincrease.com", + "haplesshydrant.com", + "harrenmedia.com", + "harrenmedianetwork.com", + "hayweb.net", + "hb.afl.rakuten.co.jp", + "hb.vntsm.com", + "hbb.afl.rakuten.co.jp", + "hbopenbid.pubmatic.com", + "heap.com", + "heimi-lwx.com", + "hellobar.com", + "helpcollar.com", + "hentaicounter.com", + "herbalaffiliateprogram.com", + "hexcan.com", + "hexusads.fluent.ltd.uk", + "heyos.com", + "hf5rbejvpwds.com", + "hfc195b.com", + "hgads.com", + "highnoongear.com", + "hightrafficads.com", + "hilariouszinc.com", + "histats.com", + "historytrade.com", + "hit-parade.com", + "hit.ua", + "hit.webcentre.lycos.co.uk", + "hitbox.com", + "hitcounters.miarroba.com", + "hitlist.ru", + "hitlounge.com", + "hitometer.com", + "hits-i.iubenda.com", + "hits.europuls.eu", + "hits.informer.com", + "hits.puls.lv", + "hits.sh", + "hits.theguardian.com", + "hits4me.com", + "hitslink.com", + "hittail.com", + "hlok.qertewrt.com", + "hocgeese.com", + "hollowafterthought.com", + "homelycrown.com", + "homepageking.de", + "honorableland.com", + "hostedads.realitykings.com", + "hotjar.com", + "hotlog.ru", + "hotrank.com.tw", + "hoverowl.com", + "hs-analytics.net", + "hs-banner.com", + "hsadspixel.net", + "hsleadflows.net", + "hsn.uqhv.net", + "htlbid.com", + "httpool.com", + "hubspotlinks.com", + "hueads.com", + "hueadsortb.com", + "hueadsxml.com", + "hurricanedigitalmedia.com", + "hustlercoach.com", + "hydraconcept.com", + "hydramedia.com", + "hyperbanner.net", + "hypertracker.com", + "hyprmx.com", + "hystericalcloth.com", + "i-i.lt", + "i305175.net", + "ia.iinfo.cz", + "iad.anm.co.uk", + "iadnet.com", + "ibillboard.com", + "icptrack.com", + "icywinter.com", + "id5-sync.com", + "idealadvertising.net", + "idevaffiliate.com", + "idtargeting.com", + "ientrymail.com", + "iesnare.com", + "ifa.tube8live.com", + "ilbanner.com", + "ilead.itrack.it", + "illustriousoatmeal.com", + "image2.pubmatic.com", + "image3.pubmatic.com", + "image4.pubmatic.com", + "image6.pubmatic.com", + "imagecash.net", + "images-pw.secureserver.net", + "img.prohardver.hu", + "imgpromo.easyrencontre.com", + "immensehoney.com", + "imonomy.com", + "imp.i312864.net", + "impossibleexpansion.com", + "imprese.cz", + "impressionmedia.cz", + "impressionmonster.com", + "improvedigital.com", + "imrworldwide.com", + "inclk.com", + "incognitosearches.com", + "incoming-telemetry.thunderbird.net", + "incoming.telemetry.mozilla.org", + "indexexchange.com", + "indexstats.com", + "indexww.com", + "indieclick.com", + "industrybrains.com", + "inetlog.ru", + "infinite-ads.com", + "infinityads.com", + "infoevent.startappservice.com", + "infolinks.com", + "inmobi.com", + "inner-active.com", + "inner-active.mobi", + "innovid.com", + "inquisitiveinvention.com", + "insgly.net", + "insidepsych.net", + "insightexpress.com", + "insightexpressai.com", + "inskinad.com", + "inspectlet.com", + "install.365-stream.com", + "instantmadness.com", + "insticator.com", + "intelliads.com", + "intelligenceadx.com", + "interactive.forthnet.gr", + "intercom-clicks.com", + "intergi.com", + "internalcondition.com", + "internetfuel.com", + "interreklame.de", + "ioam.de", + "ip.ro", + "ip193.cn", + "iperceptions.com", + "ipredictive.com", + "ipstack.com", + "irchan.com", + "ireklama.cz", + "is-tracking-pixel-api-prod.appspot.com", + "islandwebhelp.com", + "itop.cz", + "its-that-easy.com", + "ivwbox.de", + "ivykiosk.com", + "iyfbodn.com", + "iyfnzgb.com", + "j93557g.com", + "jads.co", + "jamexport.com", + "jcount.com", + "jdoqocy.com", + "jewelrysprings.com", + "jinkads.de", + "joetec.net", + "joyoussurprise.com", + "js-agent.newrelic.com", + "js-api.otherlevels.com", + "js-tags.otherlevels.com", + "js.iterable.com", + "js.users.51.la", + "jsecoin.com", + "jsrdn.com", + "jubilantglimmer.com", + "juiceblocks.com", + "juicyads.com", + "juicyads.me", + "jumptap.com", + "jungroup.com", + "justicejudo.com", + "justpremium.com", + "justrelevant.com", + "justwowjars.com", + "k.iinfo.cz", + "kameleoon.eu", + "kanoodle.com", + "kargo.com", + "kernellife.com", + "kickoffo.site", + "kindads.com", + "kindlereunion.com", + "kissmetrics.com", + "kittentacos.com", + "kittycatking.com", + "klclick.com", + "klclick1.com", + "kliks.nl", + "knitstamp.com", + "knorex.com", + "knottyswing.com", + "komoona.com", + "kompasads.com", + "kontera.com", + "kost.tv", + "kpu.samsungelectronics.com", + "krxd.net", + "ktu.sv2.biz", + "kueezrtb.com", + "kvsadman.com", + "l1.britannica.com", + "lakesecure.com", + "lameletters.com", + "landkarts.com", + "larati.net", + "largebrass.com", + "laughcloth.com", + "launchbit.com", + "layer-ad.de", + "layer-ads.de", + "lazybumblebee.com", + "lbn.ru", + "lead02.com", + "leadboltads.net", + "leadclick.com", + "leadinfo.net", + "leadingedgecash.com", + "leadplace.fr", + "leadspace.com", + "leadzupc.com", + "leaplunchroom.com", + "leftliquid.com", + "lemmatechnologies.com", + "lemnisk.co", + "lever-analytics.com", + "lfeeder.com", + "lfstmedia.com", + "lgsmartad.com", + "li.alibris.com", + "li.azstarnet.com", + "li.dailycaller.com", + "li.gatehousemedia.com", + "li.gq.com", + "li.hearstmags.com", + "li.livingsocial.com", + "li.mw.drhinternet.net", + "li.onetravel.com", + "li.patheos.com", + "li.pmc.com", + "li.realtor.com", + "li.ziffimages.com", + "liadm.com", + "libraryfacts.com", + "lifeimpressions.net", + "liftdna.com", + "ligatus.com", + "ligatus.de", + "lightspeedcash.com", + "lightstep.medium.systems", + "lijit.com", + "limecodesign.com", + "link-booster.de", + "link.axios.com", + "link.beelivery.com", + "link.email.davidlloydclubs.co.uk", + "link.email.usmagazine.com", + "link.go.chase", + "link.sbstck.com", + "link.team.hyperoptic.com", + "link.theatlantic.com", + "link.uk.expediamail.com", + "linkbuddies.com", + "linkexchange.com", + "linkprice.com", + "linkrain.com", + "linkreferral.com", + "links-ranking.de", + "links.email.crunchbase.com", + "links.housekeep.com", + "links.prosservice.fr", + "links.zoopla.co.uk", + "linksoutside.com", + "linkstable.com", + "linkstorms.com", + "linkswaper.com", + "linksynergy.com", + "linktarget.com", + "linkvertise.com", + "liquidad.narrowcastmedia.com", + "litix.io", + "live.trmzum.com", + "liveadexchanger.com", + "liveintent.com", + "livelylaugh.com", + "liverail.com", + "livingsleet.com", + "lizardslaugh.com", + "lkqd.com", + "lnks.gd", + "loading321.com", + "loadsurprise.com", + "locked4.com", + "lockerdome.com", + "locolava.com", + "log.btopenworld.com", + "log.logrocket.io", + "log.pinterest.com", + "log.videocampaign.co", + "logger.snackly.co", + "logs.roku.com", + "logs.spilgames.com", + "logsss.com", + "logua.com", + "look.djfiln.com", + "look.ichlnk.com", + "look.opskln.com", + "look.ufinkln.com", + "loopme.com", + "loudlunch.com", + "lowest-prices.eu", + "lucidmedia.com", + "luckyorange.com", + "ludicrousarch.com", + "lyricshook.com", + "lytics.io", + "lzjl.com", + "m.trb.com", + "m2.ai", + "m32.media", + "m4n.nl", + "m6r.eu", + "mackeeperapp.mackeeper.com", + "madclient.uimserv.net", + "madcpms.com", + "madinad.com", + "madisonavenue.com", + "madvertise.de", + "magicadz.co", + "magicaljoin.com", + "magsrv.com", + "mail-ads.google.com", + "maltiverse.lt.acemlnc.com", + "manageadv.cblogs.eu", + "mantisadnetwork.com", + "mapcommand.com", + "marinsm.com", + "markedmeasure.com", + "marketing.888.com", + "marketing.desertcart.com", + "marketing.net.brillen.de", + "marketing.net.home24.de", + "marketing.net.occhiali24.it", + "marketing.nyi.net", + "marketing.osijek031.com", + "marketingcloudapis.com", + "marketingsolutions.yahoo.com", + "marketo.com", + "marlowpillow.sjv.io", + "marriedbelief.com", + "mas.sector.sk", + "matchcraft.com", + "matheranalytics.com", + "mathtag.com", + "matomo.activate.cz", + "matomo.crossiety.app", + "mautic.com", + "max.i12.de", + "maximiser.net", + "maxonclick.com", + "mbs.megaroticlive.com", + "mcdlks.com", + "mcs-va.tiktok.com", + "mcs-va.tiktokv.com", + "meadowlullaby.com", + "measlymiddle.com", + "measure.office.com", + "measuremap.com", + "meatydime.com", + "media-adrunner.mycomputer.com", + "media.funpic.de", + "media.net", + "media01.eu", + "media6degrees.com", + "mediaarea.eu", + "mediabridge.cc", + "mediafuse.com", + "mediageneral.com", + "mediaiqdigital.com", + "mediamath.com", + "mediamgr.ugo.com", + "mediaplazza.com", + "mediaplex.com", + "mediascale.de", + "mediaserver.bwinpartypartners.it", + "mediasmart.io", + "mediasquare.fr", + "mediatext.com", + "mediavine.com", + "mediavoice.com", + "mediax.angloinfo.com", + "mediaz.angloinfo.com", + "medleyads.com", + "medyanetads.com", + "meetrics.net", + "megacash.de", + "megastats.com", + "megawerbung.de", + "meltmilk.com", + "memorizeneck.com", + "mercuryace.com", + "merequartz.com", + "messagelists.com", + "metadsp.co.uk", + "metaffiliation.com", + "metajaws.com", + "metanetwork.com", + "methodcash.com", + "metrics-logger.spot.im", + "metrics.api.drift.com", + "metrics.articulate.com", + "metrics.cnn.com", + "metrics.foxnews.com", + "metrics.getrockerbox.com", + "metrics.gfycat.com", + "metrics.govexec.com", + "metrics.icloud.com", + "metrics.mzstatic.com", + "metrilo.com", + "mfadsrvr.com", + "mg2connext.com", + "mgid.com", + "microstatic.pl", + "microticker.com", + "milotree.com", + "mineinvoice.com", + "minewhat.com", + "minibilling.com", + "mintegral.com", + "mintfunnel.co", + "mittencattle.com", + "mix2ads.com", + "mixedreading.com", + "mixpanel.com", + "mkto-ab410147.com", + "mktoresp.com", + "ml314.com", + "mlm.de", + "mlsend.com", + "mltrk.io", + "mmismm.com", + "mmstat.com", + "mmtro.com", + "mntzrlt.net", + "moartraffic.com", + "moat.com", + "moatads.com", + "moatpixel.com", + "mobclix.com", + "mobfox.com", + "mobileanalytics.us-east-1.amazonaws.com", + "mobilefuse.com", + "modernpricing.com", + "mon-va.byteoversea.com", + "mon.byteoversea.com", + "monarchads.com", + "monetate.net", + "monetizer101.com", + "monkeyapes.com", + "monsterpops.com", + "mookie1.com", + "mopub.com", + "morefolks.com", + "motionlessmeeting.com", + "motionspots.com", + "mousestats.com", + "movad.net", + "movemeal.com", + "mparticle.com", + "mpstat.us", + "mr-rank.de", + "mrskincash.com", + "mstrlytcs.com", + "mtrcs.samba.tv", + "mtree.com", + "munchkin.marketo.net", + "mundanenail.com", + "mushroomgods.com", + "musiccounter.ru", + "muteknife.com", + "muwmedia.com", + "mxptint.net", + "myads.company", + "myads.net", + "myads.telkomsel.com", + "myaffiliateprogram.com", + "mybbc-analytics.files.bbci.co.uk", + "mybloglog.com", + "mybuys.com", + "mycounter.ua", + "mydas.mobi", + "mylead-tracking.tracknow.info", + "mylead.global", + "mylink-today.com", + "mypagerank.net", + "mypowermall.com", + "mysketchpad.com", + "mystat-in.net", + "mystat.pl", + "mytop-in.net", + "n69.com", + "naj.sk", + "nakedly.ai", + "nappyattack.com", + "nappyneck.com", + "nastydollars.com", + "nativeroll.tv", + "navegg.com", + "navigator.io", + "navrcholu.cz", + "ncaudienceexchange.com", + "ndparking.com", + "nebulacrescent.com", + "nedstatbasic.net", + "needyneedle.com", + "neighborlywatch.com", + "nend.net", + "neocounter.neoworx-blog-tools.net", + "nervoussummer.com", + "net-filter.com", + "netaffiliation.com", + "netagent.cz", + "netclickstats.com", + "netdirect.nl", + "netech.postaffiliatepro.com", + "netmera-web.com", + "netmera.com", + "netmng.com", + "netpool.netbookia.net", + "netshelter.net", + "neudesicmediagroup.com", + "newads.bangbros.com", + "newnet.qsrch.com", + "newnudecash.com", + "news-cdn.site", + "newsadsppush.com", + "newsbotnet.com", + "newt1.adultadworld.com", + "newt1.adultworld.com", + "nexac.com", + "nexage.com", + "ng3.ads.warnerbros.com", + "nitroclicks.com", + "nmtracking.netflix.com", + "noiselessplough.com", + "nondescriptcrowd.com", + "nondescriptnote.com", + "nondescriptstocking.com", + "novem.pl", + "nowaymail.com", + "npttech.com", + "nr-data.net", + "nr.mmcdn.com", + "nr.static.mmcdn.com", + "ns1p.net", + "ntv.io", + "ntvk1.ru", + "nullitics.com", + "nuseek.com", + "nzaza.com", + "o2.mouseflow.com", + "o333o.com", + "oafishobservation.com", + "oas.benchmark.fr", + "oas.repubblica.it", + "oas.roanoke.com", + "oas.toronto.com", + "oas.uniontrib.com", + "oascentral.chicagobusiness.com", + "oascentral.fortunecity.com", + "oascentral.register.com", + "objecthero.com", + "obscenesidewalk.com", + "oceancloudhosts.com", + "oclasrv.com", + "odbierz-bony.ovp.pl", + "oewa.at", + "offaces-butional.com", + "offer.fyber.com", + "offer.sponsorpay.com", + "offerforge.com", + "offermatica.com", + "offshoregeology.com", + "ogads-pa.googleapis.com", + "oglasi.posjetnica.com", + "ogury.com", + "ojrq.net", + "omg10.com", + "omnijay.com", + "omniture.com", + "omtrdc.net", + "onaudience.com", + "onclickads.net", + "onegg.site", + "onestat.com", + "onestatfree.com", + "online-metrix.net", + "online.miarroba.com", + "onlinecash.com", + "onlinecashmethod.com", + "onlinerewardcenter.com", + "onlinestarten.net", + "onscroll.com", + "onthe.io", + "opads.us", + "open.oneplus.net", + "openad.tf1.fr", + "openad.travelnow.com", + "openads.friendfinder.com", + "openads.org", + "openadsnetwork.com", + "openbid.pubmatic.com", + "openx.angelsgroup.org.uk", + "openx.cairo360.com", + "openx.net", + "openx.skinet.cz", + "openx.smcaen.fr", + "openx2.kytary.cz", + "operationchicken.com", + "opienetwork.com", + "opmnstr.com", + "oppuz.com", + "optimallimit.com", + "optimizely.com", + "optimost.com", + "optionsnomad.com", + "optmd.com", + "optmnstr.com", + "optmstr.com", + "optnmstr.com", + "optnx.com", + "orbsrv.com", + "orientedargument.com", + "orionember.com", + "ota.cartrawler.com", + "otto-images.developershed.com", + "ourdreamstaticpages.pages.dev", + "outbrain.com", + "overconfidentfood.com", + "overkick.com", + "overture.com", + "ow.pubmatic.com", + "owebmoney.ru", + "owlsr.us", + "owneriq.net", + "oxado.com", + "oxcash.com", + "oxen.hillcountrytexas.com", + "p-n.io", + "p7cloud.net", + "paa-reporting-advertising.amazon", + "page-checker.eu", + "pagead.l.google.com", + "pagefair.com", + "pagerank-ranking.de", + "pageranktop.com", + "painstakingpickle.com", + "paleleaf.com", + "panickypancake.com", + "panoramicplane.com", + "parachutehome.sjv.io", + "parchedsofa.com", + "parentpicture.com", + "parsely.com", + "parsimoniouspolice.com", + "partner-ads.com", + "partner.pelikan.cz", + "partnerad.l.google.com", + "partnerads.ysm.yahoo.com", + "partnercash.de", + "partnerhut.com", + "partnerlinks.io", + "partners.priceline.com", + "partplanes.com", + "passeura.com", + "paychat.fuse-cloud.com", + "paycounter.com", + "paypopup.com", + "pbnet.ru", + "pbterra.com", + "pc-tc.s3-eu-west-1.amazonaws.com", + "pcash.imlive.com", + "peep-auktion.de", + "peer39.com", + "pennyweb.com", + "pepperjamnetwork.com", + "percentmobile.com", + "perfectaudience.com", + "perfiliate.com", + "performancerevenue.com", + "performancerevenues.com", + "performancing.com", + "permutive.com", + "personagraph.com", + "pgl.example.com", + "pgl.example0101", + "pgmediaserve.com", + "pgpartner.com", + "pheedo.com", + "phoenix-adrunner.mycomputer.com", + "piano.io", + "pimproll.com", + "ping.ublock.org", + "pipedream.wistia.com", + "pippio.com", + "piquantpigs.com", + "pix.spot.im", + "pixel.condenastdigital.com", + "pixel.keywee.co", + "pixel.sojern.com", + "pixel.watch", + "pixel.yabidos.com", + "placed.com", + "placeframe.com", + "placidactivity.com", + "plausible.avris.it", + "plausibleio.workers.dev", + "play4traffic.com", + "playhaven.com", + "pleasantpump.com", + "plista.com", + "plotrabbit.com", + "pltraffic8.com", + "pluckypocket.com", + "plugrush.com", + "pocketfaucet.com", + "poemprompt.com", + "pointlessprofit.com", + "pointroll.com", + "pokkt.com", + "polishedfolly.com", + "polo.feathr.co", + "popads.net", + "popcash.net", + "popmixradio.com", + "popmyads.com", + "popplantation.com", + "popub.com", + "popunder.ru", + "popunhot1.blogspot.com", + "popup.msn.com", + "popupmoney.com", + "popupnation.com", + "popuptraffic.com", + "porngraph.com", + "porntrack.com", + "possibleboats.com", + "possiblepencil.com", + "post.spmailtechno.com", + "postback.iqm.com", + "postrelease.com", + "ppc.adhere.marchex.com", + "pr-star.de", + "praddpro.de", + "prchecker.info", + "prebid.org", + "predictad.com", + "premium-offers.com", + "presetrabbits.com", + "prettyeasycafe.com", + "previousplayground.com", + "prf.hn", + "priceypies.com", + "pricklydebt.com", + "prideproms.com", + "primetime.net", + "privatecash.com", + "prmtracking.com", + "pro-market.net", + "probablepartner.com", + "processplantation.com", + "proext.com", + "profero.com", + "profitrumour.com", + "programattik.com", + "projectwonderful.com", + "promo.badoink.com", + "promobenef.com", + "promos.bwin.it", + "promos.fling.com", + "promote.pair.com", + "pronetadvertising.com", + "propellerads.com", + "propellerclick.com", + "proper.io", + "props.id", + "protectcrev.com", + "protectpool.com", + "protectsubrev.com", + "protestcopy.com", + "proton-tm.com", + "protraffic.com", + "provenpixel.com", + "prpops.com", + "prsitecheck.com", + "prufenzo.xyz", + "pstmrk.it", + "pub.chez.com", + "pub.club-internet.fr", + "pub.hardware.fr", + "pub.network", + "pub.realmedia.fr", + "pubdirecte.com", + "publicidad.elmundo.es", + "publicidees.com", + "publicsofa.com", + "pubmine.com", + "pubnative.net", + "puffyloss.com", + "puffypaste.com", + "puffypull.com", + "puffypurpose.com", + "pureclarity.net", + "pushance.com", + "pushengage.com", + "pushno.com", + "pushtrack.co", + "px.dynamicyield.com", + "px.gfycat.com", + "pxf.io", + "pxl-mailtracker.com", + "pxl.iqm.com", + "pymx5.com", + "q.azcentral.com", + "q1connect.com", + "qa-analytics.com", + "qctop.com", + "ql.tc", + "qnsr.com", + "qrlsx.com", + "quantcast.com", + "quantcount.com", + "quantserve.com", + "quantummetric.com", + "quarterserver.de", + "quickkoala.io", + "quietknowledge.com", + "quiltruler.com", + "quinst.com", + "quirkysugar.com", + "quisma.com", + "quizzicalzephyr.com", + "r.drinksdirect.net", + "r.logrocket.io", + "r.marketing.dubaisothebys.com", + "r.msn.com", + "r.scoota.co", + "r.sibmail.havasit.com", + "r1.arts-mail.com", + "r1.visualwebsiteoptimizer.com", + "r2.visualwebsiteoptimizer.com", + "r3.visualwebsiteoptimizer.com", + "rabbitrifle.com", + "rackforstorage.com", + "radar.cedexis.com", + "radiate.com", + "radiateprose.com", + "rads.realadmin.pl", + "railwayreason.com", + "rambunctiousflock.com", + "rampidads.com", + "randkuj.xyz", + "rankchamp.de", + "ranking-charts.de", + "ranking-hits.de", + "ranking-links.de", + "rankingscout.com", + "rankyou.com", + "rapidcounter.com", + "raresummer.com", + "rate.ru", + "ratings.lycos.com", + "rayjump.com", + "rcadserver.com", + "re-direct.pl", + "reachjunction.com", + "reactx.com", + "readingguilt.com", + "readymoon.com", + "realcastmedia.com", + "realclever.com", + "realclix.com", + "realmedia-a800.d4p.net", + "realsrv.com", + "realtechnetwork.com", + "realtracker.com", + "rebelhen.com", + "rebelswing.com", + "rec5.visualwebsiteoptimizer.com", + "recapture.io", + "receptivereaction.com", + "recoco.it", + "reconditerake.com", + "record.bonniergaming.com", + "record.mrwin.com", + "redirectingat.com", + "redirectvoluum.com", + "redrection.pro", + "redshell.io", + "reduxmedia.com", + "referralware.com", + "referrer.disqus.com", + "regularplants.com", + "reklam.rfsl.se", + "reklama.mironet.cz", + "reklamcsere.hu", + "reklamdsp.com", + "relmaxtop.com", + "rememberdiscussion.com", + "remox.com", + "report-1.appmetrica.webvisor.com", + "report-2.appmetrica.webvisor.com", + "report-partners.appmetrica.yandex.net", + "report.ap.yandex-net.ru", + "report.appmetrica.yandex.net", + "republika.onet.pl", + "resalag.com", + "rescuerhino.com", + "resonantbrush.com", + "resonate.com", + "responsiveads.com", + "restrainstorm.com", + "retargeter.com", + "rev.iq", + "revcatch.com", + "revcontent.com", + "reveal.clearbit.com", + "revenuedirect.com", + "revenuehits.com", + "revive.haskovo.net", + "revive.netriota.hu", + "revive.plays.bg", + "revprotect.com", + "revstats.com", + "rexadvert.xyz", + "reyden-x.com", + "rhombusads.com", + "rhythmone.com", + "richaudience.com", + "richmails.com", + "richstring.com", + "rightstats.com", + "ringplant.com", + "ringsrecord.com", + "ritzyrepresentative.com", + "rlcdn.com", + "rle.ru", + "rmads.msn.com", + "rmedia.boston.com", + "roaddynamics.com", + "roar.com", + "robotreplay.com", + "rockabox.co", + "rockagainst.com", + "rockstarwriter.com", + "rok.com.com", + "rollconnection.com", + "rose.ixbt.com", + "rotabanner.com", + "roxr.net", + "rpt-ads.vidaahub.com", + "rqtrk.eu", + "rs6.net", + "rsc-ads.vidaahub.com", + "rsc-mntz.vidaahub.com", + "rta.dailymail.co.uk", + "rtb.gumgum.com", + "rtbadzesto.com", + "rtbflairads.com", + "rtbplatform.net", + "rtbpop.com", + "rtbpopd.com", + "rtmark.net", + "rtxplatform.com", + "ru4.com", + "rubiconproject.com", + "rum-http-intake.logs.datadoghq.com", + "rum-http-intake.logs.datadoghq.eu", + "runads.com", + "rundsp.com", + "ruralrobin.com", + "s.adroll.com", + "s.dmmew.com", + "s20dh7e9dh.com", + "s2d6.com", + "sabio.us", + "sadloaf.com", + "safeoffers.pro", + "sail-horizon.com", + "samplesamba.com", + "samsungacr.com", + "samsungads.com", + "sanalytics.disneyplus.com", + "sanity-dataplane.rudderstack.com", + "savoryorange.com", + "sbird.xyz", + "sbx.pagesjaunes.fr", + "sc-analytics.appspot.com", + "scambiobanner.aruba.it", + "scanscout.com", + "scarcesign.com", + "scaredsnakes.com", + "scaredsong.com", + "scarfsmash.com", + "scatteredheat.com", + "scintillatingscissors.com", + "scintillatingsilver.com", + "scissorsstatement.com", + "scopelight.com", + "scorecardresearch.com", + "scratch2cash.com", + "screechingfurniture.com", + "screechingstocking.com", + "screechingstove.com", + "scrubswim.com", + "seadform.net", + "searchmarketing.com", + "searchramp.com", + "secre.jp", + "secretspiders.com", + "secure.webconnect.net", + "securedopen-bp.com", + "securemetrics.apple.com", + "securemetrics.apple.com.cn", + "sedoparking.com", + "sedotracker.com", + "segment-cdn.producthunt.com", + "selectivesummer.com", + "semasio.net", + "sendmepixel.com", + "seraphichorizon.com", + "serendipityecho.com", + "serv0.com", + "servclick1move.com", + "serve.tercept.com", + "servedby-buysellads.com", + "servedbyadbutler.com", + "servedbyopenx.com", + "servethis.com", + "services.hearstmags.com", + "sessioncam.com", + "sexcounter.com", + "sexlist.com", + "sextracker.com", + "shadowmade.com", + "shakegoldfish.com", + "shareasale.com", + "sharethrough.com", + "shd247.click", + "sher.index.hu", + "shesubscriptions.com", + "shinystat.com", + "shinystat.it", + "shiveringspot.com", + "shiverscissors.com", + "shockinggrass.com", + "shoppingads.com", + "showads.pubmatic.com", + "shredform.com", + "shrillspoon.com", + "shxtrk.com", + "sicksmash.com", + "sidebar.angelfire.com", + "signalayer.com", + "signalszone.com", + "sillyscrew.com", + "silvermob.com", + "simpleanalytics.io", + "simplesafari.com", + "simpli.fi", + "simulateswing.com", + "sincerebuffalo.com", + "sinoa.com", + "sitedataprocessing.com", + "siteimproveanalytics.com", + "siteimproveanalytics.io", + "siteintercept.qualtrics.com", + "sitemeter.com", + "sixscissors.com", + "sixsigmatraffic.com", + "sizmek.com", + "skimresources.com", + "skisofa.com", + "skroutza.skroutz.gr", + "skylink.vn", + "slimesupplies.net", + "slopeaota.com", + "smaato.com", + "smart-data-systems.com", + "smart-traffik.com", + "smart-traffik.io", + "smart4ads.com", + "smartadserver.com", + "smartclip.net", + "smartlook.com", + "smartstream.tv", + "smartyads.com", + "smashquartz.com", + "smashsurprise.com", + "smetrics.10daily.com.au", + "smetrics.bestbuy.com", + "smetrics.ctv.ca", + "smetrics.fedex.com", + "smetrics.foxnews.com", + "smetrics.walgreens.com", + "smetrics.washingtonpost.com", + "smilewanted.com", + "smilingcattle.com", + "smoggysnakes.com", + "smrtb.com", + "snapads.com", + "snazzypoodle.com", + "snoobi.com", + "socialspark.com", + "softclick.com.br", + "soggysponge.com", + "soicos.com", + "sombersea.com", + "sombersquirrel.com", + "sombersurprise.com", + "somniture.stuff.co.nz", + "somoaudience.com", + "sonobi.com", + "sortable.com", + "sourcepoint.vice.com", + "sovrn.com", + "sp-a-q-f.ib-game.jp", + "spacash.com", + "spaceleadster.com", + "sparklingshelf.com", + "sparkstudios.com", + "speakol.com", + "specificmedia.co.uk", + "specificpop.com", + "speedomizer.com", + "speedshiftmedia.com", + "spellingthoughts.com", + "spezialreporte.de", + "spiffymachine.com", + "spinbox.techtracker.com", + "spinbox.versiontracker.com", + "spinnaker-js.com", + "spirebaboon.com", + "sponsorads.de", + "sponsorpro.de", + "spookysleet.com", + "spotlessstamp.com", + "spotscenered.info", + "spotx.tv", + "spotxchange.com", + "springbot.com", + "springserve.com", + "sprysummit.com", + "spulse.net", + "spylog.com", + "spywarelabs.com", + "spywords.com", + "srvmath.com", + "srvtrck.com", + "srwww1.com", + "sshowads.pubmatic.com", + "sskzlabs.com", + "st.dynamicyield.com", + "st.pubmatic.com", + "stack-sonar.com", + "stackadapt.com", + "stakingsmile.com", + "stalesummer.com", + "starffa.com", + "starkscale.com", + "starrynets.com", + "starsmarter.com", + "startapp.com", + "stat-track.com", + "stat.cliche.se", + "stat.dyna.ultraweb.hu", + "stat.pl", + "stat.webmedia.pl", + "stat.xiaomi.com", + "stat.zenon.net", + "stat24.com", + "stat24.meta.ua", + "statcounter.com", + "statdynamic.com", + "static-tracking.klaviyo.com", + "static.fmpub.net", + "static.itrack.it", + "static.kameleoon.com", + "staticads.btopenworld.com", + "statistik-gallup.net", + "statm.the-adult-company.com", + "stats.blogger.com", + "stats.hyperinzerce.cz", + "stats.merriam-webster.com", + "stats.mirrorfootball.co.uk", + "stats.nextgen-email.com", + "stats.olark.com", + "stats.pusher.com", + "stats.rdphv.net", + "stats.self.com", + "stats.stb-ottow.de", + "stats.townnews.com", + "stats.wordpress.com", + "stats.wp.com", + "stats2.self.com", + "stats4all.com", + "statserv.net", + "statsie.com", + "statxpress.com", + "steadfastsound.com", + "steadfastsystem.com", + "steelhouse.com", + "steelhousemedia.com", + "stickyadstv.com", + "stiffgame.com", + "stimulatingsneeze.com", + "stomachscience.com", + "stopstomach.com", + "storetail.io", + "storygize.net", + "strack.pubmatic.com", + "straightnest.com", + "stretchsquirrel.com", + "studycooking.com", + "stupendoussleet.com", + "stupendoussnow.com", + "subscribe.hearstmags.com", + "succeedscene.com", + "successbuffet.com", + "sugoicounter.com", + "sulkycook.com", + "summerobject.com", + "sumo.com", + "sumome.com", + "sunsetcampfires.com", + "superawesome.tv", + "superchichair.com", + "superclix.de", + "superficialsquare.com", + "supersonicads.com", + "superstats.com", + "supertop.ru", + "supertop100.com", + "supply.colossusssp.com", + "supportwaves.com", + "surfmusik-adserver.de", + "surveygizmobeacon.s3.amazonaws.com", + "sw88.espn.com", + "swan-swan-goose.com", + "swankysquare.com", + "swingslip.com", + "swordgoose.com", + "synonymoussticks.com", + "systemssummit.com", + "t.appsflyer.com", + "t.bawafx.com", + "t.carta.com", + "t.co", + "t.eloqua.com", + "t.email.superdrug.com", + "t.en25.com", + "t.firstpromoter.com", + "t.insigit.com", + "t.irtyd.com", + "t.leady.com", + "t.mmtrkr.com", + "t.news.browns-restaurants.co.uk", + "t.notif-colissimo-laposte.info", + "t.pie.org", + "t.podcast.co", + "t.pubmatic.com", + "t.salesmatemail.com", + "t.vacations.disneydestinations.com", + "t.visit.disneydestinations.com", + "t.visitorqueue.com", + "t.x.co", + "t1.rorystravelclub.co.uk", + "t1.rorytravelclub-news.co.uk", + "taboola.com", + "tag-demo.mention-me.com", + "tag.mention-me.com", + "tagcommander.com", + "tagger.opecloud.com", + "tags.tiqcdn.com", + "tagtoo.com", + "tailsweep.com", + "tailsweep.se", + "takethatad.com", + "tamgrt.com", + "tangibleteam.com", + "tangyamount.com", + "tapad.com", + "tapfiliate.com", + "tapinfluence.com", + "tapjoy.com", + "tappx.com", + "targad.de", + "target.microsoft.com", + "targeting.api.drift.com", + "targeting.nzme.arcpublishing.com", + "targeting.voxus.tv", + "targetingnow.com", + "targetnet.com", + "targetpoint.com", + "tatsumi-sys.jp", + "tawdryson.com", + "tcads.net", + "teads.tv", + "tealeaf.com", + "tealium.cbsnews.com", + "tealium.com", + "tealiumiq.com", + "tedioustooth.com", + "teenrevenue.com", + "telaria.com", + "telemetry.dropbox.com", + "telemetry.goodlifefitness.com", + "telemetry.malwarebytes.com", + "telemetry.v.dropbox.com", + "temelio.com", + "tend.io", + "tendertest.com", + "ter-jrnl-oc.vidaahub.com", + "terriblethumb.com", + "text-link-ads.com", + "textad.sexsearch.com", + "textads.biz", + "textlinks.com", + "tfag.de", + "the-ozone-project.com", + "theadex.com", + "theadhost.com", + "theadsparks.com", + "thebugs.ws", + "thecrazychili.com", + "themangotea.com", + "themoneytizer.com", + "therapistla.com", + "thewavebeats.com", + "thinkitten.com", + "thirdparty.bnc.lt", + "thirdrespect.com", + "thomastorch.com", + "throtle.io", + "thruport.com", + "thunderhead.com", + "tia.timeinc.net", + "ticketaunt.com", + "ticklesign.com", + "ticksel.com", + "tics.techdirt.com", + "tidaltv.com", + "tidysprite.com", + "tinybar.com", + "tinybluewhale.com", + "tinytendency.com", + "tiresomethunder.com", + "tkbo.com", + "tls.telemetry.swe.quicinc.com", + "tlvmedia.com", + "tm.br.de", + "tnkexchange.com", + "tns-counter.ru", + "toolforthought.com", + "top-casting-termine.de", + "top-site-list.com", + "top.list.ru", + "top.mail.ru", + "top100-images.rambler.ru", + "top100.mafia.ru", + "top123.ro", + "top20free.com", + "toplist.cz", + "toplist.pornhost.com", + "toplista.mw.hu", + "toplistcity.com", + "topsir.com", + "topsite.lv", + "topsites.com.br", + "topstats.com", + "totemcash.com", + "touchclarity.com", + "tour.brazzers.com", + "tr.api.fanbyte.com", + "track-eu.customer.io", + "track.adform.net", + "track.anchorfree.com", + "track.canva.com", + "track.contently.com", + "track.customer.io", + "track.effiliation.com", + "track.flexlinks.com", + "track.flexlinkspro.com", + "track.lettingaproperty.com", + "track.mailalert.io", + "track.mailerlite.com", + "track.miro.com", + "track.nationalgunrights.org", + "track.privacyatclearbit.com", + "track.pubmatic.com", + "track.segmetrics.io", + "track.smtpmessage.com", + "track.software-codes.com", + "track.spe.schoolmessenger.com", + "track.ultravpn.com", + "track.unear.net", + "track.vcdc.com", + "track.viewdeos.com", + "track1.viewdeos.com", + "trackalyzer.com", + "trackedlink.net", + "trackedweb.net", + "tracker.bannerflow.com", + "tracker.cdnbye.com", + "tracker.icerocket.com", + "tracker.metricswave.com", + "tracker.mmdlv.it", + "tracker.samplicio.us", + "tracking.epicgames.com", + "tracking.hyros.com", + "tracking.ibxlink.com", + "tracking.intentsify.io", + "tracking.intl.miui.com", + "tracking.jiffyworld.com", + "tracking.markethero.io", + "tracking.miui.com", + "tracking.netalerts.io", + "tracking.olx-st.com", + "tracking.orixa-media.com", + "tracking.shopstyle.com", + "tracking.thinkabt.com", + "tracking.wetter.at", + "tracking01.walmart.com", + "tracking101.com", + "tracking22.com", + "trackingsoft.com", + "trackmysales.com", + "tradeadexchange.com", + "tradedoubler.com", + "traffic-exchange.com", + "traffic.hyteck.de", + "trafficfactory.biz", + "trafficforce.com", + "trafficholder.com", + "traffichunt.com", + "trafficjunky.net", + "trafficleader.com", + "trafficrouter.io", + "trafficshop.com", + "trafficspaces.net", + "trafficstrategies.com", + "trafficswarm.com", + "trafficz.com", + "traffiq.com", + "trafic.ro", + "traktrafficflow.com", + "tranquilplume.com", + "travis.bosscasinos.com", + "trck.a8.net", + "trck.mtrgt.id", + "trcklion.com", + "treasuredata.com", + "trekdata.com", + "tremendoustime.com", + "tremorhub.com", + "trendcounter.com", + "trendmd.com", + "trialfire.com", + "tribalfusion.com", + "triplelift.com", + "triptease.io", + "trk.4ff.pro", + "trk.bc.shutterfly.com", + "trk.pinterest.com", + "trk.sayerfinancial.com", + "trk.techtarget.com", + "trk1.avdlink.net", + "trk42.net", + "trkn.us", + "trkoptimizer.com", + "trkpnt.ongage.net", + "trmit.com", + "truckstomatoes.com", + "truculentrate.com", + "truehits.net", + "truehits1.gits.net.th", + "truehits2.gits.net.th", + "trust.titanhq.com", + "trustx.org", + "tsyndicate.com", + "tsyndicate.net", + "tubemogul.com", + "tumbleicicle.com", + "turboadv.com", + "turn.com", + "twelvedawn.com", + "twittad.com", + "twyn.com", + "tynt.com", + "typicalteeth.com", + "tyroo.com", + "uarating.com", + "ucfunnel.com", + "udkcrj.com", + "udncoeln.com", + "uib.ff.avast.com", + "ukoffzeh.com", + "ultimateclixx.com", + "ultramercial.com", + "ultraoranges.com", + "unaccountablepie.com", + "unarmedindustry.com", + "unbecominglamp.com", + "understoodocean.com", + "undertone.com", + "unidentifiedanalytics.web.app", + "unloadyourself.com", + "unruly.co", + "unrulymedia.com", + "untd.com", + "unusualtitle.com", + "unwieldyhealth.com", + "unwieldyimpulse.com", + "upgradeyoga.com", + "upu.samsungelectronics.com", + "urbanlaurel.com", + "url9467.comms-2.zoopla.co.uk", + "urlcash.net", + "us.a1.yimg.com", + "userreplay.com", + "userreplay.net", + "users.maxcluster.net", + "utils.mediageneral.net", + "utl-1.com", + "uu.domainforlite.com", + "v1.cnzz.com", + "v1adserver.com", + "valerie.forbes.com", + "validclick.com", + "valuead.com", + "valueclick.com", + "valueclickmedia.com", + "valuecommerce.com", + "vanfireworks.com", + "vcommission.com", + "veille-referencement.com", + "velismedia.com", + "venetrigni.com", + "vengefulgrass.com", + "ventivmedia.com", + "venturead.com", + "vericlick.com", + "vertamedia.com", + "verticalmass.com", + "vervewireless.com", + "vgnp3trk.com", + "vibrantsundown.com", + "vid.pubmatic.com", + "vidcpm.com", + "video-stats.video.google.com", + "videoadex.com", + "videoadstech.org", + "videoegg.com", + "videostats.kakao.com", + "vidora.com", + "view4cash.de", + "viglink.com", + "vilenexus.com", + "virtualvincent.com", + "visiblemeasures.com", + "visistat.com", + "visitbox.de", + "visual-pagerank.fr", + "visualrevenue.com", + "vivads.net", + "vivtracking.com", + "vmmpxl.com", + "voicefive.com", + "volatilevessel.com", + "voluum.com", + "voluumtrk2.com", + "vpon.com", + "vrs.cz", + "vtracy.de", + "vungle.com", + "w55c.net", + "wa.and.co.uk", + "waardex.com", + "warmafterthought.com", + "washbanana.com", + "wdads.sx.atl.publicus.com", + "wdfl.co", + "web-stat.com", + "web.informer.com", + "web2.deja.com", + "webads.co.nz", + "webads.nl", + "webanalytics.zohodcm.com", + "webcash.nl", + "webcontentassessor.com", + "webcounter.cz", + "webcounter.goweb.de", + "webgains.com", + "weborama.com", + "weborama.fr", + "webpower.com", + "webreseau.com", + "webseoanalytics.com", + "webstat.channel4.com", + "webstat.com", + "webstat.net", + "webtrackerplus.com", + "webtraffic.se", + "webtraxx.de", + "webxcdn.com", + "welved.com", + "werbung.meteoxpress.com", + "wetrack.it", + "whaleads.com", + "wheredoyoucomefrom.ovh", + "whirlwealth.com", + "whiskyqueue.com", + "whisperingcascade.com", + "whisperingcrib.com", + "whisperingsummit.com", + "whoisonline.net", + "wholepagecache.com", + "wickedreports.com", + "widget.educationdynamics.com", + "widget.privy.com", + "wikia-ads.wikia.com", + "wikiquotations.com", + "win.iqm.com", + "window.nixnet.cz", + "wintricksbanner.googlepages.com", + "wirecomic.com", + "wirypaste.com", + "wisepops.com", + "witch-counter.de", + "wittypopcorn.com", + "wizaly.com", + "wl.spotify.com", + "wlmarketing.com", + "wondoads.de", + "woopra.com", + "worldrealize.com", + "worldwide-cash.net", + "worriednumber.com", + "wowfunnow.com", + "writerhubs.com", + "wt-eu02.net", + "wt.bankmillennium.pl", + "www-banner.chat.ru", + "www-google-analytics.l.google.com", + "www.dnps.com", + "www.kaplanindex.com", + "www.photo-ads.co.uk", + "www8.glam.com", + "wwwpromoter.com", + "x-traceur.com", + "x6.yakiuchi.com", + "xad.com", + "xapads.com", + "xchange.ro", + "xertive.com", + "xfreeservice.com", + "xg4ken.com", + "xiti.com", + "xplusone.com", + "xponsor.com", + "xpu.samsungelectronics.com", + "xq1.net", + "xtendmedia.com", + "xtracker.logimeter.com", + "xxxcounter.com", + "xxxmyself.com", + "y.ibsys.com", + "yab-adimages.s3.amazonaws.com", + "yadro.ru", + "yandexmetrica.com", + "yepads.com", + "yesads.com", + "yesadvertising.com", + "yieldads.com", + "yieldlab.net", + "yieldmanager.net", + "yieldmo.com", + "yieldoptimizer.com", + "yieldtraffic.com", + "yldbt.com", + "ymetrica1.com", + "yoads.net", + "yoggrt.com", + "youcandrawanything.com", + "youradexchange.com", + "ypu.samsungelectronics.com", + "zangocash.com", + "zanox-affiliate.de", + "zanox.com", + "zantracker.com", + "zarget.com", + "zdbb.net", + "zedo.com", + "zemanta.com", + "zencudo.co.uk", + "zenkreka.com", + "zenzuu.com", + "zephyrlabyrinth.com", + "zeus.developershed.com", + "zeusclicks.com", + "zeydoo.com", + "zion-telemetry.api.cnn.io", + "zippingcare.com", + "zlp6s.pw", + "zm232.com", + "zmedia.com", + "zonewedgeshaft.com", + "zpu.samsungelectronics.com", + "zqtk.net", + "zzhc.vnet.cn", + } +) diff --git a/src/engine/browser_launch.py b/src/engine/browser_launch.py deleted file mode 100644 index 3039318..0000000 --- a/src/engine/browser_launch.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Launch/context-флаги для Chromium, снижающие детект автоматизации на уровне ниже -JS-фингерпринта (который уже покрывает browserforge, см. engine/fingerprint.py). -Портировано из scrapling.engines.constants / _browsers._base.BaseSessionMixin. -""" - -HARMFUL_ARGS = ( - # Playwright сам добавляет эти флаги по умолчанию — они одни из самых дешёвых - # и надёжных сигналов автоматизации, поэтому их нужно гасить через ignore_default_args - "--enable-automation", - "--disable-popup-blocking", - "--disable-component-update", - "--disable-default-apps", - "--disable-extensions", -) - -DEFAULT_ARGS = ( - "--no-pings", - "--no-first-run", - "--disable-infobars", - "--disable-breakpad", - "--no-service-autorun", - "--homepage=about:blank", - "--password-store=basic", - "--disable-hang-monitor", - "--no-default-browser-check", - "--disable-session-crashed-bubble", - "--disable-search-engine-choice-screen", -) - -STEALTH_ARGS = ( - "--test-type", - "--mute-audio", - "--disable-sync", - "--hide-scrollbars", - "--disable-logging", - "--start-maximized", # обход headless-детекта по размеру окна - "--enable-async-dns", - "--use-mock-keychain", - "--disable-translate", - "--disable-voice-input", - "--window-position=0,0", - "--disable-wake-on-wifi", - "--ignore-gpu-blocklist", - "--enable-tcp-fast-open", - "--enable-web-bluetooth", - "--disable-cloud-import", - "--disable-print-preview", - "--disable-dev-shm-usage", - "--metrics-recording-only", - "--disable-crash-reporter", - "--disable-partial-raster", - "--disable-gesture-typing", - "--disable-checker-imaging", - "--disable-prompt-on-repost", - "--force-color-profile=srgb", - "--font-render-hinting=none", - "--aggressive-cache-discard", - "--disable-cookie-encryption", - "--disable-domain-reliability", - "--disable-threaded-animation", - "--disable-threaded-scrolling", - "--enable-simple-cache-backend", - "--disable-background-networking", - "--enable-surface-synchronization", - "--disable-image-animation-resync", - "--disable-renderer-backgrounding", - "--disable-ipc-flooding-protection", - "--prerender-from-omnibox=disabled", - "--safebrowsing-disable-auto-update", - "--disable-offer-upload-credit-cards", - "--disable-background-timer-throttling", - "--disable-new-content-rendering-timeout", - "--run-all-compositor-stages-before-draw", - "--disable-client-side-phishing-detection", - "--disable-backgrounding-occluded-windows", - "--disable-layer-tree-host-memory-pressure", - "--autoplay-policy=user-gesture-required", - "--disable-offer-store-unmasked-wallet-cards", - "--disable-blink-features=AutomationControlled", - "--disable-component-extensions-with-background-pages", - "--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance", - "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4", - "--disable-features=AudioServiceOutOfProcess,TranslateUI,BlinkGenPropertyTrees", -) - -_WEBRTC_ARGS = ( - "--webrtc-ip-handling-policy=disable_non_proxied_udp", - "--force-webrtc-ip-handling-policy", -) - - -def launch_args(locale: str | None = None, block_webrtc: bool = True) -> list[str]: - """Флаги запуска Chromium: скорость + анти-детект + (опционально) защита от WebRTC-утечки - реального IP мимо прокси + (если передана locale) выставление языка браузера на уровне - лаунча, а не только через JS-контекст — иначе Web Workers/Intl внутри браузера остаются - на языке хоста, даже когда browserforge уже подменил navigator.language на странице.""" - args = list(DEFAULT_ARGS) + list(STEALTH_ARGS) - - if block_webrtc: - args += list(_WEBRTC_ARGS) - - if locale: - base_lang = locale.split("-")[0].lower() - accept_lang = f"{locale},{base_lang}" if base_lang != locale.lower() else locale - args += [f"--lang={locale}", f"--accept-lang={accept_lang}"] - - return args - - -def stealth_context_options() -> dict: - """Context-опции, не пересекающиеся с тем, что уже выставляет browserforge - (user_agent/viewport/device_scale_factor) — их сюда специально не кладём, чтобы - fingerprint оставался единственным источником истины для этих полей.""" - return { - "color_scheme": "dark", # обходит проверку prefersLightColor в creepjs - "is_mobile": False, - "has_touch": False, - "ignore_https_errors": True, - } diff --git a/src/engine/constants.py b/src/engine/constants.py new file mode 100644 index 0000000..0fdc1ae --- /dev/null +++ b/src/engine/constants.py @@ -0,0 +1,97 @@ +# Disable loading these resources for speed +EXTRA_RESOURCES = { + "font", + "image", + "media", + "beacon", + "object", + "imageset", + "texttrack", + "websocket", + "csp_report", + "stylesheet", +} + +HARMFUL_ARGS = ( + # This will be ignored to avoid detection more and possibly avoid the popup crashing bug abuse: https://issues.chromium.org/issues/340836884 + "--enable-automation", + "--disable-popup-blocking", + "--disable-component-update", + "--disable-default-apps", + "--disable-extensions", +) + +DEFAULT_ARGS = ( + # Speed up chromium browsers by default + "--no-pings", + "--no-first-run", + "--disable-infobars", + "--disable-breakpad", + "--no-service-autorun", + "--homepage=about:blank", + "--password-store=basic", + "--disable-hang-monitor", + "--no-default-browser-check", + "--disable-session-crashed-bubble", + "--disable-search-engine-choice-screen", +) + +STEALTH_ARGS = ( + # Explanation: https://peter.sh/experiments/chromium-command-line-switches/ + # Generally this will make the browser faster and less detectable + # "--incognito", + "--test-type", + "--mute-audio", + "--disable-sync", + "--hide-scrollbars", + "--disable-logging", + "--start-maximized", # For headless check bypass + "--enable-async-dns", + "--use-mock-keychain", + "--disable-translate", + "--disable-voice-input", + "--window-position=0,0", + "--disable-wake-on-wifi", + "--ignore-gpu-blocklist", + "--enable-tcp-fast-open", + "--enable-web-bluetooth", + "--disable-cloud-import", + "--disable-print-preview", + "--disable-dev-shm-usage", + # '--disable-popup-blocking', + "--metrics-recording-only", + "--disable-crash-reporter", + "--disable-partial-raster", + "--disable-gesture-typing", + "--disable-checker-imaging", + "--disable-prompt-on-repost", + "--force-color-profile=srgb", + "--font-render-hinting=none", + "--aggressive-cache-discard", + "--disable-cookie-encryption", + "--disable-domain-reliability", + "--disable-threaded-animation", + "--disable-threaded-scrolling", + "--enable-simple-cache-backend", + "--disable-background-networking", + "--enable-surface-synchronization", + "--disable-image-animation-resync", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--prerender-from-omnibox=disabled", + "--safebrowsing-disable-auto-update", + "--disable-offer-upload-credit-cards", + "--disable-background-timer-throttling", + "--disable-new-content-rendering-timeout", + "--run-all-compositor-stages-before-draw", + "--disable-client-side-phishing-detection", + "--disable-backgrounding-occluded-windows", + "--disable-layer-tree-host-memory-pressure", + "--autoplay-policy=user-gesture-required", + "--disable-offer-store-unmasked-wallet-cards", + "--disable-blink-features=AutomationControlled", + "--disable-component-extensions-with-background-pages", + "--enable-features=NetworkService,NetworkServiceInProcess,TrustTokens,TrustTokensAlwaysAllowIssuance", + "--blink-settings=primaryHoverType=2,availableHoverTypes=2,primaryPointerType=4,availablePointerTypes=4", + "--disable-features=AudioServiceOutOfProcess,TranslateUI,BlinkGenPropertyTrees", +) diff --git a/src/engine/navigation.py b/src/engine/navigation.py index 1c4fe00..679bb0d 100644 --- a/src/engine/navigation.py +++ b/src/engine/navigation.py @@ -1,42 +1,35 @@ """ -Мелкие хелперы навигации, портированные из scrapling.engines.toolbelt.navigation -и scrapling.engines.toolbelt.proxy_rotation — адаптированы под patchright/async. - -В отличие от полного Session/PagePool из scrapling.engines._browsers._base, тут -браузер запускается и закрывается на каждый запрос (см. main.py), поэтому -переиспользовать нечего — берём только то, что относится к одной навигации: -блокировку ресурсов/доменов на странице и детект прокси-ошибок для retry. +Блокировка ресурсов/доменов на странице и разбор строки прокси — портировано из +scrapling.engines.toolbelt.navigation (async-вариант, проект целиком async). """ -from typing import Callable, Optional +import logging from urllib.parse import urlparse -from patchright.async_api import Route +from msgspec import Struct, structs, convert, ValidationError +from playwright.async_api import Route -# ресурсы, которые можно дропать ради скорости — портировано из -# scrapling.engines.constants.EXTRA_RESOURCES -EXTRA_RESOURCES = { - "font", "image", "media", "beacon", "object", - "imageset", "texttrack", "websocket", "csp_report", "stylesheet", -} +from engine._types import Dict, Set, Tuple, Optional, Callable +from engine.constants import EXTRA_RESOURCES -# признаки прокси-ошибки в тексте исключения — портировано из -# scrapling.engines.toolbelt.proxy_rotation._PROXY_ERROR_INDICATORS -_PROXY_ERROR_INDICATORS = ( - "net::err_proxy", "net::err_tunnel", "connection refused", - "connection reset", "connection timed out", "failed to connect", - "could not resolve proxy", -) +log = logging.getLogger(__name__) -def is_proxy_error(error: Exception) -> bool: - """Похоже ли исключение на сбой прокси (а не на обычную ошибку навигации).""" - msg = str(error).lower() - return any(indicator in msg for indicator in _PROXY_ERROR_INDICATORS) +class ProxyDict(Struct): + server: str + username: str = "" + password: str = "" def _is_domain_blocked(hostname: str, domains: frozenset) -> bool: - """Матчинг хоста и его родительских доменов за O(1) на каждый уровень — - портировано из scrapling.engines.toolbelt.navigation._is_domain_blocked.""" + """Check if a hostname matches any blocked domain using O(1) frozenset lookups. + + Walks up the hostname's suffix chain: for "tracker.ads.doubleclick.net", + checks "tracker.ads.doubleclick.net", "ads.doubleclick.net", "doubleclick.net". + + :param hostname: The hostname to check. + :param domains: A frozenset of blocked domain names. + :return: True if the hostname or any of its parent domains is in the blocked set. + """ if hostname in domains: return True idx = hostname.find(".") @@ -48,18 +41,24 @@ def _is_domain_blocked(hostname: str, domains: frozenset) -> bool: return False -def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[set] = None) -> Callable: - """Обработчик route, блокирующий типы ресурсов и/или домены — портировано из - scrapling.engines.toolbelt.navigation.create_async_intercept_handler.""" +def create_async_intercept_handler(disable_resources: bool, blocked_domains: Optional[Set[str]] = None) -> Callable: + """Create an async route handler that blocks both resource types and specific domains. + + :param disable_resources: Whether to block default resource types. + :param blocked_domains: Set of domain names to block requests to. + :return: An async route handler function. + """ disabled_resources = EXTRA_RESOURCES if disable_resources else set() domains = frozenset(blocked_domains) if blocked_domains else frozenset() - async def handler(route: Route) -> None: + async def handler(route: Route): if route.request.resource_type in disabled_resources: + log.debug('Blocking background resource "%s" of type "%s"', route.request.url, route.request.resource_type) await route.abort() elif domains: hostname = urlparse(route.request.url).hostname or "" if _is_domain_blocked(hostname, domains): + log.debug('Blocking request to blocked domain "%s" (%s)', hostname, route.request.url) await route.abort() else: await route.continue_() @@ -67,3 +66,39 @@ def create_intercept_handler(disable_resources: bool, blocked_domains: Optional[ await route.continue_() return handler + + +def construct_proxy_dict(proxy_string: str | Dict[str, str] | Tuple) -> Dict: + """Validate a proxy and return it in the acceptable format for Playwright + Reference: https://playwright.dev/python/docs/network#http-proxy + + :param proxy_string: A string or a dictionary representation of the proxy. + :return: + """ + if isinstance(proxy_string, str): + proxy = urlparse(proxy_string) + if proxy.scheme not in ("http", "https", "socks4", "socks5") or not proxy.hostname: + raise ValueError("Invalid proxy string!") + + try: + result = { + "server": f"{proxy.scheme}://{proxy.hostname}", + "username": proxy.username or "", + "password": proxy.password or "", + } + if proxy.port: + result["server"] += f":{proxy.port}" + return result + except ValueError: + # Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc... + raise ValueError("The proxy argument's string is in invalid format!") + + elif isinstance(proxy_string, dict): + try: + validated = convert(proxy_string, ProxyDict) + result_dict = structs.asdict(validated) + return result_dict + except ValidationError as e: + raise TypeError(f"Invalid proxy dictionary: {e}") + + raise TypeError(f"Invalid proxy string: {proxy_string}") diff --git a/src/engine/page_pool.py b/src/engine/page_pool.py new file mode 100644 index 0000000..5d2e5cc --- /dev/null +++ b/src/engine/page_pool.py @@ -0,0 +1,97 @@ +""" +Пул страниц браузерного контекста — портировано из scrapling.engines._browsers._page. +""" +from threading import RLock +from dataclasses import dataclass + +from playwright.async_api._generated import Page as AsyncPage + +from engine._types import Optional, List, Literal + +PageState = Literal["ready", "busy", "error"] # States that a page can be in + + +@dataclass +class PageInfo: + """Information about the page and its current state""" + + __slots__ = ("page", "state", "url") + page: AsyncPage + state: PageState + url: Optional[str] + + def mark_busy(self, url: str = ""): + """Mark the page as busy""" + self.state = "busy" + self.url = url + + def mark_ready(self): + """Mark the page as ready to be reused by the next request""" + self.state = "ready" + self.url = "" + + def mark_error(self): + """Mark the page as having an error""" + self.state = "error" + + def __repr__(self): + return f'Page(URL="{self.url!r}", state={self.state!r})' + + def __eq__(self, other_page): + """Comparing this page to another page object.""" + if other_page.__class__ is not self.__class__: + return NotImplemented + return self.page == other_page.page + + +class PagePool: + """Manages a pool of browser pages/tabs with state tracking""" + + __slots__ = ("max_pages", "pages", "_lock") + + def __init__(self, max_pages: int = 5): + self.max_pages = max_pages + self.pages: List[PageInfo] = [] + self._lock = RLock() + + def add_page(self, page: AsyncPage) -> PageInfo: + """Add a new page to the pool, marked busy for the request that created it""" + with self._lock: + if len(self.pages) >= self.max_pages: + raise RuntimeError(f"Maximum page limit ({self.max_pages}) reached") + + page_info = PageInfo(page, "busy", "") + self.pages.append(page_info) + return page_info + + def get_ready_page(self) -> Optional[PageInfo]: + """Take the first ready page out of the pool's free pages, marking it busy, or return None""" + with self._lock: + for page_info in self.pages: + if page_info.state == "ready": + page_info.mark_busy() + return page_info + return None + + def remove_page(self, page_info: PageInfo): + """Forget a page, whether it's still in the pool or not""" + with self._lock: + if page_info in self.pages: + self.pages.remove(page_info) + + def clear(self) -> List[PageInfo]: + """Forget every page and return them so the caller can close them""" + with self._lock: + pages, self.pages = self.pages, [] + return pages + + @property + def pages_count(self) -> int: + """Get the total number of pages""" + return len(self.pages) + + @property + def busy_count(self) -> int: + """Get the number of busy pages""" + with self._lock: + return sum(1 for p in self.pages if p.state == "busy") diff --git a/src/engine/proxy_rotation.py b/src/engine/proxy_rotation.py new file mode 100644 index 0000000..87e61f4 --- /dev/null +++ b/src/engine/proxy_rotation.py @@ -0,0 +1,106 @@ +""" +Ротация прокси и детект прокси-ошибок — портировано из scrapling.engines.toolbelt.proxy_rotation. +""" +from threading import Lock + +from engine._types import Callable, Dict, List, Tuple, ProxyType + +RotationStrategy = Callable[[List[ProxyType], int], Tuple[ProxyType, int]] +_PROXY_ERROR_INDICATORS = { + "net::err_proxy", + "net::err_tunnel", + "connection refused", + "connection reset", + "connection timed out", + "failed to connect", + "could not resolve proxy", +} + + +def _get_proxy_key(proxy: ProxyType) -> str: + """Generate a unique key for a proxy (for dicts it's server plus username).""" + if isinstance(proxy, str): + return proxy + server = proxy.get("server", "") + username = proxy.get("username", "") + return f"{server}|{username}" + + +def is_proxy_error(error: Exception) -> bool: + """Check if an error is proxy-related. Works for both HTTP and browser errors.""" + error_msg = str(error).lower() + return any(indicator in error_msg for indicator in _PROXY_ERROR_INDICATORS) + + +def cyclic_rotation(proxies: List[ProxyType], current_index: int) -> Tuple[ProxyType, int]: + """Default cyclic rotation strategy - iterates through proxies sequentially, wrapping around at the end.""" + idx = current_index % len(proxies) + return proxies[idx], (idx + 1) % len(proxies) + + +class ProxyRotator: + """ + A thread-safe proxy rotator with pluggable rotation strategies. + + Supports: + - Cyclic rotation (default) + - Custom rotation strategies via callable + - Both string URLs and Playwright-style dict proxies + """ + + __slots__ = ("_proxies", "_proxy_to_index", "_strategy", "_current_index", "_lock") + + def __init__( + self, + proxies: List[ProxyType], + strategy: RotationStrategy = cyclic_rotation, + ): + """ + Initialize the proxy rotator. + + :param proxies: List of proxy URLs or Playwright-style proxy dicts. + - String format: "http://proxy1:8080" or "http://user:pass@proxy:8080" + - Dict format: {"server": "http://proxy:8080", "username": "user", "password": "pass"} + :param strategy: Rotation strategy function. Takes (proxies, current_index) and returns (proxy, next_index). Defaults to cyclic_rotation. + """ + if not proxies: + raise ValueError("At least one proxy must be provided") + + if not callable(strategy): + raise TypeError(f"strategy must be callable, got {type(strategy).__name__}") + + self._strategy = strategy + self._lock = Lock() + + # Validate and store proxies + self._proxies: List[ProxyType] = [] + self._proxy_to_index: Dict[str, int] = {} # O(1) lookup by unique key (server + username) + for i, proxy in enumerate(proxies): + if isinstance(proxy, (str, dict)): + if isinstance(proxy, dict) and "server" not in proxy: + raise ValueError("Proxy dict must have a 'server' key") + + self._proxy_to_index[_get_proxy_key(proxy)] = i + self._proxies.append(proxy) + else: + raise TypeError(f"Invalid proxy type: {type(proxy)}. Expected str or dict.") + + self._current_index = 0 + + def get_proxy(self) -> ProxyType: + """Get the next proxy according to the rotation strategy.""" + with self._lock: + proxy, self._current_index = self._strategy(self._proxies, self._current_index) + return proxy + + @property + def proxies(self) -> List[ProxyType]: + """Get a copy of all configured proxies.""" + return list(self._proxies) + + def __len__(self) -> int: + """Return the total number of configured proxies.""" + return len(self._proxies) + + def __repr__(self) -> str: + return f"ProxyRotator(proxies={len(self._proxies)})" diff --git a/src/engine/session.py b/src/engine/session.py new file mode 100644 index 0000000..f5acf8c --- /dev/null +++ b/src/engine/session.py @@ -0,0 +1,391 @@ +""" +Управление контекстом/пулом страниц — портировано из scrapling.engines._browsers._base. + +Только async-сессия (проект целиком async, см. patchright.async_api в src/api/actions.py) и без +методов детекта Cloudflare (`_detect_cloudflare`/`_challenge_cleared` в апстриме зависят от +scrapling.parser.Selector и дублируют antibot.orchestrator.pass_challenges, который в проекте +уже решает эту задачу — см. src/api/actions.py:execute). +""" +import logging +from time import time +from re import search as re_search +from asyncio import sleep as asyncio_sleep, Lock +from contextlib import asynccontextmanager, suppress + +from playwright.async_api._generated import Page as AsyncPage +from playwright.async_api import ( + Frame as AsyncFrame, + Response as AsyncPlaywrightResponse, + BrowserContext as AsyncBrowserContext, +) +from playwright._impl._errors import Error as PlaywrightError + +from engine.page_pool import PageInfo, PagePool +from engine.validators import validate, PlaywrightConfig, StealthConfig +from engine.navigation import construct_proxy_dict, create_async_intercept_handler +from engine._types import ( + Any, + Awaitable, + Dict, + List, + Set, + Optional, + Callable, + TYPE_CHECKING, + cast, + overload, + Tuple, + ProxyType, + AsyncGenerator, +) +from engine.constants import STEALTH_ARGS, HARMFUL_ARGS, DEFAULT_ARGS + +log = logging.getLogger(__name__) + + +class AsyncSession: + _config: "PlaywrightConfig | StealthConfig" + _context_options: Dict[str, Any] + if TYPE_CHECKING: + _build_context_with_proxy: Callable[..., Dict[str, Any]] + + def __init__(self, max_pages: int = 1): + self.max_pages = max_pages + self.page_pool = PagePool(max_pages) + self._max_wait_for_page = 60 + self.playwright: Any = None + self.context: Any = None + self.browser: Any = None + self._is_alive = False + self._lock = Lock() + + async def start(self) -> None: + pass + + async def close_pages(self) -> None: + """Close every open tab in the session's pool. The next request opens a fresh tab.""" + for page_info in self.page_pool.clear(): + with suppress(Exception): + await cast(AsyncPage, page_info.page).close() + + async def close(self): + """Close all resources""" + if not self._is_alive: # pragma: no cover + return + + await self.close_pages() + if self.context: + await self.context.close() + self.context = None # pyright: ignore + + if self.browser: + await self.browser.close() + self.browser = None + + if self.playwright: + await self.playwright.stop() + self.playwright = None # pyright: ignore + + self._is_alive = False + + async def __aenter__(self): + await self.start() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def _initialize_context( + self, config: PlaywrightConfig | StealthConfig, ctx: AsyncBrowserContext + ) -> AsyncBrowserContext: + """Initialize the browser context.""" + if config.init_script: # pragma: no cover + await ctx.add_init_script(path=config.init_script) + + # Аналог `init_script`, но для инлайнового JS (например, InjectFunction(fingerprint) из + # browserforge — он отдаёт готовый скрипт строкой, а не путь к файлу), не апстримное + # поле scrapling. + if config.init_script_content: + await ctx.add_init_script(config.init_script_content) + + if config.cookies: # pragma: no cover + await ctx.add_cookies(config.cookies) + + return ctx + + async def _get_page( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + blocked_domains: Optional[Set[str]] = None, + context: Optional[AsyncBrowserContext] = None, + ) -> PageInfo: # pragma: no cover + """Get a ready page from the pool, or open a new one""" + ctx = context if context is not None else self.context + if TYPE_CHECKING: + assert ctx is not None, "Browser context not initialized" + + async with self._lock: + page_info = self.page_pool.get_ready_page() if context is None else None + if page_info is None and context is None and self.page_pool.pages_count >= self.max_pages: + # At max capacity with the persistent context, so wait for a busy page to become ready + start_time = time() + while time() - start_time < self._max_wait_for_page: + await asyncio_sleep(0.05) + page_info = self.page_pool.get_ready_page() + if page_info is not None: + break + else: + raise TimeoutError( + f"No pages finished to clear place in the pool within the {self._max_wait_for_page}s timeout period" + ) + + if page_info is None: + page_info = self.page_pool.add_page(await ctx.new_page()) + + page = cast(AsyncPage, page_info.page) + page.set_default_navigation_timeout(timeout) + page.set_default_timeout(timeout) + await page.set_extra_http_headers(extra_headers or {}) + await page.unroute_all(behavior="ignoreErrors") + if disable_resources or blocked_domains: + await page.route("**/*", create_async_intercept_handler(disable_resources, blocked_domains)) + return page_info + + def get_pool_stats(self) -> Dict[str, int]: + """Get statistics about the current page pool""" + return { + "total_pages": self.page_pool.pages_count, + "busy_pages": self.page_pool.busy_count, + "max_pages": self.max_pages, + } + + @staticmethod + async def _wait_for_networkidle(page: AsyncPage | AsyncFrame, timeout: Optional[int] = None): + """Wait for the page to become idle (no network activity) even if there are never-ending requests.""" + try: + await page.wait_for_load_state("networkidle", timeout=timeout) + except (PlaywrightError, Exception): + pass + + async def _wait_for_page_stability(self, page: AsyncPage | AsyncFrame, load_dom: bool, network_idle: bool): + await page.wait_for_load_state(state="load") + if load_dom: + await page.wait_for_load_state(state="domcontentloaded") + if network_idle: + await self._wait_for_networkidle(page) + + @staticmethod + def _create_response_handler( + page_info: PageInfo, + response_container: List, + xhr_pattern: Optional[str] = None, + xhr_container: Optional[List] = None, + ) -> Callable[[AsyncPlaywrightResponse], Awaitable[None]]: + """Create an async response handler that captures the final navigation response and optionally XHR/fetch responses. + + :param page_info: The PageInfo object containing the page + :param response_container: A list to store the final response (mutable container) + :param xhr_pattern: Optional regex pattern to match XHR/fetch response URLs + :param xhr_container: Optional list to store captured XHR/fetch responses + :return: A callback function for page.on("response", ...) + """ + + async def handle_response(finished_response: AsyncPlaywrightResponse) -> None: + if ( + finished_response.request.resource_type == "document" + and finished_response.request.is_navigation_request() + and finished_response.request.frame == page_info.page.main_frame + ): + response_container[0] = finished_response + elif ( + xhr_pattern + and xhr_container is not None + and finished_response.request.resource_type in ("xhr", "fetch") + and re_search(xhr_pattern, finished_response.url) + ): + xhr_container.append(finished_response) + + return handle_response + + @asynccontextmanager + async def _page_generator( + self, + timeout: int | float, + extra_headers: Optional[Dict[str, str]], + disable_resources: bool, + proxy: Optional[ProxyType] = None, + blocked_domains: Optional[Set[str]] = None, + ) -> AsyncGenerator["PageInfo", None]: + """Acquire a page - either from persistent context or fresh context with proxy.""" + if proxy: + # Rotation mode: create fresh context with the provided proxy + if not self.browser: # pragma: no cover + raise RuntimeError("Browser not initialized for proxy rotation mode") + context_options = self._build_context_with_proxy(proxy) + context: AsyncBrowserContext = await self.browser.new_context(**context_options) + + page_info = None + try: + context = await self._initialize_context(self._config, context) + page_info = await self._get_page( + timeout, extra_headers, disable_resources, blocked_domains, context=context + ) + yield page_info + finally: + if page_info is not None: + self.page_pool.remove_page(page_info) + await context.close() + else: + # Standard mode: use PagePool with persistent context + page_info = await self._get_page(timeout, extra_headers, disable_resources, blocked_domains) + try: + yield page_info + finally: + if page_info.state == "error" or page_info.page.is_closed(): + with suppress(Exception): + await page_info.page.close() + self.page_pool.remove_page(page_info) + else: + page_info.mark_ready() + + +class BaseSessionMixin: + _config: "PlaywrightConfig | StealthConfig" + + @overload + def __validate_routine__(self, params: Dict, model: type[StealthConfig]) -> StealthConfig: ... + + @overload + def __validate_routine__(self, params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... + + def __validate_routine__( + self, params: Dict, model: type[PlaywrightConfig] | type[StealthConfig] + ) -> PlaywrightConfig | StealthConfig: + # Dark color scheme bypasses the 'prefersLightColor' check in creepjs + self._context_options: Dict[str, Any] = {"color_scheme": "dark", "device_scale_factor": 2} + self._browser_options: Dict[str, Any] = { + "args": DEFAULT_ARGS, + "ignore_default_args": HARMFUL_ARGS, + } + if "__max_pages" in params: + params["max_pages"] = params.pop("__max_pages") + + config = validate(params, model=model) + self._headers_keys = ( + {header.lower() for header in config.extra_headers.keys()} if config.extra_headers else set() + ) + + return config + + def __generate_options__(self, extra_flags: Tuple | None = None) -> None: + config: PlaywrightConfig | StealthConfig = self._config + self._context_options.update( + { + "proxy": config.proxy, + "timezone_id": config.timezone_id, + "extra_http_headers": config.extra_headers, + } + ) + if config.locale and config.cdp_url: + # Launch flags can't be set on remote browsers, so the detectable context option is the best effort left + self._context_options["locale"] = config.locale + if config.useragent: + self._context_options["user_agent"] = config.useragent + + if not config.cdp_url: + flags = self._browser_options["args"] + if config.extra_flags or extra_flags: + flags = list(set(tuple(flags) + tuple(config.extra_flags or extra_flags or ()))) + + if config.dns_over_https: + doh_flag = "--dns-over-https-templates=https://cloudflare-dns.com/dns-query" + if isinstance(flags, list): + flags.append(doh_flag) + else: + flags = list(flags) + [doh_flag] + + if config.locale: + # The context `locale` option patches the main thread only, so Web Workers keep the browser's real + # language and WAFs like Cloudflare flag the mismatch. Launch flags set it browser-wide instead, + # so workers, `Intl`, and the `Accept-Language` header all follow natively. + base_lang = config.locale.split("-")[0].lower() + accept_lang = f"{config.locale},{base_lang}" if base_lang != config.locale.lower() else config.locale + flags = (flags if isinstance(flags, list) else list(flags)) + [ + f"--lang={config.locale}", + f"--accept-lang={accept_lang}", + ] + + self._browser_options.update( + { + "args": flags, + "headless": config.headless, + "channel": "chrome" if config.real_chrome else "chromium", + } + ) + if config.executable_path: + self._browser_options["executable_path"] = config.executable_path + + self._user_data_dir = config.user_data_dir + else: + self._browser_options = {} + + if config.additional_args: + self._context_options.update(config.additional_args) + + def _build_context_with_proxy(self, proxy: Optional[ProxyType] = None) -> Dict[str, Any]: + """ + Build context options with a specific proxy for rotation mode. + + :param proxy: Proxy URL string or Playwright-style proxy dict to use for this context. + :return: Dictionary of context options for browser.new_context(). + """ + + context_options = self._context_options.copy() + + # Override proxy if provided + if proxy: + context_options["proxy"] = construct_proxy_dict(proxy) + + return context_options + + +class StealthySessionMixin(BaseSessionMixin): + def __validate__(self, **params): + self._config = self.__validate_routine__(params, model=StealthConfig) + self._context_options.update( + { + "is_mobile": False, + "has_touch": False, + # I'm thinking about disabling it to rest from all Service Workers' headache, but let's keep it as it is for now + "service_workers": "allow", + "ignore_https_errors": True, + "screen": {"width": 1920, "height": 1080}, + "viewport": {"width": 1920, "height": 1080}, + "permissions": ["geolocation", "notifications"], + } + ) + self.__generate_stealth_options() + + def __generate_stealth_options(self) -> None: + config = cast(StealthConfig, self._config) + flags: Tuple[str, ...] = tuple() + if not config.cdp_url: + flags = tuple(DEFAULT_ARGS) + tuple(STEALTH_ARGS) + + if config.block_webrtc: + flags += ( + "--webrtc-ip-handling-policy=disable_non_proxied_udp", + "--force-webrtc-ip-handling-policy", # Ensures the policy is enforced + ) + if not config.allow_webgl: + flags += ( + "--disable-webgl", + "--disable-webgl-image-chromium", + "--disable-webgl2", + ) + if config.hide_canvas: + flags += ("--fingerprinting-canvas-image-data-noise",) + + super(StealthySessionMixin, self).__generate_options__(flags) diff --git a/src/engine/session_types.py b/src/engine/session_types.py new file mode 100644 index 0000000..06f91b3 --- /dev/null +++ b/src/engine/session_types.py @@ -0,0 +1,80 @@ +""" +TypedDict'ы для kwargs сессии и `fetch()` — портировано и урезано из +scrapling.engines._browsers._types (без RequestsSession/GetRequestParams/DataRequestParams, +которые относятся к HTTP-фетчеру scrapling и тянут curl_cffi; без `solve_cloudflare` — +см. engine/validators.py). +""" +from engine._types import ( + Dict, + List, + Set, + Tuple, + Sequence, + Optional, + Callable, + TypedDict, + SetCookieParam, + SelectorWaitStates, +) +from engine.proxy_rotation import ProxyRotator + + +class StealthSession(TypedDict, total=False): + max_pages: int + headless: bool + disable_resources: bool + network_idle: bool + load_dom: bool + wait_selector: Optional[str] + wait_selector_state: SelectorWaitStates + cookies: Sequence[SetCookieParam] | None + google_search: bool + wait: int | float + timezone_id: str | None + page_action: Optional[Callable] + page_setup: Optional[Callable] + proxy: Optional[str | Dict[str, str] | Tuple] + proxy_rotator: Optional[ProxyRotator] + extra_headers: Optional[Dict[str, str]] + timeout: int | float + init_script: Optional[str] + init_script_content: Optional[str] + user_data_dir: str + selector_config: Optional[Dict] + additional_args: Optional[Dict] + locale: Optional[str] + real_chrome: bool + cdp_url: Optional[str] + useragent: Optional[str] + extra_flags: Optional[List[str]] + blocked_domains: Optional[Set[str]] + block_ads: bool + retries: int + retry_delay: int | float + capture_xhr: str | None + executable_path: Optional[str] + dns_over_https: bool + allow_webgl: bool + hide_canvas: bool + block_webrtc: bool + + +class PlaywrightFetchParams(TypedDict, total=False): + load_dom: bool + wait: int | float + network_idle: bool + google_search: bool + timeout: int | float + disable_resources: bool + wait_selector: Optional[str] + page_action: Optional[Callable] + page_setup: Optional[Callable] + selector_config: Optional[Dict] + extra_headers: Optional[Dict[str, str]] + wait_selector_state: SelectorWaitStates + blocked_domains: Optional[Set[str]] + proxy: Optional[str | Dict[str, str]] + + +class StealthFetchParams(PlaywrightFetchParams, total=False): + pass diff --git a/src/engine/stealthy.py b/src/engine/stealthy.py new file mode 100644 index 0000000..cd8d051 --- /dev/null +++ b/src/engine/stealthy.py @@ -0,0 +1,198 @@ +""" +Stealthy-браузерная сессия (persistent context + пул страниц) — портировано и урезано из +scrapling.engines._browsers._stealth.AsyncStealthySession. + +Отличия от апстрима: +- Только async-версия (проект целиком async). +- `fetch()` не строит `scrapling.Response` (это тянет `scrapling.parser.Selector` — тяжёлый + HTML-парсер, который в проекте не используется: html/cookies/user_agent достаются вручную + через сырой Playwright API). Вместо этого, если передан `page_action`, `fetch()` возвращает + РЕЗУЛЬТАТ `page_action(page)` напрямую (в апстриме он отбрасывается) — так `page_action` + становится единственной точкой, где происходит выполнение action-очереди + (`api.actions.execute`) и извлечение html/cookies/user_agent для ответа API. Если + `page_action` не передан, возвращается сам `page`. +- Без решения Cloudflare-челленджей (`_cloudflare_solver`/`solve_cloudflare`) — в проекте это + делает `antibot.orchestrator.pass_challenges`, вызываемый из `page_action`. +""" +import logging +from asyncio import sleep as asyncio_sleep + +from playwright.async_api import Page, Locator +from patchright.async_api import async_playwright +from typing_extensions import Unpack + +from engine.page_pool import PageInfo +from engine.proxy_rotation import is_proxy_error +from engine.session import AsyncSession, StealthySessionMixin +from engine.session_types import StealthSession, StealthFetchParams +from engine.validators import validate_fetch as _validate, StealthConfig +from engine._types import Any, List, Optional, ProxyType + +log = logging.getLogger(__name__) + + +class AsyncStealthySession(AsyncSession, StealthySessionMixin): + """An async Stealthy Browser session manager with page pooling.""" + + __slots__ = ( + "_config", + "_context_options", + "_browser_options", + "_user_data_dir", + "_headers_keys", + ) + + def __init__(self, **kwargs: Unpack[StealthSession]): + """A Browser session manager with page pooling, it's using a persistent browser Context by default with a temporary user profile directory. + + :param headless: Run the browser in headless/hidden (default), or headful/visible mode. + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + :param blocked_domains: A set of domain names to block requests to. Subdomains are also matched. + :param useragent: Pass a useragent string to be used. Otherwise the browser's own default is used. + :param cookies: Set cookies for the next request. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000 + :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning. + :param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need + (in this project: `api.actions.execute`). Its return value is what `fetch()` returns. + :param page_setup: A function that takes the `page` object, runs before navigation. Use it to register event listeners or routes that must be set up before the page loads. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param init_script: An absolute path to a JavaScript file to be executed on page creation for all pages in this session. + :param locale: Specify user locale, for example, `en-GB`, `de-DE`, etc. + :param timezone_id: Changes the timezone of the browser. Defaults to the system timezone. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`. + :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. + :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting. + :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak. + :param allow_webgl: Enabled by default. Disabling it disables WebGL and WebGL 2.0 support entirely. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP. + :param google_search: Enabled by default, Scrapling will set a Google referer header. + :param extra_headers: A dictionary of extra headers to add to the request. + :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only. + :param user_data_dir: Path to a User Data Directory, which stores browser session data like cookies and local storage. The default is to create a temporary directory. + :param extra_flags: A list of additional browser flags to pass to the browser on launch. + :param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than the settings above. + """ + self.__validate__(**kwargs) + super().__init__(max_pages=self._config.max_pages) + + async def start(self) -> None: + """Create a browser for this instance and context.""" + if not self.playwright: + self.playwright = await async_playwright().start() + try: + if self._config.cdp_url: + self.browser = await self.playwright.chromium.connect_over_cdp(endpoint_url=self._config.cdp_url) + if not self._config.proxy_rotator: + assert self.browser is not None + self.context = await self.browser.new_context(**self._context_options) + elif self._config.proxy_rotator: + self.browser = await self.playwright.chromium.launch(**self._browser_options) + else: + persistent_options = ( + self._browser_options | self._context_options | {"user_data_dir": self._user_data_dir} + ) + self.context = await self.playwright.chromium.launch_persistent_context(**persistent_options) + + if self.context: + self.context = await self._initialize_context(self._config, self.context) + + self._is_alive = True + except Exception: + # Clean up playwright if browser setup fails + await self.playwright.stop() + self.playwright = None + raise + else: + raise RuntimeError("Session has been already started") + + async def fetch(self, url: str, **kwargs: Unpack[StealthFetchParams]) -> Any: + """Opens up the browser and navigates to `url`. + + If `page_action` is given, returns its return value (see class docstring). Otherwise returns + the Playwright `page` navigated to `url`, still open (belongs to the session's page pool/context). + + :param url: The Target url. + :param google_search: Enabled by default, sets a Google referer header. + :param timeout: The timeout in milliseconds used in all operations and waits through the page. + :param wait: The time (milliseconds) to wait after everything finishes before returning. + :param page_action: A function that takes the `page` object, runs after navigation, and does the automation you need. Its return value is propagated as `fetch()`'s return value. + :param page_setup: A function that takes the `page` object, runs before navigation. + :param extra_headers: A dictionary of extra headers to add to the request. + :param disable_resources: Drop requests for unnecessary resources for a speed boost. + :param blocked_domains: A set of domain names to block requests to. + :param wait_selector: Wait for a specific CSS selector to be in a specific state. + :param wait_selector_state: The state to wait for the selector given with `wait_selector`. + :param network_idle: Wait for the page until there are no network connections for at least 500 ms. + :param load_dom: Enabled by default, wait for all JavaScript on page(s) to fully load and execute. + :param proxy: Static proxy to override rotator and session proxy. A new browser context will be created and used with it. + :return: `page_action`'s return value, or the navigated `page` if no `page_action` was given. + """ + static_proxy = kwargs.pop("proxy", None) + + params = _validate(kwargs, self, StealthConfig) + if not self._is_alive: # pragma: no cover + raise RuntimeError("Context manager has been closed") + + request_headers_keys = {h.lower() for h in params.extra_headers.keys()} if params.extra_headers else set() + referer = ( + "https://www.google.com/" if (params.google_search and "referer" not in request_headers_keys) else None + ) + + for attempt in range(self._config.retries): + proxy: Optional[ProxyType] = None + if self._config.proxy_rotator and static_proxy is None: + proxy = self._config.proxy_rotator.get_proxy() + else: + proxy = static_proxy + + async with self._page_generator( + params.timeout, params.extra_headers, params.disable_resources, proxy, params.blocked_domains + ) as page_info: + page_info: PageInfo + page: Page = page_info.page + if params.page_setup: + try: + await params.page_setup(page) + except Exception as e: # pragma: no cover + log.error(f"Error executing page_setup: {e}") + + try: + first_response = await page.goto(url, referer=referer) + await self._wait_for_page_stability(page, params.load_dom, params.network_idle) + + if not first_response: + raise RuntimeError(f"Failed to get response for {url}") + + result = await params.page_action(page) if params.page_action else None + + if params.wait_selector: + try: + waiter: Locator = page.locator(params.wait_selector) + await waiter.first.wait_for(state=params.wait_selector_state) + await self._wait_for_page_stability(page, params.load_dom, params.network_idle) + except Exception as e: # pragma: no cover + log.error(f"Error waiting for selector {params.wait_selector}: {e}") + + await page.wait_for_timeout(params.wait) + + return result if params.page_action else page + + except Exception as e: + page_info.mark_error() + if attempt < self._config.retries - 1: + if is_proxy_error(e): + log.warning( + f"Proxy '{proxy}' failed (attempt {attempt + 1}) | Retrying in {self._config.retry_delay}s..." + ) + else: + log.warning( + f"Attempt {attempt + 1} failed: {e}. Retrying in {self._config.retry_delay}s..." + ) + await asyncio_sleep(self._config.retry_delay) + else: + log.error(f"Failed after {self._config.retries} attempts: {e}") + raise + + raise RuntimeError("Request failed") # pragma: no cover diff --git a/src/engine/validators.py b/src/engine/validators.py new file mode 100644 index 0000000..8ef45b5 --- /dev/null +++ b/src/engine/validators.py @@ -0,0 +1,244 @@ +""" +Валидация конфигурации сессии/fetch — портировано из scrapling.engines._browsers._validators. + +Без поля/логики `solve_cloudflare`: детект и прохождение Cloudflare в проекте делает +antibot.orchestrator.pass_challenges (вызывается из api.actions.execute), а не сама сессия. +""" +from pathlib import Path +from typing import Annotated +from functools import lru_cache +from urllib.parse import urlparse +from dataclasses import dataclass, fields + +from msgspec import Struct, Meta, convert, ValidationError + +from engine._types import ( + Any, + Dict, + List, + Set, + Tuple, + Optional, + Callable, + Sequence, + overload, + SetCookieParam, + SelectorWaitStates, +) +from engine.proxy_rotation import ProxyRotator +from engine.navigation import construct_proxy_dict +from engine.session_types import PlaywrightFetchParams, StealthFetchParams + + +# Custom validators for msgspec +@lru_cache(8) +def _is_invalid_file_path(value: str, label: str = "Init script") -> bool | str: # pragma: no cover + """Fast file path validation""" + path = Path(value) + if not path.exists(): + return f"{label} path not found: {value}" + if not path.is_file(): + return f"{label} is not a file: {value}" + if not path.is_absolute(): + return f"{label} is not an absolute path: {value}" + return False + + +@lru_cache(2) +def _is_invalid_cdp_url(cdp_url: str) -> bool | str: + """Fast CDP URL validation""" + if not cdp_url.startswith(("ws://", "wss://", "http://", "https://")): + return "CDP URL must use 'ws://', 'wss://', 'http://', or 'https://' scheme" + + netloc = urlparse(cdp_url).netloc + if not netloc: # pragma: no cover + return "Invalid hostname for the CDP URL" + return False + + +# Type aliases for cleaner annotations +PagesCount = Annotated[int, Meta(ge=1, le=50)] +RetriesCount = Annotated[int, Meta(ge=1, le=10)] +Seconds = Annotated[float, Meta(ge=0)] + + +class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True): + """Configuration struct for validation""" + + max_pages: PagesCount = 1 + headless: bool = True + disable_resources: bool = False + network_idle: bool = False + load_dom: bool = True + wait_selector: Optional[str] = None + wait_selector_state: SelectorWaitStates = "attached" + cookies: Sequence[SetCookieParam] | None = [] + google_search: bool = True + wait: Seconds = 0 + timezone_id: str | None = "" + page_action: Optional[Callable] = None + page_setup: Optional[Callable] = None + proxy: Optional[str | Dict[str, str] | Tuple] = None # The default value for proxy in Playwright's source is `None` + proxy_rotator: Optional[ProxyRotator] = None + extra_headers: Optional[Dict[str, str]] = None + timeout: Seconds = 30000 + init_script: Optional[str] = None + init_script_content: Optional[str] = None + user_data_dir: str = "" + selector_config: Optional[Dict] = {} + additional_args: Optional[Dict] = {} + locale: str | None = None + real_chrome: bool = False + cdp_url: Optional[str] = None + useragent: Optional[str] = None + extra_flags: Optional[List[str]] = None + blocked_domains: Optional[Set[str]] = None + block_ads: bool = False + retries: RetriesCount = 3 + retry_delay: Seconds = 1 + capture_xhr: str | None = None + executable_path: Optional[str] = None + dns_over_https: bool = False + + def __post_init__(self): # pragma: no cover + """Custom validation after msgspec validation""" + if self.page_action and not callable(self.page_action): + raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}") + if self.page_setup and not callable(self.page_setup): + raise TypeError(f"page_setup must be callable, got {type(self.page_setup).__name__}") + if self.proxy and self.proxy_rotator: + raise ValueError( + "Cannot use 'proxy_rotator' together with 'proxy'. " + "Use either a static proxy or proxy rotation, not both." + ) + if self.proxy: + self.proxy = construct_proxy_dict(self.proxy) + if self.cdp_url: + cdp_msg = _is_invalid_cdp_url(self.cdp_url) + if cdp_msg: + raise ValueError(cdp_msg) + + if not self.cookies: + self.cookies = [] + if not self.extra_flags: + self.extra_flags = [] + if not self.selector_config: + self.selector_config = {} + if not self.additional_args: + self.additional_args = {} + if not self.capture_xhr: + self.capture_xhr = None + + if self.init_script is not None: + validation_msg = _is_invalid_file_path(self.init_script) + if validation_msg: + raise ValueError(validation_msg) + + if self.executable_path is not None: + validation_msg = _is_invalid_file_path(self.executable_path, "Browser executable") + if validation_msg: + raise ValueError(validation_msg) + + if self.block_ads: + from engine.ad_domains import AD_DOMAINS + + if self.blocked_domains: + self.blocked_domains = self.blocked_domains | set(AD_DOMAINS) + else: + self.blocked_domains = set(AD_DOMAINS) + + +class StealthConfig(PlaywrightConfig, kw_only=True, frozen=False, weakref=True): + allow_webgl: bool = True + hide_canvas: bool = False + block_webrtc: bool = False + + +@dataclass +class _fetch_params: + """A dataclass of all parameters used by `fetch` calls""" + + google_search: bool + timeout: Seconds + wait: Seconds + page_action: Optional[Callable] + page_setup: Optional[Callable] + extra_headers: Optional[Dict[str, str]] + disable_resources: bool + wait_selector: Optional[str] + wait_selector_state: SelectorWaitStates + network_idle: bool + load_dom: bool + blocked_domains: Optional[Set[str]] + selector_config: Dict + + +def validate_fetch( + method_kwargs: Dict | PlaywrightFetchParams | StealthFetchParams, + session: Any, + model: type[PlaywrightConfig] | type[StealthConfig], +) -> _fetch_params: # pragma: no cover + result: Dict[str, Any] = {} + overrides: Dict[str, Any] = {} + kwargs_dict: Dict[str, Any] = dict(method_kwargs) + + # Get all field names that _fetch_params needs + fetch_param_fields = {f.name for f in fields(_fetch_params)} + + for key in fetch_param_fields: + if key in kwargs_dict: + overrides[key] = kwargs_dict[key] + elif hasattr(session, "_config") and hasattr(session._config, key): + result[key] = getattr(session._config, key) + + if overrides: + validated_config = validate(overrides, model) + # Extract ONLY the fields that were actually overridden (not all fields) + # This prevents validated defaults from overwriting session config values + validated_dict = { + field: getattr(validated_config, field) for field in overrides.keys() if hasattr(validated_config, field) + } + + # Start with session defaults, then overwrite with validated overrides + result.update(validated_dict) + + result.setdefault("blocked_domains", None) + + return _fetch_params(**result) + + +# Cache default values for each model to reduce validation overhead +models_default_values = {} + +for _model in (StealthConfig, PlaywrightConfig): + _defaults = {} + if hasattr(_model, "__struct_defaults__") and hasattr(_model, "__struct_fields__"): + for field_name, default_value in zip(_model.__struct_fields__, _model.__struct_defaults__): # type: ignore + # Skip factory defaults - these are msgspec._core.Factory instances + if type(default_value).__name__ != "Factory": + _defaults[field_name] = default_value + + models_default_values[_model.__name__] = _defaults.copy() + + +def _filter_defaults(params: Dict, model: str) -> Dict: + """Filter out parameters that match their default values to reduce validation overhead.""" + defaults = models_default_values[model] + return {k: v for k, v in params.items() if k not in defaults or v != defaults[k]} + + +@overload +def validate(params: Dict, model: type[StealthConfig]) -> StealthConfig: ... + + +@overload +def validate(params: Dict, model: type[PlaywrightConfig]) -> PlaywrightConfig: ... + + +def validate(params: Dict, model: type[PlaywrightConfig] | type[StealthConfig]) -> PlaywrightConfig | StealthConfig: + try: + # Filter out params with the default values (no need to validate them) to speed up validation + filtered = _filter_defaults(params, model.__name__) + return convert(filtered, model) + except ValidationError as e: + raise TypeError(f"Invalid argument type: {e}") from e diff --git a/src/main.py b/src/main.py index 0008447..1df6b16 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,5 @@ -import asyncio import json import logging -import traceback import uuid from pathlib import Path from typing import Annotated @@ -12,16 +10,13 @@ from fastapi import Cookie from fastapi import FastAPI from fastapi import Request from fastapi import Response -from patchright.async_api import ProxySettings -from patchright.async_api import async_playwright +from patchright.async_api import Page, ProxySettings -from antibot.humanize import wait_for_page_stability -from engine import actions -from engine.browser_launch import HARMFUL_ARGS, launch_args, stealth_context_options -from engine.fingerprint import apply_fingerprint, context_options_for, load_or_create_fingerprint -from engine.geoip import resolve_geo -from engine.navigation import create_intercept_handler, is_proxy_error -from engine.schemas import SolveRequest +from api import actions +from api.fingerprint import browser_kwargs_for, load_or_create_fingerprint +from api.geoip import resolve_geo +from api.schemas import SolveRequest +from engine import AsyncStealthySession logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -40,6 +35,29 @@ def parse_proxy(proxy_url: str) -> ProxySettings: return settings +def make_page_action(payload: SolveRequest): + """Замыкание для AsyncStealthySession.fetch(page_action=...): выполняет очередь действий + (включая прохождение капчи/антибота через api.actions.execute) и достаёт из уже + навигированной страницы всё, что нужно для ответа API. Возврат этой функции — то, что + вернёт fetch() (см. engine/stealthy.py).""" + + async def page_action(page: Page) -> dict: + actions_result = await actions.execute( + page, payload.actions, payload.captcha, + antibot_name=payload.captcha_type, + success_locator=payload.captcha_success_locator, + detect_locator=payload.captcha_detect_locator, + ) + return { + "actions": actions_result, + "cookies": await page.context.cookies(), + "html": await page.inner_html("html"), + "user_agent": await page.evaluate("() => navigator.userAgent"), + } + + return page_action + + app = FastAPI() @@ -73,20 +91,18 @@ async def solve( response.set_cookie("session_id", session_id) logger.info("Setup new session with session_id: %s", session_id) - scr_w, scr_h = payload.screen.split("x") - scr_size = int(scr_w), int(scr_h) proxy = parse_proxy(payload.proxy) # профиль (cookies/local storage/fingerprint.json) хранится на диске по # session_id — если он уже приходил в куках, ниже подгружаем существующий # fingerprint из этой папки; если нет (новый session_id) — генерируем новый - # и сохраняем туда же (см. engine/fingerprint.py:load_or_create_fingerprint) + # и сохраняем туда же (см. api/fingerprint.py:load_or_create_fingerprint) user_data_dir = Path(__file__).parent / f"../extra/user_data_dir/{session_id}" if not user_data_dir.exists(): user_data_dir.mkdir(parents=True) # geo — до создания fingerprint'а, чтобы производная от прокси locale - # (см. engine/geoip.py) попала в сам fingerprint, а не подменяла его + # (см. api/geoip.py) попала в сам fingerprint, а не подменяла его # задним числом geo = await resolve_geo(proxy) locale = geo.get("locale") if geo else None @@ -97,118 +113,65 @@ async def solve( # TODO: если есть session_id, то user_agent нужно взять из настроек пользователя if locale: fp_kwargs["locale"] = locale + + scr_size = (1920, 1080) fingerprint = load_or_create_fingerprint( user_data_dir, # только нижняя граница: реальный монитор физически не может быть # меньше открытого на нём окна браузера (viewport ниже берём из - # payload.screen напрямую, не отсюда). load_or_create_fingerprint - # генерирует с strict=True, так что это ограничение либо честно - # соблюдается, либо сразу падает ValueError — датасет browserforge - # больше не может тихо подсунуть сюда, например, мобильный экран + # fingerprint'а, не отсюда) screen=Screen(min_width=scr_size[0], min_height=scr_size[1]), **fp_kwargs, ) - ctx_kwargs = { - # базовые анти-детект опции ниже — fingerprint накладывается поверх - # и выигрывает при конфликте по user_agent/viewport/device_scale_factor - **stealth_context_options(), - **context_options_for( - fingerprint, - viewport={"width": scr_size[0], "height": scr_size[1]}, - ), - } - + # UA/заголовки/init-script (подделка navigator/screen/WebGL/...) /viewport/screen/ + # device_scale_factor под fingerprint — единым куском kwargs (см. + # api/fingerprint.py:browser_kwargs_for), тем же путём, что и остальная конфигурация + # сессии (proxy/cookies/locale), а не отдельным вызовом после запуска контекста. + fp_browser_kwargs = browser_kwargs_for(fingerprint) permissions = ["notifications"] - if geo: - if geo.get("timezone_id"): - ctx_kwargs["timezone_id"] = geo["timezone_id"] - if geo.get("geolocation"): - ctx_kwargs["geolocation"] = geo["geolocation"] - permissions.append("geolocation") - ctx_kwargs["permissions"] = permissions + if geo and geo.get("geolocation"): + fp_browser_kwargs["additional_args"]["geolocation"] = geo["geolocation"] + permissions.append("geolocation") + fp_browser_kwargs["additional_args"]["permissions"] = permissions nav_timeout_ms = payload.timeout * 1000 if payload.timeout else 60_000 - default_timeout_ms = payload.timeout * 1000 if payload.timeout else 30_000 blocked_domains = set(payload.blocked_domains) if payload.blocked_domains else None - referer = "https://www.google.com/" if payload.google_search else None - retries = max(1, payload.retries) - async with async_playwright() as p: - context = await p.chromium.launch_persistent_context( - user_data_dir=str(user_data_dir), - proxy=proxy, - headless=True, - channel="chrome", - args=launch_args(locale=locale), - ignore_default_args=HARMFUL_ARGS, - **ctx_kwargs, - ) - try: - await apply_fingerprint(context, fingerprint) - page = context.pages[0] if context.pages else await context.new_page() - if payload.cookies: - await context.add_cookies(payload.cookies) - page.set_default_timeout(default_timeout_ms) - page.set_default_navigation_timeout(nav_timeout_ms) - if payload.disable_resources or blocked_domains: - await page.route("**/*", create_intercept_handler(payload.disable_resources, blocked_domains)) + kwargs = dict( + headless=False, + proxy=proxy, + user_data_dir=str(user_data_dir), + real_chrome=True, + locale=locale, + timezone_id=geo.get("timezone_id") if geo else None, + cookies=payload.cookies, + disable_resources=payload.disable_resources, + blocked_domains=blocked_domains, + timeout=nav_timeout_ms, + retries=max(1, payload.retries), + retry_delay=payload.retry_delay, + wait=payload.wait, + google_search=payload.google_search + ) - status = error = user_agent = None - cookies = html = None - actions_result = None - - for attempt in range(retries): - try: - await page.goto(url=payload.url, referer=referer) - await wait_for_page_stability(page, load_dom=True, network_idle=False) - - actions_result = await actions.execute( - page, payload.actions, payload.captcha, - antibot_name=payload.captcha_type, - success_locator=payload.captcha_success_locator, - detect_locator=payload.captcha_detect_locator, - ) - if payload.wait: - await page.wait_for_timeout(payload.wait) - - status = not "error" in actions_result[-1] - error = ["Something error", None][status] - cookies = await context.cookies() - html = await page.inner_html('html') - user_agent = await page.evaluate("() => navigator.userAgent") - # TODO: добавить как опцию. передачу скрина в base64. - # await page.screenshot( - # path=( - # f"/home/sokol/PycharmProjects/WebRoboApi/extra/screenshots/" - # f"{datetime.timestamp(datetime.now())}.png" - # ) - # ) - break - except Exception as e: - if attempt < retries - 1: - kind = "прокси" if is_proxy_error(e) else "запрос" - logger.warning( - "Попытка %s/%s не удалась (%s: %s), retry через %.1fs", - attempt + 1, retries, kind, e, payload.retry_delay, - ) - await asyncio.sleep(payload.retry_delay) - else: - raise - finally: - await context.close() + async with AsyncStealthySession(**kwargs) as engine: + result = await engine.fetch(payload.url, page_action=make_page_action(payload)) finally: BROWSER_SLOTS += 1 - result = { + status = not "error" in result["actions"][-1] + error = ["Something error", None][status] + + result_payload = { "status": ["err", "ok"][status], - "user_agent": user_agent, - "cookies": cookies, - "actions": actions_result, + "user_agent": result["user_agent"], + "cookies": result["cookies"], + "actions": result["actions"], } if payload.html: - result["html"] = html + result_payload["html"] = result["html"] if error: - result["error"] = error - logger.info("Outgoing payload:\n%s", json.dumps(result, indent=2, sort_keys=True, ensure_ascii=True)) - return result + result_payload["error"] = error + logger.info("Outgoing payload:\n%s", json.dumps(result_payload, indent=2, sort_keys=True, ensure_ascii=True)) + return result_payload