Scrapy 爬取 5 秒盾站点，速度可以这么快！

# -*- coding: utf-8 -*-from urllib.parse import urlencodeimport scrapy
class CloudflareSpider(scrapy.Spider):    name = 'cloudflare_spider'    def __init__(self):        super().__init__()        self.headers = {            'authority': 'xxxxx',            'accept': 'application/json, text/plain, */*',            'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',            'cache-control': 'no-cache',            'pragma': 'no-cache',            'referer': 'https://xxxxx/feed',            'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',            'sec-ch-ua-mobile': '?0',            'sec-ch-ua-platform': '"macOS"',            'sec-fetch-dest': 'empty',            'sec-fetch-mode': 'cors',            'sec-fetch-site': 'same-origin',            'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',        }        self.url = 'https://xxxx/xxx/xxx/'            def start_requests(self):        for page in range(1, 100):            params = {                'page': page,                'posts_to_load': '5',                'sort': 'top',            }            proxies = {                "http": "http://127.0.0.1:59292",                "https": "http://127.0.0.1:59292",            }            full_url = self.url + '?' + urlencode(params)            yield scrapy.Request(                url=full_url,                headers=self.headers,                callback=self.parse,                dont_filter=True,                meta={"proxies": proxies}            )                def parse(self, response, **kwargs):        print(response.text)

from scrapy.crawler import CrawlerProcessfrom scrapy.utils.project import get_project_settings
def run_spider():    process = CrawlerProcess(get_project_settings())    process.crawl('cloudflare_spider')    process.start()    if __name__ == "__main__":    run_spider()

HTTPERROR_ALLOWED_CODES = [403]

from scrapy.http import HtmlResponsefrom tls_client import Session

class DownloaderMiddleware(object):
    def __init__(self):        self.session: Session = Session(            client_identifier="chrome_104"        )
    def process_request(self, request, spider):        proxies = request.meta.get("proxies") or None        headers = request.headers.to_unicode_dict()        if request.method == "GET":            response = self.session.get(                url=request.url,                headers=headers,                proxy=proxies,                timeout_seconds=60,            )        else:            response = self.session.post(                url=request.url,                headers=headers,                proxy=proxies,                timeout_seconds=60,            )        return HtmlResponse(            url=request.url,            status=response.status_code,            body=response.content,            encoding="utf-8",            request=request,        )

"""Download handlers for http and https schemes"""
import loggingfrom time import timefrom urllib.parse import urldefrag
from tls_client import Sessionfrom twisted.internet import threadsfrom twisted.internet.error import TimeoutErrorfrom scrapy.http import HtmlResponsefrom scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
logger = logging.getLogger(__name__)

class CloudFlareDownloadHandler(HTTP11DownloadHandler):
    def __init__(self, settings, crawler=None):        super().__init__(settings, crawler)        self.session: Session = Session(            client_identifier="chrome_104"        )
    @classmethod    def from_crawler(cls, crawler):        return cls(crawler.settings, crawler)
    def download_request(self, request, spider):        from twisted.internet import reactor        timeout = request.meta.get("download_timeout") or 10        # request details        url = urldefrag(request.url)[0]        start_time = time()
        # Embedding the provided code asynchronously        d = threads.deferToThread(self._async_download, request)
        # set download latency        d.addCallback(self._cb_latency, request, start_time)        # check download timeout        self._timeout_cl = reactor.callLater(timeout, d.cancel)        d.addBoth(self._cb_timeout, url, timeout)        return d
    def _async_download(self, request):        timeout = int(request.meta.get("download_timeout"))        proxies = request.meta.get("proxies") or None        headers = request.headers.to_unicode_dict()        if request.method == "GET":            response = self.session.get(                url=request.url,                headers=headers,                proxy=proxies,                timeout_seconds=timeout,            )        else:            response = self.session.post(                url=request.url,                headers=headers,                proxy=proxies,                timeout_seconds=timeout,            )        return HtmlResponse(            url=request.url,            status=response.status_code,            body=response.content,            encoding="utf-8",            request=request,        )
    def _cb_timeout(self, result, url, timeout):        if self._timeout_cl.active():            self._timeout_cl.cancel()            return result        raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
    def _cb_latency(self, result, request, start_time):        request.meta["download_latency"] = time() - start_time        return result