Scrapy框架教程：深度爬取并持久化保存图片

scrapy startproject daimg

cd daimg
scrapy genspider daimgpc www.xxx.com

ROBOTSTXT_OBEY = False 
LOG_LEVEL = 'ERROR'
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"

start_urls = ["http://www.XXXX.com/photo/world/"]

列表= response.xpath('/html/body/div[5]/ul/li')
       for i in 列表:
           标题=i.xpath('./a/@title').extract_first()+'.jpg'
           详情页=i.xpath('./a/@href').extract_first()

start_urls = ["http://www.XXXX.com/photo/world/"]
    多页url模板='http://www.XXXX.com/photo/world/list_69_%d.html'
    
         if self.页码<4:
            新url=format(self.多页url模板%self.页码)
            self.页码+=1
            yield scrapy.Request(url=新url,callback=self.parse)

yield scrapy.Request(url=详情页,callback=self.详情页解析)
    
    def 详情页解析(self,response):
        图片地址=response.xpath('/html/body/div[4]/div[1]/ul[2]/img/@src').extract_first()
        print(图片地址)

图片名称 = scrapy.Field()
    图片地址 = scrapy.Field()

from ..items import DaimgItem

itme对象=DaimgItem()
           itme对象['图片名称']=标题
           yield scrapy.Request(url=详情页,callback=self.详情页解析,meta={'item':itme对象})




meta=response.meta
item=meta['item']
item['图片地址']=图片地址
yield item

from scrapy.pipelines.images import ImagesPipeline
class DaimgPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        请求地址=item['图片地址']
       yield scrapy.Request(url=请求地址,meta={'名称':item['图片名称']})
    def file_path(self, request, response=None, info=None, *, item=None):
        保存名称=request.meta['名称']
        return 保存名称
    def item_completed(self, results, item, info):
        return item

IMAGES_STORE='aiyou'

ITEM_PIPELINES = {
   "daimg.pipelines.DaimgPipeline": 300,
}

scrapy crawl daimgpc

CONCURRENT_REQUESTS = 32

COOKIES_ENABLED = False

RETRY_ENABLED = False

DOWNLOAD_TIMEOUT = 10