使用scrapy-redis爬取豆果美食分类及详情页数据

#使用scrapy_redis组件自己的调度器   setting文件设置
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 确保所有spider通过redis共享相同的重复过滤。
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 可选 不清理redis队列，允许暂停/恢复抓取。 允许暂定,redis数据不丢失
SCHEDULER_PERSIST = True

# REDIS 主机和端口
REDIS_HOST = '127.0.0.1'  # 写自己电脑的ip地址
REDIS_PORT = 6379

import scrapy
from ..items import TteightItem
from scrapy_redis.spiders import RedisSpider
import re

class DdouguoSpider(RedisSpider):
    name = 'ddouguo'
    allowed_domains = ['www.douguo.com']

    # start_urls = ['https://www.douguo.com/fenlei']
    redis_key = 'ddouguo:start_urls'

    # lpush ddouguo:start_urls https://www.douguo.com/fenlei

    def parse(self, response):
        # print(response.text)
        item = TteightItem()
        sub_url = re.findall('<li><a  href="(.*?)" title=".*?" target="_blank">.*?</a></li>', response.text, re.S)
        for i in sub_url:
            sub_url = 'https://www.douguo.com' + i
            # print(sub_url)
            # print(type(sub_url))
            title = sub_url.split('/')[4]
            item['title'] = title
            print(title)
            yield scrapy.Request(sub_url, meta={'item': item}, callback=self.sub_parse, dont_filter=True)

    def sub_parse(self, response):
        print(response.text)
        item = response.meta['item']
        sub_title = re.findall('<a class="cookname text-lips " href="/cookbook/.*?.html" target="_blank">(.*?)</a>',
                               response.text, re.S)
        print(sub_title)
        sub_major = re.findall('<p class="major">(.*?)</p>', response.text, re.S)
        print(sub_major)
        for sub_title, sub_major in zip(sub_title, sub_major):
            item['sub_title'] = sub_title
            item['sub_major'] = sub_major
            # print(sub_major)
            yield item

import scrapy


class TteightItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    sub_title = scrapy.Field()
    sub_major = scrapy.Field()

import pymysql

class MysqlTteightPipeline:
    def __init__(self):
        self.dg = pymysql.connect(
            user='用户名',
            password='用户密码',
            database='数据库名称'
        )
        self.cursor = self.dg.cursor()

    def process_item(self, item, spider):
        sql = """insert into ddouguo(title, sub_title, sub_major)
        values (%s, %s, %s)"""
        try:
            self.cursor.execute(sql,(item['title'],item['sub_title'],item['sub_major']))
            self.dg.commit()
        except Exception as e:
            print('失败')

        return item

    def __del__(self):
        self.dg.close()

import os

os.system('scrapy crawl ddouguo')