scrapy-redis

 git clone https://github.com/rmax/scrapy-redis.git

PIDER_MODULES = [‘example.spiders‘]
NEWSPIDER_MODULE = ‘example.spiders‘

USER_AGENT = ‘scrapy-redis (+https://github.com/rolando/scrapy-redis)‘

# 指定RFPDupeFilter方法给request对象去重
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 指定Scheduler队列
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 队列中的内容是否持久化保存，False在redis关闭的时候清空redis
SCHEDULER_PERSIST = True


ITEM_PIPELINES = {
    ‘example.pipelines.ExamplePipeline‘: 300,
    # scrapy_redis实现 items保存到redis的pipline
    ‘scrapy_redis.pipelines.RedisPipeline‘: 400,
}

# 指定redis地址
# REDIS_URl = ‘redis://192.168.1.103:6379‘
# 另一种形式 指定redis地址
REDIS_HOST = ‘localhost‘
REDIS_PORT = ‘6379‘