Scrapy-redis组件去重

pip3 install -i https://pypi.douban.com/simple scrapy-redis

DUPEFILTER_KEY = ‘dupefilter:%(timestamp)s‘
DUPEFILTER_CLASS = ‘scrapy_redis.dupefilter.RFPDupeFilter‘

REDIS_HOST = ‘ip‘                            
REDIS_PORT = 端口号                                   
REDIS_PARAMS  = {‘password‘:‘密码‘}                                 
REDIS_ENCODING = "utf-8"
或                            
# REDIS_URL = ‘redis://user:密码@ip:端口‘   （优先于以上配置）

class RedisDupeFilter(RFPDupeFilter):
    @classmethod
    def from_settings(cls, settings):
        server = get_redis_from_settings(settings)
        key = defaults.DUPEFILTER_KEY % {‘timestamp‘: ‘固定的key‘‘}
        debug = settings.getbool(‘DUPEFILTER_DEBUG‘)
        return cls(server, key=key, debug=debug)