如上篇博客,初步使用了scrapy框架了,但是只能爬取一页,或者手动的把要爬取的网址手动添加到start_url中,太麻烦
接下来介绍该如何去处理分页,手动发起分页请求

爬虫文件.py

# -*- coding: utf-8 -*-
import scrapy
from qiubaiPage.items import QiubaiproItem

class QiubaiSpider(scrapy.Spider):
    name = ‘qiubai‘
    # allowed_domains = [‘www.xxx.com‘]
    start_urls = [‘https://www.qiushibaike.com/text/‘]
    url=‘https://www.qiushibaike.com/text/page/%d/‘
    page_num=1

    # 2.基于管道的持久化存储(基于管道的持久化存储必须写下管道文件当中)
    def parse(self,response):
        div_list=response.xpath(‘//div[@id="content-left"]/div‘)
        for div in  div_list:
            try :
                author = div.xpath(‘./div[1]/a[2]/h2/text()‘)[0].extract()

            except Exception as e:
                print(e)
                continue
            content = div.xpath(‘./a[1]/div/span//text()‘).extract()
            content = ‘‘.join(content)

            # 实例话一个item对象(容器)
            item = QiubaiproItem()
            item[‘author‘] = author
            item[‘content‘] = content
            # 返回给pipline去持久化存储
            yield item

        if self.page_num<10:  #发起请求的条件
            self.page_num+=1
            url=(self.url%self.page_num)
            #手动发起请求,调用parse再去解析
            yield scrapy.Request(url=url,callback=self.parse)


items.py

import scrapy
class QiubaiproItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    author=scrapy.Field()
    content=scrapy.Field()

pipline.py

class QiubaipagePipeline(object):
    f = None

    # 开启爬虫时执行程序执行一次,重写父类的方法,可以开启数据库等,要记得参数有一个spider不要忘记了
    def open_spider(self, spider):
        self.f = open(‘./qiushibaike.txt‘, ‘w‘, encoding=‘utf-8‘)

    # 提取处理数据(保存数据)
    def process_item(self, item, spider):
        self.f.write(item[‘author‘] + ‘:‘ + item[‘content‘] + ‘\n‘)
        return item

    # .关闭爬虫时执行也是只执行一次,重写父类方法,可以关闭数据库等,重写父类要要有参数spider,不要忘记了
    def colse_spider(self, spider):
        self.f.close()

注意:要基于管道存储要记得去settings.py把注释放开

  def start_requests(self):
        for u in self.start_urls:
           yield scrapy.Request(url=u,callback=self.parse)

def start_requests(self):
        #请求的url
        post_url = ‘http://fanyi.baidu.com/sug‘
        # post请求参数
        formdata = {
            ‘kw‘: ‘wolf‘,
        }
        # 发送post请求
        yield scrapy.FormRequest(url=post_url, formdata=formdata, callback=self.parse)

　　对于cookies的处理就是不用处理,直接去settings.py把cookies的相关配置放开就行