多线程爬虫

import requests
import threading
import pymongo
from threading import Lock
from queue import Queue
from lxml import etree


# 爬虫类，负责采集数据的,(创建父线程类)
class CrawlThread(threading.Thread):
    def __init__(self, name, pageQueue, dataQueue):
        # 爬虫类重写父类方法，但是又得调父类的一些方法，所有调用下父类方法
        super().__init__()
        # 实例属性的初始化与赋值爬虫名字
        self.threadname = name
        # 实例属性的初始化与赋值url队列
        self.pageQueue = pageQueue
        # 实例属性的初始化与赋值headers，直接写死
        self.headers = {
              ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36‘
        }
        self.dataQueue = dataQueue

    def run(self):
        base_url = ‘https://www.xiaohua.com/duanzi?page=%s‘
        while 1:
            try:
                # block = True表示堵塞，没有东西时，一直等，只到等到有东西,False相反
                page = self.pageQueue.get(block=False)
                url = base_url%page
                print(‘%s正在采集数据‘%self.threadname)
                # 响应数据
                response = requests.get(url=url, headers=self.headers)
                # put方法：Queue.put(item,[block[, timeout]])：将item消息写入队列，block默认值为True
                self.dataQueue.put(response.text)
                print(‘%s提交数据完毕‘%self.threadname)
            except:
                break


# 解析类，负责数据解析,(创建父线程类)
class ParseThread(threading.Thread):

    def __init__(self, name, dataQueue, lock):
        super().__init__()
        self.threadname = name
        self.dataQueue = dataQueue
        self.lock = lock

    def run(self):
        while 1:
            try:
                html = self.dataQueue.get(block=False)
                print(‘%s正在解析数据‘%self.threadname)
                # 调用方法并传参到具体的解析过程
                self.parse(html)
                print(‘%s解析数据完毕‘%self.threadname)
            except:
                break

    def parse(self, html):
        # 具体的解析过程
        tree = etree.HTML(html)
        div_list = tree.xpath(‘//div[@class="one-cont"]‘)
        for div in div_list:
            item = {}
            author = div.xpath(‘./div/div/a/i/text()‘)[0]
            item[‘author‘] = author
            # 上锁
            with self.lock:
                self.save(item)


    def save(self, item):
        # 建立数据库连接,数据库存储
        conn = pymongo.MongoClient(‘localhost‘, 27017)
        db = conn.jokes_dxc_author
        table = db.xhua
        table.insert_one(item)

def main():
    # 实例化队列，存放url
    pageQueue = Queue()
    for i in range(1, 101):
        pageQueue.put(i)
    # 存放脏数据
    dataQueue = Queue()
    # 创建锁
    lock = Lock()

    # 开启爬虫线程(爬虫名字)
    crawlList = [‘爬虫1号‘,‘爬虫2号‘,‘爬虫3号‘]
    # 循环赋值爬虫名字
    ThreadCrawl = []
    for var in crawlList:
        c = CrawlThread(var, pageQueue, dataQueue)
        c.start()
        ThreadCrawl.append(c)
    for var in ThreadCrawl:
        var.join()

    # 开启解析线程
    parseList = [‘解析一号‘,‘解析二号‘,‘解析三号‘]
    ThreadParse = []
    for var in parseList:
        p = ParseThread(var, dataQueue, lock)
        p.start()
        ThreadParse.append(p)
    for p in ThreadParse:
        p.join()

if __name__ == ‘__main__‘:
    main()