降低耦合性获取微博数据

  1 import json
  2 import queue
  3 import re
  4 import threading
  5 
  6 import requests
  7 import xlrd
  8 from lxml import etree
  9 from pymongo import MongoClient
 10 
 11 proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
 12             "host": "http-dyn.abuyun.com",
 13             "port": "9020",
 14             "user": "H6VZC52B4BF2986D",
 15             "pass": "7850C72DC876E723",
 16         }
 17 
 18 
 19 class WB:
 20     def __init__(self):
 21         self.start_temp_url = "https://m.weibo.cn/api/container/getIndex?type=uid&value={}"
 22         self.headers = {
 23             "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"
 24         }
 26         self.proxies = {
 27             "http": proxyMeta,
 28             "https": proxyMeta,
 29         }
 30         self.save_content_q = queue.Queue()
 31         self.url_q = queue.Queue()
 32         self.client = MongoClient(host=‘localhost‘, port=27017)
 33         # 使用的mongo的数据表
 34         self.db = self.client.WbImageSet
 35 
 36     def save_data_mongodb(self, collect_name, data):
 37         # 定义一个去重的图标集名称
 38         self.collect_name = self.db[collect_name]
 39         history_record = self.collect_name.find_one({"_id": data[‘id‘]})
 40         if history_record:
 41             # 数据库中已经存在数据
 42             pass
 43         else:
 44             # 数据库中不存在，插入数据
 45             self.collect_name.update_one({‘_id‘: data[‘id‘]}, {‘$set‘: data}, upsert=True)
 46 
 47     def get_author_info(self, url):
 48         """
 49         获取博主信息
 50         :return:
 51         """
 52         response = requests.get(url=url, headers=self.headers, proxies=self.proxies, verify=True)
 53         self.data = response.content.decode(‘utf8‘)
 54         content = json.loads(self.data).get(‘data‘)
 55         max_content = content.get(‘userInfo‘).get(‘statuses_count‘)
 56         return max_content
 57 
 58     def get_containerid(self):
 59         """获取微博主页的containerid，爬取微博内容时需要此id"""
 60         content = json.loads(self.data).get(‘data‘)
 61         for data in content.get(‘tabsInfo‘).get(‘tabs‘):
 62             if (data.get(‘tab_type‘) == ‘weibo‘):
 63                 containerid = data.get(‘containerid‘)
 64         return containerid
 65 
 66     def get_url(self, containerid, max_content):
 67         for x in range(int(max_content)//10):
 68             wb_content_url = ‘https://m.weibo.cn/api/container/getIndex?type=uid&value=‘ + self.name + ‘&containerid=‘+containerid+‘&page=‘+str(x)
 69             self.url_q.put(wb_content_url)
 70 
 71     def get_wb_content(self):
 72         """获取微博内容信息,并保存到文本中，内容包括：每条微博的内容、微博详情页面地址、点赞数、评论数、转发数等"""
 73         num = 0
 74         while True:
 75             try:
 76                 if self.url_q.empty():
 77                     break
 78                 weibo_url = self.url_q.get()
 79                 response = requests.get(url=weibo_url, headers=self.headers, proxies=self.proxies, verify=True)
 80                 content = json.loads(response.content.decode(‘utf8‘)).get(‘data‘)
 81                 cards = content.get(‘cards‘)
 82                 if (len(cards) > 0):
 83                     for j in range(len(cards)):
 84                         num += 1
 85                         card_type = cards[j].get(‘card_type‘)
 86                         item = {}
 87                         if (card_type == 9):
 88                             mblog = cards[j].get(‘mblog‘)
 89                             scheme = cards[j].get(‘scheme‘)  # 微博地址
 90                             print(scheme)
 91                             print("--正在抓取--{}--第{}条微博--".format(self.name, num))
 92                             text_id = mblog.get("id")  # 微博id 用于去重
 93                             text = mblog.get(‘text‘)  # 微博内容
 94                             # 以下为获取标题的正则匹配
 95                             html = etree.HTML(text)
 96                             x = html.xpath(‘//text()‘)
 97                             title = ‘,‘.join(x)
 98                             title = title.replace(‘\r‘, ‘‘).replace(‘\n‘, ‘‘).replace(‘\t‘, ‘‘)
 99                             title = re.sub(‘(#.*#)‘, ‘‘, title)
100                             title = re.sub(‘@‘, ‘‘, title)
101                             title = re.sub(‘ ‘, ‘‘, title)
102                             pictures = mblog.get(‘pics‘)  # 正文配图，返回list
103                             pic_urls = []  # 存储图片url地址
104                             if pictures:
105                                 for picture in pictures:
106                                     pic_url = picture.get(‘large‘).get(‘url‘)
107                                     pic_urls.append(pic_url)
108                             if pic_urls == []:
109                                 continue
110                             item[‘id‘] = text_id
111                             item[‘category‘] = self.category
112                             item[‘author‘] = self.name
113                             item[‘title‘] = title
114                             item[‘url‘] = pic_urls
115                             item[‘select‘] = 0
116                             # 去重逻辑
117                             self.save_data_mongodb(self.name, item)
118             except Exception as e:
119                 print(e)
120 
121     def run(self):
122         while True:
123             if wb_content_q.empty():
124                 break
125             dict_wb = wb_content_q.get()
126             self.category = dict_wb[‘category‘]
127             self.name = dict_wb[‘name‘]
128             self.wb_id = dict_wb[‘id‘]
129             max_content = self.get_author_info(self.start_temp_url.format(self.wb_id))
130             containerid = self.get_containerid()
131             self.get_url(containerid, max_content)
132 
133             Threads_caiji = []
134             for x in range(5):
135                 t1 = threading.Thread(target=self.get_wb_content)
136                 Threads_caiji.append(t1)
137             for a in Threads_caiji:
138                 a.start()
139                 a.join()
140 
141 
142 class Excel_path:
143 
144     @staticmethod
145     def get_excel_info(path,num):
146         """
147         从excel中读取数据 所有链接切割把博主的id分割出来，按照 id:category 的方式保存为字典
148         :param path:  excel路径
149         :return:返回从excel中读取的字典
150         """
151         excel_sheet = xlrd.open_workbook(path).sheet_by_index(num)
152         category_name = excel_sheet.col_values(0)[1:]
153         wb_url = excel_sheet.col_values(2)[1:]
154         name = excel_sheet.col_values(1)[1:]
155         for i in wb_url:
156             item = {}
157             a = wb_url.index(i)
158             c = i.split(‘?‘)[0]
159             d = c.split(‘/‘)[-1]
160             item[‘name‘] = name[a]
161             item[‘category‘] = category_name[a]
162             item[‘id‘] = d
163             print(item)
164             wb_content_q.put(item)
165 
166 
167 if __name__ == ‘__main__‘:
168     wb_content_q = queue.Queue()
169     excel_path = ‘D:\\gongsi_code\\ImageSpider\\微博\\Image_set\\数据源.xlsx‘
170     excel_index = 0
171     Excel_path.get_excel_info(excel_path, int(excel_index))
172     WB().run()

 1 import os
 2 import queue
 3 import re
 4 import threading
 5 import time
 6 from concurrent.futures.thread import ThreadPoolExecutor
 7 
 8 import pymongo
 9 import requests
10 import xlrd
11 
12 
13 class WbDownload:
14     def __init__(self):
15         self.client = pymongo.MongoClient(host=‘localhost‘, port=27017)
16         self.db = self.client.WbImageSet
17         self.info_q = queue.Queue()
18 
19     def get_info(self, collection_name):
20         self.collection = self.db[collection_name]
21         a = self.collection.find({"select": 0})
22         for index in a:
23             self.info_q.put(index)
24 
25     def save_(self):
26         while True:
27             """保存到本地"""
28             if self.info_q.empty():
29                 break
30             image = self.info_q.get()
31             category_name = image[‘category‘]
32             upload_time = time.strftime("%Y-%m-%d", time.localtime())
33             rule = re.compile(r‘\s*‘, re.S)
34             rule2 = re.compile(r‘\W*‘, re.S)
35             title = rule.sub(‘‘, image[‘title‘])
36             title = rule2.sub(‘‘, title)
37             path = ‘D:/微博/‘ + category_name + ‘/‘ + str(upload_time) + ‘/‘ + title
38             if os.path.exists(path):
39                 continue
40             else:
41                 os.makedirs(path)
42             with open(path + ‘/content.txt‘, ‘w‘, encoding=‘utf8‘)as fb:
43                 fb.write(str([image[‘title‘]]))
44             for x in image[‘url‘]:
45                 x_index = image[‘url‘].index(x)
46                 if x[-4:-1:] == ‘gif‘:
47                     continue
48                 with open(path + ‘/{}.jpg‘.format(str(x_index)), ‘wb‘) as f:
49                     response = requests.get(url=x)
50                     f.write(response.content)
51             self.collection.update_one({"id": id}, {"$set": {"select": 1}})
52             print(‘-----  ‘ + title + ‘  写入完成  ------‘)
53 
54 
55     def run(self):
56         while True:
57             if name_q.empty():
58                 break
59             name = name_q.get()
60             self.get_info(name)
61             Threads = []
62             for i in range(20):
63                 t_down = threading.Thread(target=self.save_)
64                 t_down.start()
65                 Threads.append(t_down)
66             for t in Threads:
67                 t.join()
68 
69 
70 
71 
72 class Excel_path:
73 
74     @staticmethod
75     def get_excel_info(path,num):
76         """
77         从excel中读取数据 所有链接切割把博主的id分割出来，按照 id:category 的方式保存为字典
78         :param path:  excel路径
79         :return:返回从excel中读取的字典
80         """
81         excel_sheet = xlrd.open_workbook(path).sheet_by_index(num)
82         name = excel_sheet.col_values(1)[1:]
83         for x in name:
84             print(x)
85             name_q.put(x)
86 
87 
88 if __name__ == ‘__main__‘:
89     name_q = queue.Queue()
90     path = ‘D:\\gongsi_code\\ImageSpider\\微博\\Image_set\\数据源.xlsx‘
91     excel_index = 0
92     Excel_path.get_excel_info(path, excel_index)
93     with ThreadPoolExecutor(5) as executor:
94             wb = WbDownload()
95             executor.submit(wb.run)