29、煲剧狂人

 1 from gevent import monkey
 2 monkey.patch_all()
 3 
 4 from bs4 import BeautifulSoup
 5 import gevent,requests,csv
 6 from gevent.queue import Queue
 7 
 8 url_list = [‘http://www.mtime.com/top/tv/top100/‘]
 9 for i in range(2,11):
10     url_list.append(‘http://www.mtime.com/top/tv/top100/index-{}.html‘.format(i))
11 
12 work = Queue()
13 
14 for url in url_list:
15     work.put_nowait(url)
16 
17 def pachong():
18     while not work.empty():
19         url = work.get_nowait()
20         res = requests.get(url)
21         items = BeautifulSoup(res.text,‘html.parser‘).find_all(‘div‘,class_=‘mov_con‘)
22         for item in items:
23             title = item.find(‘h2‘).text.strip()
24             director = ‘null‘
25             actor    = ‘null‘
26             remarks  = ‘null‘
27             tag_ps = item.find_all(‘p‘)
28             for tag_p in tag_ps:
29                 if tag_p.text[:2] == ‘导演‘:
30                     director = tag_p.text[3:].strip()
31                 elif tag_p.text[:2] == ‘主演‘:
32                     actor = tag_p.text[3:].strip().replace(‘\t‘,‘‘)
33                 elif tag_p[‘class‘]:
34                     remarks = tag_p.text.strip()
35             with open(‘top100.csv‘,‘a‘,newline=‘‘,encoding=‘utf-8-sig‘) as csv_file:
36                 writer = csv.writer(csv_file)
37                 writer.writerow([title,director,actor,remarks])
38 
39 task_list = []
40 
41 for x in range(3):
42     task = gevent.spawn(pachong)
43     task_list.append(task)
44 
45 with open(‘top100.csv‘,‘w‘,newline=‘‘,encoding=‘utf-8-sig‘) as csv_file:
46     writer = csv.writer(csv_file)
47     writer.writerow([‘电视剧名‘,‘导演‘,‘主演‘,‘简介‘])
48 
49 gevent.joinall(task_list)