[python应用]python简单图片抓取

 1 import requests
 2 import threading
 3 import queue
 4 from subprocess import Popen,PIPE
 5 from bs4 import BeautifulSoup as bs
 6 import urllib
 7 import base64
 8 
 9 
10 queue=queue.Queue()
11 
12 class Jiandan(threading.Thread):
13         def __init__(self,queue):
14             threading.Thread.__init__(self)
15             self._queue=queue
16             
17         def run(self):
18             while not self._queue.empty():
19                 url=self._queue.get_nowait()
20                 self.spider(url)
21                 
22         def spider(self,url):
23             headers={}
24             r=requests.get(url)
25             #选择lxml解析器
26             soup=bs(r.content,‘lxml‘)
27             #查找html标签name为img的元素获取到它
28             imgs=soup.find_all(name=‘img‘,attrs={})
29             
30             for img in imgs:
31                 if ‘onload‘ in str(img):
32                     img=img[‘org_src‘]
33                 else:
34                     img=img[‘src‘]
35                 
36                 name=img.split(‘/‘)[-1]
37                 
38                 #获取的url没http:所以要加上http协议才能访问下载
39                 img="http:"+img
40                 
41                 #存储图片的路径地址
42                 lu="C:\\Users\\xhds1\\Desktop\\img\\"+name
43                 print(lu)
44                 urlretrieve(img,lu)  #urlretrieve模块下载图片
45                # print(name)
46                 
47 def main():
48     sss=""
49     for i in range(137,139):
50         riqi="20200121-%s"%(i)
51         
52         #使用了base64模块进行URL编码  这里遇到的问题是编码后就成为了字节流类型 
53         #查了资料才得知必须转换成字符串类型才行 纠结了好久
54         strbs=base64.b64encode(riqi.encode(encoding="utf-8")).decode("utf-8") 
55       
56         queue.put("https://jandan.net/pic/"+strbs+"#comments")
57                   
58     threads=[]
59     thread_count=5
60     
61     for i in range(thread_count):
62         threads.append(Jiandan(queue))
63     for t in threads:
64         t.start()
65     for t in threads:
66         t.join()
67         
68 if __name__==‘__main__‘:
69     main()