python—爬虫伪装成浏览器的三种方法

 1 # 方法一：通过opener添加headers
 2 import urllib.request
 3 url = ‘http://www.qiushibaike.com/‘
 4 
 5 # 头文件格式header=(‘User-Agent‘,具体用户代理值) 元组形式，元组内包含(key,value)
 6 headers=(‘User-Agent‘,‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0‘)
 7 # 创建opener对象
 8 opener = urllib.request.build_opener()
 9 opener.addheaders = [headers]
10 # (1)直接使用opener进行爬虫
11 data = opener.open(url).read()
12 # print(data)
13 print(len(data))
14 # (2)将opener安装为全局,全局生效后，再使用urlopen
15 urllib.request.install_opener(opener) 
16 data = urllib.request.urlopen(url).read()
17 print(len(data))

 1 # 方法二：批量添加headers 有时候只添加user-agent这一个header不行，需要多添加几个header
 2 headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0","Content-Type":"application/javascript",
 3 }
 4 #创建opener对象
 5 opener = urllib.request.build_opener()
 6 opener.addheaders = headers.items()  #header.items() 字典(Dictionary).items() 函数以列表返回可遍历的(键, 值) 元组数组。正好符合addheaders属性要求的值的类型。[(key1,value1),(key2,value2),(key3,value3)]
 7 # (1)直接使用opener进行爬虫
 8 data = opener.open(url).read()
 9 # print(data)
10 print(len(data))
11 # (2)将opener安装为全局,全局生效后，再使用urlopen
12 urllib.request.install_opener(opener) 
13 data = urllib.request.urlopen(url).read()
14 print(len(data))

1 # 方法三：通过Request添加headers
2 req = urllib.request.Request(url)
3 req.add_header(‘User-Agent‘, ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0‘)
4 req_data = urllib.request.urlopen(req).read()
5 print(len(req_data))