爬虫（GET）——爬取多页的html

 1 # coding:utf-8
 2 
 3 import urllib.request
 4 
 5 def loadPage(fullurl,filename):
 6     """作用：根据url发送请求，获取服务器响应请求"""
 7     ua_headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36"}
 8     print("正在下载" + filename)
 9 
10     request = urllib.request.Request(fullurl, headers=ua_headers )
11     response = urllib.request.urlopen(request)
12     return response.read()
13 
14 
15 def writePage(html, filename):
16     """
17     作用：将html内容写入到本地
18     html：服务器相应文件内容
19     """
20     print("正在写入" + filename)
       # 新建文件，存储html信息
21     with open(filename, "w") as f:
22         f.write(str(html))
23 
24 
25 def tiebaSpider(url, beginpage, endpage):
26     """
27     作用：贴吧爬虫调度器，负责组合处理每个页面的url
28     url:贴吧url的前部分
29     beginPage： 起始页
30     endPage： 结束页
31     :return:
32     """
       # 构造每页的url和文件名
33     for page in range(beginpage, endpage+1):
34         pn = (page-1)*50
35         fullurl = url + "&" + str(pn)
36         filename = "第" + str(page) + "页.html"
37 
38         html = loadPage(fullurl, filename)
39         writePage(html, filename)
40     print("完成！")
41 
42 
43 if __name__ == "__main__":
44     kw = input("请输入要爬取的贴吧名： ")
45     beginPage = int(input("请输入起始页： "))
46     endPage = int(input("请输入结束页： "))
47 
48     url = "http://tieba.baidu.com/f?"
49     kw = urllib.parse.urlencode({"kw": kw})
50 
51     url = url + kw
52 
53     tiebaSpider(url, beginPage, endPage)