网络爬虫（5）--小实战

 1 # coding=utf-8
 2 __author__ = ‘f403‘
 3 from urllib.request import urlopen
 4 from urllib.error import HTTPError
 5 from bs4 import BeautifulSoup
 6 import re
 7 rootUrl = "http://www.heibanke.com/lesson/crawler_ex00/"
 8 
 9 def getUrl(url=""):
10     try:
11         html = urlopen(rootUrl+url)
12         if html is None:
13             print("html is empty")
14             return  None
15         else:
16             try:
17                 bs = BeautifulSoup(html.read().decode(‘utf8‘),"lxml")
18                 #text = bs.find("h3").get_text()
19 
20                 text = bs.find(‘h3‘).get_text()
21                 return text
22             except AttributeError as e:
23                 return None
24     except HTTPError as e:
25         return None
26 
27 if __name__ ==‘__main__‘:
28     txt = getUrl()
29     print(txt)
30     while txt is not None:
31         pattern = re.compile("[0-9]+")
32         num = pattern.search(txt)
33         if num is not None:
34             print(num.group())
35             txt = getUrl(num.group())
36         else:
37             txt = None
38 
39     print("end")