python 爬poj.org的题目

 1 #-*-coding:utf8-*-
 2 
 3 import requests
 4 import re
 5 import sys
 6 reload(sys)
 7 sys.setdefaultencoding("utf-8")
 8 
 9 class spider(object):
10     def __init__(self):
11         print "开始爬虫"
12 
13     def getsource(self,url):
14         html = requests.get(url)
15         return html.text
16 
17     def changepage(self,url,total_page):
18         now_page = 1
19         page_group = []
20 
21         for i in range(now_page,total_page+1):
22             link = re.sub(‘(\d+)‘,‘%s‘%i,url,re.S)
23             page_group.append(link)
24 
25         return page_group
26 
27 
28 if __name__ ==‘__main__‘:
29 
30     problem = []
31     url = ‘http://poj.org/problemlist?volume=1‘
32 
33 
34     pojspider = spider()
35     all_link = pojspider.changepage(url,10)
36 
37     for link in all_link:
38         print link
39         htm = pojspider.getsource(link)
40 
41         problem = re.findall(‘href=problem?(.*?)</a>‘,htm,re.S)
42 
43         for i in problem:
44             if i[0]==‘?‘:
45                 print i
46                 f = open(‘info.txt‘,‘a‘)
47                 f.writelines(i+‘\n‘)
48                 f.close()