正则表达式

1 import re
2 secret = "1232354xxYouxx1231xxandxx120938xxmexx123412"
3 b = re.findall("xx.*xx",secret)
4 c = re.findall("xx.*?xx",secret)

1 s2 = "asdfxxIxx123xxlovexx"
2 q = re.search("xx(.*?)xx123xx(.*?)xx",s2).group(2)
3 p = re.findall("xx(.*?)xx123xx(.*?)xx",s2)
4 print(q)
5 print(p[0][1])

1 import re
2 html = "<ul>12312<a>12312313<a>3123</ul>"
3 text_fied = re.findall("<ul>(.*?)</ul>",html,re.S)[0]
4 the_text = re.findall("<a>(.*?)<a>",text_fied,re.S)
5 print(the_text)

1 import requests
2 html = requests.get(‘http://tieba.baidu.com/f?kw=python&fr=ala0&tpl=5‘)
3 print(html.text)

 1 #_*_coding:utf-8_*_
 2 import requests
 3 import re
 4 
 5 import sys#命令提示符的默认格式为gbk，而网页默认的是utf-8，需要转换
 6 reload(sys)
 7 sys.setdefaultencoding(‘utf-8‘)
 8 
 9 class ImoocSpider(object): #创建一个类，含有一个构造方法
10     def __init__(self):
11         print "start spider"
12 
13     def ChangPage(self,url,totalNum):#改变页码，利用sub功能
14         allLink=[]
15         for pageNum in range(1,1+totalNum):
16             allLink.append(re.sub("\d+","%s"%pageNum,url,re.S))
17         return allLink
18 
19     def GetSource(self,link):
20         return  requests.get(link).text
21 
22     def GetEveryClass(self,html):
23         return re.findall(‘<li id=".*?</li>‘,html,re.S)
24 
25     def GetInfo(self,eachInfo):
26         info={}
27         info["title"]=re.search(‘title="(.*?)"‘,eachInfo,re.S).group(1)
28         info["content"] = re.search(‘<p(.*?)>(.*?)</p>‘, eachInfo, re.S).group(2)#选择的是第二个匹配的内容
29         timeandlevel=re.findall(‘<em>(.*?)</em>‘,eachInfo,re.S)#findall生成的是一个列表，所以下面将两个数据分别存在字典里面
30         info["classtime"] =timeandlevel[0]
31         info["learnlevel"] =timeandlevel[1]
32         return info
33 
34     def saveInfo(self,classInfo): #将爬虫出来的数据写到文件中
35         file=open("info.txt","ab")
36         for each in classInfo:
37             file.write(‘title:‘ + each[‘title‘] + ‘\n‘)
38             file.write(‘content:‘+each[‘content‘]+‘\n‘)
39             file.write(‘classtime:‘ + each[‘classtime‘] + ‘\n‘)
40             file.write(‘learnlevel:‘ + each[‘learnlevel‘] + ‘\n‘)
41             file.write(‘\n\n‘)
42 
43         file.close()
44 
45 
46 if __name__ ==‘__main__‘:#脚本运行时，程序开始的地方
47     classInfo = []
48     url="http://www.jikexueyuan.com/course/?pageNum=1"
49     spider=ImoocSpider() #实例化这个类
50     allLink=spider.ChangPage(url,20) #调用这个类，并且赋值
51     for link in allLink:
52         print "开始处理链接"+link
53         html=spider.GetSource(link)
54         everyClass=spider.GetEveryClass(html)
55         for each in everyClass:
56             info=spider.GetInfo(each)
57             classInfo.append(info)
58     spider.saveInfo(classInfo)