Python_结合Re正则模块爬虫

import re
import requests

def parse_page(url):
    headers = {
        ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36‘
    }
    respose = requests.get(url,headers).text
    titles = re.findall(r‘<div\sclass="cont">.*?<b>(.*?)</b>‘,respose,re.DOTALL)
    authors = re.findall(r‘<p\sclass="source">.*?<a\s.*?>(.*?)</a>‘,respose,re.DOTALL)
    dynaties = re.findall(r‘<p\sclass="source">.*?<a\s.*?><a\s.*?>(.*?)</a>‘,respose,re.DOTALL)
    content_tags = re.findall(r‘div\sclass="contson"\sid=.*?>(.*?)</div>‘,respose,re.DOTALL)
    contents = []
    for content in content_tags:
        # 替换<b>
        text = re.sub(r‘<.*?>‘,‘‘,content)
        contents.append(text.strip())

    poems = []
    # zip进行解包
    for values in zip(titles,authors,dynaties,contents):
        titles,authors,dynaties,content = values
        poem = {
            ‘titles‘:titles,
            ‘authors‘:authors,
            ‘dynaties‘:dynaties,
            ‘content_tags‘:content
        }
        poems.append(poem)

    for poem in poems:
        print(poem)
        print(‘*‘*40)

def main():
    for x in range(1,10):
        url = ‘https://www.gushiwen.cn/default_%d.aspx‘%x
        parse_page(url)

if __name__ == ‘__main__‘:
    main()

Python_结合Re正则模块爬虫

原文：https://www.cnblogs.com/Teachertao/p/14191209.html