Python基础之爬取小说

  1 import requests  # 请求包  用于发起网络请求
  2 from bs4 import BeautifulSoup  # 解析页面内容帮助包
  3 import time
  4 
  5 """
  6 说明：爬取小说
  7 步骤：1. 先爬取所有章节，及章节明细对应的URL
  8 2. 解析单个章节的内容
  9 3. 保存
 10 """
 11 
 12 
 13 def get_data(url: str = None):
 14     """
 15     获取数据
 16     :param url: 请求网址
 17     :return:返回请求的页面内容
 18     """
 19     # 请求头，模拟浏览器，否则请求会返回418
 20     header = {
 21         ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ‘
 22                       ‘Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363‘}
 23     resp = requests.get(url=url, headers=header)  # 发送请求
 24 
 25     if resp.status_code == 200:
 26         if resp.encoding != resp.apparent_encoding:
 27             # 如果返回的编码和页面显示编码不一致，直接获取text会出现乱码，需要转码
 28             return resp.content.decode(encoding=resp.apparent_encoding)
 29         else:
 30             # 如果返回成功，则返回内容
 31             return resp.text
 32     else:
 33         # 否则，打印错误状态码，并返回空
 34         print(‘返回状态码：‘, resp.status_code)
 35         return
 36 
 37 
 38 def parse_chapters(html: str = None):
 39     """
 40     爬取章节列表
 41     :param html:
 42     :return:
 43     """
 44     if html is None:
 45         return
 46     else:
 47         chapters = []
 48         bs = BeautifulSoup(html, features=‘html.parser‘)
 49         ul_chapters = bs.find(‘ul‘, class_=‘chapter-list clearfix‘)
 50         # print(ul_chapters)
 51         li_chapters = ul_chapters.find_all(‘li‘, class_=‘col-4‘)  # 此处需要注意，页面源码查看是有空格，但是BeautifulSoup转换后空格消失
 52         for li_chapter in li_chapters:
 53             a_tag = li_chapter.find(‘a‘)
 54             # print(a_tag)
 55             a_href = a_tag.get(‘href‘)  # 此处也可以用a_tag[‘href‘]
 56             a_text = a_tag.get_text()  # 获取内容：章节标题
 57             chapters.append({‘title‘: a_text, ‘href‘: a_href})
 58 
 59         return chapters
 60 
 61 
 62 def get_and_parser_single_chapter(chapters: list = []):
 63     """
 64     获取单个章节
 65     :param chapters: 章节列表
 66     :return:
 67     """
 68     for (index, chapter) in enumerate(chapters, 1):
 69         title = chapter.get(‘title‘)
 70         href = chapter.get(‘href‘)
 71         while True:
 72             print(‘开始第%d章爬取‘ % index)
 73             html = get_data(href)
 74             if html is not None:
 75                 content = parse_single_chapter(html)
 76                 save_data(title, content)  # 保存数据
 77                 print(‘第%d章爬取成功‘ % index)
 78                 break
 79             else:
 80                 print(‘第%d章爬取失败‘ % index)
 81                 time.sleep(2)
 82 
 83 
 84 def parse_single_chapter(html: str = None):
 85     """
 86     解析单个章节内容
 87     :param html:
 88     :return:
 89     """
 90     bs = BeautifulSoup(html, features=‘html.parser‘)
 91     div_reader_box = bs.find(‘div‘, class_=‘reader_box‘)
 92     div_title = div_reader_box.find(‘div‘, class_=‘title_txtbox‘)
 93     title = div_title.get_text()  # 获取标题
 94     div_book_info = div_reader_box.find(‘div‘, class_=‘bookinfo‘)
 95     book_info = div_book_info.get_text()
 96     div_content = div_reader_box.find(‘div‘, class_=‘content‘)
 97     content = ‘‘
 98     p_tags = div_content.find_all(‘p‘)
 99     for p_tag in p_tags:
100         content =content + p_tag.get_text() + ‘\r\n‘
101     # content = div_content.get_text()
102     return title + ‘\n‘ + book_info + ‘\n‘ + content
103 
104 
105 def save_data(name, content):
106     """
107     保存数据
108     :param name: 文件名
109     :param content: 文件内容
110     :return:
111     """
112     with open(‘妙手小医仙\\‘ + name + ‘.txt‘, ‘w‘, encoding=‘utf-8‘) as f:
113         f.write(content)
114 
115 
116 url = ‘http://book.zongheng.com/showchapter/1102448.html‘
117 print(‘开始时间>>>>>‘, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
118 html_chapters = get_data(url)
119 chapters = parse_chapters(html_chapters)
120 get_and_parser_single_chapter(chapters)
121 print(‘结束时间>>>>>‘, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
122 print(‘done‘)