抓取biqukan

#python3.7
‘‘‘
功能：实现www.biqukan.com/1_1094/5403177.html小说下载为txt
v1.0
‘‘‘
import requests,sys,time
from lxml import etree
##0.获取所有章节url
def get_url_list(catalog_url):
    res=requests.get(catalog_url)
    #time.sleep(1)
    if res.status_code==200:
        text=res.text
        html=etree.HTML(text)
        hrefs=html.xpath(‘//div[@class="listmain"]/dl/dt[2]/following-sibling::*/a/@href‘)
        #print(‘hrefs‘)
        return hrefs
    return None

##1.获取页面
def get_one_page(url):
    res=requests.get(url)
    #time.sleep(1)
    if res.status_code==200:
        return res.text
    return None

##2.解析页面
def parse_one_page(text):
    html=etree.HTML(text)
    title=html.xpath(‘//div[@class="content"]/h1//text()‘)
    content=html.xpath(‘//div[@class="showtxt"]//text()‘) #去掉换行
    contents=‘‘.join(content).replace(‘\xa0‘*8,‘\n‘*2)#把列表转换为一整段文本，并把8个空格换为2个换行
    #print(title,contents)
    
    #print(title)
    return title,contents #返回多个参数，相当于返回一个元组return(title,content)

##3.保存内面
def write_to_file(title,contents):
    with open(‘一念永恒.txt‘,‘a‘,encoding=‘utf-8‘)as f:
        f.write(title[0]+‘\n‘+contents+‘\n‘)
        
        #for content in contents:
        #    f.write(content)
        
##主函数
def main():
    #0.获取章节列表的网址
    catalog_url=‘https://www.biqukan.com/1_1094/‘
    urls=get_url_list(catalog_url)
    #print(urls)
    #1.把网址传入详情抓取页面，并保存
    for i in range(len(urls)):
        rel_url=‘https://www.biqukan.com‘+urls[i]
        print(rel_url)
        text=get_one_page(rel_url)
        #接收2步return的多个参数写法a,b=(x,y)
        title,contents=parse_one_page(text)
        #print(title,contents)
        #写入txt
        write_to_file(title[0],contents)
        #显示下载进度
        sys.stdout.write("  已下载:%.3f%%" %  float(i/len(urls)) + ‘\r‘)
        sys.stdout.flush()
        
    

##执行入口    
if __name__==‘__main__‘:
    main()