爬取豆瓣电影

import  requests
import  time
from  lxml  import  etree
import  json


#获取网页函数
def  getpage(url):
    try:
        headers={‘User-Agent‘:‘Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.117 Mobile Safari/537.36‘}
        res=requests.get(url,headers=headers)
        if res.status_code==200:
            return res.text
        else:
            return None
    except:
        return None
    
    
#解析网页函数
def  parsepage(html):
    ht=etree.HTML(html)
    items=ht.xpath(‘//div[@class="item"]‘)
    for  item  in  items:
        res={
        ‘title‘:item.xpath(‘.//span[@class="title"]/text()‘),
        ‘index‘:item.xpath(‘.//div[@class="item"]//em/text()‘),
        ‘score‘:item.xpath(‘.//span[@class="rating_num"]/text()‘),
        ‘actor‘:item.xpath(‘.//p[@class=""]/text()‘),
        ‘image‘:item.xpath(‘.//img[@width="100"]/@src‘)
        }
        yield  res
        
    

#写入文件
def  writefile(item):
    with  open(‘豆瓣.json‘,‘a‘,encoding=‘utf-8‘)  as  f:
        print(‘正在写入数据{}...‘.format(item[‘title‘]))
        f.write(json.dumps(item,ensure_ascii=False))
        f.write(‘\n‘)
        
#定义一个主函数   
def  main(offset):
    url=‘https://movie.douban.com/top250?start={}‘.format(offset)
    html=getpage(url)
    print(‘正在解析程序.....‘)
    if  html:
        for  i in  parsepage(html):
            writefile(i)
            
            

if __name__=="__main__":
    for  i in  range(0,250,25):
        main(offset=i)
        time.sleep(2)