python爬虫爬取豆瓣电视剧数据

import re
import requests
from bs4 import BeautifulSoup
import time
import random
import string
import logging
import json
import jsonpath
import pandas as pd

import pdb

User_Agents = [
    ‘Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50‘,
    ‘Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50‘,
    ‘Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0‘,
    ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1‘,
    ‘Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11‘,
    ‘Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11‘,
    ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11‘,
]


class DoubanSpider(object):
    ‘‘‘豆瓣爬虫‘‘‘

    def __init__(self):
        # 基本的URL
        self.base_url = ‘https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_‘ + ‘start={start}‘
        self.full_url = self.base_url
        self.tv_detailurl = ‘https://movie.douban.com/j/subject_abstract?subject_id=‘

    def download_tvs(self, offset):
        # offset控制一次下载的量，resp返回的响应体
        self.headers = {‘User-Agent‘: random.choice(User_Agents)}
        self.full_url = self.base_url.format(start=offset)
        resp = None
        try:
            resp = requests.get(self.full_url, headers=self.headers)
        except Exception as e:
            print(logging.error(e))
        return resp

    def get_tvs(self, resp):
        # resp响应体
        # movies爬取到的电影信息
        print(‘get_tvs‘)
        print(resp)
        tv_urls = []
        if resp:
            if resp.status_code == 200:
                html = resp.text
                unicodestr = json.loads(html)
                tv_list = unicodestr[‘subjects‘]
                for item in tv_list:
                    data = re.findall(r‘[0-9]+‘, str(item[‘url‘]))
                    tv_urls.append(self.tv_detailurl + str(data[0]))
            print(‘tv_urls‘)
            return tv_urls
        return None

    def download_detailtvs(self, tv_urls):
        tvs = []
        for item in tv_urls:
            self.headers = {‘User-Agent‘: random.choice(User_Agents)}
            resp = requests.get(item, headers=self.headers)
            html = resp.text
            unicodestr = json.loads(html)
            tvs.append(unicodestr[‘subject‘])
        return tvs


def main():
    spider = DoubanSpider()
    offset =0
    data = {‘title‘:[], ‘types‘:[], ‘directors‘:[], ‘actors‘:[], ‘release_year‘:[], ‘region‘:[], ‘star‘:[], ‘episodes_count‘:[],‘rate‘:[]}
    tv_file = pd.DataFrame(data)
    tv_file.to_csv(‘res_url.csv‘,mode=‘w‘,index=None)
    while True:
        reps = spider.download_tvs(offset)
        tv_urls = spider.get_tvs(reps)
        tvss = spider.download_detailtvs(tv_urls)
        for tvsss in tvss:
            ‘‘‘
            #pdb.set_trace()
            tvsss=re.sub(r‘\\u200e‘,‘‘,tvsss)
            tvsss = re.sub(r‘\‘‘, ‘\"‘, tvsss)
            tvsss = re.sub(r‘\‘‘, ‘\"‘, tvsss)
            #将short_comment去掉
            tvsss = re.sub(r‘(\"short_comment\").*(\"directors\")‘, ‘\"directors\"‘,tvsss)
            #将true,false改为"True","False"
            tvsss = re.sub(r‘True‘, ‘\"True\"‘, tvsss)
            tvsss = re.sub(r‘False‘, ‘\"False\"‘, tvsss)
            #给所有的list加上双引号
            print(tvsss)
            #将: [转化为: "[
            tvsss = re.sub(r‘: \[‘, ‘: "[‘, tvsss)
            #jiang ],zhuanhuawei ]",
            tvsss=re.sub(r‘\],‘,‘]",‘,tvsss)
            # 以上正确
            print(tvsss)
            #将director的内容改为单引号
            r1 = re.findall(r‘(?<=directors": "\[).*?(?=\]\")‘, tvsss)
            #正确
            if r1:
                r2 = re.sub(r‘\"‘, ‘\‘‘, r1[0])
                r3 = re.sub(r‘\"‘, ‘\‘‘, r2)
                tvsss = re.sub(r‘(?<=directors\": \"\[).*?(?=\]\")‘, r3, tvsss)
            #zhengque
            #将actors的内容改为单引号
            print(tvsss)
            r1 = re.findall(r‘(?<=actors\": \"\[).*?(?=\]\")‘, tvsss)
            print("actors")
            print(r1)
            if r1:
                r2 = re.sub(r‘\"‘, ‘\‘‘, r1[0])
                r3 = re.sub(r‘\"‘, ‘\‘‘, r2)
                tvsss = re.sub(r‘(?<=actors\": \"\[)[\s\S]*?(?=\]\")‘, r3, tvsss)
            #将剧情types改为单引号
            print(tvsss)
            r1 = re.findall(r‘(?<=types": "\[).*?(?=\]\")‘, tvsss)
            if r1:
                r2 = re.sub(r‘\"‘, ‘\‘‘, r1[0])
                r3 = re.sub(r‘\"‘, ‘\‘‘, r2)
                tvsss = re.sub(r‘(?<=types\": \"\[).*?(?=\]\")‘, r3, tvsss)
            # 正确
            #将二维的数据转化为一维的
            types=str(tvs[‘types‘])
            actor = str(tvs[‘actors‘])
            director = str(tvs[‘directors‘])
            types=re.sub(r‘\‘‘,‘‘,types)
            actor = re.sub(r‘\‘‘, ‘‘, actor)
            director = re.sub(r‘\‘‘, ‘‘, director)
            types = re.sub(r‘\‘‘, ‘‘, types)
            actor= re.sub(r‘\‘‘, ‘‘, actor)
            director = re.sub(r‘\‘‘, ‘‘, director)
            types=types.strip(‘[]‘)
            actor=actor.strip(‘[]‘)
            director=director.strip(‘[]‘)

            data2={‘title‘:tvs[‘title‘], ‘types‘:types, ‘directors‘:director, ‘actors‘:actor, ‘release_year‘:tvs[‘release_year‘], ‘region‘:tvs[‘region‘], ‘star‘:tvs[‘star‘], ‘episodes_count‘:tvs[‘episodes_count‘],‘rate‘:tvs[‘rate‘]}
            print(data2)
            tv_file=pd.DataFrame(data2,index=[0])
            #tv_file = pd.DataFrame(data)
            # pdb.set_trace()
            tv_file.to_csv(‘res_url.csv‘, mode=‘a‘, index=None,header=None)
        offset += 20
        id = offset
        # 控制访问速速
        time.sleep(10)


if __name__ == ‘__main__‘:
    main()