python爬虫

# -*- coding: utf-8 -*-
import urllib.request,re,time,random,gzip
from bs4 import BeautifulSoup

def get_stock_code():
    import tushare as ts
    df_stock = ts.get_sz50s()
    code_list = df_stock[‘code‘]
    return code_list


def saveFile(data,i):  
    path = "E:\\projects\\paper_"+str(i+1)+".txt"  
    file = open(path,‘wb‘)  
    page = ‘第‘+str(i+1)+‘\n‘  
    file.write(page.encode(‘gbk‘))  
    for d in data:  
        d = str(d)+‘\n‘  
        file.write(d.encode(‘gbk‘))  
    file.close()  
  
def ungzip(data):  
    try:  
        data = gzip.decompress(data)  
    except:  
        print("δ?????????????...")  
    return data  
  
#CSDN??????  
class CSDNSpider:  
    def __init__(self,pageIdx=1,url="http://blog.csdn.net/fly_yr/article/list/1"):  
        #??????  
        self.pageIdx = pageIdx  
        self.url = url[0:url.rfind(‘/‘) + 1] + str(pageIdx)  
        self.headers = {  
            #"Connection": "keep-alive",  
            ‘User-Agent‘:‘Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6‘
            #"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",  
            #"Accept-Encoding": "gzip, deflate, sdch",  
            #"Accept-Language": "zh-CN,zh;q=0.8",  
            #"Host": "data.eastmoney.com"  
        }  
        
    def readData_tonghuashun_report(self,urlweb,code):
        import os,sys,re
        ret=""
        req = urllib.request.Request(url=urlweb, headers=self.headers)
        res = urllib.request.urlopen(req)  
        data = res.read()  
        data = ungzip(data)  
        data = data.decode(‘utf-8‘)
        soup=BeautifulSoup(data,"html5lib") 
        info = (soup.find(‘div‘,‘kuaizhaolefttitle aDBlue nu‘))
        info = (info.find(‘p‘)).string.split(‘ ‘)
        time = info[0]
        source = info[1].split(‘机构：‘)[1]
        source = source.split(‘\xa0‘)[0]
        content = str(soup.find(‘div‘,‘kuaizhao_contant aDBlue nu‘))
        content = content.split(‘<br/><br/>‘)
        contenttemp = ‘‘
        for i in content:
            contenttemp = contenttemp+i+‘\n‘
        contenttemp = contenttemp.split(‘>‘)[1]
        contenttemp = contenttemp.split(‘<‘)[0]
        print(contenttemp)
        article = time+‘\n‘+source+‘\n‘+contenttemp
        return article
    
    def readData_tonghuashun_news(self,urlweb,code):
        import os,sys,re
        ret=""
        req = urllib.request.Request(url=urlweb, headers=self.headers)
        res = urllib.request.urlopen(req)  
        data = res.read()  
        data = ungzip(data)  
        soup=BeautifulSoup(data,"html5lib") 
        time = (soup.find(‘span‘,‘time‘)).string
        source = soup.find(‘span‘,{"id":‘source_baidu‘})
        source = (str(source.find(‘a‘))).split(‘>‘)
        source = (source[1]).split(‘ ‘)
        sourcetemp=‘‘
        for i in source:
            i = i.split(‘\n‘)
            for j in i:
                j = j.split(‘\t‘)
                for k in j:
                    if(k!=‘‘):
                        sourcetemp = k
                        break
                if(sourcetemp!=‘‘):
                    break
            if(sourcetemp!=‘‘):
                break
        source = sourcetemp
        content = soup.find(‘div‘,‘atc-content‘)
        contents = content.find_all(‘p‘)
        contenttemp = ‘‘
        content = ‘‘
        for i in contents:
            content=content +str(i)+‘\n‘
        for s in content:
            if (s==‘。‘)or(s==‘，‘)or(s==‘\n‘)or(s.isdigit()):
                contenttemp += s
                continue
            if u‘\u4e00‘ <= s <= u‘\u9fff‘:
                contenttemp += s
        article = time+‘\n‘+source+‘\n‘+contenttemp
        return article
#         pat_1 = re.compile(ur‘[\u4e00-\u9fa5]‘)
#         for item in re.findall(pat_1,content):
#             contenttemp+=item
#         print(contenttemp)
        
    
    def readData_sina_report(self,urlweb,code):  
        ret=""
        req = urllib.request.Request(url=urlweb, headers=self.headers)
        res = urllib.request.urlopen(req)  
  
        data = res.read()  
        data = ungzip(data)  
        #data = data.decode(‘utf-8‘)
        #print(data)
        soup=BeautifulSoup(data,"html5lib") 
        #items = soup.find_all(‘div‘,‘newsContent‘)
        infos = soup.find_all(‘div‘,‘creab‘)
        count = 0
        for info in infos:
            info = info.find_all(‘span‘) 
            for i in info:
                for j in i:
                    j.find(‘a‘)
                    if "：" in j:
                        pass
                    else:
                        source = j.string
                    if "日期" in j:
                        time = j
                        time = time.split(‘：‘)[-1]
        
        
        contents = soup.find_all(‘p‘)
        contentAll = "" 
        for content in contents:
            contentAll+=str(content)
        print(contentAll)
    def readData_sina_news(self,urlweb,code):  
        ret=""
        req = urllib.request.Request(url=urlweb, headers=self.headers)
        res = urllib.request.urlopen(req)  
  
        data = res.read()  
        data = ungzip(data)  
        data = data.decode(‘utf-8‘)
        #print(data)
        soup=BeautifulSoup(data,"html5lib") 
        time = soup.find(‘span‘,‘time-source‘)
        time = (time.string).split(‘ ‘)
        print(time)
        count = 0
        for i in time:
            i = i.split(‘\n‘)
            for j in i:
                j = j.split(‘\t‘)
                for k in j:
                    if(k!="")and(count==0):
                        timetemp = k
                        count+=1
                    elif(k!="")and(count==1):
                        source = k
                        break
            
        time = timetemp
        contents = soup.find(‘div‘,‘article article_16‘)
        contents = contents.find_all(‘p‘)
        content = ‘‘
        for i in range(len(contents)-1):
            
            try:
                
                content+=contents[i+1].string+‘\n‘
            except:
                pass
        article = code+‘\n‘+time+‘\n‘+source+‘\n‘+content
        print(article)
        return article
    
    
    
    
    def readData_eastmoney_news(self,urlweb):  
        ret=""
        req = urllib.request.Request(url=urlweb, headers=self.headers)
        res = urllib.request.urlopen(req)  
        data = res.read()  
        data = ungzip(data)  
        data = data.decode(‘utf-8‘)
        soup=BeautifulSoup(data,"html5lib") 
        time = soup.find_all(‘div‘,‘time‘)
        #getTime
        time = time[0].string
        sources = soup.find(‘div‘,‘source‘)
        trs=sources.find("img")
        source = str(trs.attrs["alt"])
        contents = soup.find_all(‘p‘)
        contentStr = ‘‘
        for i in range(len(contents)):
            try:
                contentStr += contents[i].string
                contentStr += ‘\n‘
            except:
                pass
        artical = time+‘\n‘+source+‘\n‘+contentStr
        return artical
    
    def readData_eastmoney_report(self,urlweb):  
        ret=""
        req = urllib.request.Request(url=urlweb, headers=self.headers)
        res = urllib.request.urlopen(req)  
        data = res.read()  
        data = ungzip(data)  
        #data = data.decode(‘utf-8‘)
        soup=BeautifulSoup(data,"html5lib") 
        infos = soup.find(‘div‘,‘report-infos‘)
        infos = infos.find_all(‘span‘)
        time = infos[1].string
        time = time.split(‘ ‘)
        timetemp = ""
        for i in time:
            i = i.split(‘\n‘)
            for j in i:
                j = j.split(‘\t‘)
                for k in j:
                    if(k!=""):
                        timetemp = k
                        break
                if(timetemp!=""):
                    break
            if(timetemp!=""):
                break
        time = timetemp
        source = infos[2].string
        source = source.split(‘ ‘)
        for i in source:
            i = i.split(‘\n‘)
            for j in i:
                j = j.split(‘\t‘)
                for k in j:
                    if(k!=""):
                        sourcetemp = k
                        break
        source = sourcetemp
        #getTime
        
        contents = soup.find_all(‘p‘)
        contentStr = ‘‘
        for i in range(len(contents)):
            try:
                contentStr += contents[i].string
                contentStr += ‘\n‘
            except:
                pass
        artical = time+‘\n‘+source+‘\n‘+contentStr
        return artical
    
    
                 
                 
    def getAllUrl(self,url):
        import re
        import requests
        r = requests.get(url)
        data = r.text
        link_list =re.findall(r"(?<=href=\").+?(?=\")|(?<=href=\‘).+?(?=\‘)" ,data)
        url_list = []
        for url in link_list:
            
            url_list.append(url)
        return url_list
    
    def getAllUrl2(self,url):
        cs = CSDNSpider()  
        url_list = []
        for i in cs.getAllUrl(‘http://finance.eastmoney.com/yaowen.html‘):
            url_list.append(i)
        for i in cs.getAllUrl(‘http://finance.eastmoney.com/pinglun.html‘):
            url_list.append(i)
        for i in cs.getAllUrl(‘http://stock.eastmoney.com/bidu.html‘):
            url_list.append(i)
        return url_list
            


def save_file_sina():
    import os
    import csv
    cs = CSDNSpider()
    code_list = get_stock_code()
    for code in code_list:
        code = str(code)
        URL="http://money.finance.sina.com.cn/corp/view/vCB_AllNewsStock.php?symbol=sh"+code+"&Page=1"
        print(URL)
        url_list = cs.getAllUrl(URL)
        time = []
        source = []
        count = 0
        for i in url_list:
            #artical = cs.readData_sina(i,code)
            try:
                if ‘search‘ in i:
                    artical = cs.readData_sina_report(i,code)
                else:
                    artical = cs.readData_sina_news(i,code)
                artical = artical.split(‘\n‘)
                code = artical[0]
                timetemp = artical[1].split(‘ ‘)[0]
                timetemp = timetemp.split(‘日‘)[0]
                if "年" in timetemp:
                    timetemp = timetemp.replace(‘年‘,‘-‘)
                    timetemp = timetemp.replace(‘月‘,‘-‘)
                sourcetemp = artical[2]
                
                path = ‘abc/sina/‘+code+‘/‘+timetemp+‘/‘
        
                print(path)
                if timetemp in time:
                    if sourcetemp in source:
                        print(‘ss3‘)
                        csvFile = open(path+‘/‘+sourcetemp+‘.txt‘, "a")
                        writer = csv.writer(csvFile)
                        print(artical)
                        writer.writerow(artical)
                        csvFile.close()
                        print(‘end3‘)
                    else:
                        print(‘ss1‘)
                        source.append(sourcetemp)
                        csvfile = open(path+‘/‘+sourcetemp+‘.txt‘, ‘w‘)
                        writer = csv.writer(csvfile)
                        print(artical)
                        writer.writerow(artical)
                        csvfile.close()
                        print(‘end1‘)
                else:
                    os.makedirs(path)
                    csvfile = open(path+‘/‘+sourcetemp+‘.txt‘, ‘w‘)
                    print(‘ss2‘)
                    time.append(timetemp)
                    #csvfile = open(path, ‘w‘)
                    writer = csv.writer(csvfile)
                    print(artical)
                    writer.writerow(artical)
                    csvfile.close()
                    print(‘end2‘)
                print(count)
                count = count+1
            except:
                pass
            




def save_file_eastmoney():
    import os
    import csv
    cs = CSDNSpider()  
    #cs.readData2()
    code_list = get_stock_code()
    for code in code_list:
        code = str(code)
        url = "http://quote.eastmoney.com/sh"+code+".html"
        url_list = cs.getAllUrl(url)
        time = []
        mechanism = []
        count = 0
        for url in url_list:
            try:        
                if "news" in url:
                    artical = cs.readData_eastmoney_news(url)
                elif "report" in url:
                    artical = cs.readData_eastmoney_report(url)
                else:
                    continue                
                artical = artical.split(‘\n‘)
                timetemp = artical[0].split(‘日‘)[0]
                if "年" in timetemp:
                    timetemp = timetemp.replace(‘年‘,‘-‘)
                    timetemp = timetemp.replace(‘月‘,‘-‘)
                mechanismtemp = artical[1]
                path = ‘abc/eastmoney/‘+code+‘/‘+timetemp
            
                print(path)
                if timetemp in time:
                    if mechanismtemp in mechanism:
                        print(‘ss3‘)
                        csvFile = open(path+‘/‘+mechanismtemp+‘.txt‘, "a")
                        writer = csv.writer(csvFile)
                        print(artical)
                        writer.writerow(artical)
                        csvFile.close()
                        print(‘end3‘)
                    else:
                        print(‘ss1‘)
                        mechanism.append(mechanismtemp)
                        csvfile = open(path+‘/‘+mechanismtemp+‘.txt‘, ‘w‘)
                        writer = csv.writer(csvfile)
                        print(artical)
                        writer.writerow(artical)
                        csvfile.close()
                        print(‘end1‘)
                else:
                    os.makedirs(path)
                    csvfile = open(path+‘/‘+mechanismtemp+‘.txt‘, ‘w‘)
                    print(‘ss2‘)
                    time.append(timetemp)
                    #csvfile = open(path, ‘w‘)
                    writer = csv.writer(csvfile)
                    print(artical)
                    writer.writerow(artical)
                    csvfile.close()
                    print(‘end2‘)
                print(count)
                count = count+1
            except:
                pass
cs = CSDNSpider()
cs.readData_tonghuashun_report("http://search.10jqka.com.cn/snapshot/report_pdf/ea02eda3880f930e.html",‘100000‘)
#cs.readData_sina_news(‘http://cj.sina.com.cn/article/detail/5966752440/177997‘,‘60000‘)
#cs.readData_sina2(‘http://finance.sina.com.cn/stock/hyyj/2017-02-28/doc-ifyavvsk3874186.shtml‘, ‘10000‘)
#cs.readData_sina(‘http://finance.sina.com.cn/stock/hyyj/2016-11-23/doc-ifxxwrwk1751619.shtml‘)