Python高级应用程序设计任务

import requests
from requests.exceptions import RequestException #爬取异常函数
from bs4 import BeautifulSoup
import os  #操作系统接口模块 用来写入爬出数据
import csv #爬出数据存为csv文件
import time

def get_one_page(url): #获取网页url 进行爬取

    print(‘进行爬取‘+url)
    headers={‘User-Agent‘:‘User-Agent:Mozilla/5.0‘} #头文件的user-agent
    try:
        response = requests.get(url,headers=headers)
        if response.status_code == 200:
            return response.content
        return None
    except RequestException:
        return None

def parse_one_page(html):#解析清理网页

    soup = BeautifulSoup(html,  "lxml")
    info = soup.find(‘div‘,  class_=‘wdetail‘)
    rows=[]
    tr_list = info.find_all(‘tr‘)[1:]       # 使用从第二个tr开始取
    for index,  tr in enumerate(tr_list):     # enumerate可以返回元素的位置及内容
        td_list = tr.find_all(‘td‘)
        date = td_list[0].text.strip().replace("\n", "")  # 取每个标签的text信息，并使用replace()函数将换行符删除
        weather = td_list[1].text.strip().replace("\n", "").split("/")[0].strip()
        temperature_high = td_list[2].text.strip().replace("\n",  "").split("/")[0].strip()
        temperature_low = td_list[2].text.strip().replace("\n",  "").split("/")[1].strip()

        rows.append((date,weather,temperature_high,temperature_low))
    return rows





cities = [‘beijing‘,‘shanghai‘,‘tianjin‘,‘chongqing‘,‘akesu‘,‘anning‘,‘anqing‘,
         ‘anshan‘,‘anshun‘,‘anyang‘,‘baicheng‘,‘baishan‘,‘baiyiin‘,‘bengbu‘,‘baoding‘,
         ‘baoji‘,‘baoshan‘,‘bazhong‘,‘beihai‘,‘benxi‘,‘binzhou‘,‘bole‘,‘bozhou‘,
         ‘cangzhou‘,‘changde‘,‘changji‘,‘changshu‘,‘changzhou‘,‘chaohu‘,‘chaoyang‘,
         ‘chaozhou‘,‘chengde‘,‘chengdu‘,‘chenggu‘,‘chengzhou‘,‘chibi‘,‘chifeng‘,‘chishui‘,
         ‘chizhou‘,‘chongzuo‘,‘chuxiong‘,‘chuzhou‘,‘cixi‘,‘conghua‘,
         ‘dali‘,‘dalian‘,‘dandong‘,‘danyang‘,‘daqing‘,‘datong‘,‘dazhou‘,
         ‘deyang‘,‘dezhou‘,‘dongguan‘,‘dongyang‘,‘dongying‘,‘douyun‘,‘dunhua‘,
         ‘eerduosi‘,‘enshi‘,‘fangchenggang‘,‘feicheng‘,‘fenghua‘,‘fushun‘,‘fuxin‘,
         ‘fuyang‘,‘fuyang1‘,‘fuzhou‘,‘fuzhou1‘,‘ganyu‘,‘ganzhou‘,‘gaoming‘,‘gaoyou‘,
         ‘geermu‘,‘gejiu‘,‘gongyi‘,‘guangan‘,‘guangyuan‘,‘guangzhou‘,‘gubaotou‘,
         ‘guigang‘,‘guilin‘,‘guiyang‘,‘guyuan‘,‘haerbin‘,‘haicheng‘,‘haikou‘,
         ‘haimen‘,‘haining‘,‘hami‘,‘handan‘,‘hangzhou‘,‘hebi‘,‘hefei‘,‘hengshui‘,
         ‘hengyang‘,‘hetian‘,‘heyuan‘,‘heze‘,‘huadou‘,‘huaian‘,‘huainan‘,‘huanggang‘,
         ‘huangshan‘,‘huangshi‘,‘huhehaote‘,‘huizhou‘,‘huludao‘,‘huzhou‘,‘jiamusi‘,
         ‘jian‘,‘jiangdou‘,‘jiangmen‘,‘jiangyin‘,‘jiaonan‘,‘jiaozhou‘,‘jiaozou‘,
         ‘jiashan‘,‘jiaxing‘,‘jiexiu‘,‘jilin‘,‘jimo‘,‘jinan‘,‘jincheng‘,‘jingdezhen‘,
         ‘jinghong‘,‘jingjiang‘,‘jingmen‘,‘jingzhou‘,‘jinhua‘,‘jining1‘,‘jining‘,
         ‘jinjiang‘,‘jintan‘,‘jinzhong‘,‘jinzhou‘,‘jishou‘,‘jiujiang‘,‘jiuquan‘,‘jixi‘,
         ‘jiyuan‘,‘jurong‘,‘kaifeng‘,‘kaili‘,‘kaiping‘,‘kaiyuan‘,‘kashen‘,‘kelamayi‘,
         ‘kuerle‘,‘kuitun‘,‘kunming‘,‘kunshan‘,‘laibin‘,‘laiwu‘,‘laixi‘,‘laizhou‘,
         ‘langfang‘,‘lanzhou‘,‘lasa‘,‘leshan‘,‘lianyungang‘,‘liaocheng‘,‘liaoyang‘,
         ‘liaoyuan‘,‘lijiang‘,‘linan‘,‘lincang‘,‘linfen‘,‘lingbao‘,‘linhe‘,‘linxia‘,
         ‘linyi‘,‘lishui‘,‘liuan‘,‘liupanshui‘,‘liuzhou‘,‘liyang‘,‘longhai‘,‘longyan‘,
         ‘loudi‘,‘luohe‘,‘luoyang‘,‘luxi‘,‘luzhou‘,‘lvliang‘,‘maanshan‘,‘maoming‘,
         ‘meihekou‘,‘meishan‘,‘meizhou‘,‘mianxian‘,‘mianyang‘,‘mudanjiang‘,‘nanan‘,
         ‘nanchang‘,‘nanchong‘,‘nanjing‘,‘nanning‘,‘nanping‘,‘nantong‘,‘nanyang‘,
         ‘neijiang‘,‘ningbo‘,‘ningde‘,‘panjin‘,‘panzhihua‘,‘penglai‘,‘pingdingshan‘,
         ‘pingdu‘,‘pinghu‘,‘pingliang‘,‘pingxiang‘,‘pulandian‘,‘puning‘,‘putian‘,‘puyang‘,
         ‘qiannan‘,‘qidong‘,‘qingdao‘,‘qingyang‘,‘qingyuan‘,‘qingzhou‘,‘qinhuangdao‘,
         ‘qinzhou‘,‘qionghai‘,‘qiqihaer‘,‘quanzhou‘,‘qujing‘,‘quzhou‘,‘rikaze‘,‘rizhao‘,
         ‘rongcheng‘,‘rugao‘,‘ruian‘,‘rushan‘,‘sanmenxia‘,‘sanming‘,‘sanya‘,‘xiamen‘,
         ‘foushan‘,‘shangluo‘,‘shangqiu‘,‘shangrao‘,‘shangyu‘,‘shantou‘,‘ankang‘,‘shaoguan‘,
         ‘shaoxing‘,‘shaoyang‘,‘shenyang‘,‘shenzhen‘,‘shihezi‘,‘shijiazhuang‘,‘shilin‘,
         ‘shishi‘,‘shiyan‘,‘shouguang‘,‘shuangyashan‘,‘shuozhou‘,‘shuyang‘,‘simao‘,
         ‘siping‘,‘songyuan‘,‘suining‘,‘suizhou‘,‘suzhou‘,‘tacheng‘,‘taian‘,‘taicang‘,
         ‘taixing‘,‘taiyuan‘,‘taizhou‘,‘taizhou1‘,‘tangshan‘,‘tengchong‘,‘tengzhou‘,
         ‘tianmen‘,‘tianshui‘,‘tieling‘,‘tongchuan‘,‘tongliao‘,‘tongling‘,‘tonglu‘,‘tongren‘,
         ‘tongxiang‘,‘tongzhou‘,‘tonghua‘,‘tulufan‘,‘weifang‘,‘weihai‘,‘weinan‘,‘wendeng‘,
         ‘wenling‘,‘wenzhou‘,‘wuhai‘,‘wuhan‘,‘wuhu‘,‘wujiang‘,‘wulanhaote‘,‘wuwei‘,‘wuxi‘,‘wuzhou‘,
         ‘xian‘,‘xiangcheng‘,‘xiangfan‘,‘xianggelila‘,‘xiangshan‘,‘xiangtan‘,‘xiangxiang‘,
         ‘xianning‘,‘xiantao‘,‘xianyang‘,‘xichang‘,‘xingtai‘,‘xingyi‘,‘xining‘,‘xinxiang‘,‘xinyang‘,
         ‘xinyu‘,‘xinzhou‘,‘suqian‘,‘suyu‘,‘suzhou1‘,‘xuancheng‘,‘xuchang‘,‘xuzhou‘,‘yaan‘,‘yanan‘,
         ‘yanbian‘,‘yancheng‘,‘yangjiang‘,‘yangquan‘,‘yangzhou‘,‘yanji‘,‘tantai‘,‘yanzhou‘,‘yibin‘,
         ‘yichang‘,‘yichun‘,‘yichun1‘,‘yili‘,‘yinchuan‘,‘yingkou‘,‘yulin1‘,‘yulin‘,‘yueyang‘,‘yongkang‘,
         ‘yongzhou‘,‘yuxi‘,‘changchun‘,‘zaozhuang‘,‘zhangjiajie‘,‘zhangjiakou‘,‘changsha‘,‘changle‘,
         ‘zhangzhou‘,‘zhuhai‘,‘zhengzhou‘,‘zunyi‘,‘fuqing‘,‘foshan‘,‘‘] #获取城市名称
years = [‘2011‘,‘2012‘,‘2013‘,‘2014‘,‘2015‘,‘2016‘,‘2017‘,‘2018‘]
months = [‘01‘,‘02‘,‘03‘,‘04‘,‘05‘,‘06‘, ‘07‘, ‘08‘,‘09‘,‘10‘,‘11‘,‘12‘]

if __name__ == ‘__main__‘:#主函数
    # os.chdir()  # 设置工作路径
    for city in cities:
        with open(city + ‘_weather.csv‘, ‘a‘, newline=‘‘) as f:
            writer = csv.writer(f)
            writer.writerow([‘date‘,‘weather‘,‘temperature_high‘,‘temperature_low‘])
            for year in years:
                for month in months:
                    url = ‘http://www.tianqihoubao.com/lishi/‘+city+‘/month/‘+year+month+‘.html‘
                    html = get_one_page(url)
                    content=parse_one_page(html)
                    writer.writerows(content)
                    print(city+year+‘年‘+month+‘月‘+‘数据爬取完毕!‘)
                    time.sleep(2)

import csv

from matplotlib import pyplot as plt

from datetime import datetime

#读取CSV文件数据

filename=‘zhangzhou_weather.csv‘

with open(filename) as f: #打开这个文件，并将结果文件对象存储在f中

    reader=csv.reader(f)  #创建一个阅读器reader

    header_row=next(reader) #返回文件中的下一行

    dates,highs,lows=[],[],[]      #声明存储日期，最值的列表

    for row in reader:

        current_date=datetime.strptime(row[0],‘%Y-%m-%d‘)  #将日期数据转换为datetime对象

        dates.append(current_date)    #存储日期

        high=int(row[1])    #将字符串转换为数字

        highs.append(high)   #存储温度最大值

        low=int(row[3])

        lows.append(low)    #存储温度最小值

 

#根据数据绘制图形

fig=plt.figure(dpi=128,figsize=(10,6))

plt.plot(dates,highs,c=‘Orange‘,alpha=0.5)#实参alpha指定颜色的透明度，0表示完全透明，1（默认值）完全不透明

plt.plot(dates,lows,c=‘green‘,alpha=0.5)

plt.fill_between(dates,highs,lows,facecolor=‘gray‘,alpha=0.1) #给图表区域填充颜色

plt.title(‘zhangzhou high and low temperature-2011‘,fontsize=24)

plt.xlabel(‘‘,fontsize=16)

plt.ylabel(‘Temperature(F)‘,fontsize=16)

plt.tick_params(axis=‘both‘,which=‘major‘,labelsize=16)

fig.autofmt_xdate()  #绘制斜的日期标签

plt.show()