web crawling(plus5) news crawling and proxy

#Author：Mini
#！/usr/bin/env python
import urllib.request
import urllib.error
import re
data=urllib.request.urlopen("http://news.sina.com.cn/").read()
data1=data.decode("utf-8","ignore")
pat=‘ href="(http://news.sina.com.cn/.*?)">‘
allurl=re.compile(pat).findall(data1)
for i in range(0,len(allurl)):
    try:
        print(str(i)+"\n\ntime")
        thisurl=allurl[i]
        fh="E:/m/"+str(i)+".html"
        urllib.request.urlretrieve(thisurl,fh)
        print("success!")
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
            if hasattr(e, "reason"):
                print(e.reason)


************************************

import urllib.request
import re
import  urllib.error

def use_proxy(url,proxy_addr):
    proxy=urllib.request.ProxyHandler({"http":proxy_addr})
    opener1=urllib.request.build_opener(proxy,urllib.request.HTTPHandler)
    urllib.request.install_opener(opener1)
proxy_addr="220.161.37.21:8118"
url="http://blog.csdn.net/"
headers=("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36Query String Parametersview sourceview URL encoded")
opener=urllib.request.build_opener()
opener.addheaders=[headers]
urllib.request.install_opener(opener)
data=use_proxy(url,proxy_addr)
data=urllib.request.urlopen(url).read().decode("utf-8","ignore")
print(len(data))

pat=‘ <h3  class="csdn-tracking-statistics" data-mod="popu_430" data-poputype="feed"  data-feed-show="false"  data-dsm="post"><a href="(.*?)"‘
res=re.compile(pat).findall(data)
for i in range(0,len(res)):
 try:
    fil="E:/m/"+str(i)+".html"
    urllib.request.urlretrieve(res,filename=fil)
    print(str(i),"\n\ntime")
 except urllib.error.URLError as e:
    if hasattr(e, "code"):
        print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)

web crawling(plus5) news crawling and proxy

原文：http://www.cnblogs.com/rabbittail/p/7623041.html