python3 requests 抓取乱码问题

遇到此问题后 设置r.encoding=‘gbk‘或r.encoding=‘gb2312‘后可以了

代码如下：

#encoding:UTF-8

import urllib  

import re 

import sys

import requests

import chardet 

url = ‘http://baidu.com‘

UserAgent = ‘Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko‘

accept = ‘*/*‘

acceptLanguage = ‘zh-Hans-CN, zh-Hans; q=0.8, en-US; q=0.5, en; q=0.3‘

acceptEncoding = ‘gzip, deflate, sdch‘

Cookie = ‘AD_RS_COOKIE=20083361; _trs_ua_s_1=em41_6_j0ywubts; _gscu_1771678062=91030129ufqfm619; _gscs_1771678062=9103012938hyzj19|pv:1; _gscbrs_1771678062=1; _trs_uv=23au_6_j0ywubts‘

Host = ‘www.stats.gov.cn‘

CacheControl = ‘no-cache‘

Connection = ‘Keep-Alive‘

headers = { ‘User-Agent‘ : UserAgent,‘accept‘:accept,‘accept-Encoding‘:acceptEncoding,‘Accept-Language‘:acceptLanguage,‘Cookie‘:Cookie,‘Host‘:Host,‘Cache-Control‘ : CacheControl,‘Connection‘:Connection}

s = requests.Session()

s.headers.update(headers)

req = s.get(url)

req.encoding=‘gb2312‘

#req.encoding= ‘utf-8‘

#non_bmp_map = dict.fromkeys(range(0x10000, sys.maxunicode + 1), 0xfffd)

#print(req.text.translate(non_bmp_map))

‘‘‘

if req.encoding == ‘ISO-8859-1‘:

    encodings = requests.utils.get_encodings_from_content(req.text)

    if encodings:

        encoding = encodings[0]

    else:

        encoding = req.apparent_encoding

encode_content = req.content.decode(encoding, ‘replace‘).encode(‘utf-8‘, ‘replace‘)

print(encode_content);

‘‘‘

#print(req.text.encode(encoding=‘UCS-2‘,errors=‘UTF-8‘))

#print sys.getfilesystemencoding() 

print (req.text)  

python3 requests 抓取乱码问题

原文：http://volvo9yue.blog.51cto.com/12265053/1912826