链家网上关于福州二手房每平方米价格的爬取及数据处理

import requests#导入库
from bs4 import BeautifulSoup
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import matplotlib
from scipy.optimize import leastsq
from numpy import genfromtxt
url=‘https://fz.lianjia.com/xiaoqu/‘
headers={
‘User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64)AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.18363‘
}#伪装爬虫
resp=requests.get(url)#发送requests请求
#print(resp.text)
html=resp.text
soup=BeautifulSoup(html,‘lxml‘)
#print(soup)
infos=soup.find(‘ul‘,{‘class‘:‘listContent‘}).find_all(‘li‘)#find查询
#print(infos)
for info in infos:
    name=info.find(‘div‘,{‘class‘:‘title‘}).find(‘a‘).get_text()
    #print(name)
    price=info.find(‘div‘,{‘class‘:‘xiaoquListItemPrice‘}).find(‘div‘,{‘class‘:‘totalPrice‘}).find(‘span‘).get_text()
    #print(price)
    data=(name,price)
    #print(data)
    D=pd.DataFrame(data,index=["简介","平方单价"])
    #print(D.T)
    with open(r‘C:\Users\86150\Desktop\house/abc.scv‘,‘a‘,encoding=‘utf-8‘) as f:
        f.write(‘{},{}\n‘.format(nameprice))#保存到Excel

#数据清洗
print(‘\n====各列是否有缺失值情况如下：====‘)
print(D.isnull()) 
print(D.duplicated())
print(D.isna().head()) 
print(D.describe())

#条形图
import matplotlib.pyplot as plt
plt.figure(figsize=(10, 10), dpi=80)
# 柱子总数
N = 30
# 包含每个柱子对应值的序列
values = (19441,7853,17715,20261,11373,18850,24968,17773,18653,23799,18027,22596,20842,
          15882,28645,21279,2726,15182,26430,2790,1542,1919,2939,18762,26299,18808,16852,8694,31261,20561)
# 包含每个柱子下标的序列
index = np.arange(N)
# 柱子的宽度
width = 0.45
p2 = plt.bar(index, values, width, label="num", color="#87CEFA")
# 设置横轴标签
plt.xlabel(‘region‘)
# 设置纵轴标签
plt.ylabel(‘unit price‘)
# 添加标题
plt.title(‘Second-hand house square unit price ‘)
# 添加纵横轴的刻度
plt.xticks(index, (‘1‘,‘2‘,‘3‘,‘4‘,‘5‘,‘6‘,‘7‘,‘8‘,‘9‘,‘10‘,‘11‘,‘12‘,‘13‘,‘14‘,‘15‘,‘16‘,‘17‘,‘18‘,‘19‘,‘20‘,‘21‘,
                   ‘22‘,‘23‘,‘24‘,‘25‘,‘26‘,‘27‘,‘28‘,‘29‘,‘30‘))
# plt.yticks(np.arange(0, 10000, 10))
# 添加图例
plt.legend(loc="upper right")
plt.show()

#折线图
from pylab import *
mpl.rcParams[‘font.sans-serif‘] = [‘SimHei‘]  # 添加这条可以让图形显示中文

x_axis_data = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30]
y_axis_data = [19441,7853,17715,20261,11373,18850,24968,17773,18653,23799,18027,22596,20842,15882,
               28645,21279,2726,15182,26430,2790,1542,1919,2939,18762,26299,18808,16852,8694,31261,20561]

# plot中参数的含义分别是横轴值，纵轴值，线的形状，颜色，透明度,线的宽度和标签
plt.plot(x_axis_data, y_axis_data, ‘ro-‘, color=‘#4169E1‘, alpha=0.8, linewidth=1, label=‘二手房单价‘)

# 显示标签，如果不加这句，即使在plot中加了label=‘一些数字‘的参数，最终还是不会显示标签
plt.legend(loc="upper right")
plt.ylabel(‘平方单价‘)

plt.show()
# plt.savefig(‘demo.jpg‘)  # 保存该图片


#一元二次
chinese=matplotlib.font_manager.FontProperties(fname=‘C:/Windows/Fonts/simsun.ttc‘)

source_data=genfromtxt(‘C:/house.csv‘,dtype=float,delimiter=‘,‘)
data=np.delete(source_data,0,axis=0)
price=data[:,0]
number=data[:,1]
X=np.array(number)
Y=np.array(price)
def func(p,x):
    a,b,c=p
    return a*x*x+b*x+c
def error(p,x,y):
    return func(p,x)-y
p0=[1,0]
def main():
    plt.figure(figsize=(8,6))
    p0=[1,30,30]
    Para=leastsq(error,p0,args=(X,Y))
    a,b,c=Para[0]
    print("a=",a,"b=",b,"c=",c)
    plt.scatter(X,Y,color="green",label="样本数据",linewidth=2)
    x=np.linspace(1,30,30)
    y=a*x*x+b*x+c
    plt.plot(x,y,color="red",label="拟合曲线",linewidth=2)
    plt.legend(prop=chinese)
    plt.title("Second-hand house square unit price")
    plt.grid()
    plt.show()
main()