import re
import bs4
import numpy as np
import pandas as pd
import requests as req
from bs4 import BeautifulSoup


# 返回网页源码的函数
def getHTMLText(url):
    try:
        # 使用requests库爬取网页源码2
        r = req.get(
            url, headers={‘user-agent‘: ‘Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36‘})
        # 判断是否爬取过程中出现错误
        r.raise_for_status()
        #  设置源码的编码格式
        r.encoding = r.apparent_encoding
        # 返回源码
        return r.text
    except:
        return "爬取失败"

# 解析页面源码并组成集合返回爬取数据
def XiamiMusicList(html):
    List = []
    # 使用bs4库中的BeautifulSoup类解析html文本
    soup = BeautifulSoup(html, "html.parser")

    trs = soup.find_all("tr","bg")
    for i in trs:
        # 招聘岗位
        a1 = i.find_all("a")[0].text.replace("\n", "")
        # 招聘链接
        a1href = i.find_all("a")[0].attrs[‘href‘].replace("\n", "")
        # 唯一id
        recruit_id = i.find_all("a")[0].attrs[‘href‘].split(
            "=")[-1].replace("\n", "")
        # 公司名称
        a2 = i.find_all("a")[1].text.replace("\n", "")
        # 公司地址
        a3 = i.find_all("a")[2].text.replace("\n", "")
        # 薪资
        a4 = i.find_all("a")[3].text.replace("\n", "")
        # 学历
        a5 = i.find_all("a")[4].text.replace("\n", "")
        # 发布日期
        a6 = i.find_all("a")[5].text.replace("\n", "")
        # 组装集合
        temp = [recruit_id, a1, a2, a3, a4, a5, a6, a1href]
        List.append(temp)
    return List
   

# 数据持久化
def weiteExcel(musicList, MusicListPath):

    # 将数据转化为DataFrame格式
    df = pd.DataFrame(musicList, columns=[
                      "recruit_id", "Occupation", "company", "locationi", "salary", "Education", "createtime", "recruit_href"])

    # 输出excel的对象
    writer = pd.ExcelWriter(MusicListPath)

    # 将数据写入到excel表格中
    df.to_excel(writer, sheet_name="recruitList")

    # 关闭输出流
    writer.save()

#  主函数
def main():
    
    Path = "https://www.xmrc.com.cn/net/info/Resultg.aspx?a=a&g=g&recordtype=1&searchtype=1&keyword="

    # 搜索关键词
    keyword = "python"

    url = "&releasetime=365&worklengthflag=0&sortby=updatetime&ascdesc=Desc&PageIndex="
    # 页数关键词// 修改此项值能获取指定页面数据
    page = "1"
    html = getHTMLText(Path+keyword+url+page)
    # print(html)

    # 存放路径
    recruitListPath = "recruitList.xlsx"

    # 获取爬取数组
    List = XiamiMusicList(html)

    # 将数据保存至文件中
    weiteExcel(List, recruitListPath)

main()

#数据清洗与可视化
import requests as req
from bs4 import BeautifulSoup
import bs4
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
import seaborn as sns

# 解决seaborn中文乱码问题
sns.set_style(‘whitegrid‘,{‘font.sans-serif‘:[‘simhei‘,‘Arial‘]})

# 读取文件
recruitList = pd.read_excel("recruitList.xlsx", sheet_name="recruitList")
print(recruitList.head(5))

# 数据清洗
recruitList[‘Occupation‘] = recruitList[‘Occupation‘].map(str.strip)
print(recruitList[‘Occupation‘].head(5))

print(recruitList[‘recruit_id‘].head(5))

# 数据清洗
recruitList[‘company‘] = recruitList[‘company‘].map(str.strip)
print(recruitList[‘company‘].head(5))
recruitList[‘locationi‘] = recruitList[‘locationi‘].map(str.strip)
print(recruitList[‘locationi‘].head(5))

# 数据清洗
# 对空字段填充一个默认值
recruitList[‘salary‘] = recruitList[‘salary‘].fillna(‘5000-7000‘)
recruitList[‘salary‘] = recruitList[‘salary‘].map(str.strip)
print(recruitList[‘salary‘].head(5))

# 数据清洗
# 对空字段填充一个默认值
recruitList[‘Education‘] = recruitList[‘Education‘].fillna(‘没要求‘)
recruitList[‘Education‘] = recruitList[‘Education‘].map(str.strip)
print(recruitList[‘Education‘].head(5))

#学历要求可视化
sns.boxplot(x=‘Education‘,y=‘recruit_id‘, data=recruitList)