首页 > 其他 > 详细

证监会处罚公告爬取

时间:2020-05-23 22:09:00      阅读:77      评论:0      收藏:0      [点我收藏+]

域名:http://www.csrc.gov.cn/pub/zjhpublic/3300/3313/index_7401.htm

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import re


class CfSpider(CrawlSpider):
name = cf
allowed_domains = [csrc.gov.cn]
start_urls = [http://www.csrc.gov.cn/pub/zjhpublic/3300/3313/index_7401.htm]

rules = (
Rule(LinkExtractor(allow=r‘/G\d+/\d+/t\d+_\d+\.htm‘), callback=parse_item),
# Rule(LinkExtractor(allow=r‘/3300/3313/index_7401_.*?\.htm‘),follow=True), # 不起作用,所以重写了start_requests
)

def start_requests(self):
current_page = 0
while current_page < 67:
if current_page == 0:
url = http://www.csrc.gov.cn/pub/zjhpublic/3300/3313/index_7401
next_url = url+".html"
else:
url = http://www.csrc.gov.cn/pub/zjhpublic/3300/3313/index_7401_{}.htm
next_url = url.format(str(current_page))
yield scrapy.Request(
url=next_url,
callback=self.parse,
)
current_page += 1


def parse_item(self, response):
item = dict()
item["title"] = response.xpath("//span[@id=‘lTitle‘]/text()").extract_first()
item["pub_title"] = re.findall(r"<span>(20\d+年\d{2}月\d{2}日)</span>",response.body.decode(),re.S)
item["pub_title"] = item["pub_title"][0] if item["pub_title"] else None
item["index_number"] = response.xpath("//table[@id=‘headContainer‘]//tr[1]//td[@colspan=‘2‘]//td[1]/text()").extract_first()
item["href"] = response.url
yield item

证监会处罚公告爬取

原文:https://www.cnblogs.com/nuochengze/p/12944333.html

(0)
(0)
   
举报
评论 一句话评论(0
关于我们 - 联系我们 - 留言反馈 - 联系我们:wmxa8@hotmail.com
© 2014 bubuko.com 版权所有
打开技术之扣,分享程序人生!