scrapy-scrapyd-scrapydweb

# -*- coding: utf-8 -*-
import scrapy
from news.items import NewsItem
import sqlite3
import smtplib
from email.mime.text import MIMEText

class SouhunewsSpider(scrapy.Spider):
    name = ‘souhunews‘
    allowed_domains = [‘sohu.com‘]
    start_urls = [‘‘‘http://sohu.com/c/8/1463?
                  spm=smpc.null.side-nav.16.1581303075427Zowrm4P‘‘‘]
   

    def parse(self, response):
        conn = sqlite3.connect(‘scrapy.db‘)
        c = conn.cursor()
        try:
            c.execute(‘DROP TABLE books‘)
        except:
            pass    
        
        c.execute(‘‘‘CREATE TABLE books(title text primary key,link text,fro text)‘‘‘)
        conn.commit()
        conn.close()
        
        it = []
        for x in response.xpath("//div[@data-role=‘news-item‘]"):
            item = NewsItem()
            item[‘link‘] = x.css("h4>a::attr(href)").get()
            item[‘title‘] = x.xpath("h4/a/text()").get()    
            item[‘fro‘] = x.xpath("div/span[@class=‘name‘]/a/text()").get()
            yield item

import scrapy
class NewsItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    link = scrapy.Field()#链接
    title = scrapy.Field()#标题
    fro = scrapy.Field()#来源

import sqlite3
import smtplib
from email.mime.text import MIMEText

class NewsPipeline(object):
    def __init__(self):
        pass
        
    def process_item(self, item, spider):
        
        return item

class SQLitePipeline(object):

    #打开数据库
    def open_spider(self, spider):
        db_name = spider.settings.get(‘SQLITE_DB_NAME‘, ‘scrapy.db‘)

        self.db_conn = sqlite3.connect(db_name)
        self.db_cur = self.db_conn.cursor()       

    #关闭数据库
    def close_spider(self, spider):
        self.db_conn.commit()
        self.db_conn.close()
        conn = sqlite3.connect(r‘E:\testpy\news\scrapy.db‘)
        
        c = conn.cursor()
        c.execute(‘‘‘SELECT * FROM books‘‘‘)
        data = c.fetchall()
        conn.commit()
        conn.close()

        txt = ‘‘
        for d in data:
            txt = txt+‘‘‘<p>标题：{}</p><p><a href = ‘{}‘>链接</a></p><p>来源：{}</p>‘‘‘.format(d[0],d[1],d[2])

        #邮件模块
        user = ‘#########@qq.com‘
        pwd = ‘#############‘#注意不是密码
        to = ‘#############@qq.com‘

        msg = MIMEText(txt,‘html‘,‘utf-8‘)
        msg[‘Subject‘] = ‘News‘
        msg[‘From‘] = user
        msg[‘To‘] = to

        s = smtplib.SMTP()
        s.connect(‘smtp.qq.com‘,25)
        s.login(user,pwd)
        s.sendmail(user,to,msg.as_string())
        s.quit()
        
    #对数据进行处理
    def process_item(self, item, spider):
        self.insert_db(item)
        return item

    #插入数据
    def insert_db(self, item):
        values = (
            (item[‘title‘][2:-2]).strip(),
            item[‘link‘][2:],
            item[‘fro‘]
        )

        sql = ‘INSERT INTO books VALUES(?,?,?)‘
        self.db_cur.execute(sql, values)

ITEM_PIPELINES = {
    ‘news.pipelines.SQLitePipeline‘: 400,
}#对应管道中的SQLite数据库操作

SQLITE_DB_NAME = ‘scrapy.db‘

[settings]
default = news.settings

[deploy:demo]
url = http://localhost:6800/
project = news