Learning Scrapy《精通Python爬虫框架Scrapy》 02：示例代码中的一些典型问题

 1 import datetime
 2 import urlparse
 3 import socket
 4 
 5 from scrapy.loader.processors import MapCompose, Join
 6 from scrapy.linkextractors import LinkExtractor
 7 from scrapy.spiders import CrawlSpider, Rule
 8 from scrapy.loader import ItemLoader
 9 from scrapy.http import FormRequest
10 
11 from properties.items import PropertiesItem
12 
13 
14 class LoginSpider(CrawlSpider):
15     name = ‘login‘
16     allowed_domains = ["web"]
17 
18     # Start with a login request
19     def start_requests(self):
20         return [
21             FormRequest(
22                 "http://web:9312/dynamic/login",
23                 formdata={"user": "user", "pass": "pass"}
24             )]
25 
26     # Rules for horizontal and vertical crawling
27     rules = (
28         Rule(LinkExtractor(restrict_xpaths=‘//*[contains(@class,"next")]‘)),
29         Rule(LinkExtractor(restrict_xpaths=‘//*[@itemprop="url"]‘),
30              callback=‘parse_item‘)
31     )
32 
33     def parse_item(self, response):
34         """ This function parses a property page.
35 
36         @url http://web:9312/properties/property_000000.html
37         @returns items 1
38         @scrapes title price description address image_urls
39         @scrapes url project spider server date
40         """
41 
42         # Create the loader using the response
43         l = ItemLoader(item=PropertiesItem(), response=response)
44 
45         # Load fields using XPath expressions
46         l.add_xpath(‘title‘, ‘//*[@itemprop="name"][1]/text()‘,
47                     MapCompose(unicode.strip, unicode.title))
48         l.add_xpath(‘price‘, ‘.//*[@itemprop="price"][1]/text()‘,
49                     MapCompose(lambda i: i.replace(‘,‘, ‘‘), float),
50                     re=‘[,.0-9]+‘)
51         l.add_xpath(‘description‘, ‘//*[@itemprop="description"][1]/text()‘,
52                     MapCompose(unicode.strip), Join())
53         l.add_xpath(‘address‘,
54                     ‘//*[@itemtype="http://schema.org/Place"][1]/text()‘,
55                     MapCompose(unicode.strip))
56         l.add_xpath(‘image_urls‘, ‘//*[@itemprop="image"][1]/@src‘,
57                     MapCompose(lambda i: urlparse.urljoin(response.url, i)))
58 
59         # Housekeeping fields
60         l.add_value(‘url‘, response.url)
61         l.add_value(‘project‘, self.settings.get(‘BOT_NAME‘))
62         l.add_value(‘spider‘, self.name)
63         l.add_value(‘server‘, socket.gethostname())
64         l.add_value(‘date‘, datetime.datetime.now())
65 
66         return l.load_item()

from urllib.parse import urljoin
.
.
.
.........lambda i: urljoin(response.url, i)