import scrapy
from my_scrapy.items import MyScrapyItem
from fake_useragent import UserAgent
 
 
class SpiderSpider(scrapy.Spider):
    # 爬虫名称
    name = 'spider'
    # 域名限制，允许爬取的范围
    # allowed_domains = ['https://quotes.toscrape.com/']
 
    base_url = 'https://quotes.toscrape.com/page/{}/'
 
    # 初始请求的页面
    start_urls = ['https://quotes.toscrape.com/']
 
    headers = [{'user-agent': UserAgent().random}]
    cookies = {'name': 'zqh'}
 
    def start_request(self):
        for page in range(1, 6):
            url = self.base_url.format(page)
            yield scrapy.Request(url=url, callback=self.parse, headers=self.headers, cookies=self.cookies)
 
    def parse(self, response):
        # text = response.text
 
        quotes = response.xpath('//div[@class="quote"]')
        for quote in quotes:
            # 旧方法 get()为新方法
            # text = quote.xpath('./span[@class = "text"]/text()').extract_first()
 
            # 实例化对象
            item = MyScrapyItem()
 
            # 利用xpth进行爬取
            text = quote.xpath('./span[@class = "text"]/text()').get()
            author = quote.xpath('.//small[@class="author"]/text()').get()
            Tags = quote.xpath('.//a[@class="tag"]/text()').getall()
            item['text'] = text
            item['author'] = author
            item['Tag'] = Tags
 
            # 迭代出去
            yield item
 
        # 简洁的拼接
        # yield from response.follow_all(response.css('.pager .next a::attr("href")'), callback=self.parse)

 Response对象

腾讯招聘案例