# -*- coding: utf-8 -*-
import scrapy
import requests
import os
import sys
import io
from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request


sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='gb18030')


class CnblogsSpider(scrapy.Spider):
    name = 'cnblogs'
    allowed_domains = ['http://www.duia.com/']
    start_urls = ['http://www.duia.com/']

    def parse(self, response):
        hxs = HtmlXPathSelector(response)
        all_tags_href = hxs.select('//a[@class="sku-mr"]/@href').extract()
        print(len(all_tags_href))

        for href in all_tags_href:
            # ret = requests.get(href)
            # print(ret)
            # href是'http://www.duia.com/sku/1','http://www.duia.com/sku/2'这样字符串
            yield Request(url=href, callback=self.second_level)

    def second_level(self, response):
        hxs = HtmlXPathSelector(response)
        all_tags_href = hxs.select('//div[@class="container"]//ul/li/a/@href').extract()
        print(len(all_tags_href))