import scrapy
import time
import re
from copy import deepcopy

from weinanweb.items import WeinanwebItem


class WnwebSpider(scrapy.Spider):
    name = 'wnweb'
    allowed_domains = ['weinan.hsw.cn']
    start_urls = ['http://weinan.hsw.cn/weinannews']

    def parse(self, response):
        item = WeinanwebItem()
        List = response.xpath("//div[@class='yaoweilist']/ul[@class='list_tpbt']/li")
        for li in List:
            item['title'] = li.xpath(".//p[@class='titc fz18c']/a/text()").extract_first()
            item['title_url'] = li.xpath("./p/a/@href").extract()[0]
            item['abstract'] = li.xpath("./p[@class='absc fz14c']/text()").extract_first()
            item['content'] = []
            item['image_urls'] = []
            # print(item['title'])
            yield scrapy.Request(
                url=item['title_url'],
                meta={"item": deepcopy(item)},
                callback=self.content_info
            )
        # 列表翻页的处理
        new_url = response.xpath("//div[@class='page']/a[last()]").extract_first()
        time.sleep(10)
        if new_url:
            yield scrapy.Request(
                new_url,
                callback=self.parse)


        # 有一种新闻页
    def content_info(self, response):
        time.sleep(0.2)
        item = response.meta['item']
        # 新闻页有两种结构 if判断  分别取不同的 页面内容
        if response.xpath("//div[@class='contentBox cf']").extract():
            # 小图模式
            item['content'] += response.xpath("//div[@class='contentBox cf']").extract()
            item['image_urls'] += response.xpath("//div[@class='contentBox cf']//img/@src").extract()
            yield item
            # 小图模式没找到翻页的
            # next_content_url = response.xpath("").extract_first()
            # if next_content_url:
            #     yield scrapy.Request(next_content_url,
            #                          meta={'item': item},
            #                          callback=self.content_info)

        elif response.xpath("//div[@id='photoView']/img/@src").extract():
            # 大图模式
            item['content'] += response.xpath("//div[@class='bd main']//p/text()").extract()
            item['image_urls'] += response.xpath("//div[@id='photoView']/img/@src").extract()
            next_content_url = response.xpath("//div[@id='news_more_page_div_id']"
                                              "/a[@id='nexpageid']/@href").extract_first()
            if next_content_url:
                yield scrapy.Request(next_content_url,
                                     meta={'item': item},
                                     callback=self.content_info)
        else:
            pass

        yield item

import scrapy
import json

from scrapy.exporters import JsonItemExporter
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
from weinanweb import settings


class WeinanwebPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        """
        你需要重写 get_media_requests() 方法，
        并对各个图片URL返回一个Request:      
        """
        for image_url in item['image_urls']:
            yield scrapy.Request(image_url)

    def item_completed(self, results, item, info):
        """
        当一个单独项目中的所有图片请求完成时（要么完成下载，要么因为某种原因下载失败），
                 item_completed() 方法将被调用。       
        """
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("Item contains no images")

        yield item


class ItemPipeline(object):
    def __init__(self):
        self.file = open("Qianhua.json", 'wb')

    def process_item(self, item, spider):
        text = json.dumps(dict(item), ensure_ascii=False, indent=2)

        self.file.write(text.encode('utf-8'))
        print('正在写入——————————>>>')

    def close(self):
        self.file.close()

import scrapy


class WeinanwebItem(scrapy.Item):   
    image_urls = scrapy.Field()
    image = scrapy.Field()

    title = scrapy.Field()
    abstract = scrapy.Field()
    content = scrapy.Field()
    title_url = scrapy.Field()