1 # -*- coding:utf-8 -*-
 2 
 3 import json
 4 import requests
 5 
 6 from lxml import etree
 7 import codecs
 8 
 9 
10 siteurl = "http://news.163.com/special/shijiuda_roll/"
11 user_agent="Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36"
12 headers = {"User-Agent":user_agent}
13 
14 url = siteurl 
15 html = requests.get(url,headers = headers)
16 
17 selector= etree.HTML(html.text)
18         
19 items_time = selector.xpath('//body//div//ul[@class="list_f14d"]//li/span/text()')
20 items_url = selector.xpath('//body//div//ul[@class="list_f14d"]//li/a/@href')
21 items_title =selector.xpath('//body//div//ul[@class="list_f14d"]//li/a/text()')
22 items = [[a,b,c] for a,b,c in zip(items_time,items_url,items_title)]
23 
24 item_dict= {}
25 item_list = []
26 items_url_old =[]
27 
28 
29 for item in items:
30     if item[1] is None :
31           pass
32     if item[1] is not None and item[1] not in items_url_old:
33         new_html= requests.get(item[1],headers = headers)
34         selector1=etree.HTML(new_html.text)
35         text = selector1.xpath('//body//div[@class ="post_text"]//p/text()')
36         item_dict['time']=item[0]
37         item_dict['url']=item[1]
38         item_dict['title']=item[2]
39         item_dict['text']=text
40         items_url_old.append(item[1])
41         item_list.append(item_dict)