抓取新浪社会新闻源代码

一轮江月明

视频地址如下，我觉得讲的深入浅出，分享给大家，欢迎大家讨论。

https://edu.hellobi.com/course/81/play/lesson/1761

1. import requests
   
2. from bs4 import BeautifulSoup
   
3. import re
   
4. import json
   
5. from datetime import datetime
   
6. import pandas
   
7. 
   
8. #取得评论数的函数
   
9. def getCommentCounts(newsurl):
   
10.     m = re.search('doc-i(.+).shtml',newsurl)
    
11.     newsid = m.group(1)
    
12.     commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=sh&newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'
    
13.     comments = requests.get(commentURL.format(newsid))
    
14.     comments.encoding ='utf-8'
    
15.     jd = json.loads(comments.text.strip('var data='))
    
16.     return (jd['result']['count']['total'])
    
17. 
    
18. #在正文页面取得包含标题、新闻来源、时间、正文、编辑姓名、评论数的一个字典result
    
19. def getNewsDetail(newsurl):
    
20.     result = {}
    
21.     res = requests.get(newsurl)
    
22.     res.encoding = 'utf-8'
    
23.     soup = BeautifulSoup(res.text,'html.parser')
    
24.     result['title'] = soup.select('#artibodyTitle')[0].text
    
25.     result['newssource'] = soup.select('.time-source a')[0].text
    
26.     timesource = soup.select('.time-source')[0].contents[0].strip()
    
27.     result['dt'] = datetime.strptime(timesource,'%Y年%m月%d日%H:%M')
    
28.     result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
    
29.     result['editor'] = soup.select('.article-editor')[0].text.lstrip('责任编辑：')
    
30.     result['comments'] = getCommentCounts(newsurl)
    
31.     return result
    
32. 
    
33. '''
    
34. #在新闻标题列表页面如何取得第一次加载的时间、标题以及链接
    
35. res_1 = requests.get('http://news.sina.com.cn/society/')
    
36. res_1.encoding = 'utf-8'
    
37. soup =BeautifulSoup(res_1.text,'html.parser')
    
38. 
    
39. for news in soup.select('.news-item'):
    
40.     if len(news.select('h2'))>0:
    
41.         h2 = news.select('h2')[0].text
    
42.         time = news.select('.time')[0].text
    
43.         a = news.select('a')[0]['href']
    
44.         print (time,h2,a)      
    
45. '''
    
46. 
    
47. #在新闻标题列表页面如何取得滚动后加载的所有新闻链接，并调用getNewsDetail(newsurl)函数
    
48. def parseListLinks(url):
    
49.     res = requests.get(url)
    
50.     jd = json.loads(res.text.lstrip('   newsloadercallback(').rstrip(');'))
    
51.     newsdetails = []
    
52.     for each in jd['result']['data']:
    
53.         newsdetails.append(getNewsDetail(each['url']))#这里调用了上面的getNewsDetail(newsurl)函数
    
54.     return newsdetails
    
55. 
    
56. 
    
57. 
    
58. url = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=shxw&cat_2==zqsk||=qwys||=shwx||=fz-shyf&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page={}'
    
59. news_total = []
    
60. for i in range(1,6):
    
61.     newsurl = url.format(i)
    
62.     newsary = parseListLinks(newsurl)
    
63.     news_total.extend(newsary)
    
64. df = pandas.DataFrame(news_total)
    
65. 
    
66. df.to_excel('news.xlsx')
    
67. 
    
68. import sqlite3
    
69. with sqlite3.connect('news.sqlite')as db:
    
70.     df.to_sql('news',con = db)
    
71. 
    
72. import sqlite3
    
73. with sqlite3.connect('news.sqlite')as db:
    
74.     df2 = pandas.read_sql_query('SELECT * FROM NEWS',con = db)
    
75. 
    
76. 
    
77. 
    

复制代码