[已解决]依然还是知乎某用户主页的“更多”按钮的问题

无下限HENTAI

上回其实已经问过了，由@SixPy 大神给了一个简单的解答。当时觉得会了，所以就没再问。但这两天照之前的想法试了一下，发现失败了。
http://bbs.fishc.com/forum.php?m ... peid%26typeid%3D393
之前是发现每次点击“更多”的时候会POST一个start，这个start的值是他的主页上最后一条记录的'data-time'。

url

start

于是我就想着找到最后一个data-time，POST到那个url就行了，然后就写了如下代码

1. import urllib.request as ur
   
2. import urllib.parse as up
   
3. from http.cookiejar import CookieJar
   
4. import re
   
5. from bs4 import BeautifulSoup
   
6. import os
   
7. import time
   
8. 
   
9. url = 'http://www.zhihu.com/'
   
10. 
    
11. #构造opener
    
12. cj = CookieJar()
    
13. opener = ur.build_opener(ur.HTTPCookieProcessor(cj))
    
14. opener.addheader = [('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36')]
    
15. 
    
16. #获取_xsrf
    
17. def get_xsrf():
    
18.     url = 'http://www.zhihu.com/'
    
19.     response = opener.open(url)
    
20.     html = response.read()
    
21.     soup = BeautifulSoup(html,'html.parser')
    
22.     val = soup.find(attrs = {'name':'_xsrf'})
    
23.     value = val['value']
    
24.     return value
    
25. 
    
26. #登录知乎
    
27. url2 = ''.join([url,'login/email'])
    
28. data = {}
    
29. data['_xsrf'] = get_xsrf()
    
30. data['password'] = 密码
    
31. data['captcha_type'] = 'cn'
    
32. data['remember_me'] = 'true'
    
33. data['email'] = 账号
    
34. response2 = opener.open(url2,up.urlencode(data).encode('utf-8'))
    
35. 
    
36. #轮子哥主页！
    
37. url_lunzi = 'https://www.zhihu.com/people/excited-vczh'
    
38. response3 = opener.open(url_lunzi)
    
39. html2 = response3.read().decode('utf-8')
    
40. soup2 = BeautifulSoup(html2,'html.parser')
    
41. 
    
42. #寻找他赞过的答案
    
43. answer = []
    
44. while 1:
    
45.     if len(answer) < 30:
    
46.         a = soup2.find_all(name='div',
    
47.                             class_="zm-profile-section-item zm-item clearfix")
    
48.         start = a[-1]['data-time']
    
49.         url_more = 'https://www.zhihu.com/people/excited-vczh/activities'
    
50.         data_more = {'start':start}
    
51.         response3 = opener.open(url_more, up.urlencode(data_more).encode('utf-8'))
    
52.         html3 = response3.read().decode('utf-8')
    
53.         soup3 = BeautifulSoup(html3,'html.parser')
    
54.         allanswer = soup3.find_all(name = 'div',
    
55.                                  class_ = "zm-profile-section-main zm-profile-section-activity-main zm-profile-activity-page-item-main",
    
56.                                 )
    
57.         for i in range(len(allanswer)-1,-1,-1):
    
58.             if allanswer[i] in answer:
    
59.                 del allanswer[i]
    
60.             elif '赞同了回答' not in allanswer[i].span.next_sibling:
    
61.                 del allanswer[i]
    
62.         answer = allanswer
    
63.     else:
    
64.         break
    
65. 
    
66. 
    
67. #下载图片
    
68. route = 'D:\\vczh_pic'
    
69. try:
    
70.     os.mkdir(route)
    
71. except:
    
72.     pass
    
73. finally:
    
74.     os.chdir(route)
    
75. for each in answer:
    
76.     link = each.span.next_sibling.next_sibling['href']
    
77.     link = ''.join(['http://www.zhihu.com',link])
    
78.     res = opener.open(link).read().decode('utf-8')
    
79.     soup = BeautifulSoup(res,'html.parser')
    
80.     pic = soup.find_all(attrs = {'data-rawwidth':'1280','class':'origin_image zh-lightbox-thumb lazy'})
    
81.     for each_pic in pic:
    
82.         pic_name = re.sub(r'https://pic\d\.zhimg\.com/','',each_pic['data-actualsrc'])
    
83.         with open(pic_name,'wb') as f:
    
84.             src = each_pic['data-actualsrc']
    
85.             pic_res = ur.urlopen(src).read()
    
86.             f.write(pic_res)

复制代码

这个代码我分段测试过，登录和筛选记录以及下载图片都是正常的，问题还是出在了想点击“更多”时
下面是出的问题

1. Traceback (most recent call last):
   
2.   File "C:\Users\Administrator\Desktop\轮带逛.py", line 71, in <module>
   
3.     response3 = opener.open(url_more, up.urlencode(data_more).encode('utf-8'))
   
4.   File "D:\Python34\lib\urllib\request.py", line 469, in open
   
5.     response = meth(req, response)
   
6.   File "D:\Python34\lib\urllib\request.py", line 579, in http_response
   
7.     'http', request, response, code, msg, hdrs)
   
8.   File "D:\Python34\lib\urllib\request.py", line 507, in error
   
9.     return self._call_chain(*args)
   
10.   File "D:\Python34\lib\urllib\request.py", line 441, in _call_chain
    
11.     result = func(*args)
    
12.   File "D:\Python34\lib\urllib\request.py", line 587, in http_error_default
    
13.     raise HTTPError(req.full_url, code, msg, hdrs, fp)
    
14. urllib.error.HTTPError: HTTP Error 403: Forbidden

复制代码

实在是束手无策了，难道必须要用selenium才行吗？应该怎么修改呢？

最佳答案

月排行榜 / 总排行榜

SixPy

2016-8-27 23:36:06

你翻页太快，zhihu有反爬虫机制~

跳转到最佳答案楼层

无下限HENTAI

顺道问一句。。。BeautifulSoup的find功能是不是只能找最开始的一个，不能从后往前找，如果想找最后一个就必须find_all然后取-1？

SixPy

你翻页太快，zhihu有反爬虫机制~

无下限HENTAI

SixPy 发表于 2016-8-27 23:36
你翻页太快，zhihu有反爬虫机制~

感觉好像不是。。。我在else上面，也就是answer=allanswer那一句下面加了一句time.sleep(10)，但还是报了一样的错误信息