[已解决]请前辈指点 python56课 爬不出来图

wow7jiao

请问是不是煎鸡蛋加了反爬虫,把jandan.net/ooxx/page-82#comments前面的http://去掉料.所以Request无法解析??

1. import urllib.request
   
2. import os
   
3. 
   
4. def url_open(url):
   
5.     req = urllib.request.Request(url)
   
6.     req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.65 Safari/537.36')
   
7.     response = urllib.request.urlopen(req)
   
8.     html = response.read()
   
9.     print(url)
   
10. 
    
11.     return html
    
12. 
    
13. 
    
14. def get_page(url):
    
15.     html = url_open(url).decode('utf-8')
    
16. 
    
17. 
    
18. 
    
19.     a = html.find('current-comment-page') + 23
    
20.     b = html.find(']', a)
    
21. 
    
22.     return html[a:b]
    
23.         
    
24. def find_imgs(url):
    
25.     url = url[7:]
    
26.    
    
27.     html = url_open(url).decode('utf-8')
    
28.     img_addrs = []
    
29. 
    
30.     a = html.find('img src=')
    
31.    
    
32.     while a != -1:
    
33.         b = html.find('.jpg', a, a+255)
    
34.         if b != -1:
    
35.             img_addrs.append(html[a+9:b+4])
    
36.         else:
    
37.             b = a + 9
    
38. 
    
39.         a = html.find('img src=', b)
    
40. 
    
41.     ##return img_addrs
    
42.     for each in img_addrs:
    
43.         print(each)
    
44.         
    
45.         
    
46. 
    
47. def save_imgs(folder, img_addrs):
    
48.     pass
    
49.    
    
50. def download_mm(folder='OOXX', pages=10):
    
51.     os.mkdir(folder)
    
52.     os.chdir(folder)
    
53. 
    
54. 
    
55. 
    
56. 
    
57.     url = "http://jandan.net/ooxx/"
    
58.     page_num = int(get_page(url))
    
59. 
    
60.     for i in range(pages):
    
61.         page_num -= 1
    
62.         page_url = url + 'page-' + str(page_num) + '#comments'
    
63.         ##print(page_url)
    
64.         img_addrs = find_imgs(page_url)
    
65. 
    
66.         save_imgs(folder, img_addrs)
    
67. 
    
68. 
    
69. if __name__ == '__main__':
    
70.     download_mm()

复制代码

下面是出错提示:
>>>
= RESTART: C:\Users\Administrator\Desktop\056轮一只爬虫的自我修养4：OOXX\download_mm.py =
http://jandan.net/ooxx/
Traceback (most recent call last):
  File "C:\Users\Administrator\Desktop\056轮一只爬虫的自我修养4：OOXX\download_mm.py", line 67, in <module>
    download_mm()
  File "C:\Users\Administrator\Desktop\056轮一只爬虫的自我修养4：OOXX\download_mm.py", line 61, in download_mm
    img_addrs = find_imgs(page_url)
  File "C:\Users\Administrator\Desktop\056轮一只爬虫的自我修养4：OOXX\download_mm.py", line 27, in find_imgs
    html = url_open(url).decode('utf-8')
  File "C:\Users\Administrator\Desktop\056轮一只爬虫的自我修养4：OOXX\download_mm.py", line 5, in url_open
    req = urllib.request.Request(url)
  File "D:\Python36\lib\urllib\request.py", line 329, in __init__
    self.full_url = url
  File "D:\Python36\lib\urllib\request.py", line 355, in full_url
    self._parse()
  File "D:\Python36\lib\urllib\request.py", line 384, in _parse
    raise ValueError("unknown url type: %r" % self.full_url)
ValueError: unknown url type: 'jandan.net/ooxx/page-82#comments'

最佳答案

月排行榜 / 总排行榜

°蓝鲤歌蓝

2018-1-22 21:51:32

°蓝鲤歌蓝 发表于 2018-1-24 12:10
不好意思是 mztu

http://www.mzitu.com/ 网络有点卡，一直打错很抱歉

跳转到最佳答案楼层

°蓝鲤歌蓝

°蓝鲤歌蓝 发表于 2018-1-24 12:10
不好意思是 mztu

http://www.mzitu.com/ 网络有点卡，一直打错很抱歉

ComeAsYouAre

建议换网站

zhuzhuwei

是的，去掉html=html[7:]就可以了
但是我find('img src')可以成功，find('.jpg')不能成功

°蓝鲤歌蓝

讲道路，这个网站的图片已经加密了，你刚学爬虫的话就换个网站吧，以后再来挑战。http://www.meiztu.com/就很不错

wow7jiao

°蓝鲤歌蓝 发表于 2018-1-23 20:38
讲道路，这个网站的图片已经加密了，你刚学爬虫的话就换个网站吧，以后再来挑战。http://www.meiztu.com/就 ...

你想说的是meizitu.com吧，我去爬了还是一样出问题

1. import urllib.request
   
2. import os
   
3. import random
   
4. 
   
5. 
   
6. 
   
7. def url_open(url):
   
8.     req = urllib.request.Request(url)
   
9.     req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.65 Safari/537.36')
   
10. 
    
11. 
    
12. 
    
13.     response = urllib.request.urlopen(req)
    
14.     html = response.read()
    
15. 
    
16.     return html
    
17. 
    
18. 
    
19. def get_page(url):
    
20.     html = url_open(url).decode('utf-8')
    
21. 
    
22.     a = html.find('thisclass') + 11
    
23.     b = html.find('>', a)
    
24. 
    
25.    
    
26. 
    
27.     return html[a:b]
    
28. 
    
29. 
    
30. def find_imgs(url):
    
31.     html = url_open(url).decode('utf-8')
    
32.     img_addrs = []
    
33. 
    
34.     a = html.find('img src=')
    
35. 
    
36.     while a != -1:
    
37.         b = html.find('.jpg', a, a+255)
    
38.         if b != -1:
    
39.             img_addrs.append(html[a+9:b+4])
    
40.         else:
    
41.             b = a + 9
    
42. 
    
43.         a = html.find('img src=', b)
    
44. 
    
45.     return img_addrs
    
46. 
    
47. 
    
48. def save_imgs(folder, img_addrs):
    
49.     for each in img_addrs:
    
50.         filename = each.split('/')[-2]
    
51.         with open(filename, 'wb') as f:
    
52.             img = url_open(each)
    
53.             f.write(img)
    
54. 
    
55. 
    
56. def download_mm(folder='OOXX', pages=10):
    
57.     os.mkdir(folder)
    
58.     os.chdir(folder)
    
59. 
    
60.     url = "www.meizitu.com/a/pure.html"
    
61.     page_num = int(get_page(url))
    
62. 
    
63.     for i in range(pages):
    
64.         page_num += 1
    
65.         page_url = 'www.meizitu.com/a/pure_' + str(page_num) + 'html'
    
66.         img_addrs = find_imgs(page_url)
    
67.         save_imgs(folder, img_addrs)
    
68. 
    
69. if __name__ == '__main__':
    
70.     download_mm()
    

复制代码

°蓝鲤歌蓝

wow7jiao 发表于 2018-1-24 11:53
你想说的是meizitu.com吧，我去爬了还是一样出问题

网址打错了

°蓝鲤歌蓝

°蓝鲤歌蓝 发表于 2018-1-24 12:08
网址打错了

不好意思是 mztu

°蓝鲤歌蓝

°蓝鲤歌蓝 发表于 2018-1-24 12:08
网址打错了

不好意思是 mztu

wow7jiao

°蓝鲤歌蓝 发表于 2018-1-24 12:11
http://www.mzitu.com/ 网络有点卡，一直打错很抱歉

图下来了，图片可以打开，内容很惊喜

1. import urllib.request
   
2. import os
   
3. 
   
4. 
   
5. 
   
6. def url_open(url):
   
7.     req = urllib.request.Request(url)
   
8.     req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.65 Safari/537.36')
   
9. 
   
10.    
    
11.     response = urllib.request.urlopen(req)
    
12.     html = response.read()
    
13. 
    
14.     return html
    
15. 
    
16. 
    
17. 
    
18. 
    
19. def find_imgs(url):
    
20.     html = url_open(url).decode('utf-8')
    
21.     img_addrs = []
    
22. 
    
23.     a = html.find('src=')
    
24. 
    
25.     while a != -1:
    
26.         b = html.find('.jpg', a, a+255)
    
27.         if b != -1:
    
28.             img_addrs.append(html[a+5:b+4])
    
29.         else:
    
30.             b = a + 9
    
31. 
    
32.         a = html.find('src=', b)
    
33. 
    
34.     return img_addrs
    
35. 
    
36. 
    
37. def save_imgs(folder, img_addrs):
    
38.     for each in img_addrs:
    
39.         filename = each.split('/')[-1]
    
40.         with open(filename, 'wb') as f:
    
41.             img = url_open(each)
    
42.             f.write(img)
    
43. 
    
44. 
    
45. def download_mm(folder='OOXX', pages=10):
    
46.     os.mkdir(folder)
    
47.     os.chdir(folder)
    
48. 
    
49.     url = "http://www.mzitu.com/mm/"
    
50.     page_num = 1
    
51. 
    
52.     for i in range(pages):
    
53.         page_num += 1
    
54.         page_url = url + 'page/' + str(page_num) + '/'
    
55.         img_addrs = find_imgs(page_url)
    
56.         save_imgs(folder, img_addrs)
    
57. 
    
58. if __name__ == '__main__':
    
59.     download_mm()
    

复制代码

°蓝鲤歌蓝

wow7jiao 发表于 2018-1-24 13:02
图下来了，图片可以打开，内容很惊喜

还可以换一些网站发现更多的惊喜，后面可以尝试爬取小说，电影排行榜，视频等等。