【业余党】我也来分享一个妹纸图爬虫，大家多指点

Pear1994

有点烂，，，凑合着看吧

1. import requests
   
2. import os
   
3. from lxml import etree
   
4. import re
   
5. 
   
6. 
   
7. headers = {
   
8.     'Referer': 'http://www.mmjpg.com/mm/',
   
9.     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'
   
10. }
    
11. 
    
12. path = os.getcwd() + '\img'
    
13. 
    
14. 
    
15. def get_index():
    
16.     r = requests.get('http://www.mmjpg.com', headers=headers)
    
17.     r.encoding = 'utf-8'
    
18.     html = r.text
    
19.     selector = etree.HTML(html)
    
20.     content = selector.xpath('//li/a')[0]
    
21.     num = content.xpath('@href')[0]
    
22.     num = num.split('/')
    
23.     num = int(num[-1])
    
24.     return range(1, num + 1)
    
25. 
    
26. 
    
27. def get_page(url):
    
28.     r = requests.get(url, headers=headers)
    
29.     r.encoding = 'utf-8'
    
30.     html = r.text
    
31.     # print(html)
    
32.     selector = etree.HTML(html)
    
33.     page = selector.xpath('//div[@id="page"]/a/text()')[-2]
    
34.     return(page)
    
35. 
    
36. 
    
37. def get_img(url):
    
38.     r = requests.get(url, headers=headers)
    
39.     r.encoding = 'utf-8'
    
40.     html = r.text
    
41.     selector = etree.HTML(html)
    
42.     try:
    
43.         content = selector.xpath('//div[@id="content"]/a/img')[0]
    
44.         img_url = content.xpath('@src')[0]
    
45.         title = content.xpath('@alt')[0]
    
46.         sav_img(title, img_url)
    
47.     except Exception as e:
    
48.         print('Erro!!!')
    
49.         pass
    
50. 
    
51. 
    
52. def sav_img(name, url):
    
53.     name = name + '.jpg'
    
54.     if name in os.listdir(path):
    
55.         print('重复文件')
    
56.     else:
    
57.         r = requests.get(url, headers=headers)
    
58.         with open(name, 'wb') as f:
    
59.             f.write(r.content)
    
60.         print(name)
    
61. 
    
62. 
    
63. def main(index):
    
64.     main_url = 'http://www.mmjpg.com/mm/%s' % index
    
65.     page = get_page(main_url)
    
66.     if os.path.exists(path) != True:
    
67.         os.mkdir(path)
    
68.     else:
    
69.         os.chdir(path)
    
70.     # print(path)
    
71.     for i in range(1, int(page) + 1):
    
72.         url = '%s/%s' % (main_url, i)
    
73.         try:
    
74.             get_img(url)
    
75.         except Exception as e:
    
76.             raise e
    
77. 
    
78. 
    
79. if __name__ == '__main__':
    
80.     indexs = get_index()
    
81.     for index in reversed(indexs):
    
82.         main(index)
    

复制代码

fccnb

懒洋羊

alltolove

pony马

向这样的爬虫，我一般只看爬取的URL

壹个闲人

感谢分享

Yaohr

59kkkkk

小猫猫打怪升级

为什么我运行的都是闪一下就没了呢？