Python爬虫，抓取代理地址，可配合其他工具使用

fl93

RT


做完了python入门的所有作业

老甲鱼视频里说做一个自动抓代理地址的爬虫

经过尝试，代码在最后

主要作为配合其他工具代理使用

纯新手，请老司机们刀下留情

有任何问题和建议请留言，期待你的回复

1. import urllib.request
   
2. import urllib.parse
   
3. import chardet
   
4. import gzip
   
5. import json
   
6. import re
   
7. from bs4 import BeautifulSoup
   
8. 
   
9. 
   
10. 
    
11. def get_proxylist(proxy_info=None):
    
12.     '''
    
13.     get_proxylist() is a func to get proxy ip addresses from http://www.gatherproxy.com/
    
14.     it will return proxy informations with a list
    
15.     each element in this list is a dictionary
    
16. 
    
17. 
    
18.     proxy_info is the proxy information to hiding your real ip address
    
19.     if you can not connect to http://www.gatherproxy.com or you get block by this server
    
20.     parameter format is like blow:
    
21.     "ip:port" --> "192.168.1.1:8080"
    
22. 
    
23.    
    
24.     you can access by folowing keywords
    
25. 
    
26.     important keyword:
    
27.     "PROXY_COUNTRY"  --> Proxy location
    
28.     "PROXY_IP"       --> Proxy service IP
    
29.     "PROXY_PORT"     --> Need to covert from Hex to Decimal(now is automatically coverted)
    
30.     "PROXY_STATUS"   --> OK is the default status
    
31.     "PROXY_UPTIMELD" --> Max capacity and how many people are using this proxy
    
32.    
    
33.     '''
    
34.    
    
35.     url = 'http://www.gatherproxy.com/'
    
36. 
    
37.     header = {
    
38.     'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.281 Safari/537.36'
    
39.     }
    
40.     try:
    
41.    
    
42.         if proxy_info:
    
43.             proxy_support = urllib.request.ProxyHandler({'http':proxy_info})
    
44.             req = urllib.request.Request(url=url, headers=header)
    
45.             opener = urllib.request.build_opener(proxy_support)
    
46.             
    
47.         else:
    
48.             req = urllib.request.Request(url=url, headers=header)
    
49.             opener = urllib.request.build_opener()
    
50. 
    
51.         response = opener.open(req)
    
52.    
    
53.         html = response.read()
    
54.         
    
55.         encode = chardet.detect(html)['encoding']
    
56.         #print('当前编码为:%s\n'%encode)
    
57.         if encode == 'GB2312':
    
58.             encode = 'GBK'
    
59.             
    
60.         #if not encode:
    
61.         #    html = gzip.decompress(html)
    
62.         #    encode = chardet.detect(html)['encoding']
    
63.         #    print('当前编码为:%s\n'%encode)
    
64. 
    
65.             
    
66.         html = html.decode(encode)
    
67.         soup = BeautifulSoup(html, 'html.parser')
    
68. 
    
69.         proxy_list = list()
    
70.         
    
71.         proxy_js = soup.find_all(type="text/javascript")
    
72.         for each_js in proxy_js:
    
73.             tmp = re.search(r'gp.insertPrx\(({.+\})\)', each_js.text)
    
74.             if tmp:
    
75.                 tmp = json.loads(tmp.group(1))
    
76.                 port_hex = tmp.get("PROXY_PORT")
    
77.                 tmp["PROXY_PORT"] = int(port_hex, 16)
    
78.                 proxy_list.append(tmp)
    
79.         
    
80.         return proxy_list
    
81. 
    
82.     except ConnectionResetError as e:
    
83.         print('Your ip address has been blocked by this server! please try again with proxy_info parameter filled\n')
    
84.         print('Error Information is: %s' %e)
    
85.    
    
86.     except urllib.error.URLError as e:
    
87.         print('Failed to connect to the proxy ip! please try again with new proxy information\n')
    
88.         print('Error Information is: %s' %e)
    

复制代码