|
50鱼币
看了小甲鱼《一只爬虫的自我修养:OOXX》,无论是用自己写的代码还是小甲鱼的源代码都无法成功获取图片,错误代码:HTTP Error 503: Service Temporarily Unavailable,用小甲鱼的代码则是出现[WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应,连接尝试失败。去掉IP代理也是一样,求解原因
代码如下:
import urllib.request
import os
import random
def open_url(url):
print(url)
req = urllib.request.Request(url)
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36')
proxies = ['1.182.183.2:8888', '36.7.172.18:82','42.248.207.88:8888']
proxy = random.choice(proxies)
proxy_support = urllib.request.ProxyHandler({'http':proxy})
opener = urllib.request.build_opener(proxy_support)
urllib.request.install_opener(opener)
response = urllib.request.urlopen(url)
html = response.read()
print(html)
return html
def get_page(url):
html = open_url(url).decode('utf-8')
a = html.find('current-comment-page')+23
b = html.find(']',a)
print(html[a:b])
return html[a:b]
def find_imgs(url):
html = open_url(url).decode('utf-8')
img_addrs=[]
a = html.find('a href=')
while a != -1:
b = html.find('\" target',a)
if b != -1:
img_addrs.append(html[a+8:b])
else:
b = a + 8
a = html.find('a href=',b)
return img_addrs
def save_imgs(folder,img_addrs):
for each in img_addrs:
filename = each.split('/')[-1]
with open(filename,'wb') as f:
img = open_url(each)
print(each)
f.write(img)
def download_mm(folder='ooxx',pages=10):
os.mkdir(folder)
os.chdir(folder)
url = 'http://jandan.net/ooxx/'
page_num = int(get_page(url))
for i in range(pages):
page_num -= i
page_url = url + 'page-' + str(page_num) + '#comments'
img_addrs = find_imgs(page_url)
save_imgs(folder,img_addrs)
if __name__=='__main__':
download_mm()
时间过了这么久了,网页的地址,图片的地址可能都有变化,还有就是之前看一个帖子说煎蛋网加入了反爬虫机制,所以直接用原来的方法爬,肯定会出问题的。
以下是解决方案,
可以添加一些代码将程序伪装成人为操作
比如加入浏览器的一些代码(小甲鱼老师在55课隐藏有讲)
- data['type'] = 'AUTO'
- data['i'] = content
- data['doctype'] = 'json'
- data['xmlVersion'] = '1.8'
- data['keyfrom'] = 'fanyi.web'
- data['ue'] = 'UTF-8'
- data['action'] = 'FY_BY_CLICKBUTTON'
- data['typoResult'] = 'true'
复制代码
或者换一个网站,比如【 www.chunmm.com】。。。仅供参考,试试就行啊,记住就测试一下代码能不能用就行啊。。。
刚写的代码,测试可用【仅供参考】
- import urllib.request as u
- import os
- def url_open(url):
- req = u.Request(url)
- req.add_header('User-Agent' , 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0')
- response = u.urlopen(url)
- html = response.read()
- return html
- def find_imgs(url):
- html = url_open(url).decode('utf-8')
- img_addrs = []
- a = html.find('img id="maxImg1" src=')
- b = html.find('.jpg', a , a+255)
- if b != 1:
- img_addrs.append(html[a+22:b+4])
- print(img_addrs[0])
- return img_addrs[0]
-
- def save_imgs(folder, img_address):
- filename = img_address.split('/')[-1]
- with open(filename, 'wb') as f:
- img = url_open(img_address)
- f.write(img)
- def download_mm(folder = 'jiandan', pages = 10):
- # 生成文件夹
- os.mkdir(folder)
- # 改变工作目录
- os.chdir(folder)
- url = 'http://www.symmz.com/xinggan/3384-'
- #page_num = int(get_page(url))
-
- for i in range(1,pages):
- #page_num -= i
- page_url = url + str(i) + '.html'
- print(page_url)
- img_address = find_imgs(page_url)
- save_imgs(folder, img_address)
- if __name__ == '__main__':
- download_mm()
复制代码
|
最佳答案
查看完整内容
时间过了这么久了,网页的地址,图片的地址可能都有变化,还有就是之前看一个帖子说煎蛋网加入了反爬虫机制,所以直接用原来的方法爬,肯定会出问题的。
以下是解决方案,
可以添加一些代码将程序伪装成人为操作
比如加入浏览器的一些代码(小甲鱼老师在55课隐藏有讲)
或者换一个网站,比如【www.chunmm.com】。。。仅供参考,试试就行啊,记住就测试一下代码能不能用就行啊。。。
刚写的代码,测试可用【仅供参考】
|