[已解决]爬虫程序下载图片，但图片地址没有http:的前缀，显示url错误

当回首遇上转身

系统版本 windows7    python版本3.5.2

网站图片源码没有http：前缀该如何处理？
参考于：小甲鱼python爬虫教程《论一只爬虫的自我修养4：OOXX》

1. import urllib.request
   
2. import os
   
3. 
   
4. 
   
5. def url_open(url):
   
6.     req =urllib.request.Request(url)
   
7.     req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36')
   
8.     response = urllib.request.urlopen(url)
   
9.     html = response.read()
   
10. 
    
11.     print(url)
    
12.     return html
    
13. 
    
14.    
    
15. def get_page(url):
    
16.     html = url_open(url).decode('utf-8')
    
17. 
    
18.     a = html.find('current-comment-page') + 23
    
19.     b = html.find(']',a)
    
20. 
    
21.     return html[a:b]
    
22. 
    
23.    
    
24. def find_imgs(url):
    
25.     html = url_open(url).decode('utf-8')
    
26.     img_addrs = []
    
27. 
    
28.     a = html.find('img src=')
    
29.    
    
30.     while a!= -1:
    
31.         b = html.find('.jpg', a, a+255)
    
32.         if b != -1:
    
33.             img_addrs.append(html[a+9:b+4])
    
34.         else:
    
35.             b = a + 9
    
36.             
    
37.         a = html.find('img src=',b)
    
38. 
    
39.     for each in img_addrs:
    
40.         print(each)
    
41. 
    
42.     return img_addrs
    
43. 
    
44.    
    
45. def save_imgs(folder, img_addrs):
    
46.     for each in img_addrs:
    
47.         filename = each.split('/')[-1]
    
48.         with open(filename, 'wb') as f:
    
49.             img = url_open(each)
    
50.             f.write(img)
    
51. 
    
52. 
    
53. def download_mm(folder='OOXX', pages=10):
    
54.     os.mkdir(folder)
    
55.     os.chdir(folder)
    
56. 
    
57.     url = "http://jandan.net/ooxx/"
    
58.     page_num = int(get_page(url))
    
59. 
    
60.     for i in range(pages):
    
61.         page_num -= i
    
62.         page_url = url + 'page-' + str(page_num) + '#comments'
    
63.         img_addrs = find_imgs(page_url)
    
64.         save_imgs(folder, img_addrs)
    
65. 
    
66.         
    
67. if __name__ == '__main__':
    
68.     download_mm()
    

复制代码

显示错误

1. 
   
2. ============ RESTART: E:\python\practice\2017.4.30\download_mm.py ============
   
3. [url]http://jandan.net/ooxx/[/url]
   
4. [url]http://jandan.net/ooxx/page-26#comments[/url]
   
5. //wx2.sinaimg.cn/mw600/661eb95cly1ff3biqf7yqj20iz0sgq7o.jpg
   
6. Traceback (most recent call last):
   
7.   File "E:\python\practice\2017.4.30\download_mm.py", line 69, in <module>
   
8.     download_mm()
   
9.   File "E:\python\practice\2017.4.30\download_mm.py", line 65, in download_mm
   
10.     save_imgs(folder, img_addrs)
    
11.   File "E:\python\practice\2017.4.30\download_mm.py", line 50, in save_imgs
    
12.     img = url_open(each)
    
13.   File "E:\python\practice\2017.4.30\download_mm.py", line 7, in url_open
    
14.     req =urllib.request.Request(url)
    
15.   File "C:\Users\Administrator\AppData\Local\Programs\Python\Python35\lib\urllib\request.py", line 269, in __init__
    
16.     self.full_url = url
    
17.   File "C:\Users\Administrator\AppData\Local\Programs\Python\Python35\lib\urllib\request.py", line 295, in full_url
    
18.     self._parse()
    
19.   File "C:\Users\Administrator\AppData\Local\Programs\Python\Python35\lib\urllib\request.py", line 324, in _parse
    
20.     raise ValueError("unknown url type: %r" % self.full_url)
    
21. ValueError: unknown url type: '//wx2.sinaimg.cn/mw600/661eb95cly1ff3biqf7yqj20iz0sgq7o.jpg'
    
22. >>>

复制代码

最佳答案

月排行榜 / 总排行榜

ooxx7788

2017-5-3 12:49:50

自己在前面加上http呗

33行
img_addrs.append('http:'+tml[a+9:b+4])

跳转到最佳答案楼层

hbkhata

my python code:

import urllib.request
import os

def url_open(url):
    req = urllib.request.Request(url)
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.81 Safari/537.36')
    response = urllib.request.urlopen(url)
    html = response.read()
    return html


def get_page(url):

    html =url_open(url).decode('utf-8')
   

    a = html.find('current-comment-page')+23
    b = html.find(']',a)

    return html[a:b]
                  

def find_imgs(url):
    html =  url_open(url).decode('utf-8')

    img_addrs = []

    a = html.find('img src=')

    while a !=-1:

         b = html.find('.jpg',a,a+255)

         if b!= -1:
             img_addrs.append(html[a+9:b+4])
         else:
             b=a+9

         a = html.find('img src=',b)

    return img_addrs



def save_img(folder,img_addrs):
    for each in img_addrs:
        filenames = each.split('/')[-1]
        with open(filenames,'wb')as f:
            img = url_open(each)
            f.write(img)

def download_mm(folder='OOXX',pages=10):
    os.mkdir(folder)
    os.chdir(folder)


    url = "http://jandan.net/ooxx/"
    page_num = int(get_page(url))

    for i in range(pages):
        
        page_num -=i
        
        page_url = url + 'page-' + str(page_num) + '#comments'

        img_addrs = find_imgs(page_url)

        save_img(folder,img_addrs)



if __name__ == '__main__':
    download_mm()


出現
ValueError: unknown url type: '//wx2.sinaimg.cn/mw600/661eb95cly1ff3biqf7yqj20iz0sgq7o.jpg'

請教如何解決阿

ooxx7788

自己在前面加上http呗

33行
img_addrs.append('http:'+tml[a+9:b+4])

当回首遇上转身

ooxx7788 发表于 2017-5-3 12:49
自己在前面加上http呗

33行

我自行解决了，不过还是送你一个最佳吧

730327319

当回首遇上转身 发表于 2017-5-11 13:51
我自行解决了，不过还是送你一个最佳吧

你怎么解决的？？？

赵柯

楼主解决了，也应该把方法发出来啊，给小白普及下。