今天就跟大家聊聊有关如何使用Python爬虫抓取网页图片,可能很多人都不太了解,为了让大家更加了解,小编给大家总结了以下内容,希望大家根据这篇文章可以有所收获。
代码
__author__ = "JentZhang"
import urllib.request
import os
import random
import re
def url_open(url):
'''
打开网页
:param url:
:return:
'''
req = urllib.request.Request(url)
req.add_header('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36')
# 应用代理
'''
proxyies = ["111.155.116.237:8123","101.236.23.202:8866","122.114.31.177:808"]
proxy = random.choice(proxyies)
proxy_support = urllib.request.ProxyHandler({"http": proxy})
opener = urllib.request.build_opener(proxy_support)
urllib.request.install_opener(opener)
'''
response = urllib.request.urlopen(url)
html = response.read()
return html
def save_img(folder, img_addrs):
'''
保存图片
:param folder: 要保存的文件夹
:param img_addrs: 图片地址(列表)
:return:
'''
# 创建文件夹用来存放图片
if not os.path.exists(folder):
os.mkdir(folder)
os.chdir(folder)
for each in img_addrs:
filename = each.split('/')[-1]
try:
with open(filename, 'wb') as f:
img = url_open("http:" + each)
f.write(img)
except urllib.error.HTTPError as e:
# print(e.reason)
pass
print('完毕!')
def find_imgs(url):
'''
获取全部的图片链接
:param url: 连接地址
:return: 图片地址的列表
'''
html = url_open(url).decode("utf-8")
img_addrs = re.findall(r'src="(.+?\.gif)', html)
return img_addrs
def get_page(url):
'''
获取当前一共有多少页的图片
:param url: 网页地址
:return:
'''
html = url_open(url).decode('utf-8')
a = html.find("current-comment-page") + 23
b = html.find("]</span>", a)
return html[a:b]
def download_mm(url="http://jandan.net/ooxx/", folder="OOXX", pages=1):
'''
主程序(下载图片)
:param folder:默认存放的文件夹
:param pages: 下载的页数
:return:
'''
page_num = int(get_page(url))
for i in range(pages):
page_num -= i
page_url = url + "page-" + str(page_num) + "#comments"
img_addrs = find_imgs(page_url)
save_img(folder, img_addrs)
if __name__ == "__main__":
download_mm()
看完上述内容,你们对如何使用Python爬虫抓取网页图片有进一步的了解吗?如果还想了解更多知识或者相关内容,请关注天达云行业资讯频道,感谢大家的支持。