当我们想要获取百度图片的时候,面对一张张图片,一次次的点击右键下载十分麻烦。python爬虫可以实现批量下载,根据我们下载网站位置、图片位置、图片下载数量、图片下载位置等需求进行批量下载,本文演示python爬虫批量爬取百度图片的代码过程。
1、导入相关库
import requests#爬虫库 import re#正则表达式库 import os#系统库 import time#时间库
2、确定百度图片链接
url='http://image.baidu.com/search/index?tn=baiduimage&fm=result&ie=utf-8&word='#百度链接
3、解码
# 解码def decode(url): for key, value in str_table.items(): url = url.replace(key, value) return url.translate(char_table)
4、百度图片下拉
def buildUrls(word): word = urllib.parse.quote(word) url = r"http://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&fp= result&queryWord={word}&cl=2&lm=-1&ie=utf-8&oe=utf-8&st=-1&ic=0&word={word}&face=0&istype=2nc=1&pn={pn}&rn=60" urls = (url.format(word=word, pn=x) for x in itertools.count(start=0, step=60)) return urls re_url = re.compile(r'"objURL":"(.*?)"')
5、使用正则表达式获取图片数量
totalnum = re.findall('<div id="resultInfo" style="font-size: 13px;">(.*?)</div>', string) print("百度图片"+totalnum[0])img_url_regex = '"thumbURL":"(.*?)",'#正则匹配式count=0#总共下载 的图片数index=0#链接后面的序号page=0#当前搜集的页while(1): strhtml=requests.get(url+str(index),headers=headers)#get方式获取数据 string=str(strhtml.text) print("已爬取网页")
6、利用正则表达式找到图片url
print("第"+str(page+1)+"页共收集到"+str(len(pic_url))+"张图片")
7、创建文件路径下载
def mkDir(dirName): dirpath = os.path.join(sys.path[0], dirName) if not os.path.exists(dirpath): os.mkdir(dirpath) return dirpathif __name__ == '__main__': with open('list.txt', 'r', encoding="utf-8") as f: result = f.read() keys = result.split('\n') key_words = list(enumerate(keys, start=1)) for key in key_words: word = key[1] dirpath = mkDir(word) imgType = 'jpg' strtag = word numIMGS = 3 urls = buildUrls(word) index = 0 #print("= = " * 25) for url in urls: print("正在请求:", url) html = requests.get(url, timeout=10).content.decode('utf-8') imgUrls = resolveImgUrl(html) # print(imgUrls) if len(imgUrls) == 0: # 没有图片则结束 break for url in imgUrls: if downImgs(url, dirpath, strtag + ' ' + str(index + 1), imgType): index += 1 # print("已下载 %s 张" % index) # 双 break 跳出下载循环 if index == numIMGS: break if index == numIMGS: # print('您一共下载了 %s 张图片' % index) # print('程序正在终止') break
以上就是python爬虫批量爬取百度图片过程,大家直接套入代码使用就可以了。
神龙|纯净稳定代理IP免费测试>>>>>>>>天启|企业级代理IP免费测试>>>>>>>>IPIPGO|全球住宅代理IP免费测试