爬虫之利用Scrapy进行图片的爬取

一：Scrapy自带管道方法爬取

scrapy genspider image360 image360.com

ITEM_PIPELINES = { # 'reptile.pipelines.ReptilePipeline': 300, 'scrapy.pipelines.images.ImagesPipeline':300, }

​ 新增图片保存路径

# 下载图片时需配置路径 IMAGES_STORE = "images"

class Image360(scrapy.Item): # 定义图片url image_urls = scrapy.Field()

import scrapy

from ..items import Image360

class Image360Spider(scrapy.Spider): name = 'image360' # allowed_domains = ['image360.com'] start_urls = ['https://image.so.com/zjl?ch=beauty&sn=30&listtype=new&temp=1']

def parse(self, response): # 实例化item对象 item = Image360()

# 获取图片保存的列表信息并转换为python格式 content = response.json()["list"]

# 获取图片url image_urls = [msg["imgurl"] for msg in content]

# 处理数据 item["image_urls"] = image_urls

yield item

scrapy crawl image360

​ 执行后会在项目目录下生成一个二级文件夹，用scrapy自带的管道类爬取图片非差方便，但我们可以发现，图片的名称是由一长串数字组成的，只是因为scrapy对我们的文件名进行了加密(images.py中的file_path方法中可看到加密函数，加密方式为hash的sha1加密)，所以我们需要自定义管道类来进行保存。

二：自定义管道方法爬取

scrapy genspider image360 image360.com

class Image360(scrapy.Item): # 定义图片url image_urls = scrapy.Field() # 定义图片名字 image_name = scrapy.Field()

import scrapy # 思路 # 获取保存图片的url # 请求进行下载

from ..items import Image360

class Image360Spider(scrapy.Spider): name = 'image360' # allowed_domains = ['image360.com'] start_urls = ['https://image.so.com/zjl?ch=beauty&sn=30&listtype=new&temp=1']

def parse(self, response): # 实例化item对象 item = Image360()

# 获取图片保存的列表信息并转换为python格式 content = response.json()["list"]

# 获取图片名称 image_name = [msg["title"] for msg in content]

# 获取图片url image_urls = [msg["imgurl"] for msg in content]

# 处理数据 item["image_name"] = img_name item["image_urls"] = image_urls

yield item

import scrapy from scrapy.pipelines.images import ImagesPipeline

class Image360Pipeline(ImagesPipeline): # 重写get_media_requests方法是为了获取item def get_media_requests(self,item,info): # 此时获取到的item是一个列表，需要循环获取每一个url，并将相应的response对象传给file_path for url in item["image_urls"]: yield scrapy.Request(url=url,meta={"index":item["image_urls"].index(url),"item":item})

# 重写file_path方法是为了修改文件名 def file_path(self,request,response=None,info=None): # 获取item item = request.meta["item"] # 获取索引 index = request.meta["index"] return f'full/{item["image_name"][index]}.jpg'

ITEM_PIPELINES = { 'reptile.pipelines.Image360Pipeline': 300, }