1.准备工作(依赖包)：redis和mysql请自行准备，此案例涉及到爬取数据的保存和去重(通过redis)

<dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.5.6</version> </dependency>

<dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>4.12</version> </dependency>

<dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.47</version> </dependency>

<dependency> <groupId>org.mybatis</groupId> <artifactId>mybatis</artifactId> <version>3.4.6</version> </dependency>

<dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.38</version> </dependency>

<dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>1.2.12</version> </dependency>

<dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>1.7.25</version> </dependency>

<dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <version>1.7.25</version> <scope>test</scope> </dependency>

<dependency> <groupId>redis.clients</groupId> <artifactId>jedis</artifactId> <version>3.0.1</version> </dependency>

2.配置文件

    2.1.db.properties

driver=com.mysql.jdbc.Driver url=jdbc:mysql:///spider?useUnicode=true&characterEncoding=UTF-8 username=root password=admin

    2.2.log4j.properties

# Global logging configuration log4j.rootLogger=ERROR, stdout # MyBatis logging configuration… log4j.logger.com.test.spider=TRACE # Console output… log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%5p [%t] – %m%n

   2.3.mybatis.xml

<?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE configuration PUBLIC "-//mybatis.org//DTD Config 3.0//EN" "http://mybatis.org/dtd/mybatis-3-config.dtd"> <configuration> <!–属性配置–> <properties resource="db.properties"/> <environments default="development"> <environment id="development"> <!–事务管理器–> <transactionManager type="JDBC"/> <!–数据源–> <dataSource type="POOLED"> <property name="driver" value="${driver}"/> <property name="url" value="${url}"/> <property name="username" value="${username}"/> <property name="password" value="${password}"/> </dataSource> </environment> </environments>

<mappers> <mapper resource="com/test/spider/mapper/UserMapper.xml"/> </mappers> </configuration>

3.实体类News:com.test.spider.bean;

package com.test.spider.bean;

import java.util.Date;

public class News { private Integer id; private String title; private String intro; private String source; private String vurl; private Date publishTime; @Override public String toString() { return "News{" + "id=" + id + ", title='" + title + ''' + ", intro='" + intro + ''' + ", source='" + source + ''' + ", vurl='" + vurl + ''' + ", publishTime=" + publishTime + '}'; }

public Integer getId() { return id; }

public void setId(Integer id) { this.id = id; }

public String getTitle() { return title; }

public void setTitle(String title) { this.title = title; }

public String getIntro() { return intro; }

public void setIntro(String intro) { this.intro = intro; }

public String getSource() { return source; }

public void setSource(String source) { this.source = source; }

public String getVurl() { return vurl; }

public void setVurl(String vurl) { this.vurl = vurl; }

public Date getPublishTime() { return publishTime; }

public void setPublishTime(Date publishTime) { this.publishTime = publishTime; } }

4.NewsMapper接口：com.test.spider.mapper;

package com.test.spider.mapper;

import com.test.spider.bean.News; import org.apache.ibatis.annotations.Param; import java.util.List;

public interface NewsMapper { Integer batchSave(@Param("news") List<News> list); }

5.NewsMapper.xml配置文件：com.test.spider.mapper;

<?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd"> <mapper namespace="com.test.spider.mapper.NewsMapper"> <insert id="batchSave"> insert into news(title,intro,source,vurl,publish_time) VALUES <foreach collection="news" separator="," item="item"> (#{item.title},#{item.intro},#{item.source},#{item.vurl},#{item.publishTime}) </foreach> </insert> </mapper>

6.MybatisUtil工具类：com.test.spider.util;

package com.test.spider.util;

import org.apache.ibatis.io.Resources; import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.session.SqlSessionFactoryBuilder;

import java.io.IOException; import java.io.InputStream;

public class MybatisUtil {

private static SqlSessionFactory factory = null;

static{ String resource = "mybatis.xml"; InputStream in = null; try { in = Resources.getResourceAsStream(resource); } catch (IOException e) { e.printStackTrace(); } factory = new SqlSessionFactoryBuilder().build(in); }

public static SqlSession getSession(){ return factory.openSession(); } }

7.EntertainmentSpider主要功能：com.test.spider;

package com.test.spider;

import com.alibaba.fastjson.JSON; import com.sun.org.apache.bcel.internal.generic.NEW; import com.test.spider.bean.News; import com.test.spider.mapper.NewsMapper; import com.test.spider.util.MybatisUtil; import jdk.nashorn.internal.scripts.JD; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.apache.ibatis.session.SqlSession; import org.junit.Test; import redis.clients.jedis.Jedis;

import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map;

/** * Created by thinkpad on 2019/9/18. */ public class EntertainmentSpider {

public void main() throws Exception { Integer page = 0; while(true){ //1.确定Url String indexUrl = "https://pacaio.match.qq.com/irs/rcd?cid=146&token=49cbb2154853ef1a74ff4e53723372ce&ext=ent&page="+page+"&callback=__jp7"; //2.发送请求,获取数据 //获取httpclient对象 CloseableHttpClient httpClient = HttpClients.createDefault(); //创建请求方式对象 HttpGet get = new HttpGet(indexUrl); //执行请求操作 CloseableHttpResponse response = httpClient.execute(get); if(response.getStatusLine().getStatusCode() == 200){ HttpEntity entity = response.getEntity(); String res = EntityUtils.toString(entity, "utf-8"); //3.解析数据 res = toJsonString(res); //截取字符串 //转为map对象 Map<String, Object> map = jsonToMap(res); //页面没有数据使,结束循环 if(Integer.parseInt(map.get("datanum").toString()) == 0){ break; } //转换为新闻对象 List<News> list = mapToBean(((List<Map>) map.get("data"))); //4.保存数据 //保存新闻数据 if(list.size()>0){ Integer count = saveNews(list); } System.out.println(page); } page++; } System.out.println("执行结束…."); }

//截取结果字符串 public String toJsonString(String src){ int start = src.indexOf("{"); int end = src.lastIndexOf("}")+1; return src.substring(start,end); }

//json字符串转换为java对象 public Map<String,Object> jsonToMap(String src){ return JSON.parseObject(src, HashMap.class); }

//新闻数据转换为新闻对象 public List<News> mapToBean(List<Map> src) throws ParseException {

SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

List<News> list = new ArrayList<News>();

for (Map map : src) { //获取url的值 String url = map.get("vurl").toString();

if(!checkUrl(url)){ continue; }

News news = new News(); news.setTitle(map.get("title").toString()); news.setIntro(map.get("intro").toString()); news.setSource(map.get("source").toString()); news.setVurl(url); String publish_time = map.get("publish_time").toString(); news.setPublishTime(format.parse(publish_time)); list.add(news); } return list; }

//保存新闻的数据 public int saveNews(List<News> list){

SqlSession session = MybatisUtil.getSession(); NewsMapper mapper = session.getMapper(NewsMapper.class); Integer res = mapper.batchSave(list); session.commit(); session.close(); return res; }

//判断url是否已经在redis中保存 public Boolean checkUrl(String url){ Jedis jedis = new Jedis("localhost",6379); Long count = jedis.sadd("bigdata:0701:spider:news:url", url); jedis.close(); return count>0?true:false; } }

8.App（测试类）:com.test.spider；

package com.test.spider;

import com.test.spider.EntertainmentSpider;

import java.util.Timer; import java.util.TimerTask;

public class App {

public static void main(String[] args) {

final EntertainmentSpider spider = new EntertainmentSpider(); //创建定时器 Timer timer = new Timer(); timer.schedule( new TimerTask() { @Override public void run() { try { spider.main();

} catch (Exception e) { e.printStackTrace(); } } }, 0, 1200000L ); } }