1.导入相关jar包

1.作用于页面解析<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.14.2</version>
</dependency>2.http请求<dependency><groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
</dependency>

2.获取页面源码

//1.创建对象
CloseableHttpClient httpClient = HttpClients.createDefault();

//设置请求地址http://yun.itheima.com/search?keys=java
URIBuilder uriBuilder = new URIBuilder(“https://xy.51job.com/default-xs.php”);
//设置请求信息
// uriBuilder.setParameter(“callback”,”jQuery1192927″);
// uriBuilder.setParameter(“page”,”1″);
// uriBuilder.setParameter(“productId”,”100008348530″);

//2.访问地址
HttpGet httpGet = new HttpGet(uriBuilder.build());
// 模拟浏览器浏览（user-agent的值可以通过浏览器浏览，查看发出请求的头文件获取）
httpGet.setHeader(“user-agent”, “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36”);
//3.发起请求
CloseableHttpResponse response = httpClient.execute(httpGet);
//4.解析响应
if (response.getStatusLine().getStatusCode() == 200) {
// HttpEntity表示http的request和resposne实体，它由消息头和消息体组成。
// 从HttpEntity中可以获取http请求头和回应头，也可以获取http请求体和回应体信息。
//HttpEntity的使用，与@RequestBody 、@ResponseBody类似。
HttpEntity entity = response.getEntity();
String content = EntityUtils.toString(entity, “gb2312”);
//解析页面获取数据
tests(content);
// System.out.println(content);
}
//5.关闭response
response.close();
httpClient.close();

3.解析页面  1.通过jQuery 语法来获取页面信息

//1.解析Url地址
Document doc = Jsoup.parse(string);
//2.使用标签选择器，获取title
Elements spuEles = doc.select(“div.ctxt > div.cell > div.e”);
System.out.println(String.valueOf(spuEles));
for (Element spuEle : spuEles) {
//获取招聘标题
String topic = spuEle.select(“[target]”).attr(“title”);
System.out.println(“==================topic================” + topic);
//获取招聘公司url
String url = spuEle.select(“[target]”).attr(“href”);
}

4.解析页面  2.获取页面js 数据  进行 爬取

//获取数据
String iphone = iphone();
System.err.println(iphone);
if(!””.equals(iphone)){
//掐头
String replace = iphone.replace(“fetchJSON_comment98(“, “”);
//去尾
String data = replace.substring(0, replace.lastIndexOf(“;”) – 1);

JSONObject action = JSONObject.parseObject(data);

JSONArray comments = action.getJSONArray(“comments”);

List<Comments> commentsList = comments.toJavaList(Comments.class);

for (Comments string : commentsList) {
System.out.println(“===============用户====================”+string);
commentService.insert(string);
}
if(page <= 100){
System.err.println(page);
insert();
}

}else {

System.out.println(“===========没有用户==============”);
}