Jsoup初探

简介

jsoup is an open-source Java library of methods designed to extract and manipulate data stored in HTML documents.

Jsoup就是一个用java语言编写用来获取和操控html页面及元素的开源Jar包。
我们使用的一般是1.7的版本，博主用的是jsoup-1.7.2.jar

文档

jsoup有官方的中文使用文档，比较简单jsoup Cookbook(中文版)

用法

一小段代码爬取我的博客文章标题列表，带您简单了解它的用法
选择器截图：

主页选择器
package com.blog.jsoup;

import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class BlogSpider {

public static void main(String[] args) {
  try {
    Document doc = Jsoup.connect("http://www.jianshu.com/users/7a8e041a132c/latest_articles").get();
    //通过选择器获取文章列表，选择器dom元素直接通过Chrome开发者工具检查元素Copy -> Copy Selector即可获取
    Elements articleList = doc.select("#list-container > ul > li > div > h4 > a");
    for (int i = 0; i < articleList.size(); i++) {
      System.out.println(articleList.get(i).text());
    }
  } catch (IOException e) {
    e.printStackTrace();
  }

}

}

运行结果如下：

运行结果

细心的读者可能会发现进入我的博客主页之后最新文章列表不止爬取显示的九篇，而是会不断加载更多，这是因为访问这个页面时生成的静态页面只请求了第一页的数据，而随着用户下拉，异步加载了更多的数据，这个是在浏览器生成静态页面之后做的，所以在这里没有办法拿到。这说明了很重要的一点，Jsoup访问页面所拿到的html是我们传统意义上查看网页源代码的数据，而不是你在审查元素的那些数据，所以，想要拿到异步加载的数据需要去解析异步访问的js，然后通过抓包来查看访问的servlet看看能不能模拟访问拿到更多数据。

访问https页面

由于https的证书加密了，直接通过上述方法去访问并提取html页面也会报错

javax.net.ssl.SSLHandshakeException: java.security.cert.CertificateException: No subject alternative DNS name matching m.v.qq.com found.
at com.sun.net.ssl.internal.ssl.Alerts.getSSLException(Alerts.java:174)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1747)
at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:241)
at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:235)
at com.sun.net.ssl.internal.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1209)
at com.sun.net.ssl.internal.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:135)
at com.sun.net.ssl.internal.ssl.Handshaker.processLoop(Handshaker.java:593)
at com.sun.net.ssl.internal.ssl.Handshaker.process_record(Handshaker.java:529)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:943)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1188)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1215)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1199)
at sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:476)
at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:166)
at sun.net.www.protocol.https.HttpsURLConnectionImpl.connect(HttpsURLConnectionImpl.java:133)
at org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:439)
at org.jsoup.helper.HttpConnection$Response.execute(HttpConnection.java:424)
at org.jsoup.helper.HttpConnection.execute(HttpConnection.java:178)
at org.jsoup.helper.HttpConnection.get(HttpConnection.java:167)
at com.blog.jsoup.BlogSpider.main(BlogSpider.java:13)
Caused by: java.security.cert.CertificateException: No subject alternative DNS name matching m.v.qq.com found.
at sun.security.util.HostnameChecker.matchDNS(HostnameChecker.java:193)
at sun.security.util.HostnameChecker.match(HostnameChecker.java:77)
at com.sun.net.ssl.internal.ssl.X509TrustManagerImpl.checkIdentity(X509TrustManagerImpl.java:264)
at com.sun.net.ssl.internal.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:250)
at com.sun.net.ssl.internal.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1188)
... 15 more

好在java有提供相应的方法去访问：
为了阅读方便，贴出整个class

package com.blog.jsoup;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.security.SecureRandom;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import javax.security.cert.CertificateException;
import javax.security.cert.X509Certificate;

import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class BlogSpider {

public static void main(String[] args) {
try {
  Document doc = Jsoup.connect("http://www.jianshu.com/users/7a8e041a132c/latest_articles").get();
  // 通过选择器获取文章列表，选择器dom元素直接通过Chrome开发者工具检查元素Copy -> Copy Selector即可获取
  Elements articleList = doc.select("#list-container > ul > li > div > h4 > a");
  for (int i = 0; i < articleList.size(); i++) {
    System.out.println(articleList.get(i).text());
  }
} catch (IOException e) {
  e.printStackTrace();
}
try {
  SSLSocketFactory.getSocketFactory().setHostnameVerifier(new AllowAllHostnameVerifier());

  String https = "https://m.v.qq.com/x/page/xxxxxxxxx.html";
  // Document doc = Jsoup.connect(https).get();
  init();
  HttpsURLConnection conn = (HttpsURLConnection) new URL(https).openConnection();
  conn.setDoOutput(true);
  conn.setDoInput(true);
  conn.connect();
  BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
  StringBuffer sb = new StringBuffer();
  String line;
  while ((line = br.readLine()) != null)
    sb.append(line);

  Document doc = Jsoup.parse(sb.toString());
} catch (Exception e) {
  e.printStackTrace();
}

}

public static void init() {
try {
  SSLContext sc = SSLContext.getInstance("TLS");
  sc.init(null, new TrustManager[] { new X509TrustManager() {
    public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
    }

    public void checkServerTrusted(X509Certificate[] chain, String authType)

        throws CertificateException {
    }

    @Override
    public java.security.cert.X509Certificate[] getAcceptedIssuers() {
      return null;
    }

    @Override
    public void checkClientTrusted(java.security.cert.X509Certificate[] arg0, String arg1)
        throws java.security.cert.CertificateException {
      // TODO Auto-generated method stub

    }

    @Override
    public void checkServerTrusted(java.security.cert.X509Certificate[] arg0, String arg1)
        throws java.security.cert.CertificateException {
      // TODO Auto-generated method stub

    }
  } }, new SecureRandom());
  HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
  HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
    @Override
    public boolean verify(String arg0, SSLSession arg1) {
      return true;
    }

  });
} catch (Exception e) {
  e.printStackTrace();
}
}

}
最后编辑于 ：
©著作权归作者所有,转载或内容合作请联系作者
平台声明：文章内容（如有图片或视频亦包括在内）由作者上传并发布，文章内容仅代表作者本人观点，简书系信息发布平台，仅提供信息存储服务。

推荐阅读更多精彩内容

  • Android - 收藏集
    Android 自定义View的各种姿势1 Activity的显示之ViewRootImpl详解 Activity...
    passiontim阅读 174,831评论 25赞 709
  • Spring Cloud
    Spring Cloud为开发人员提供了快速构建分布式系统中一些常见模式的工具（例如配置管理，服务发现，断路器，智...
    卡卡罗2017阅读 135,288评论 19赞 139
  • walk the dog
    镇上唯一一间小旅馆，从斑驳的墙纸已能闻出破败气息，但显然并不减少在此逗留的人气。毕竟，小镇上种种醋雨酸风就是在旅馆...
    INKMAX阅读 295评论 0赞 0
  • 游四方山（考场作文）
    就这样我们走走停停，经过一段平坦的公路，便又是一条蜿蜒的山路。路的两旁不再是樱花，而是参天大树，树荫把太阳给遮住了...
    小肥皂sy阅读 847评论 5赞 4
  • 发
    了
    文字苏梦阅读 263评论 0赞 0