如何在Java中处理非UTF8html页面？

Question

我的任务是使用Java从urls中检索html字符串。

我知道如何使用HttpUrlConnection & InputStream来获取字符串。

但是，对于某些页面，我遇到了编码问题。

如果一些页面有不同的编码(例如，GB2312)，而不是UTF8，我得到的字符串只是任意字符或问号。

谁能告诉我如何解决这个问题？

谢谢

下面是我从url下载html的代码。

private String downloadHtml(String urlString) {
    URL url = null;
    InputStream inStr = null;
    StringBuffer buffer = new StringBuffer();

    try {
        url = new URL(urlString);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // Cast shouldn't fail
        HttpURLConnection.setFollowRedirects(true);
        // allow both GZip and Deflate (ZLib) encodings
        //conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); 
        String encoding = conn.getContentEncoding();
        inStr = null;

        // create the appropriate stream wrapper based on
        // the encoding type
        if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
            inStr = new GZIPInputStream(conn.getInputStream());
        } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
            inStr = new InflaterInputStream(conn.getInputStream(),
              new Inflater(true));
        } else {
            inStr = conn.getInputStream();
        }
        int ptr = 0;


        InputStreamReader inStrReader = new InputStreamReader(inStr, Charset.forName("GB2312"));

        while ((ptr = inStrReader.read()) != -1) {
            buffer.append((char)ptr);
        }
        inStrReader.close();

        conn.disconnect();
    }
    catch(Exception e) {

        e.printStackTrace();
    }
    finally {
        if (inStr != null)
            try {
                inStr.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
    }

    return buffer.toString();
}

Answer 1

通过使用InputStreamReader并指定您的字符集，如下所示：

inStr = new InputStreamReader(InputStream, Charset.forName("GB2312"));

以下代码适用于我：

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.zip.GZIPInputStream;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;

public class Foo {

public static void main(String[] args) {
    System.out.println(downloadHtml("http://baike.baidu.com/view/6000001.htm"));
}


private static String downloadHtml(String urlString) {
    URL url = null;
    InputStream inStr = null;
    StringBuffer buffer = new StringBuffer();

    try {
        url = new URL(urlString);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection(); // Cast shouldn't fail
        HttpURLConnection.setFollowRedirects(true);
        // allow both GZip and Deflate (ZLib) encodings
        //conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); 
        String encoding = conn.getContentEncoding();
        inStr = null;

        // create the appropriate stream wrapper based on
        // the encoding type
        if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
            inStr = new GZIPInputStream(conn.getInputStream());
        } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
            inStr = new InflaterInputStream(conn.getInputStream(),
              new Inflater(true));
        } else {
            inStr = conn.getInputStream();
        }
        int ptr = 0;


        InputStreamReader inStrReader = new InputStreamReader(inStr, Charset.forName("GB2312"));

        while ((ptr = inStrReader.read()) != -1) {
            buffer.append((char)ptr);
        }
        inStrReader.close();

        conn.disconnect();
    }
    catch(Exception e) {

        e.printStackTrace();
    }
    finally {
        if (inStr != null)
            try {
                inStr.close();
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
    }

    return buffer.toString();
  }

}

Answer 2

使用InputStreamReader读取inputStream，使用构造函数InputStreamReader(InputStream in，Charset cs)