用于抓取百度搜索到的第一个网站的源码代码,Java代码库,德仔网

代码语言
代码分类
【Java】用于抓取百度搜索到的第一个网站的源码
作者:Staryong / 发布于2013/11/28/ 664
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.codec.binary.StringUtils;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.io.IOUtils;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.ParseException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;

import java.io.File; 
import java.io.IOException;
import java.sql.Timestamp;
import java.lang.*;
import org.jsoup.Jsoup; 
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.awt.*;

import javax.swing.JFrame;



public class CatchTestApp {
 private final static HttpClient httpClient = new DefaultHttpClient();
 /**
  * @param args
  */
 public static void main(String[] args) {
  // TODO Auto-generated method stub 
  try{
   System.out.println("输入你的要抓取的网页名称：");
   BufferedReader in = new BufferedReader(new InputStreamReader(System.in)); 
   String s = in.readLine();
   String keyWord = s; 
   String url = LocateURL(keyWord);
   String charset = LocateCharSet(url);
   System.out.println("显示抓取到的页面编码方式："+charset);
   CatchWeb(url,keyWord,charset);
  }catch(IOException e){
   e.printStackTrace();
  }
 }
 
 //定位百度搜索到的第一条网站URL方法
 public static String LocateURL(String keyword){  
  //设置百度查询的结果
  String threadUrl ="http://www.baidu.com/s?wd="+keyword;
  try{
  Document doc = (Document)Jsoup.connect(threadUrl).get(); //用Document记录页面信息
  //System.out.println(doc);
  //定位到一条网站URL
   for(Element ele : doc.getElementById("1").select("tbody > tr")){
       if(!ele.select("td").toString().equals("")){ 
         String url = ele.select("td").get(0).select("a").attr("href"); 
         System.out.println("显示抓取到的页面URL："+url);
         threadUrl = url;
       } 
   }
  
  }catch(IOException e){
   e.printStackTrace();
  }
  return threadUrl;
 }
 
 
 //获取所要抓取网页的编码方式
  public static String LocateCharSet(String url){  
   String sub = "utf-8";
   String s = "";
   String t="text";
   try{
   Document doc = Jsoup.connect(url).get();     
  //判断编码方式
   Elements eles = doc.getElementsByTag("head").select("meta[http-equiv]");
   for(Element ele :eles){
    if(ele.attr("http-equiv").equals("Content-Type")||
      ele.attr("http-equiv").equals("Content-type")||
      ele.attr("http-equiv").equals("content-type")||
      ele.attr("http-equiv").equals("content-Type")){
     String[] str = ele.attr("content").split("[=]");
     sub = str[1]; 
    }
    
   }
   
   //判断编码方式所在 (不要用这个方法，不好)
//   for(int i=0 ; s.equals(t)!= true ; i++){  
//     String charSet = doc.getElementsByTag("head").select("meta[http-equiv]").get(i).attr("content");  
//     s=charSet.substring(0,t.length());
//     String[] str = charSet.split("[=]");
//     sub = str[1];
//   }  
   }catch(IOException e){
    e.printStackTrace();
   }
    return sub;
   }
 
 
 private static Document parseJSoupDocumentFromHtml(String url) {
  // TODO Auto-generated method stub
  return null;
 }

 //抓取网页全部内容方法
 @SuppressWarnings("deprecation")
 public static void CatchWeb(String url,String keyword,String charset){  
  try{
   HttpGet httpget = new HttpGet(url);// Get请求
   HttpResponse httpresponse = httpClient.execute(httpget); // 发送请求
   HttpEntity entity = httpresponse.getEntity();// 获取返回数据 
   String chars = EntityUtils.getContentCharSet(entity);
   System.out.println("charset:"+charset);
   if(chars != null){
    charset = chars;
   }
   System.out.println("chars:"+chars);
   String info =EntityUtils.toString(entity, charset);
   System.out.println("显示响应信息状态："+httpresponse.getStatusLine());
   System.out.println(info);
   File(info,keyword,charset);  
  }catch(ParseException e){
   e.printStackTrace();
  }catch(IOException e){
   e.printStackTrace();
  }  
 }
 
 //将抓取的页面信息输出到txt文档
 public static void File(String info,String keyword,String charset){
  File file = new File("D:/抓获网页源码库/"+keyword+"网页源码.txt");
   try{
    FileOutputStream out = new FileOutputStream(file);
    byte b[]= info.getBytes(charset);
    System.out.println("数据正在写入文本文档中......");
    out.write(b);
    out.close();
    System.out.println("数据保存成功！");
   }catch(Exception e){
    e.printStackTrace();
   }
   
 }
 
}
试试其它关键字
　搜索源码　
同语言下
可能有用的
Staryong贡献的其它代码(3)