Go语言视频零基础入门到精通

javalearner

以下是一个java爬虫程序，它能从指定主页开始，按照指定的深度抓取该站点域名下的网页并维护简单索引。

参数：private static int webDepth = 2;//爬虫深度。

主页的深度为1，设置深度后超过该深度的网页不会抓取。     private int intThreadNum = 10;//线程数。开启的线程数。    抓取时也会在程序源文件目录下生成一个report.txt文件记录爬虫的运行情况，并在抓取结束后生成一个fileindex.txt文件维护网页文件索引。    本程序用到了多线程(静态变量和同步)，泛型，文件操作，URL类和连接，Hashtable类关联数组，正则表达式及其相关类。

     运行时需使用命令行参数，第一个参数应使用http://开头的有效URL字符串作为爬虫的主页，第二个参数（可选）应输入可转换为int型的字符串（用 Integer.parseInt(String s)静态方法可以转换的字符串，如3）作为爬虫深度，如果没有，则默认深度为2。
  
   
   
     
   

     
   
  
本程序的不足之处是：只考虑了href= href=" href="后加绝对url的这三种情况(由于url地址在网页源文件中情况比较复杂，有时处理也会出现错误)，还有相对url和 window.open("的情况没有考虑。异常处理程序也只是简单处理。如果读者有改进办法可以把源代码帖出，不胜感激。 附上源代码如下（保存名为GetWeb.java）：   import java.io.File;
import java.io.BufferedReader;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.Hashtable; public class GetWeb {
  private int webDepth = 2;//爬虫深度
  private int intThreadNum = 10;//线程数
  private String strHomePage = "";//主页地址
  private String myDomain;//域名
  private String fPath = "web";//储存网页文件的目录名
  private ArrayList<String> arrUrls = new ArrayList<String>();//存储未处理URL
  private ArrayList<String> arrUrl = new ArrayList<String>();//存储所有URL供建立索引
  private Hashtable<String,Integer> allUrls = new Hashtable<String,Integer>();//存储所有URL的网页号
  private Hashtable<String,Integer> deepUrls = new Hashtable<String,Integer>();//存储所有URL深度
  private int intWebIndex = 0;//网页对应文件下标，从0开始
  private String charset = "GB2312";
  private String report = "";
  private long startTime;
  private int webSuccessed = 0;
  private int webFailed = 0;
  
  
  public GetWeb(String s)
  {
   this.strHomePage = s;
  }
  public GetWeb(String s,int i)
  {
   this.strHomePage = s;
   this.webDepth = i;
  }
  
  public synchronized void addWebSuccessed()
  {
   webSuccessed++;
  }
  public synchronized void addWebFailed()
  {
   webFailed++;
  }
  public synchronized void addReport(String s)
  {
   try
   {
    report += s;
    PrintWriter pwReport = new PrintWriter(new FileOutputStream("report.txt"));
    pwReport.println(report);
    pwReport.close();
   }
   catch(Exception e)
   {
    System.out.println("生成报告文件失败!");
   }
  }
  public synchronized String getAUrl()
  {
   String tmpAUrl = arrUrls.get(0);
   arrUrls.remove(0);
   return tmpAUrl;
  }
  public synchronized String getUrl()
  {
   String tmpUrl = arrUrl.get(0);
   arrUrl.remove(0);
   return tmpUrl;
  }
  public synchronized Integer getIntWebIndex()
  {
   intWebIndex++;
   return intWebIndex;
  }
  /**
   * @param args
   */
  public static void main(String[] args)
  {
   if (args.length == 0 || args[0].equals(""))
   {
    System.out.println("No input!");
    System.exit(1);
   }
   else if(args.length == 1)
   {
    GetWeb gw = new GetWeb(args[0]);
    gw.getWebByHomePage();
   }
   else
   {
    GetWeb gw = new GetWeb(args[0],Integer.parseInt(args[1]));
    gw.getWebByHomePage();
   }
  }
  public void getWebByHomePage()
  {
   startTime = System.currentTimeMillis();
   this.myDomain = getDomain();
   if (myDomain == null)
   {
    System.out.println("Wrong input!");
    //System.exit(1);
    return;
   }
   System.out.println("Homepage = " + strHomePage);
   addReport("Homepage = " + strHomePage + "!
");
   System.out.println("Domain = " + myDomain);
   addReport("Domain = " + myDomain + "!
");
   arrUrls.add(strHomePage);
   arrUrl.add(strHomePage);
   allUrls.put(strHomePage,0);
   deepUrls.put(strHomePage,1);
   File fDir = new File(fPath);
         if(!fDir.exists())
         {
          fDir.mkdir();
         }
   System.out.println("Start!");
   this.addReport("Start!
");
   String tmp = getAUrl();
   this.getWebByUrl(tmp,charset,allUrls.get(tmp)+"");
   int i = 0;
   for (i=0;i<intThreadNum;i++)
   {
    new Thread(new Processer(this)).start();
   }
   while (true)
   {
    if(arrUrls.isEmpty() && Thread.activeCount() == 1)
    {
     long finishTime = System.currentTimeMillis();
     long costTime = finishTime-startTime;
     System.out.println("




Finished!");
     addReport("




Finished!
");
     System.out.println("Start time = " + startTime + "   " + "Finish time = " + finishTime + "   " + "Cost time = " + costTime + "ms");
     addReport("Start time = " + startTime + "   " + "Finish time = " + finishTime + "   " + "Cost time = " + costTime + "ms" + "
");
     System.out.println("Total url number = " + (webSuccessed+webFailed) + "   Successed: " + webSuccessed + "   Failed: " + webFailed);
     addReport("Total url number = " + (webSuccessed+webFailed) + "   Successed: " + webSuccessed + "   Failed: " + webFailed + "
");
     
     String strIndex = "";
     String tmpUrl = "";
     while (!arrUrl.isEmpty())
     {
      tmpUrl = getUrl();
      strIndex += "Web depth:" + deepUrls.get(tmpUrl) + "   Filepath: " + fPath + "/web" + allUrls.get(tmpUrl) + ".htm" + "   url:" + tmpUrl + "

";
     }
     System.out.println(strIndex);
     try
     {
      PrintWriter pwIndex = new PrintWriter(new FileOutputStream("fileindex.txt"));
      pwIndex.println(strIndex);
      pwIndex.close();
     }
     catch(Exception e)
     {
      System.out.println("生成索引文件失败!");
     }
     break;
    }
   }
  }
  public void getWebByUrl(String strUrl,String charset,String fileIndex)
  {
   try
   {
    //if(charset==null||"".equals(charset))charset="utf-8";
    System.out.println("Getting web by url: " + strUrl);
    addReport("Getting web by url: " + strUrl + "
");
    URL url = new URL(strUrl);
    URLConnection conn = url.openConnection();
    conn.setDoOutput(true);
    InputStream is = null;
    is = url.openStream();
   
    String filePath = fPath + "/web" + fileIndex + ".htm";
    PrintWriter pw = null;
    FileOutputStream fos = new FileOutputStream(filePath);
    OutputStreamWriter writer = new OutputStreamWriter(fos);
    pw = new PrintWriter(writer);
    BufferedReader bReader = new BufferedReader(new InputStreamReader(is));
    StringBuffer sb = new StringBuffer();
    String rLine = null;
    String tmp_rLine = null;
    while ( (rLine = bReader.readLine()) != null)
    {
     tmp_rLine = rLine;
     int str_len = tmp_rLine.length();
     if (str_len > 0)
     {
      sb.append("
" + tmp_rLine);
      pw.println(tmp_rLine);
      pw.flush();
      if (deepUrls.get(strUrl) < webDepth)getUrlByString(tmp_rLine,strUrl);
     }
     tmp_rLine = null;
    }
    is.close();
    pw.close();
    System.out.println("Get web successfully! " + strUrl);
    addReport("Get web successfully! " + strUrl + "
");
    addWebSuccessed();
   }
   catch (Exception e)
   {
    System.out.println("Get web failed!       " + strUrl);
    addReport("Get web failed!       " + strUrl + "
");
    addWebFailed();
   }
  }
  public String getDomain()
  {
   String reg = "(?<=http\://[a-zA-Z0-9]{0,100}[.]{0,1})[^.\s]*?\.(com|cn|net|org|biz|info|cc|tv)";
   Pattern p = Pattern.compile(reg,Pattern.CASE_INSENSITIVE);
   Matcher m = p.matcher(strHomePage);
   boolean blnp = m.find();
   if (blnp == true)
   {
    return m.group(0);
   }
   return null;
  }
  public void getUrlByString(String inputArgs,String strUrl)
  {
   String tmpStr = inputArgs;
   String regUrl = "(?<=(href=)["]?["]?)[http://][^\s""\?]*(" + myDomain + ")[^\s"">]*";
   Pattern p = Pattern.compile(regUrl,Pattern.CASE_INSENSITIVE);
   Matcher m = p.matcher(tmpStr);
   boolean blnp = m.find();
   //int i = 0;
   while (blnp == true)
   {
    if (!allUrls.containsKey(m.group(0)))
    {
     System.out.println("Find a new url,depth:" + (deepUrls.get(strUrl)+1) + " "+ m.group(0));
     addReport("Find a new url,depth:" + (deepUrls.get(strUrl)+1) + " "+ m.group(0) + "
");
     arrUrls.add(m.group(0));
     arrUrl.add(m.group(0));
     allUrls.put(m.group(0),getIntWebIndex());
     deepUrls.put(m.group(0),(deepUrls.get(strUrl)+1));
    }
    tmpStr = tmpStr.substring(m.end(),tmpStr.length());
    m = p.matcher(tmpStr);
    blnp = m.find();
   }
  }
  
  class Processer implements Runnable
  {
      GetWeb gw;
      public Processer(GetWeb g)
      {
          this.gw = g;
      }
      public void run()
      {
       //Thread.sleep(5000);
       while (!arrUrls.isEmpty())
       {
        String tmp = getAUrl();
        getWebByUrl(tmp,charset,allUrls.get(tmp)+"");
       }
      }
  }
}

  
  
   
   

     
   

     
   
  

复制代码

源码下载：http://file.javaxxz.com/2014/10/28/235830734.zip