package?com.crawl;

import?java.io.File;
import?java.io.IOException;
import?java.util.ArrayList;
import?java.util.HashMap;
import?java.util.List;
import?java.util.Map;

import?org.jsoup.Jsoup;
import?org.jsoup.nodes.Document;
import?org.jsoup.nodes.Element;
import?org.jsoup.select.Elements;?

import?com.crawl.util.FileUtils;

public?class?CrawlWebSite?{
?
	private?final?static?String?charSet?=?“utf-8“;
	private?final?static?String?rootDir?=?“C:/crawlWebSite/“;
	//private?final?static?String?rootUrl?=?“http://www.inmarsat.com/“;
	private?final?static?String?rootUrl?=?“http://www.jhcomn.com/“;
	
	private?final?static?int?timeOut?=?30000;	
	
	/**網(wǎng)站上相對地址與絕對地址的映射*/
	private?static?Map?absRelativeUrlMap?=?new?HashMap（）;	
	/**網(wǎng)站上的url與最終本地使用的url映射*/
	private?static?Map?urlmapMap?=?new?HashMap（）;	
	/**網(wǎng)站上的cssjs*/
	private?static?Map?cssjsmapMap?=?new?HashMap（）;	
	private?static?List?allFiles?=?new?ArrayList（）;	
	/**過濾掉不爬取的內(nèi)容格式*/
	public?static?final?String?filterExtArray?[]??=?{“rar““zip““bmp““dib““gif““jfif““jpe““jpeg““jpg““png““tif““tiff““ico““pdf““doc““docx““xls““xlsx“};?
	/**
	?*?@param?args
	?*?@throws?IOException?
	?*/
	public?static?void?main（String[]?args）??{
		
			System.out.println（“start....“）;
		
			//獲取所有urls
			getSubUrls（rootUrlrootUrl）;
		?
			//test?
			//absRelativeUrlMap.put（“http://www.inmarsat.com/?p=3999“?“http://www.inmarsat.com/?p=3999“）;
			
			//保存文件
			for（String?absUrl?:?absRelativeUrlMap.keySet（））{
				
				String?content;
				try?{
					content?=?readContent（absUrl）;
				}?catch?（IOException?e）?{
					System.err.println（“url=“+absUrl+“?頁面無效！“）;
					continue;
				}
				if（!absUrl.startsWith（rootUrl））{
					continue;
				}
				String?filePath?=?absUrl.substring（rootUrl.length（））;
				filePath?=?FileUtils.parseFilePath（filePath）;
				?
				//urlmapMap.put（absRelativeUrlMap.get（absUrl）?filePath）;//脫機運行和在服務器運行有所不同。。。
				urlmapMap.put（absRelativeUrlMap.get（absUrl）?rootDir.concat（filePath））;//脫機運行。。。
				FileUtils.writeFile（content?rootDir.concat（filePath）charSet）;
			}
			??
			System.out.println（“--------------------“）;?
			//更新文件中的url
			getAllFiles（new?File（rootDir））;?
			Document?doc?=?null;
			for（File?file?:?allFiles）{
				try?{
					doc?=?Jsoup.parse（file?“utf8“rootUrl）;
					dealCssJsFile（doc）;
					replaceUrl（doc）;
				}?catch?（Exception?e）?{?
					e.printStackTrace（）;
				}??
				String?newContent?=?doc.html（）;
				FileUtils.writeFile（newContent?file.getAbsolutePath（）charSet）;
			}
			
			System.out.println（“finished.“）;
		
		?
	}
		
	
	/**
	?*?獲取指定url頁面中的所有鏈接
	?*?@param?url
	?*?@return
	?*?@throws?IOException?
	?*/
	public?static?void?getSubUrls（String?absUrlString?relativeUrl）???{
		//?TODO?Auto-generated?method?stub
		??
		if（absRelativeUrlMap.get（absUrl）!=null?||?filter（absUrl））{
			retu

?屬性????????????大小?????日期????時間???名稱
-----------?---------??----------?-----??----

?????文件????????360??2014-03-04?20:30??crawl-website\.classpath

?????文件????????389??2014-03-04?20:01??crawl-website\.project

?????文件????????629??2014-03-04?20:01??crawl-website\.settings\org.eclipse.jdt.core.prefs

?????文件?????278273??2013-06-05?09:45??crawl-website\lib\jsoup-1.7.1.jar

?????文件???????9347??2014-03-05?20:22??crawl-website\src\com\crawl\CrawlWebSite.java

?????文件???????5574??2014-03-05?20:20??crawl-website\src\com\crawl\util\FileUtils.java

?????文件???????1150??2014-03-05?13:36??crawl-website\src\com\crawl\util\StringUtil.java

?????目錄??????????0??2014-03-05?13:36??crawl-website\src\com\crawl\util

?????目錄??????????0??2014-03-05?20:35??crawl-website\src\com\crawl

?????目錄??????????0??2014-03-04?20:02??crawl-website\src\com

?????目錄??????????0??2014-03-04?20:01??crawl-website\.settings

?????目錄??????????0??2014-03-04?20:30??crawl-website\lib

?????目錄??????????0??2014-03-04?20:02??crawl-website\src

?????目錄??????????0??2014-03-05?20:36??crawl-website

-----------?---------??----------?-----??----

???????????????295722????????????????????14