简单的网络爬虫（获取页面中的url）

简单的网络爬虫（获取页面中的url）

package text;

import java.io.BufferedReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class SpiderNet {
	public static void main(String[] args) {
		URL url = null;
		URLConnection urlConnection = null;
		PrintWriter out = null;
		BufferedReader in = null;
		//编写正则表达式
		String regex = "http://[\\w+\\.?/?]+\\.[A-Za-z]+";
		//定义正则表达式
		Pattern pattern = Pattern.compile(regex);
		try {
			//访问链接,这里选择steam官网主页
			url = new URL("http://store.steampowered.com/");
			//建立url的连接connection
			urlConnection = url.openConnection();
			//创建新 PrintWriter,路径D:/mzsds_mzm.txt
			out = new PrintWriter(new FileWriter("D:/mzsds_mzm.txt"),true);
			//将url地址的响应读到in中
			in = new BufferedReader(new InputStreamReader(urlConnection.getInputStream()));
			String buffer = null;
			Set set = new HashSet();
			//读响应数据到.txt文件中
			while((buffer=in.readLine())!=null){
				//匹配正则表达式
				Matcher matcher = pattern.matcher(buffer);
				if(matcher.find()){
					//输出匹配内容
					set.add(matcher.group());
				}
			}
			for (String str : set) {  
			     out.println(str);  
			}  
		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}

很简单，没有筛选结果，包含了其他的链接，不是很完善。

结果输出：

http://cdn.steamstatic.com.8686c.com/steam/apps/668210/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/346110/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/672730/capsule_184x69.jpg
http://www.valvesoftware.com/jobs.html
http://translation.steampowered.com
http://www.steampowered.com
http://www.valvesoftware.com
http://store.akamai.steamstatic.com/public/css/v6/home.css
http://www.w3.org
http://store.akamai.steamstatic.com
http://store.akamai.steamstatic.com/public/shared/css/motiva_sans.css
http://cdn.steamstatic.com.8686c.com/steam/apps/434460/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/566160/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/header_logo.png
http://www.valvesoftware.com/about.html
http://store.akamai.steamstatic.com/public/css/v6/store.css
http://store.steampowered.com
http://cdn.steamstatic.com.8686c.com/steam/apps/686190/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/668630/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/spotlights/68ca004cca70f207cddabbb4/spotlight_image_english.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/465240/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/javascript/cluster.js
http://store.akamai.steamstatic.com/public/shared/images/header/globalheader_logo.png
http://twitter.com
http://store.akamai.steamstatic.com/public/javascript/home.js
http://cdn.steamstatic.com.8686c.com/steam/apps/619880/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/641990/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/css/buttons.css
http://cdn.steamstatic.com.8686c.com/steam/apps/680320/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/css/shared_responsive.css
http://store.akamai.steamstatic.com/public/shared/css/shared_global.css
http://cdn.steamstatic.com.8686c.com/steam/apps/515220/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/544610/capsule_184x69.jpg
http://steamcommunity.com
http://store.akamai.steamstatic.com/public/javascript/main.js
http://store.akamai.steamstatic.com/public/shared/javascript/shared_global.js
http://store.akamai.steamstatic.com/public/images/v5/globalheader_logo.png
http://store.akamai.steamstatic.com/public/shared/javascript/shared_responsive_adapter.js
http://cdn.steamstatic.com.8686c.com/steam/apps/555150/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/javascript/tooltip.js
http://cdn.steamstatic.com.8686c.com/steam/apps/577800/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/650780/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/header_menu_hamburger.png
http://cdn.steamstatic.com.8686c.com/steam/apps/629960/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/689090/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/373930/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/551770/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/559910/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/614570/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/635320/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/apps/435150/capsule_184x69.jpg
http://www.facebook.com
http://cdn.steamstatic.com.8686c.com/steam/apps/554620/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/shared/images/responsive/logo_valve_footer.png
http://cdn.steamstatic.com.8686c.com/steam/apps/607880/capsule_184x69.jpg
http://store.akamai.steamstatic.com/public/javascript/dynamicstore.js
http://store.akamai.steamstatic.com/public/images/blank.gif
http://cdn.steamstatic.com.8686c.com/steam/apps/449840/capsule_184x69.jpg
http://cdn.steamstatic.com.8686c.com/steam/spotlights/7830c4a8f396478f946d265b/spotlight_image_english.jpg
http://store.akamai.steamstatic.com/public/shared/images/trans.gif
http://store.akamai.steamstatic.com/public/shared/images/responsive/share_steam_logo.png
http://www.valvesoftware.com/legal.htm
http://cdn.steamstatic.com.8686c.com/steam/apps/357310/capsule_184x69.jpg