JAVA爬取百度数据

package com.bonc;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

public class DownloadBD2 {
private String inputFile;
private String outFile ;
private String outShapeFile ;
private String outImagePath ;
private Map cityCodes = new HashMap<>();

public DownloadBD2(String inputFile,String outPath) throws IOException {
	this.inputFile=inputFile;
	this.outFile = outPath+"/poi_" + System.currentTimeMillis() + ".txt";

// this.outShapeFile = outPath+"/map_bd_search_shape"+index+".sql";
this.outImagePath = outPath+"/image";
//湖南的
cityCodes.put(“娄底市”, “221”);
cityCodes.put(“长沙市”, “158”);
cityCodes.put(“岳阳市”, “220”);
cityCodes.put(“常德市”, “219”);
cityCodes.put(“张家界市”, “312”);
cityCodes.put(“怀化市”, “363”);
cityCodes.put(“株洲市”, “222”);
cityCodes.put(“永州市”, “314”);
cityCodes.put(“湘潭市”, “313”);
cityCodes.put(“湘西土家族苗族自治州”, “274”);
cityCodes.put(“益阳市”, “272”);
cityCodes.put(“衡阳市”, “159”);
cityCodes.put(“邵阳市”, “273”);
cityCodes.put(“郴州市”, “275”);
//全国的
// File bianMa = new File(“D:\hunan\bianMa.txt”);
// BufferedReader br = new BufferedReader(new FileReader(bianMa));
// String s = null;
// while((s = br.readLine())!=null){
// String name =s.split("?[0];
// String ID = s.split("?[1];
// cityCodes.put(name, ID);
// }

	}
	//br.close();
	
   

public void download() throws Exception{
	
	File in = new File(this.inputFile);
	 
	BufferedReader br = new BufferedReader(new FileReader(in));//构造一个BufferedReader类来读取文件
    String s = null;
    File outSqlFile = new File(this.outFile);
 	outSqlFile.createNewFile();
 	//System.out.println("创建输出文件");
 	FileOutputStream fot = new FileOutputStream(outSqlFile);
 	OutputStreamWriter outWriter = new OutputStreamWriter(fot,"UTF-8");  
    int r = 1;
    while((s = br.readLine())!=null){//使用readLine方法，一次读一行
    	System.out.println(s);
    	//indexOf 方法返回一个整数值,指出 String对象内子字符串的开始位置。如果没有找到子字符串,则返回-1
    	if(s.indexOf(",")>0) {
    		//System.out.println("进入判断");
    		String keyWord = s.split(",")[1];
    		String city=s.split(",")[0];
    		//System.out.println(keyWord +"     "+city);
    		if(keyWord.indexOf("厕所")>-1) {
    			continue;
    		}
    		try {
    			System.out.print(r+"-->"+city);
    			r++;
    			downloadKeyWord(city,keyWord,outWriter);
    			
    		}catch (Exception e) {
				e.printStackTrace();

// System.err.println(“获取百度信息失败:”+s);
}
}
}
br.close();
outWriter.close();
fot.close();
}

private void downloadKeyWord(String city,String keyWord,OutputStreamWriter Writer) throws Exception {
	//http://api.map.baidu.com/?qt=s&c=131&wd=餐饮&rn=1
	String cityCode = this.cityCodes.get(city);

// String cityCode = “158”;
System.out.println(cityCode);
if(cityCode==null) {
System.out.println(city);
return;

	}
	int rn = 30;
	 String urlStr = "http://api.map.baidu.com/?qt=s&c="+cityCode+"&wd="+keyWord+"&rn="+rn;
	 System.out.println(urlStr);
	 URL url   = new URL(urlStr);
	 HttpURLConnection  huc = (HttpURLConnection)url.openConnection(); 
	 huc.setConnectTimeout(10000); 
	 huc.setReadTimeout(10000); 
	 huc.setRequestMethod("GET"); 
	 InputStream   in = url.openStream(); 
	 InputStreamReader inr = new InputStreamReader(in);
	 BufferedReader br = new BufferedReader(inr);
	 StringBuffer sb = new StringBuffer();
	 String tmpline = "";
	 while((tmpline =br.readLine())!=null) {
		  
		 sb.append(tmpline);
	 }
	 
	 inr.close();
	 in.close();
	 
	 String jsonStr = sb.toString();
	 //System.out.println(city+":"+keyWord+"#"+jsonStr.toString());
	 int n =0;
	 while(jsonStr.contains("\"ext\"")) {
		 String tmpImage = jsonStr.substring(jsonStr.indexOf("\"image\""));
		 tmpImage =tmpImage.substring(0,tmpImage.indexOf("\",")+2);
 
		 int ext = jsonStr.indexOf("\"ext\"");
		 int ext_type = jsonStr.indexOf("\"ext_type\"",ext)+10;
		 if(ext0) {
			 geo= geo.substring(geo.indexOf("|1-")+3);
			 geo= geo.substring(0, geo.indexOf(";"));
			String[]  pointStrArr = geo.split(",");
			if(pointStrArr.length%2==0) {
				shpStr.append("[");
				for(int i =0;i

// e.printStackTrace();
return “default.jpg”;
}
return uid+".jpg";
}

public static void main(String[] args) throws IOException {
	// TODO Auto-generated method stub 		
	
	DownloadBD2 dd = new DownloadBD2("E:\\dichanxiaoqu\\村级地名.txt","E:\\data\\");
	try {
		dd.download();
	} catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}	
}

}