Jsoup获取全国地区数据（省市县镇村）

  1 import java.io.BufferedWriter;
  2 import java.io.File;
  3 import java.io.FileWriter;
  4 import java.io.IOException;
  5 import java.util.HashMap;
  6 import java.util.Map;
  7 
  8   
  9 import org.jsoup.Jsoup;
 10 import org.jsoup.nodes.Document;
 11 import org.jsoup.nodes.Element;
 12 import org.jsoup.select.Elements;
 13 
 14 /**
 15  * 全国省市县镇村数据爬取
 16  * @author liushaofeng
 17  * @date 2015-10-11 上午12:19:39
 18  * @version 1.0.0
 19  */
 20 public class JsoupTest
 21 {
 22     private static Map<Integer, String> cssMap = new HashMap<Integer, String>();
 23 
 24     static
 25     {
 26         cssMap.put(1, "provincetr");// 省
 27         cssMap.put(2, "citytr");// 市
 28         cssMap.put(3, "countytr");// 县
 29         cssMap.put(4, "towntr");// 镇
 30         cssMap.put(5, "villagetr");// 村
 31     }
 32 
 33     public static void main(String[] args) throws IOException
 34     {
 35         int level = 1;
 36 
 37         // 获取全国各个省级信息
 38         Document connect = connect("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2013/");
 39         Elements rowProvince = connect.select("tr." + cssMap.get(level));
 40         for (Element provinceElement : rowProvince)// 遍历每一行的省份城市
 41         {
 42             Elements select = provinceElement.select("a");
 43             for (Element province : select)// 每一个省份(四川省)
 44             {
 45                 System.out.println(province.text());
 46                 parseNextLevel(province, level + 1);
 47             }
 48         }
 49     }
 50 
 51     private static void parseNextLevel(Element parentElement, int level) throws IOException
 52     {
 53         try
 54         {
 55             Thread.sleep(500);
 56         } catch (InterruptedException e)
 57         {
 58             e.printStackTrace();
 59         }
 60 
 61         Document doc = connect(parentElement.attr("abs:href"));
 62         Elements newsHeadlines = doc.select("tr." + cssMap.get(level));//
 63         // 获取表格的一行数据
 64         for (Element element : newsHeadlines)
 65         {
 66             printInfo(element, level + 1);
 67             Elements select = element.select("a");
 68             if (select.size() != 0)
 69             {
 70                 parseNextLevel(select.last(), level + 1);
 71             }
 72         }
 73     }
 74 
 75     private static void printInfo(Element element, int level)
 76     {
 77         BufferedWriter bufferedWriter = null;
 78         try
 79         {
 80             bufferedWriter = new BufferedWriter(new FileWriter(new File("F:\\CityInfo.txt"), true));
 81             String str = element.select("td").last().text() + "{" + level + "}["
 82                 + element.select("td").first().text() + "]";
 83             System.out.println(str);
 84             bufferedWriter.write(str);
 85             bufferedWriter.newLine();
 86             bufferedWriter.flush();
 87         } catch (IOException e)
 88         {
 89             e.printStackTrace();
 90         } finally
 91         {
 92             if (bufferedWriter != null)
 93             {
 94                 try
 95                 {
 96                     bufferedWriter.close();
 97                 } catch (IOException e)
 98                 {
 99                     e.printStackTrace();
100                 }
101                 bufferedWriter = null;
102             }
103         }
104     }
105 
106     private static Document connect(String url)
107     {
108         if (url == null || url.isEmpty())
109         {
110             throw new IllegalArgumentException("The input url(‘" + url + "‘) is invalid!");
111         }
112         try
113         {
114             return Jsoup.connect(url).timeout(60 * 1000).get();
115         } catch (IOException e)
116         {
117             e.printStackTrace();
118             return null;
119         }
120     }
121 }