我们可以通过在java程序中模拟浏览器一样,把数据抓下来,具体方法是在java程序中set header和cookie,下面是一个例子:
public class NetConnection { public static final int MAX_HOTWORDS_FILE_SIZE = 256 * 1024; public static void main(String[] args) { send("http://tuan.aibang.com/shenzhen/new_2033549.html"); } static void send(String _url){ HttpClient http = new HttpClient(); http.getHttpConnectionManager().getParams().setConnectionTimeout( 100000); GetMethod get = new GetMethod(_url); get.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 100000); get.setFollowRedirects(false); InputStream is; String host = "www.aibang.com"; try { get.setRequestHeader("Host",host); get.setRequestHeader( "user-agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8"); get.setRequestHeader( "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); get.setRequestHeader("Accept-Language","zh-cn,zh;q=0.5"); get.setRequestHeader("Accept-Charset","GB2312,utf-8;q=0.7,*;q=0.7"); get.setRequestHeader("Connection","Keep-Alive"); get.setRequestHeader("Cookie","582081171805; cy=1; __utma=205923334.3209590505032285000.1256126987.1269858466.1270605495.11; __utmz=205923334.1256126987.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); ano=0snUeoHWygEkAAAAOTAyZjM3ZjQtNjA2NC00NWYzLWIxNWYtMjRlMDliZjAzYTM3jnk83_pKoSEk-9gwcIFx8jFOXVM1; sid=no4tpkmvedoj3gycvfz1s055; lb.dp=100729098.20480.0000; __utmb=205923334.1.10.1270605495; __utmc=205923334"); int er = 0; er = http.executeMethod(get); System.out.println("er="+er); if (er == 200) { try { is = get.getResponseBodyAsStream(); ReadFile2(is); } catch (Exception e) { System.out.println("download error="+e); } } }catch(Exception ex){ ex.printStackTrace(); } } //解析inputStream里的数据 static void ReadFile2(InputStream inputStream){ byte[] buffer = new byte[1024]; StringBuffer bab = new StringBuffer(); int nTotal = 0; try { do{ int nBytesRead = inputStream.read(buffer); System.out.println("nBytesRead="+nBytesRead); if(nBytesRead > 0){ bab.append(new String(buffer,0,nBytesRead)); nTotal += nBytesRead; }else{ System.out.println(String.format("download end. file size=%d", nTotal)); break; } }while(nTotal < MAX_HOTWORDS_FILE_SIZE); System.out.println("bab = "+bab); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams;
java如果去请求一个网页的数据,布布扣,bubuko.com
原文:http://blog.csdn.net/baidu_nod/article/details/37743147