Hi Java!

HttpBot httpBot=new HttpBot();
String html=httpBot.doGet("http://www.google.com");

package org.hijava.httpbot;
 
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.util.HashMap;
import java.util.Map;
 
/**
 * Http网络爬虫类
 *
 * @author yava
 */
public class HttpBot {
	private Map cookieMap;
	private String userAgent;
	private String encoding;
	private String host;
	private String referer;
	private int responseCode;
	private static final String separator = System.getProperty("line.separator");
	private static final String GET="GET";
	private static final String POST="POST";
 
	public HttpBot() {
		this.userAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5";
		this.encoding = "UTF-8";
		cookieMap=new HashMap();
	}
 
	public HttpBot(String host,String referer){
		this();
		this.host=host;
		this.referer=referer;
	}
	/**
	 * 以GET方式发送请求，返回页面html代码
	 * @param urlStr
	 * @return
	 */
	public String doGet(String urlStr) {
		HttpURLConnection http=getConnection(urlStr,HttpBot.GET);
		String content="";
		try {
			http.connect();
			processCookie(http);
			this.referer=urlStr;
			this.responseCode=http.getResponseCode();
			if(this.responseCode==302){
				String location=http.getHeaderField("Location");
				return doGet(location);
			}
			InputStream is=http.getInputStream();
			content=getContent(is);
			is.close();
			http.disconnect();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return content;
	}
	/**
	 * 以Post方式发送请求，返回页面html代码
	 * @param urlStr
	 * @param paramMap 请求参数
	 * @return
	 */
	public String doPost(String urlStr,Map paramMap){
		HttpURLConnection http=getConnection(urlStr,HttpBot.POST);
		http.setDoOutput(true);
		String content="";
		try {
			OutputStream os = http.getOutputStream();
			os.write(getParamBytes(paramMap));
			http.connect();
			processCookie(http);
			this.referer=urlStr;
			this.responseCode=http.getResponseCode();
			if(this.responseCode==302){
				String location=http.getHeaderField("Location");
				return doGet(location);
			}
			InputStream is=http.getInputStream();
			content=getContent(is);
			is.close();
			http.disconnect();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return content;
	}
	/**
	 * 从输入流获取网页内容
	 * @param is
	 * @return
	 */
	private String getContent(InputStream is){
		StringBuilder builder = new StringBuilder();
		try {
			BufferedReader reader = new BufferedReader(new InputStreamReader(is,encoding));
			String line;
			while ((line = reader.readLine()) != null) {
				builder.append(line).append(separator);
			}
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
 
		return builder.toString();
	}
 
	private HttpURLConnection getConnection(String urlStr,String reqMethod){
		HttpURLConnection http=null;
		try {
			URL url = new URL(urlStr);
			http = (HttpURLConnection) url.openConnection();
			http.setRequestProperty("User-Agent", this.userAgent);
			http.setRequestProperty("Host", this.host);
			http.setRequestProperty("Cookie", this.getCookie());
			http.setRequestProperty("Referer", this.referer);
			http.setRequestMethod(reqMethod);
			http.setInstanceFollowRedirects(false);
		} catch (IOException e) {
			e.printStackTrace();
		}
 
		return http;
	}
	/**
	 * 处理cookie
	 * @param http
	 */
	private void processCookie(HttpURLConnection http){
		String key = null;
		for (int i = 1; (key = http.getHeaderFieldKey(i)) != null; i++) {
			if (key.equalsIgnoreCase("set-cookie")) {
				String cookie = null;
				cookie = http.getHeaderField(i);
				int i1=cookie.indexOf("=");
				int i2=cookie.indexOf(";");
				if(i1!=-1&&i2!=-1){
					String _value=cookie.substring(i1+1, i2);
					if("EXPIRED".equalsIgnoreCase(_value)){
						continue;
					}
					String _key=cookie.substring(0, i1);
					cookieMap.put(_key, _value);
				}
			}
		}
	}
	/**
	 * 获取cookie
	 * @return
	 */
	public String getCookie(){
		String cookie="";
		for(Map.Entry entry:cookieMap.entrySet()){
			cookie=cookie+entry.getKey()+"="+entry.getValue()+";";
		}
		return cookie;
	}
 
	private byte[] getParamBytes(Map paramMap){
		String paramStr="";
		for(Map.Entry entry:paramMap.entrySet()){
			String value="";
			try {
				value=URLEncoder.encode(entry.getValue(), "UTF-8");
			} catch (UnsupportedEncodingException e) {
				e.printStackTrace();
			}
			paramStr=paramStr+entry.getKey()+"="+value+"&";
		}
		return paramStr.getBytes();
	}
}

acl SSL_ports port 443
acl Safe_ports port 80		# http
acl Safe_ports port 21		# ftp
acl Safe_ports port 443		# https
acl Safe_ports port 70		# gopher
acl Safe_ports port 210		# wais
acl Safe_ports port 1025-65535	# unregistered ports
acl Safe_ports port 280		# http-mgmt
acl Safe_ports port 488		# gss-http
acl Safe_ports port 591		# filemaker
acl Safe_ports port 777		# multiling http
acl CONNECT method CONNECT
 
auth_param basic program /usr/sbin/ncsa_auth /usr/local/squid/password
acl normal proxy_auth REQUIRED
http_access allow normal
 
# Deny requests to certain unsafe ports
http_access deny !Safe_ports
 
# Deny CONNECT to other than secure SSL ports
http_access deny CONNECT !SSL_ports
 
# And finally deny all other access to this proxy
http_access deny all
 
# Squid normally listens to port 3128
http_port 3128
 
# We recommend you to use at least the following line.
hierarchy_stoplist cgi-bin ?
 
# Uncomment and adjust the following to add a disk cache directory.
#cache_dir null /tmp
 
# Leave coredumps in the first cache dir
coredump_dir /usr/local/squid/var/cache
 
# Add any of your own refresh_pattern entries above these.
refresh_pattern ^ftp:		1440	20%	10080
refresh_pattern ^gopher:	1440	0%	1440
refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
refresh_pattern .		0	20%	4320

		String cookieVal = null;
		String key=null;
		   for (int i = 1; (key = http.getHeaderFieldKey(i)) != null; i++ ) {
	              if (key.equalsIgnoreCase("set-cookie")) {
	            	  cookieVal = http.getHeaderField(i);
	            	  cookieVal = cookieVal.substring(0, cookieVal.indexOf(";"));
	            	  sessionId=sessionId+cookieVal+";";
	              }
	           }

		http.setRequestProperty("Cookie", sessionId);