public HttpBase() 
{
         
    this(null);
}

Creates a new instance of HttpBase

 public HttpBase(Log logger) 
{
    if (logger != null) {
      this.logger = logger;
    }
    robots = new RobotRulesParser();
}

Creates a new instance of HttpBase

 public Configuration getConf() 
{
    return this.conf;
}

 public int getMaxContent() 
{
    return maxContent;
}

 public int getMaxDelays() 
{
    return maxDelays;
}

 public int getMaxThreadsPerHost() 
{
    return maxThreadsPerHost;
}

 public ProtocolOutput getProtocolOutput(Text url,
    CrawlDatum datum) 
{
    
    String urlString = url.toString();
    try {
      URL u = new URL(urlString);
      if (checkRobots) {
        try {
          if (!robots.isAllowed(this, u)) {
            return new ProtocolOutput(null, new ProtocolStatus(ProtocolStatus.ROBOTS_DENIED, url));
          }
        } catch (Throwable e) {
          // XXX Maybe bogus: assume this is allowed.
          if (logger.isTraceEnabled()) {
            logger.trace("Exception checking robot rules for " + url + ": " + e);
          }
        }
      }
      long crawlDelay = robots.getCrawlDelay(this, u);
      long delay = crawlDelay  > 0 ? crawlDelay : serverDelay;
      if (checkBlocking && maxCrawlDelay  >= 0 && delay  > maxCrawlDelay) {
        // skip this page, otherwise the thread would block for too long.
        LOGGER.info("Skipping: " + u + " exceeds fetcher.max.crawl.delay, max="
                + (maxCrawlDelay / 1000) + ", Crawl-Delay=" + (delay / 1000));
        return new ProtocolOutput(null, ProtocolStatus.STATUS_WOULDBLOCK);
      }
      String host = null;
      if (checkBlocking) {
        try {
          host = blockAddr(u, delay);
        } catch (BlockedException be) {
          return new ProtocolOutput(null, ProtocolStatus.STATUS_BLOCKED);
        }
      }
      Response response;
      try {
        response = getResponse(u, datum, false); // make a request
      } finally {
        if (checkBlocking) unblockAddr(host, delay);
      }
      int code = response.getCode();
      byte[] content = response.getContent();
      Content c = new Content(u.toString(), u.toString(),
                              (content == null ? EMPTY_CONTENT : content),
                              response.getHeader("Content-Type"),
                              response.getHeaders(), this.conf);
      if (code == 200) { // got a good response
        return new ProtocolOutput(c); // return it
      } else if (code == 410) { // page is gone
        return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.GONE, "Http: " + code + " url=" + url));
      } else if (code  >= 300 && code <  400) { // handle redirect
        String location = response.getHeader("Location");
        // some broken servers, such as MS IIS, use lowercase header name...
        if (location == null) location = response.getHeader("location");
        if (location == null) location = "";
        u = new URL(u, location);
        int protocolStatusCode;
        switch (code) {
          case 300:   // multiple choices, preferred value in Location
            protocolStatusCode = ProtocolStatus.MOVED;
            break;
          case 301:   // moved permanently
          case 305:   // use proxy (Location is URL of proxy)
            protocolStatusCode = ProtocolStatus.MOVED;
            break;
          case 302:   // found (temporarily moved)
          case 303:   // see other (redirect after POST)
          case 307:   // temporary redirect
            protocolStatusCode = ProtocolStatus.TEMP_MOVED;
            break;
          case 304:   // not modified
            protocolStatusCode = ProtocolStatus.NOTMODIFIED;
            break;
          default:
            protocolStatusCode = ProtocolStatus.MOVED;
        }
        // handle this in the higher layer.
        return new ProtocolOutput(c, new ProtocolStatus(protocolStatusCode, u));
      } else if (code == 400) { // bad request, mark as GONE
        if (logger.isTraceEnabled()) { logger.trace("400 Bad request: " + u); }
        return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.GONE, u));
      } else if (code == 401) { // requires authorization, but no valid auth provided.
        if (logger.isTraceEnabled()) { logger.trace("401 Authentication Required"); }
        return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.ACCESS_DENIED, "Authentication required: "
                + urlString));
      } else if (code == 404) {
        return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.NOTFOUND, u));
      } else if (code == 410) { // permanently GONE
        return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.GONE, u));
      } else {
        return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.EXCEPTION, "Http code=" + code + ", url="
                + u));
      }
    } catch (Throwable e) {
      e.printStackTrace(LogUtil.getErrorStream(logger));
      return new ProtocolOutput(null, new ProtocolStatus(e));
    }
}

 public String getProxyHost() 
{
    return proxyHost;
}

 public int getProxyPort() 
{
    return proxyPort;
}

 abstract protected Response getResponse(URL url,
    CrawlDatum datum,
    boolean followRedirects) throws IOException, ProtocolException

 public RobotRules getRobotRules(Text url,
    CrawlDatum datum) 
{
    return robots.getRobotRulesSet(this, url);
}

 public long getServerDelay() 
{
    return serverDelay;
}

 public int getTimeout() 
{
    return timeout;
}

 public boolean getUseHttp11() 
{
    return useHttp11;
}

 public String getUserAgent() 
{
    return userAgent;
}

 protected  void logConf() 
{
    if (logger.isInfoEnabled()) {
      logger.info("http.proxy.host = " + proxyHost);
      logger.info("http.proxy.port = " + proxyPort);
      logger.info("http.timeout = " + timeout);
      logger.info("http.content.limit = " + maxContent);
      logger.info("http.agent = " + userAgent);
      logger.info(Protocol.CHECK_BLOCKING + " = " + checkBlocking);
      logger.info(Protocol.CHECK_ROBOTS + " = " + checkRobots);
      if (checkBlocking) {
        logger.info("fetcher.server.delay = " + serverDelay);
        logger.info("http.max.delays = " + maxDelays);
      }
    }
}

 protected static  void main(HttpBase http,
    String[] args) throws Exception 
{
    boolean verbose = false;
    String url = null;
    String usage = "Usage: Http [-verbose] [-timeout N] url";
    if (args.length == 0) {
      System.err.println(usage);
      System.exit(-1);
    }
    for (int i = 0; i <  args.length; i++) { // parse command line
      if (args[i].equals("-timeout")) { // found -timeout option
        http.timeout = Integer.parseInt(args[++i]) * 1000;
      } else if (args[i].equals("-verbose")) { // found -verbose option
        verbose = true;
      } else if (i != args.length - 1) {
        System.err.println(usage);
        System.exit(-1);
      } else // root is required parameter
        url = args[i];
    }
//    if (verbose) {
//      LOGGER.setLevel(Level.FINE);
//    }
    ProtocolOutput out = http.getProtocolOutput(new Text(url), new CrawlDatum());
    Content content = out.getContent();
    System.out.println("Status: " + out.getStatus());
    if (content != null) {
      System.out.println("Content Type: " + content.getContentType());
      System.out.println("Content Length: " +
                         content.getMetadata().get(Response.CONTENT_LENGTH));
      System.out.println("Content:");
      String text = new String(content.getContent());
      System.out.println(text);
    }
}

 public byte[] processDeflateEncoded(byte[] compressed,
    URL url) throws IOException 
{
    if (LOGGER.isTraceEnabled()) { LOGGER.trace("inflating...."); }
    byte[] content = DeflateUtils.inflateBestEffort(compressed, getMaxContent());
    if (content == null)
      throw new IOException("inflateBestEffort returned null");
    if (LOGGER.isTraceEnabled()) {
      LOGGER.trace("fetched " + compressed.length
                 + " bytes of compressed content (expanded to "
                 + content.length + " bytes) from " + url);
    }
    return content;
}

 public byte[] processGzipEncoded(byte[] compressed,
    URL url) throws IOException 
{
    if (LOGGER.isTraceEnabled()) { LOGGER.trace("uncompressing...."); }
    byte[] content;
    if (getMaxContent()  >= 0) {
        content = GZIPUtils.unzipBestEffort(compressed, getMaxContent());
    } else {
        content = GZIPUtils.unzipBestEffort(compressed);
    } 
    if (content == null)
      throw new IOException("unzipBestEffort returned null");
    if (LOGGER.isTraceEnabled()) {
      LOGGER.trace("fetched " + compressed.length
                 + " bytes of compressed content (expanded to "
                 + content.length + " bytes) from " + url);
    }
    return content;
}

 public  void setConf(Configuration conf) 
{
        this.conf = conf;
        this.proxyHost = conf.get("http.proxy.host");
        this.proxyPort = conf.getInt("http.proxy.port", 8080);
        this.useProxy = (proxyHost != null && proxyHost.length()  > 0);
        this.timeout = conf.getInt("http.timeout", 10000);
        this.maxContent = conf.getInt("http.content.limit", 64 * 1024);
        this.maxDelays = conf.getInt("http.max.delays", 3);
        this.maxThreadsPerHost = conf.getInt("fetcher.threads.per.host", 1);
        this.userAgent = getAgentString(conf.get("http.agent.name"), conf.get("http.agent.version"), conf
                .get("http.agent.description"), conf.get("http.agent.url"), conf.get("http.agent.email"));
        this.serverDelay = (long) (conf.getFloat("fetcher.server.delay", 1.0f) * 1000);
        this.maxCrawlDelay = (long)(conf.getInt("fetcher.max.crawl.delay", -1) * 1000);
        // backward-compatible default setting
        this.byIP = conf.getBoolean("fetcher.threads.per.host.by.ip", true);
        this.useHttp11 = conf.getBoolean("http.useHttp11", false);
        this.robots.setConf(conf);
        this.checkBlocking = conf.getBoolean(Protocol.CHECK_BLOCKING, true);
        this.checkRobots = conf.getBoolean(Protocol.CHECK_ROBOTS, true);
        logConf();
}

 public boolean useProxy() 
{
    return useProxy;
}