/* $Id: DataCache.java 988245 2010-08-23 18:39:35Z kwright $ */

/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.manifoldcf.crawler.connectors.webcrawler;

import org.apache.manifoldcf.core.interfaces.*;
import org.apache.manifoldcf.agents.interfaces.*;
import org.apache.manifoldcf.crawler.interfaces.*;
import org.apache.manifoldcf.crawler.system.Logging;
import org.apache.manifoldcf.crawler.system.ManifoldCF;
import java.util.*;
import java.io.*;

import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.client.RedirectException;
import org.apache.http.client.CircularRedirectException;
import org.apache.http.NoHttpResponseException;
import org.apache.http.HttpException;

/** This class is a cache of a specific URL's data.  It's fetched early and kept,
* so that (1) an accurate data length can be found, and (2) we can compute a version
* checksum.
*/
public class DataCache
{
  public static final String _rcsid = "@(#)$Id: DataCache.java 988245 2010-08-23 18:39:35Z kwright $";

  // Hashmap containing the cache of files.
  // This is keyed by document identifier, and contains DocumentData objects.
  protected Map<String,DocumentData> cacheData = new HashMap<String,DocumentData>();

  /** Constructor.
  */
  public DataCache()
  {
  }


  /** Add a data entry into the cache.
  * This method is called whenever the data from a fetch is considered interesting or useful, and will
  * be thus passed on from getDocumentVersions() to the processDocuments() phase.  At the moment that's
  * usually a 200 or a 302 response.
  *@param documentIdentifier is the document identifier (url).
  *@param connection is the connection, upon which a fetch has been done that needs to be
  * cached.
  *@return a "checksum" value, to use as a version string.
  */
  public String addData(IProcessActivity activities, String documentIdentifier, IThrottledConnection connection)
    throws ManifoldCFException, ServiceInterruption
  {
    // Grab the response code, and the content-type header
    int responseCode = connection.getResponseCode();
    String contentType = connection.getResponseHeader("Content-Type");
    String referralURI = connection.getResponseHeader("Location");

    // Create a temporary file; that's what we will cache
    try
    {
      // First, get the stream.
      InputStream dataStream = connection.getResponseBodyStream();
      if (dataStream == null)
        return null;
      try
      {
        File tempFile = File.createTempFile("_webcache_","tmp");
        try
        {
          // Causes memory leaks if left around; there's no way to release
          // the record specifying that the file should be deleted, even
          // after it's removed.  So disable this and live with the occasional
          // dangling file left as a result of shutdown or error. :-(
          // tempFile.deleteOnExit();
          ManifoldCF.addFile(tempFile);

          // Transfer data to temporary file
          long checkSum = 0L;
          OutputStream os = new FileOutputStream(tempFile);
          try
          {
            byte[] byteArray = new byte[65536];
            while (true)
            {
              int amt;
              try
              {
                amt = dataStream.read(byteArray,0,byteArray.length);
              }
              catch (java.net.SocketTimeoutException e)
              {
                Logging.connectors.warn("Socket timeout exception reading socket stream: "+e.getMessage(),e);
                long currentTime = System.currentTimeMillis();
                throw new ServiceInterruption("Socket timeout: "+e.getMessage(),e,currentTime + 300000L,
                  currentTime + 12 * 60 * 60000L,-1,false);
              }
              catch (ConnectTimeoutException e)
              {
                Logging.connectors.warn("Socket connect timeout exception reading socket stream: "+e.getMessage(),e);
                long currentTime = System.currentTimeMillis();
                throw new ServiceInterruption("Socket timeout: "+e.getMessage(),e,currentTime + 300000L,
                  currentTime + 12 * 60 * 60000L,-1,false);
              }
              catch (InterruptedIOException e)
              {
                //Logging.connectors.warn("IO interruption seen",e);
                throw new ManifoldCFException("Interrupted: "+e.getMessage(),ManifoldCFException.INTERRUPTED);
              }
              catch (IOException e)
              {
                Logging.connectors.warn("IO exception reading socket stream: "+e.getMessage(),e);
                long currentTime = System.currentTimeMillis();
                throw new ServiceInterruption("Read timeout: "+e.getMessage(),e,currentTime + 300000L,
                  currentTime + 12 * 60 * 60000L,-1,false);
              }
              if (amt == -1)
                break;
              int i = 0;
              while (i < amt)
              {
                byte x = byteArray[i++];
                long bytevalue = (long)x;
                checkSum = (checkSum << 5) ^ (checkSum >> 3) ^ (bytevalue << 2) ^ (bytevalue >> 3);
              }

              os.write(byteArray,0,amt);
              // Check if job is alive before looping
              activities.checkJobStillActive();
            }
          }
          finally
          {
            os.close();
          }

          synchronized(this)
          {
            deleteData(documentIdentifier);
            cacheData.put(documentIdentifier,new DocumentData(tempFile,responseCode,contentType,referralURI));
            return new Long(checkSum).toString();
          }

        }
        catch (IOException e)
        {
          ManifoldCF.deleteFile(tempFile);
          throw e;
        }
        catch (ManifoldCFException e)
        {
          ManifoldCF.deleteFile(tempFile);
          throw e;
        }
        catch (ServiceInterruption e)
        {
          ManifoldCF.deleteFile(tempFile);
          throw e;
        }
        catch (Error e)
        {
          ManifoldCF.deleteFile(tempFile);
          throw e;
        }
      }
      finally
      {
        try
        {
          dataStream.close();
        }
        catch (java.net.SocketTimeoutException e)
        {
          Logging.connectors.warn("WEB: Socket timeout exception closing data stream, ignoring: "+e.getMessage(),e);
        }
        catch (ConnectTimeoutException e)
        {
          Logging.connectors.warn("WEB: Socket connect timeout exception closing data stream, ignoring: "+e.getMessage(),e);
        }
        catch (InterruptedIOException e)
        {
          throw e;
        }
        catch (IOException e)
        {
          // We can get this if the socket was unexpectedly closed by the server; treat this
          // as a Service Interruption.  Generally, this is ok - warn but don't do anything else.
          Logging.connectors.warn("WEB: IO exception closing data stream, ignoring: "+e.getMessage(),e);
        }
      }
    }
    catch (java.net.SocketTimeoutException e)
    {
      throw new ManifoldCFException("Socket timeout exception creating temporary file: "+e.getMessage(),e);
    }
    catch (ConnectTimeoutException e)
    {
      throw new ManifoldCFException("Socket connect timeout exception creating temporary file: "+e.getMessage(),e);
    }
    catch (InterruptedIOException e)
    {
      //Logging.connectors.warn("IO interruption seen",e);
      throw new ManifoldCFException("Interrupted: "+e.getMessage(),ManifoldCFException.INTERRUPTED);
    }
    catch (IOException e)
    {
      throw new ManifoldCFException("IO exception creating temporary file: "+e.getMessage(),e);
    }
  }

  /** Get the response code.
  *@param documentIdentifier is the document identifier.
  *@return the code.
  */
  public synchronized int getResponseCode(String documentIdentifier)
  {
    DocumentData dd = cacheData.get(documentIdentifier);
    if (dd == null)
      return IThrottledConnection.FETCH_NOT_TRIED;
    return dd.getResponseCode();
  }

  /** Get the content type.
  *@param documentIdentifier is the document identifier.
  *@return the content type, or null if there is none.
  */
  public synchronized String getContentType(String documentIdentifier)
  {
    DocumentData dd = cacheData.get(documentIdentifier);
    if (dd == null)
      return null;
    return dd.getContentType();
  }

  /** Get the referral URI.
  *@param documentIdentifier is the document identifier.
  *@return the referral URI, or null if none.
  */
  public synchronized String getReferralURI(String documentIdentifier)
  {
    DocumentData dd = cacheData.get(documentIdentifier);
    if (dd == null)
      return null;
    return dd.getReferralURI();
  }

  /** Fetch binary data length.
  *@param documentIdentifier is the document identifier.
  *@return the length.
  */
  public synchronized long getDataLength(String documentIdentifier)
  {
    DocumentData dd = cacheData.get(documentIdentifier);
    if (dd == null)
      return 0L;
    return dd.getData().length();
  }

  /** Fetch binary data entry from the cache.
  *@param documentIdentifier is the document identifier (url).
  *@return a binary data stream.
  */
  public synchronized InputStream getData(String documentIdentifier)
    throws ManifoldCFException
  {
    DocumentData dd = cacheData.get(documentIdentifier);
    if (dd == null)
      return null;
    try
    {
      return new FileInputStream(dd.getData());
    }
    catch (FileNotFoundException e)
    {
      throw new ManifoldCFException("File not found exception opening data: "+e.getMessage(),e);
    }
  }

  /** Delete specified item of data.
  *@param documentIdentifier is the document identifier (url).
  */
  public synchronized void deleteData(String documentIdentifier)
  {
    DocumentData dd = cacheData.remove(documentIdentifier);
    if (dd != null)
    {
      ManifoldCF.deleteFile(dd.getData());
    }
  }

  // Protected classes

  /** This class represents everything we need to know about a document that's getting passed from the
  * getDocumentVersions() phase to the processDocuments() phase.
  */
  protected static class DocumentData
  {
    /** The cache file for the data */
    protected File data;
    /** The response code */
    protected int responseCode;
    /** The content-type header value */
    protected String contentType;
    /** The referral URI */
    protected String referralURI;

    // More will probably go here later, but I can't think of much else at the moment.

    /** Constructor. */
    public DocumentData(File data, int responseCode, String contentType, String referralURI)
    {
      this.data = data;
      this.responseCode = responseCode;
      this.contentType = contentType;
      this.referralURI = referralURI;
    }

    /** Get the data */
    public File getData()
    {
      return data;
    }

    /** Get the response code */
    public int getResponseCode()
    {
      return responseCode;
    }

    /** Get the contentType */
    public String getContentType()
    {
      return contentType;
    }

    /** Get the referral URI */
    public String getReferralURI()
    {
      return referralURI;
    }

  }

}
