FileProductStorage.java

  1. /*
  2.  * FileProductStorage
  3.  */
  4. package gov.usgs.earthquake.distribution;

  5. import gov.usgs.earthquake.product.ByteContent;
  6. import gov.usgs.earthquake.product.Content;
  7. import gov.usgs.earthquake.product.Product;
  8. import gov.usgs.earthquake.product.ProductId;
  9. import gov.usgs.earthquake.product.io.DirectoryProductHandler;
  10. import gov.usgs.earthquake.product.io.DirectoryProductSource;
  11. import gov.usgs.earthquake.product.io.FilterProductHandler;
  12. import gov.usgs.earthquake.product.io.ObjectProductHandler;
  13. import gov.usgs.earthquake.product.io.ObjectProductSource;
  14. import gov.usgs.earthquake.product.io.ProductHandler;
  15. import gov.usgs.earthquake.product.io.ProductSource;
  16. import gov.usgs.util.Config;
  17. import gov.usgs.util.DefaultConfigurable;
  18. import gov.usgs.util.FileUtils;
  19. import gov.usgs.util.ObjectLock;
  20. import gov.usgs.util.StringUtils;

  21. import java.io.File;
  22. import java.net.URL;
  23. import java.security.MessageDigest;
  24. import java.util.ArrayList;
  25. import java.util.HashMap;
  26. import java.util.Iterator;
  27. import java.util.List;
  28. import java.util.Map;
  29. import java.util.concurrent.ExecutorService;
  30. import java.util.concurrent.Executors;
  31. import java.util.logging.Level;
  32. import java.util.logging.Logger;

  33. /**
  34.  * Store products in the file system.
  35.  *
  36.  * This implementation of ProductStorage extracts products into directories.
  37.  *
  38.  * The FileProductStorage implements the Configurable interface and can use the
  39.  * following configuration parameters:
  40.  *
  41.  * <dl>
  42.  * <dt>directory</dt>
  43.  * <dd>(Optional, default = storage) The base directory where products are
  44.  * stored. Each product is stored in a separate directory within this directory.
  45.  * </dd>
  46.  *
  47.  * <dt>verifySignatures</dt>
  48.  * <dd>(Optional, default = off) Whether or not to verify signatures:
  49.  * <dl>
  50.  * <dt>off</dt>
  51.  * <dd>no verification</dd>
  52.  *
  53.  * <dt>test</dt>
  54.  * <dd>test but accept invalid signatures</dd>
  55.  *
  56.  * <dt>anything else</dt>
  57.  * <dd>reject invalid signatures.</dd>
  58.  * </dl>
  59.  * </dd>
  60.  *
  61.  * <dt>keychain</dt>
  62.  * <dd>(Optional) List of key section names to load for signature verification.</dd>
  63.  * </dl>
  64.  *
  65.  * An attempt is made to make storage operations atomic by using read and write
  66.  * locks. While a write operation (store or remove) is in progress, read
  67.  * operations will block. It is possible for a remove operation to occur between
  68.  * the time getProduct() returns and the time when product contents are actually
  69.  * loaded from a file. Users who are concerned about this should use the
  70.  * getInMemoryProduct() method, which holds a read lock until all product files
  71.  * are read.
  72.  *
  73.  * To override the directory structure or format, override one or more of the
  74.  * following methods:
  75.  *
  76.  * <pre>
  77.  * String getProductPath(ProductId)
  78.  * ProductSource getProductSourceFormat(File)
  79.  * ProductOutput getProductHandlerFormat(File)
  80.  * </pre>
  81.  */
  82. public class FileProductStorage extends DefaultConfigurable implements
  83.         ProductStorage {

  84.     /** Logging object. */
  85.     private static final Logger LOGGER = Logger
  86.             .getLogger(FileProductStorage.class.getName());

  87.     /** Property for configured listeners */
  88.     public static final String STORAGE_LISTENER_PROPERTY = "listeners";

  89.     /** Storage path property name used by Configurable interface. */
  90.     public static final String DIRECTORY_PROPERTY_NAME = "directory";
  91.     /** Default storage path if none is provided. */
  92.     public static final String DEFAULT_DIRECTORY = "storage";

  93.     /** Property for whether or not to hash file paths. */
  94.     public static final String USE_HASH_PATHS_PROPERTY = "useHashes";
  95.     /** Do not use hashes (Default). */
  96.     public static final boolean USE_HASH_PATHS_DEFAULT = false;

  97.     /** Property for legacyStorages. */
  98.     public static final String LEGACY_STORAGES_PROPERTY = "legacyStorages";

  99.     /** Base directory for product storage. */
  100.     private File baseDirectory;

  101.     private boolean useHashes = USE_HASH_PATHS_DEFAULT;

  102.     /** Locks used to make storage operations atomic. */
  103.     private ObjectLock<ProductId> storageLocks = new ObjectLock<ProductId>();

  104.     private SignatureVerifier verifier = new SignatureVerifier();

  105.     /**
  106.      * @return the storageLocks
  107.      */
  108.     public ObjectLock<ProductId> getStorageLocks() {
  109.         return storageLocks;
  110.     }

  111.     private Map<StorageListener, ExecutorService> listeners = new HashMap<StorageListener, ExecutorService>();

  112.     /**
  113.      * A list of product storages used only for retrieving products, never for
  114.      * storing. Assists with migration between formats and other settings.
  115.      */
  116.     private final ArrayList<ProductStorage> legacyStorages = new ArrayList<ProductStorage>();

  117.     /**
  118.      * Create this digest once, and clone it later. Only used if
  119.      * <code>useHashed</code> is set to <code>true</code>.
  120.      */
  121.     private static final MessageDigest SHA_DIGEST;
  122.     static {
  123.         MessageDigest digest = null;
  124.         try {
  125.             digest = MessageDigest.getInstance("SHA");
  126.         } catch (Exception e) {
  127.             LOGGER.warning("Unable to create SHA Digest for HashFileProductStorage");
  128.             digest = null;
  129.         }
  130.         SHA_DIGEST = digest;
  131.     }

  132.     /**
  133.      * This is chosen because 16^3 = 4096 &lt; 32000, which is the ext3
  134.      * subdirectory limit.
  135.      */
  136.     public static final int DIRECTORY_NAME_LENGTH = 3;

  137.     /**
  138.      * Create a new FileProductStorage using the default storage path.
  139.      */
  140.     public FileProductStorage() {
  141.         this(new File(DEFAULT_DIRECTORY));
  142.     }

  143.     /**
  144.      * Create a new FileProductStorage.
  145.      *
  146.      * @param baseDirectory
  147.      *            the base directory for all products being stored.
  148.      */
  149.     public FileProductStorage(final File baseDirectory) {
  150.         this.baseDirectory = baseDirectory;
  151.     }

  152.     /**
  153.      * Configure this object.
  154.      *
  155.      * Expects a key named "directory".
  156.      */
  157.     public void configure(Config config) throws Exception {
  158.         String directory = config.getProperty(DIRECTORY_PROPERTY_NAME,
  159.                 DEFAULT_DIRECTORY);
  160.         baseDirectory = new File(directory);
  161.         LOGGER.config("[" + getName() + "] using storage directory "
  162.                 + baseDirectory.getCanonicalPath());

  163.         // Configure verifier
  164.         verifier.configure(config);

  165.         // Set up our configured listeners
  166.         Iterator<String> listenerIter = StringUtils.split(
  167.                 config.getProperty(STORAGE_LISTENER_PROPERTY), ",").iterator();
  168.         while (listenerIter.hasNext()) {
  169.             String listenerName = listenerIter.next();
  170.             try {
  171.                 StorageListener listener = (StorageListener) Config.getConfig()
  172.                         .getObject(listenerName);
  173.                 addStorageListener(listener);
  174.             } catch (Exception ccx) {
  175.                 throw new ConfigurationException("[" + getName()
  176.                         + "] listener \"" + listenerName
  177.                         + "\" was not properly configured. " + ccx.getMessage());
  178.             }
  179.         }

  180.         // load legacy storages
  181.         Iterator<String> legacyIter = StringUtils.split(
  182.                 config.getProperty(LEGACY_STORAGES_PROPERTY), ",").iterator();
  183.         while (legacyIter.hasNext()) {
  184.             String legacyName = legacyIter.next();
  185.             try {
  186.                 ProductStorage legacyStorage = (ProductStorage) Config
  187.                         .getConfig().getObject(legacyName);
  188.                 legacyStorages.add(legacyStorage);
  189.             } catch (Exception e) {
  190.                 throw new ConfigurationException("[" + getName()
  191.                         + "] legacy storage '" + legacyName
  192.                         + "' not properly configured. " + e.getMessage());
  193.             }
  194.         }
  195.     }

  196.     @Override
  197.     public synchronized void notifyListeners(final StorageEvent event) {
  198.         Iterator<StorageListener> listenerIter = listeners.keySet().iterator();
  199.         while (listenerIter.hasNext()) {
  200.             final StorageListener listener = listenerIter.next();
  201.             LOGGER.finer("[" + getName() + "] listener :: "
  202.                     + listener.getClass().getCanonicalName());
  203.             final ExecutorService service = listeners.get(listener);

  204.             service.submit(new Runnable() {

  205.                 public void run() {
  206.                     listener.onStorageEvent(event);
  207.                 }
  208.             });
  209.         }
  210.     }

  211.     @Override
  212.     public void addStorageListener(final StorageListener listener) {
  213.         LOGGER.finest("[" + getName() + "] adding listener :: "
  214.                 + listener.getClass().getCanonicalName());
  215.         if (!listeners.containsKey(listener)) {
  216.             ExecutorService service = Executors.newSingleThreadExecutor();
  217.             listeners.put(listener, service);
  218.         }
  219.     }

  220.     @Override
  221.     public void removeStorageListener(final StorageListener listener) {
  222.         ExecutorService service = listeners.remove(listener);

  223.         if (service != null) {
  224.             service.shutdown();
  225.         }
  226.     }

  227.     /**
  228.      * A method for subclasses to override the storage path.
  229.      *
  230.      * The returned path is appended to the base directory when storing and
  231.      * retrieving products.
  232.      *
  233.      * @param id
  234.      *            the product id to convert.
  235.      * @return the directory used to store id.
  236.      */
  237.     public String getProductPath(final ProductId id) {

  238.         if (useHashes) {
  239.             return getHashedProductPath(id);
  240.         } else {
  241.             return getNormalProductPath(id);
  242.         }
  243.     }
  244.     /**
  245.      * @param id Specific productID
  246.      * @return string buffer of hashed product path
  247.      */
  248.     protected String getHashedProductPath(final ProductId id) {
  249.         try {
  250.             MessageDigest digest;
  251.             synchronized (SHA_DIGEST) {
  252.                 digest = ((MessageDigest) SHA_DIGEST.clone());
  253.             }

  254.             String hexDigest = toHexString(digest.digest(id.toString()
  255.                     .getBytes()));

  256.             StringBuffer buf = new StringBuffer();
  257.             // start with product type, to give idea of available products and
  258.             // disk usage when looking at filesystem
  259.             buf.append(id.getType());

  260.             // sub directories based on hash
  261.             int length = hexDigest.length();
  262.             for (int i = 0; i < length; i += DIRECTORY_NAME_LENGTH) {
  263.                 String part;
  264.                 if (i + DIRECTORY_NAME_LENGTH < length) {
  265.                     part = hexDigest.substring(i, i + DIRECTORY_NAME_LENGTH);
  266.                 } else {
  267.                     part = hexDigest.substring(i);
  268.                 }
  269.                 buf.append(File.separator);
  270.                 buf.append(part);
  271.             }

  272.             return buf.toString();
  273.         } catch (CloneNotSupportedException e) {
  274.             // fall back to parent class
  275.             return getNormalProductPath(id);
  276.         }
  277.     }

  278.     /**
  279.      * Convert an array of bytes into a hex string. The string will always be
  280.      * twice as long as the input byte array, because bytes < 0x10 are zero
  281.      * padded.
  282.      *
  283.      * @param bytes
  284.      *            byte array to convert to hex.
  285.      * @return hex string equivalent of input byte array.
  286.      */
  287.     private String toHexString(final byte[] bytes) {
  288.         StringBuffer buf = new StringBuffer();
  289.         int length = bytes.length;
  290.         for (int i = 0; i < length; i++) {
  291.             String hex = Integer.toHexString(0xFF & bytes[i]);
  292.             if (hex.length() == 1) {
  293.                 buf.append('0');
  294.             }
  295.             buf.append(hex);
  296.         }
  297.         return buf.toString();
  298.     }

  299.     /**
  300.      * @param id ProductId
  301.      * @return string buffer of normal product path
  302.      */
  303.     public String getNormalProductPath(final ProductId id) {
  304.         StringBuffer buf = new StringBuffer();
  305.         buf.append(id.getType());
  306.         buf.append(File.separator);
  307.         buf.append(id.getCode());
  308.         buf.append(File.separator);
  309.         buf.append(id.getSource());
  310.         buf.append(File.separator);
  311.         buf.append(id.getUpdateTime().getTime());
  312.         return buf.toString();
  313.     }

  314.     /**
  315.      * A method for subclasses to override the storage format.
  316.      *
  317.      * When overriding this method, the method getProductSourceFormat should
  318.      * also be overridden.
  319.      *
  320.      * @param file
  321.      *            a file that should be converted into a ProductHandler.
  322.      * @return the ProductHandler.
  323.      * @throws Exception if error occurs
  324.      */
  325.     protected ProductHandler getProductHandlerFormat(final File file)
  326.             throws Exception {
  327.         return new DirectoryProductHandler(file);
  328.     }

  329.     /**
  330.      * A method for subclasses to override the storage format.
  331.      *
  332.      * When overriding this method, the method getProductHandlerFormat should
  333.      * also be overridden.
  334.      *
  335.      * @param file
  336.      *            a file that should be converted into a ProductSource.
  337.      * @return the ProductSource.
  338.      * @throws Exception if error occurs
  339.      */
  340.     protected ProductSource getProductSourceFormat(final File file)
  341.             throws Exception {
  342.         return new DirectoryProductSource(file);
  343.     }

  344.     /**
  345.      * Get the file or directory used to store a specific product.
  346.      *
  347.      * @param id
  348.      *            which product.
  349.      * @return a file or directory where the product would be stored.
  350.      */
  351.     public File getProductFile(final ProductId id) {
  352.         String path = getProductPath(id);
  353.         // remove any leading slash so path will always be within baseDirectory.
  354.         if (path.startsWith("/")) {
  355.             path = path.substring(1);
  356.         }
  357.         return new File(baseDirectory, path);
  358.     }

  359.     /**
  360.      * Get a product from storage.
  361.      *
  362.      * Calls the getProductSource method, and uses ObjectProductHandler to
  363.      * convert the ProductSource into a Product.
  364.      *
  365.      * @param id
  366.      *            the product to retrieve.
  367.      * @return the product, or null if not in this storage.
  368.      */
  369.     public Product getProduct(ProductId id) throws Exception {
  370.         ProductSource source = getProductSource(id);
  371.         if (source == null) {
  372.             return null;
  373.         } else {
  374.             return ObjectProductHandler.getProduct(source);
  375.         }
  376.     }

  377.     /**
  378.      * Get a product from storage, loading all file contents into memory.
  379.      *
  380.      * This method may cause memory problems if product contents are large.
  381.      *
  382.      * @param id
  383.      *            the product to retrieve.
  384.      * @return the loaded product.
  385.      * @throws Exception if error occurs
  386.      */
  387.     public Product getInMemoryProduct(ProductId id) throws Exception {
  388.         LOGGER.finest("[" + getName() + "] acquiring read lock for product id="
  389.                 + id.toString());
  390.         storageLocks.acquireReadLock(id);
  391.         LOGGER.finest("[" + getName() + "] acquired read lock for product id="
  392.                 + id.toString());
  393.         try {
  394.             // load product
  395.             Product product = getProduct(id);
  396.             // convert all contents to ByteContent
  397.             Map<String, Content> contents = product.getContents();
  398.             Iterator<String> iter = contents.keySet().iterator();
  399.             while (iter.hasNext()) {
  400.                 String path = iter.next();
  401.                 contents.put(path, new ByteContent(contents.get(path)));
  402.             }
  403.             // product content is all in memory
  404.             return product;
  405.         } finally {
  406.             LOGGER.finest("[" + getName()
  407.                     + "] releasing read lock for product id=" + id.toString());
  408.             storageLocks.releaseReadLock(id);
  409.             LOGGER.finest("[" + getName()
  410.                     + "] released write lock for product id=" + id.toString());
  411.         }
  412.     }

  413.     /**
  414.      * Get a ProductSource from storage.
  415.      *
  416.      * @param id
  417.      *            the product to retrieve.
  418.      * @return a ProductSource for the product, or null if not in this storage.
  419.      */
  420.     public ProductSource getProductSource(ProductId id) throws Exception {
  421.         ProductSource productSource = null;

  422.         LOGGER.finest("[" + getName() + "] acquiring read lock for product id="
  423.                 + id.toString());
  424.         // acquire lock in case storage operation in progress
  425.         storageLocks.acquireReadLock(id);
  426.         LOGGER.finest("[" + getName() + "] acquired read lock for product id="
  427.                 + id.toString());
  428.         try {
  429.             File productFile = getProductFile(id);
  430.             if (productFile.exists()) {
  431.                 productSource = getProductSourceFormat(productFile);
  432.             }
  433.             if (productSource == null) {
  434.                 Iterator<ProductStorage> legacyIter = legacyStorages.iterator();
  435.                 while (legacyIter.hasNext()) {
  436.                     ProductStorage next = legacyIter.next();
  437.                     try {
  438.                         productSource = next.getProductSource(id);
  439.                         if (productSource != null) {
  440.                             break;
  441.                         }
  442.                     } catch (Exception e) {
  443.                         LOGGER.log(Level.FINE, "[" + getName() + "] " +
  444.                                 "legacy storage getProductSource exception ", e);
  445.                     }
  446.                 }
  447.             }
  448.         } finally {
  449.             // release the lock no matter what
  450.             LOGGER.finest("[" + getName()
  451.                     + "] releasing read lock for product id=" + id.toString());
  452.             storageLocks.releaseReadLock(id);
  453.             LOGGER.finest("[" + getName()
  454.                     + "] released read lock for product id=" + id.toString());
  455.         }

  456.         return productSource;
  457.     }

  458.     /**
  459.      * Check whether a product exists in storage.
  460.      *
  461.      * @param id
  462.      *            the product to check.
  463.      * @return true if the product exists, false otherwise.
  464.      */
  465.     public boolean hasProduct(ProductId id) throws Exception {
  466.         boolean hasProduct = false;

  467.         LOGGER.finest("[" + getName() + "] acquiring read lock for product id="
  468.                 + id.toString());
  469.         // acquire lock in case storage operation in progress
  470.         storageLocks.acquireReadLock(id);
  471.         LOGGER.finest("[" + getName() + "] acquired read lock for product id="
  472.                 + id.toString());
  473.         try {
  474.             File productDirectory = getProductFile(id);
  475.             hasProduct = productDirectory.exists();
  476.             if (hasProduct) {
  477.                 // be a little more detailed...
  478.                 ProductSource source = getProductSource(id);
  479.                 if (source == null) {
  480.                     hasProduct = false;
  481.                 } else if (source instanceof DirectoryProductSource) {
  482.                     // not sure how we would get here
  483.                     // FileNotFound exception appears in logs...
  484.                     hasProduct = (new File(productDirectory,
  485.                             DirectoryProductHandler.PRODUCT_XML_FILENAME)
  486.                             .exists());
  487.                 }
  488.                 if (source != null) {
  489.                     source.close();
  490.                 }
  491.             }

  492.             if (!hasProduct) {
  493.                 // primary storage doesn't have product, check legacy storages
  494.                 Iterator<ProductStorage> legacyIter = legacyStorages.iterator();
  495.                 while (legacyIter.hasNext()) {
  496.                     ProductStorage next = legacyIter.next();
  497.                     try {
  498.                         if (next.hasProduct(id)) {
  499.                             return true;
  500.                         }
  501.                     } catch (Exception e) {
  502.                         LOGGER.log(Level.FINE, "[" + getName()
  503.                                 + "] legacy storage hasProduct exception ", e);
  504.                     }
  505.                 }
  506.             }
  507.         } finally {
  508.             LOGGER.finest("[" + getName()
  509.                     + "] releasing read lock for product id=" + id.toString());
  510.             // release lock no matter what
  511.             storageLocks.releaseReadLock(id);
  512.             LOGGER.finest("[" + getName()
  513.                     + "] released read lock for product id=" + id.toString());
  514.         }

  515.         return hasProduct;
  516.     }

  517.     /**
  518.      * Remove a product from storage.
  519.      *
  520.      * @param id
  521.      *            product to remove.
  522.      */
  523.     public void removeProduct(ProductId id) throws Exception {
  524.         String idString = id.toString();
  525.         LOGGER.finest("[" + getName()
  526.                 + "] acquiring write lock for product id=" + idString);
  527.         // acquire lock in case storage operation in progress
  528.         storageLocks.acquireWriteLock(id);
  529.         LOGGER.finest("[" + getName() + "] acquired write lock for product id="
  530.                 + idString);
  531.         try {
  532.             File productFile = getProductFile(id);
  533.             if (productFile.exists()) {
  534.                 // recursively delete the product directory
  535.                 FileUtils.deleteTree(productFile);
  536.                 // remove any empty parent directories
  537.                 FileUtils.deleteEmptyParents(productFile, baseDirectory);
  538.                 LOGGER.finer("[" + getName() + "] product removed, id=" + idString);
  539.             }
  540.             productFile = null;
  541.             // remove from any legacy storages
  542.             Iterator<ProductStorage> legacyIter = legacyStorages.iterator();
  543.             while (legacyIter.hasNext()) {
  544.                 ProductStorage next = legacyIter.next();
  545.                 try {
  546.                     next.removeProduct(id);
  547.                 } catch (Exception e) {
  548.                     LOGGER.log(Level.FINE, "[" + getName()
  549.                             + "] legacy storage remove exception ", e);
  550.                 }
  551.             }
  552.         } finally {
  553.             LOGGER.finest("[" + getName()
  554.                     + "] releasing write lock for product id=" + idString);
  555.             // release lock no matter what
  556.             storageLocks.releaseWriteLock(id);
  557.             LOGGER.finest("[" + getName()
  558.                     + "] released write lock for product id=" + idString);
  559.         }

  560.         // Notify listeners
  561.         notifyListeners(new StorageEvent(this, id, StorageEvent.PRODUCT_REMOVED));
  562.     }

  563.     /**
  564.      * Store a product in storage.
  565.      *
  566.      * Same as storeProductSource(new ObjectProductSource(product)).
  567.      *
  568.      * @param product
  569.      *            the product to store.
  570.      * @return the id of the stored product.
  571.      */
  572.     public ProductId storeProduct(Product product) throws Exception {
  573.         return storeProductSource(new ObjectProductSource(product));
  574.     }

  575.     /**
  576.      * Store a ProductSource to storage.
  577.      *
  578.      * If any exceptions occur while storing a product (other than the product
  579.      * already existing in storage) the incompletely stored product is removed.
  580.      *
  581.      * @param source
  582.      *            the ProductSource to store.
  583.      * @return the id of the stored product.
  584.      */
  585.     public ProductId storeProductSource(ProductSource source) throws Exception {
  586.         StorageProductOutput output = new StorageProductOutput();
  587.         // output acquires the storageLock during onBeginProduct, once the
  588.         // product id is known.
  589.         try {
  590.             source.streamTo(output);
  591.             // close output so file(s) are written
  592.             output.close();

  593.             ProductId id = output.getProductId();
  594.             LOGGER.finer("[" + getName() + "] product stored id=" + id
  595.                     + ", status=" + output.getStatus());

  596.             verifier.verifySignature(getProduct(id));

  597.         } catch (Exception e) {
  598.             if (!(e instanceof ProductAlreadyInStorageException)
  599.                     && !(e.getCause() instanceof ProductAlreadyInStorageException)) {
  600.                 if (e instanceof InvalidSignatureException) {
  601.                     // suppress stack trace for invalid signature
  602.                     LOGGER.warning(e.getMessage()
  603.                             + ", removing incomplete product");
  604.                 } else {
  605.                     LOGGER.log(
  606.                             Level.WARNING,
  607.                             "["
  608.                                     + getName()
  609.                                     + "] exception while storing product, removing incomplete product",
  610.                             e);
  611.                 }
  612.                 try {
  613.                     // remove incompletely stored product.
  614.                     removeProduct(output.getProductId());
  615.                 } catch (Exception e2) {
  616.                     // ignore
  617.                     LOGGER.log(Level.WARNING, "[" + getName()
  618.                             + "] exception while removing incomplete product",
  619.                             e2);
  620.                 }
  621.             }
  622.             throw e;
  623.         } finally {
  624.             // DO RELEASE THE WRITE LOCK HERE

  625.             // This leads to thread sync problems in
  626.             // SearchResponseXmlProductSource, because xml events were sent in
  627.             // one thread, leading to acquisition of a write lock, while this
  628.             // method was called in a separate thread and attempted to release
  629.             // the write lock.

  630.             // However, not releasing the lock here leads to other problems when
  631.             // hubs are receiving products via multiple receivers.

  632.             ProductId id = output.getProductId();

  633.             if (id != null) {
  634.                 // release the write lock
  635.                 LOGGER.finest("[" + getName()
  636.                         + "] releasing write lock for product id="
  637.                         + id.toString());
  638.                 storageLocks.releaseWriteLock(id);
  639.                 LOGGER.finest("[" + getName()
  640.                         + "] released write lock for product id="
  641.                         + id.toString());
  642.             }

  643.             // close underlying handler
  644.             output.close();
  645.             output.setProductOutput(null);

  646.             source.close();
  647.         }

  648.         ProductId id = output.getProductId();
  649.         // Notify our storage listeners
  650.         StorageEvent event = new StorageEvent(this, id,
  651.                 StorageEvent.PRODUCT_STORED);
  652.         notifyListeners(event);

  653.         return id;
  654.     }

  655.     /**
  656.      * Used when storing products.
  657.      *
  658.      * When onBeginProduct is called with the ProductId being stored, a
  659.      * DirectoryProductOutput is created which manages storage.
  660.      */
  661.     private class StorageProductOutput extends FilterProductHandler {

  662.         /** The stored product id. */
  663.         private ProductId id;

  664.         /** The stored product status. */
  665.         private String status;

  666.         /**
  667.          * Construct a new StorageProductOutput.
  668.          */
  669.         public StorageProductOutput() {
  670.         }

  671.         /**
  672.          * @return the product id that was stored.
  673.          */
  674.         public ProductId getProductId() {
  675.             return id;
  676.         }

  677.         /**
  678.          * @return the product status that was stored.
  679.          */
  680.         public String getStatus() {
  681.             return status;
  682.         }

  683.         /**
  684.          * The productID is stored and can be found using getProductId().
  685.          */
  686.         public void onBeginProduct(ProductId id, String status, URL trackerURL)
  687.                 throws Exception {
  688.             // save the product id for later
  689.             this.id = id;
  690.             this.status = status;

  691.             // acquire write lock for product
  692.             LOGGER.finest("[" + getName()
  693.                     + "] acquiring write lock for product id=" + id.toString());
  694.             storageLocks.acquireWriteLock(id);
  695.             // keep track that we have write lock
  696.             LOGGER.finest("[" + getName()
  697.                     + "] acquired write lock for product id=" + id.toString());
  698.             if (hasProduct(id)) {
  699.                 throw new ProductAlreadyInStorageException("[" + getName()
  700.                         + "] product already in storage");
  701.             }

  702.             // set the wrapped product output
  703.             setProductOutput(getProductHandlerFormat(getProductFile(id)));
  704.             // call the directory product output onBeginProduct method to start
  705.             // writing the product
  706.             super.onBeginProduct(id, status, trackerURL);
  707.         }

  708.         public void onEndProduct(ProductId id) throws Exception {
  709.             // call the directory product output onEndProduct method to finish
  710.             // writing the product
  711.             super.onEndProduct(id);

  712.             // DONT RELEASE THE LOCK HERE, this causes bigger problems on
  713.             // hubs...

  714.             // release the write lock
  715.             // LOGGER.finest("Releasing write lock for product id=" +
  716.             // id.toString());
  717.             // storageLocks.releaseWriteLock(id);
  718.             // keep track that we no longer have write lock
  719.             // this.haveWriteLock = false;
  720.             // LOGGER.finest("Released write lock for product id=" +
  721.             // id.toString());
  722.         }
  723.     }

  724.     /**
  725.      * Called at client shutdown to free resources.
  726.      */
  727.     public void shutdown() throws Exception {
  728.         // Remove all our listeners. Doing this will also shut down the
  729.         // ExecutorServices
  730.         Iterator<StorageListener> listenerIter = listeners.keySet().iterator();
  731.         while (listenerIter.hasNext()) {
  732.             removeStorageListener(listenerIter.next());
  733.             // Maybe we should call "listener.shutdown()" here as well?
  734.         }

  735.         // shutdown any legacy storages
  736.         Iterator<ProductStorage> legacyIter = legacyStorages.iterator();
  737.         while (legacyIter.hasNext()) {
  738.             ProductStorage next = legacyIter.next();
  739.             try {
  740.                 next.shutdown();
  741.             } catch (Exception e) {
  742.                 LOGGER.log(Level.FINE, "[" + getName()
  743.                         + "] legacy storage shutdown exception ", e);
  744.             }
  745.         }
  746.     }

  747.     /**
  748.      * Called after client configuration to begin processing.
  749.      */
  750.     public void startup() throws Exception {
  751.         // startup any legacy storages
  752.         Iterator<ProductStorage> legacyIter = legacyStorages.iterator();
  753.         while (legacyIter.hasNext()) {
  754.             ProductStorage next = legacyIter.next();
  755.             try {
  756.                 next.startup();
  757.             } catch (Exception e) {
  758.                 LOGGER.log(Level.FINE, "[" + getName()
  759.                         + "] legacy storage startup exception ", e);
  760.             }
  761.         }
  762.     }

  763.     /**
  764.      * @return the baseDirectory
  765.      */
  766.     public File getBaseDirectory() {
  767.         return baseDirectory;
  768.     }

  769.     /**
  770.      * @param baseDirectory
  771.      *            the baseDirectory to set
  772.      */
  773.     public void setBaseDirectory(File baseDirectory) {
  774.         this.baseDirectory = baseDirectory;
  775.     }

  776.     /**
  777.      * @return the rejectInvalidSignatures
  778.      */
  779.     public boolean isRejectInvalidSignatures() {
  780.         return verifier.isRejectInvalidSignatures();
  781.     }

  782.     /**
  783.      * @param rejectInvalidSignatures
  784.      *            the rejectInvalidSignatures to set
  785.      */
  786.     public void setRejectInvalidSignatures(boolean rejectInvalidSignatures) {
  787.         verifier.setRejectInvalidSignatures(rejectInvalidSignatures);
  788.     }

  789.     /**
  790.      * @return the testSignatures
  791.      */
  792.     public boolean isTestSignatures() {
  793.         return verifier.isTestSignatures();
  794.     }

  795.     /**
  796.      * @param testSignatures
  797.      *            the testSignatures to set
  798.      */
  799.     public void setTestSignatures(boolean testSignatures) {
  800.         verifier.setTestSignatures(testSignatures);
  801.     }

  802.     /**
  803.      * @return the keychain
  804.      */
  805.     public ProductKeyChain getKeychain() {
  806.         return verifier.getKeychain();
  807.     }

  808.     /**
  809.      * @param keychain
  810.      *            the keychain to set
  811.      */
  812.     public void setKeychain(ProductKeyChain keychain) {
  813.         verifier.setKeychain(keychain);
  814.     }

  815.     /**
  816.      * @return the legacyStorages.
  817.      */
  818.     public List<ProductStorage> getLegacyStorages() {
  819.         return legacyStorages;
  820.     }

  821. }