BESUncompressCache.cc

 
// This file is part of bes, A C++ back-end server implementation framework
// for the OPeNDAP Data Access Protocol.
 
// Copyright (c) 2015 OPeNDAP, Inc
// Author: Nathan Potter <npotter@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
#include "config.h"
 
#include <string>
#include <fstream>
#include <sstream>
#include <sys/stat.h>
 
#include "BESInternalError.h"
#include "BESUtil.h"
#include "BESDebug.h"
#include "TheBESKeys.h"
#include "BESUncompressCache.h"
 
using std::endl;
using std::string;
 
BESUncompressCache *BESUncompressCache::d_instance = 0;
bool BESUncompressCache::d_enabled = true;
 
const string BESUncompressCache::DIR_KEY = "BES.UncompressCache.dir";
const string BESUncompressCache::PREFIX_KEY = "BES.UncompressCache.prefix";
const string BESUncompressCache::SIZE_KEY = "BES.UncompressCache.size";
 
#define MODULE "cache"
#define prolog std::string("BESUncompressCache::").append(__func__).append("() - ")
 
unsigned long BESUncompressCache::getCacheSizeFromConfig()
{
    bool found;
    string size;
    unsigned long size_in_megabytes = 0;
    TheBESKeys::TheKeys()->get_value(SIZE_KEY, size, found);
    if (found) {
        std::istringstream iss(size);
        iss >> size_in_megabytes;
    }
    else {
        string msg = prolog+ "The BES Key " + SIZE_KEY
            + " is not set! It MUST be set to utilize the decompression cache. ";
        BESDEBUG( MODULE, msg << endl);
        throw BESInternalError(msg, __FILE__, __LINE__);
    }
    return size_in_megabytes;
}
 
string BESUncompressCache::getCacheDirFromConfig()
{
    bool found;
    string subdir = "";
    TheBESKeys::TheKeys()->get_value(DIR_KEY, subdir, found);
 
    if (!found) {
        string msg = prolog + "The BES Key " + DIR_KEY
            + " is not set! It MUST be set to utilize the decompression cache. ";
        BESDEBUG( MODULE, msg << endl);
        throw BESInternalError(msg, __FILE__, __LINE__);
    }
 
    return subdir;
}
 
string BESUncompressCache::getCachePrefixFromConfig()
{
    bool found;
    string prefix = "";
    TheBESKeys::TheKeys()->get_value(PREFIX_KEY, prefix, found);
    if (found) {
        prefix = BESUtil::lowercase(prefix);
    }
    else {
        string msg = prolog + "The BES Key " + PREFIX_KEY
            + " is not set! It MUST be set to utilize the decompression cache. ";
        BESDEBUG( MODULE, msg << endl);
        throw BESInternalError(msg, __FILE__, __LINE__);
    }
 
    return prefix;
}

string BESUncompressCache::get_cache_file_name(const string &src, bool mangle)
{
    string cache_file_name = src;
 
    if (mangle) {
        string::size_type last_dot = cache_file_name.rfind('.');
        if (last_dot != string::npos) {
            cache_file_name = cache_file_name.substr(0, last_dot);
        }
    }
    cache_file_name = BESFileLockingCache::get_cache_file_name(cache_file_name);
 
    BESDEBUG( MODULE, prolog << "cache_file_name:      '" << cache_file_name << "'" << endl);
 
    return cache_file_name;
}
 
BESUncompressCache::BESUncompressCache()
{
    BESDEBUG( MODULE, prolog << "BEGIN" << endl);
 
    d_enabled = true;
    d_dimCacheDir = getCacheDirFromConfig();
    d_dimCacheFilePrefix = getCachePrefixFromConfig();
    d_maxCacheSize = getCacheSizeFromConfig();
 
    BESDEBUG( MODULE, prolog << "Cache configuration params: " << d_dimCacheDir << ", " << d_dimCacheFilePrefix << ", " << d_maxCacheSize << endl);
 
    initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
 
    BESDEBUG( MODULE, prolog << "END" << endl);
 
}
BESUncompressCache::BESUncompressCache(const string &data_root_dir, const string &cache_dir, const string &prefix,
    unsigned long long size)
{
    BESDEBUG( MODULE, prolog << "BEGIN" << endl);
    d_enabled = true;
 
    d_dataRootDir = data_root_dir;
    d_dimCacheDir = cache_dir;
    d_dimCacheFilePrefix = prefix;
    d_maxCacheSize = size;
 
    initialize(d_dimCacheDir, d_dimCacheFilePrefix, d_maxCacheSize);
 
    BESDEBUG( MODULE, prolog << "END" << endl);
}
 
BESUncompressCache *
BESUncompressCache::get_instance(const string &data_root_dir, const string &cache_dir, const string &result_file_prefix,
    unsigned long long max_cache_size)
{
    if (d_enabled && d_instance == 0) {
        if (dir_exists(cache_dir)) {
            d_instance = new BESUncompressCache(data_root_dir, cache_dir, result_file_prefix, max_cache_size);
            d_enabled = d_instance->cache_enabled();
            if(!d_enabled){
                delete d_instance;
                d_instance = NULL;
                BESDEBUG( MODULE, prolog <<  "Cache is DISABLED"<< endl);
           }
            else {
    #ifdef HAVE_ATEXIT
                atexit(delete_instance);
    #endif
                BESDEBUG( MODULE, prolog << "Cache is ENABLED"<< endl);
           }
        }
    }
    return d_instance;
}

BESUncompressCache *
BESUncompressCache::get_instance()
{
    if (d_enabled && d_instance == 0) {
        d_instance = new BESUncompressCache();
        d_enabled = d_instance->cache_enabled();
        if(!d_enabled){
            delete d_instance;
            d_instance = NULL;
            BESDEBUG( MODULE, prolog << "Cache is DISABLED"<< endl);
        }
        else {
#ifdef HAVE_ATEXIT
            atexit(delete_instance);
#endif
            BESDEBUG( MODULE, prolog << "Cache is ENABLED"<< endl);
        }
    }
 
    return d_instance;
}
 
BESUncompressCache::~BESUncompressCache() {}

bool BESUncompressCache::is_valid(const string &cache_file_name, const string &local_id)
{
    // If the cached response is zero bytes in size, it's not valid.
    // (hmmm...)
    string datasetFileName = BESUtil::assemblePath(d_dataRootDir, local_id, true);
 
    off_t entry_size = 0;
    time_t entry_time = 0;
    struct stat buf;
    if (stat(cache_file_name.c_str(), &buf) == 0) {
        entry_size = buf.st_size;
        entry_time = buf.st_mtime;
    }
    else {
        return false;
    }
 
    if (entry_size == 0) return false;
 
    time_t dataset_time = entry_time;
    if (stat(datasetFileName.c_str(), &buf) == 0) {
        dataset_time = buf.st_mtime;
    }
 
    // Trick: if the d_dataset is not a file, stat() returns error and
    // the times stay equal and the code uses the cache entry.
 
    // TODO Fix this so that the code can get a LMT from the correct handler.
    // TODO Consider adding a getLastModified() method to the libdap::DDS object to support this
    // TODO The DDS may be expensive to instantiate - I think the handler may be a better location
    // for an LMT method, if we can access the handler when/where needed.
    if (dataset_time > entry_time) return false;
 
    return true;
}