Store the DAP metadata responses. More...

#include <GlobalMetadataStore.h>

Inheritance diagram for bes::GlobalMetadataStore:

Collaboration diagram for bes::GlobalMetadataStore:

Classes
struct	MDSReadLock
	Unlock and close the MDS item when the ReadLock goes out of scope. More...

struct	StreamDAP

struct	StreamDAS
	Instantiate with a DDS or DMR and use to write the DAS response. More...

struct	StreamDDS
	Instantiate with a DDS or DMR and use to write the DDS response. More...

struct	StreamDMR
	Instantiate with a DDS or DMR and use to write the DMR response. More...

Public Types
typedef struct MDSReadLock	MDSReadLock

Public Member Functions
bool	cache_enabled () const

virtual bool	cache_too_big (unsigned long long current_size) const
	look at the cache size; is it too large? Look at the cache size and see if it is too big.

virtual bool	create_and_lock (const std::string &target, int &fd)
	Create a file in the cache and lock it for write access.

void	disable ()
	Disable the cache.

void	dump (std::ostream &strm) const override
	dumps information about this object

void	enable ()
	Enable the cache.

virtual void	exclusive_to_shared_lock (int fd)
	Transfer from an exclusive lock to a shared lock.

std::string	get_cache_directory () const

virtual std::string	get_cache_file_name (const std::string &src, bool mangle=true)

std::string	get_cache_file_prefix () const

virtual time_t	get_cache_lmt (const std::string &fileName, const std::string &suffix)
	Get the last modified time for the cached object file.

virtual unsigned long long	get_cache_size ()
	Get the cache size.

virtual libdap::DDS *	get_dds_object (const std::string &name)
	Build a DDS object from the cached Response.

virtual libdap::DMR *	get_dmr_object (const std::string &name)
	Build a DMR object from the cached Response.

virtual bool	get_exclusive_lock (const std::string &target, int &fd)

virtual bool	get_exclusive_lock_nb (const std::string &target, int &fd)

virtual bool	get_read_lock (const std::string &target, int &fd)
	Get a read-only lock on the file if it exists.

void	initialize (const std::string &cache_dir, const std::string &prefix, unsigned long long size)
	Initialize an instance of FileLockingCache.

virtual bool	is_available_helper (const std::string &realName, const std::string &relativeName, const std::string &fileType, const std::string &suffix)
	helper function that checks if last modified time is greater than cached file

virtual MDSReadLock	is_das_available (const BESContainer &container)

virtual MDSReadLock	is_das_available (const std::string &name)
	Is the DAS response for.

virtual MDSReadLock	is_dds_available (const BESContainer &container)
	Is the DDS response for.

virtual MDSReadLock	is_dds_available (const std::string &name)
	Is the DDS response for.

virtual MDSReadLock	is_dmr_available (const BESContainer &container)

virtual MDSReadLock	is_dmr_available (const std::string &name)
	Is the DMR response for.

virtual MDSReadLock	is_dmr_available (const std::string &realName, const std::string &relativeName, const std::string &fileType)

virtual MDSReadLock	is_dmrpp_available (const BESContainer &container)

virtual MDSReadLock	is_dmrpp_available (const std::string &name)
	Is the DMR++ response for.

bool	is_unlimited () const
	Is this cache allowed to store as much as it wants?

virtual void	parse_das_from_mds (libdap::DAS *das, const std::string &name)

virtual void	purge_file (const std::string &file)
	Purge a single file from the cache.

virtual bool	remove_responses (const std::string &name)
	Remove all cached responses and objects for a granule.

virtual void	unlock_and_close (const std::string &target)

virtual void	update_and_purge (const std::string &new_file)
	Purge files from the cache.

virtual unsigned long long	update_cache_info (const std::string &target)
	Update the cache info file to include 'target'.

virtual void	write_das_response (const std::string &name, std::ostream &os)
	Write the stored DAS response to a stream.

virtual void	write_dds_response (const std::string &name, std::ostream &os)
	Write the stored DDS response to a stream.

virtual void	write_dmr_response (const std::string &name, std::ostream &os)
	Write the stored DMR response to a stream.

virtual void	write_dmrpp_response (const std::string &name, std::ostream &os)
	Write the stored DMR++ response to a stream.

Add responses to the GlobalMetadataStore
These methods use a DDS or DMR object to generate the DDS, DAS and DMR responses for DAP (2 and 4). They store those in the MDS and then update the MDS ledger file with the operation (add), the kind of object used to build the responses (DDS or DMR), name of the granule and hashes/names for each of the three files in the MDS that hold the responses. If verbose logging is on, the bes log also will hold information about the operation. If there is an error, that will always be recorded in the bes log.
virtual bool	add_responses (libdap::DDS *dds, const std::string &name)
	Add the DAP2 metadata responses using a DDS.

virtual bool	add_responses (libdap::DMR *dmr, const std::string &name)
	Add the DAP4 metadata responses using a DMR.

Static Public Member Functions
static bool	dir_exists (const std::string &dir)

Get an instance of GlobalMetadataStore
There are two ways to get an instance of GlobalMetadataStore singleton. Note If the cache_dir parameter is the empty string, get_instance() will return null for the pointer to the singleton and caching is disabled. This means that if the cache directory is not set in the bes.conf file(s), then the cache will be disabled. If the cache directory is given (or set in bes.conf) but the prefix or size is not, that's an error. If the directory is named but does not exist, it will be made. If the BES cannot make it, then an error will be signaled. Returns A pointer to a GlobalMetadataStore object; null if the cache is disabled.
static GlobalMetadataStore *	get_instance (const std::string &cache_dir, const std::string &prefix, unsigned long long size)
	Get an instance of the GlobalMetadataStore object.

static GlobalMetadataStore *	get_instance ()

Protected Member Functions
std::string	get_hash (const std::string &name)

MDSReadLock	get_read_lock_helper (const std::string &name, const std::string &suffix, const std::string &object_name)

	GlobalMetadataStore (const GlobalMetadataStore &src)

void	initialize ()
	Configure the ledger using LEDGER_KEY and LOCAL_TIME_KEY.

bool	remove_response_helper (const std::string &name, const std::string &suffix, const std::string &object_name)

bool	store_dap_response (StreamDAP &writer, const std::string &key, const std::string &name, const std::string &response_name)

void	write_ledger ()

write_response_helper
void	write_response_helper (const std::string &name, std::ostream &os, const std::string &suffix, const std::string &object_name)

void	write_response_helper (const std::string &name, std::ostream &os, const std::string &suffix, const std::string &xml_base, const std::string &object_name)
	This version looks at the first few bytes and substitutes a new value for xml:base.


	GlobalMetadataStore ()

	GlobalMetadataStore (const std::string &cache_dir, const std::string &prefix, unsigned long long size)

Static Protected Member Functions
static std::string	get_cache_dir_from_config ()

static std::string	get_cache_prefix_from_config ()

static unsigned long	get_cache_size_from_config ()

static void	insert_xml_base (int fd, std::ostream &os, const std::string &xml_base)
	like transfer_bytes(), but adds the xml:base attribute to the DMR/++

static void	transfer_bytes (int fd, std::ostream &os)

Protected Attributes
std::string	d_ledger_entry

Friends
class	DmrppMetadataStore

class	DmrppMetadataStoreTest

class	GlobalMetadataStoreTest

Detailed Description

Store the DAP metadata responses.

Provide a global persistent store for the DAP metadata responses. Using either a DDS or a DMR, write the three DAP metadata responses to the global metadata store. This class also provides methods to get those responses and write them to an output stream and a method to remove the responses.

The class maintains a ledger of 'add' and 'remove' operations.

The class is implemented as a singleton; use the get_instance() methods to get an instance of the metadata store.

BES Keys used:

DAP.GlobalMetadataStore.path: store root directory (assumes the store is using a POSIX file system)
DAP.GlobalMetadataStore.prefix: prefix for the names of items in the store
DAP.GlobalMetadataStore.size: Maximum size of the store. Zero indicates unlimited size.
DAP.GlobalMetadataStore.ledger: Name of the ledger. A relative pathname. will be interpreted as relative to the directory where the BES was started. The default name mds_ledger.txt
BES.LogTimeLocal: Use local or GMT time for the ledger entries; default is to use GMT

Note: To change the xml:base attribute in the DMR response use DMR::set_request_xml_base().

Todo: Add support for storing binary DDS and DMR objects. This will require modifications to libdap so that we can 'serialize' those and additions to some of the handlers so that they can record extra information used by their specializations of those objects.

Author: jhrg

Definition at line 89 of file GlobalMetadataStore.h.

Member Typedef Documentation

◆ MDSReadLock

typedef struct MDSReadLock bes::GlobalMetadataStore::MDSReadLock

Definition at line 213 of file GlobalMetadataStore.h.

Constructor & Destructor Documentation

◆ GlobalMetadataStore() [1/2]

GlobalMetadataStore::GlobalMetadataStore ( )

protected

Private constructors that call BESFileLockingCache's constructor; lookup cache directory, item prefix and max cache size in BESKeys

Note: Use the get_instance() methods to get a pointer to the singleton for this class. Do not use this method except in derived classes. This method either builds a valid object or throws an exception.

Parameters

cache_dir	key to find cache dir
prefix	key to find the cache prefix
size	key to find the cache size (in MBytes)

Exceptions

BESSyntaxUserError if the keys are not set in the BESKeys, if the key are either the empty string or zero, respectively, or if the cache directory does not exist.

Definition at line 401 of file GlobalMetadataStore.cc.

◆ GlobalMetadataStore() [2/2]

GlobalMetadataStore::GlobalMetadataStore	(	const std::string &	cache_dir,
		const std::string &	prefix,
		unsigned long long	size )

protected

Definition at line 407 of file GlobalMetadataStore.cc.

◆ ~GlobalMetadataStore()

virtual bes::GlobalMetadataStore::~GlobalMetadataStore ( )

inlinevirtual

Definition at line 237 of file GlobalMetadataStore.h.

Member Function Documentation

◆ add_responses() [1/2]

bool GlobalMetadataStore::add_responses	(	libdap::DDS *	dds,
		const std::string &	name )

virtual

Add the DAP2 metadata responses using a DDS.

This method adds only the DDS and DAS unless the code was compiled with the symbol SYMMETRIC_ADD_RESPONSES defined.

Parameters

name	The granule name or identifier
dds	A DDS built from the granule

Returns: True if all of the cache/store entries were written, False if any could not be written.

Definition at line 658 of file GlobalMetadataStore.cc.

◆ add_responses() [2/2]

bool GlobalMetadataStore::add_responses	(	libdap::DMR *	dmr,
		const std::string &	name )

virtual

Add the DAP4 metadata responses using a DMR.

This method adds only the DMR unless the code was compiled with the symbol SYMMETRIC_ADD_RESPONSES defined.

Parameters

name	The granule name or identifier
dmr	A DMR built from the granule

Returns: True if all of the cache/store entry was written, False if any could not be written.

Reimplemented in bes::DmrppMetadataStore.

Definition at line 700 of file GlobalMetadataStore.cc.

◆ cache_enabled()

bool BESFileLockingCache::cache_enabled ( ) const

inlineinherited

Returns: Is this cache enabled?

Definition at line 200 of file BESFileLockingCache.h.

◆ cache_too_big()

bool BESFileLockingCache::cache_too_big ( unsigned long long current_size ) const

virtualinherited

look at the cache size; is it too large? Look at the cache size and see if it is too big.

Returns: True if the size is too big, false otherwise.

Definition at line 817 of file BESFileLockingCache.cc.

◆ create_and_lock()

bool BESFileLockingCache::create_and_lock	(	const std::string &	target,
		int &	fd )

virtualinherited

Create a file in the cache and lock it for write access.

If the file does not exist, make it, open it for read-write access and get an exclusive lock on it. The locking operation blocks, although that should never happen.

Parameters

target	The name of the file to make/open/lock
fd	Value-result param that holds the file descriptor of the opened file

Returns: True if the operation was successful, false otherwise. This method will return false if the file already existed (the file won't be locked and the descriptor reference is undefined - but likely -1).

Exceptions

BESBESInternalError if any error except EEXIST is returned by open(2) or if fcntl(2) returns an error.

Definition at line 625 of file BESFileLockingCache.cc.

◆ dir_exists()

bool BESFileLockingCache::dir_exists ( const std::string & dir )

staticinherited

Does the directory exist?

Note: This is a static method, so it can be called from other static methods like those that build instances of singletons.

Parameters

dir	The pathname to test.

Returns: True if the directory exists, false otherwise

Definition at line 1190 of file BESFileLockingCache.cc.

◆ disable()

void BESFileLockingCache::disable ( )

inlineinherited

Disable the cache.

Definition at line 205 of file BESFileLockingCache.h.

◆ dump()

void BESFileLockingCache::dump ( std::ostream & strm ) const

overridevirtualinherited

dumps information about this object

Displays the pointer value of this instance along with information about this cache.

Parameters

strm	C++ i/o stream to dump the information to

Implements BESObj.

Definition at line 1205 of file BESFileLockingCache.cc.

◆ enable()

void BESFileLockingCache::enable ( )

inlineinherited

Enable the cache.

Definition at line 210 of file BESFileLockingCache.h.

◆ exclusive_to_shared_lock()

void BESFileLockingCache::exclusive_to_shared_lock ( int fd )

virtualinherited

Transfer from an exclusive lock to a shared lock.

If the file has an exclusive write lock on it, change that to a shared read lock. This is an atomic operation. If the call to fcntl(2) is protected by locking the cache, a dead lock will result given typical use of this class. This method exists to help with the situation where one process has the cache locked and is blocking on a shared read lock for a file that a second process has locked exclusively (for writing). By changing the exclusive lock to a shared lock, the first process can get its shared lock and then release the cache.

Parameters

fd	The file descriptor that is exclusively locked and which, on exit, will have a shared lock.

Definition at line 660 of file BESFileLockingCache.cc.

◆ get_cache_dir_from_config()

string GlobalMetadataStore::get_cache_dir_from_config ( )

staticprotected

Definition at line 262 of file GlobalMetadataStore.cc.

◆ get_cache_directory()

std::string BESFileLockingCache::get_cache_directory ( ) const

inlineinherited

Returns: The directory used for the an instance of BESFileLockingCache

Definition at line 191 of file BESFileLockingCache.h.

◆ get_cache_file_name()

string BESFileLockingCache::get_cache_file_name	(	const std::string &	src,
		bool	mangle = true )

virtualinherited

Returns the fully qualified file system path name for the cache file associated with this particular cache resource. This does not look in the cache to see if the fle is present, it just returns the name that will (or is) be used once/if the file is added to the cache.

Note: Names are mangled: ALl occurrences of the characters '<', '>', '=', ',', '/', '(', ')', '"', ''', ':', '?', and ' ' are replaced with the '#' character.

Parameters

src	The source name to (or in the) cache
mangle	If True, assume the name is a file pathname and mangle it. If false, do not mangle the name (assume the caller has sent a suitable string) but do turn the string into a pathname located in the cache directory with the cache prefix. The 'mangle' param is true by default.

Reimplemented in BESUncompressCache.

Definition at line 472 of file BESFileLockingCache.cc.

◆ get_cache_file_prefix()

std::string BESFileLockingCache::get_cache_file_prefix ( ) const

inlineinherited

Returns: The prefix used for items in an instance of BESFileLockingCache

Definition at line 186 of file BESFileLockingCache.h.

◆ get_cache_lmt()

time_t GlobalMetadataStore::get_cache_lmt	(	const std::string &	fileName,
		const std::string &	suffix )

virtual

Get the last modified time for the cached object file.

Parameters

name	- name of the object
suffix	- suffix of the object

Returns: The last modified time.

Definition at line 1011 of file GlobalMetadataStore.cc.

◆ get_cache_prefix_from_config()

string GlobalMetadataStore::get_cache_prefix_from_config ( )

staticprotected

Definition at line 247 of file GlobalMetadataStore.cc.

◆ get_cache_size()

unsigned long long BESFileLockingCache::get_cache_size ( )

virtualinherited

Get the cache size.

Read the size information from the cache info file and return it. This methods locks the cache.

Returns: The size of the cache.

Definition at line 829 of file BESFileLockingCache.cc.

◆ get_cache_size_from_config()

unsigned long GlobalMetadataStore::get_cache_size_from_config ( )

staticprotected

Definition at line 231 of file GlobalMetadataStore.cc.

◆ get_dds_object()

DDS * GlobalMetadataStore::get_dds_object ( const std::string & name )

virtual

Build a DDS object from the cached Response.

Read the DDS and DAS responses, build a DDS using their information and return the binary DDS response. The variables are built using the default BaseTypeFactory but the DDS object has the factory set to null when it is returned. The DDS is 'loaded' with attribute information as well, so it can be used to return the DDX response.

Note: This method uses temporary files to hold the responses and then parses them to build the DDS object

Todo: If/When the DDS can be serialized, we should be able to replace this implementation with something far better - and something that can include information in specialized BaseTypes and DDS classes.

Parameters

name	Path to the dataset, relative to the BES data root directory.

Returns: A pointer to the DDS object; the caller must delete this object.

Exceptions

BESInternalError

is thrown if

name does not have a cached DDS or DAS response.

Definition at line 1263 of file GlobalMetadataStore.cc.

◆ get_dmr_object()

DMR * GlobalMetadataStore::get_dmr_object ( const std::string & name )

virtual

Build a DMR object from the cached Response.

Read and parse a DMR response , building a binary DMR object. The object is returned with a null factory. The variables are built using the default DAP4 type factory.

Parameters

name	Name of the dataset

Returns: A pointer to the DMR object; the caller must delete this object.

Exceptions

BESInternalError

is thrown if

name does not have a cached DMR response.

Definition at line 1225 of file GlobalMetadataStore.cc.

◆ get_exclusive_lock()

bool BESFileLockingCache::get_exclusive_lock	(	const std::string &	target,
		int &	ref_fd )

virtualinherited

A blocking call to get an exclusive (write) lock on a file in the cache. Because this cache uses per-process advisory locking, it's possible to call this several times from within a single process and get a lock each time.

Note: This is used only by the purge_file() method.

Parameters

file_name	Name of the file to lock
ref_fd	Return value parameter that holds the file descriptor that is locked.

Returns: Return false if the file does not exist, otherwise block until the lock is acquired and then return true.

Exceptions

BESInternalError is thrown on any condition other than the file not existing

Definition at line 1079 of file BESFileLockingCache.cc.

◆ get_exclusive_lock_nb()

bool BESFileLockingCache::get_exclusive_lock_nb	(	const std::string &	target,
		int &	ref_fd )

virtualinherited

A non-blocking call to get an exclusive (write) lock on a file in the cache. Because this cache uses per-process advisory locking, it's possible to call this several times from within a single process and get a lock each time.

Note: This is used only by the update_and_purge() method.

Parameters

file_name	Name of the file to lock
ref_fd	Return value parameter that holds the file descriptor that is locked.

Returns: Return false if the file does not exist or the non-blocking lock acquisition fails, otherwise the lock is acquired and then return true.

Exceptions

BESInternalError is thrown on any condition other than the file not existing or the lock being unavailable.

Definition at line 920 of file BESFileLockingCache.cc.

◆ get_hash()

string GlobalMetadataStore::get_hash ( const std::string & name )

protected

Compute the SHA256 hash for the item name

Parameters

name	The name to hash

Returns: The SHA256 hash of the name.

Definition at line 486 of file GlobalMetadataStore.cc.

◆ get_instance() [1/2]

GlobalMetadataStore * GlobalMetadataStore::get_instance ( )

static

Get an instance of the GlobalMetadataStore using the default values for the cache directory, prefix and size.

Returns: A pointer to a GlobalMetadataStore object; null if the cache is disabled.

Definition at line 338 of file GlobalMetadataStore.cc.

◆ get_instance() [2/2]

GlobalMetadataStore * GlobalMetadataStore::get_instance	(	const std::string &	cache_dir,
		const std::string &	prefix,
		unsigned long long	size )

static

Get an instance of the GlobalMetadataStore object.

This class is a singleton, so the first call to any of two 'get_instance()' methods makes an instance and subsequent calls return a pointer to that instance.

Parameters

cache_dir_key	Key to use to get the value of the cache directory. If this is the empty string, return null right away.
prefix_key	Key for the item/file prefix. Each item added to the cache uses this as a prefix so cached items can be easily identified when the same directory is used for several caches or /tmp is used for the cache.
size_key	The maximum size of the data stored in the cache, in megabytes

Returns: A pointer to a GlobalMetadataStore object. If the cache is disabled, then the pointer returned will be null and the cache will be marked as not enabled. Subsequent calls will return immediately.

Definition at line 308 of file GlobalMetadataStore.cc.

◆ get_read_lock()

bool BESFileLockingCache::get_read_lock	(	const std::string &	target,
		int &	fd )

virtualinherited

Get a read-only lock on the file if it exists.

Try to get a read-only lock on the file, blocking until we can get it. If the file does not exist, return false.

Note: If this code returns false, that means the file did not exist in the cache at the time of the test. by the time the caller gets the result, the file may have been added to the cache by another process.

Parameters

target	The path of the cached file
fd	A value-result parameter set to the locked cached file. Undefined if the file could not be locked for read access.

Returns: true if the file is in the cache and has been locked, false if the file is/was not in the cache.

Exceptions

Error if the attempt to get the (shared) lock failed for any reason other than that the file does/did not exist.

Definition at line 565 of file BESFileLockingCache.cc.

◆ get_read_lock_helper()

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::get_read_lock_helper	(	const std::string &	name,
		const std::string &	suffix,
		const std::string &	object_name )

protected

Common code to acquire a read lock on a MDS item. This method locks the response for reading. When the MDSReadLock goes out of scope, the response is unlocked.

This method logs (using LOG, not VERBOSE) cache hits and misses.

Parameters

name	Granule name
suffix	One of 'dds_r', 'das_r' or 'dmr_r'
object_name	One of DDS, DAS or DMR (used for logging only)

Returns: True if the object was locked, false otherwise

Definition at line 744 of file GlobalMetadataStore.cc.

◆ initialize() [1/2]

void GlobalMetadataStore::initialize ( )

protected

Configure the ledger using LEDGER_KEY and LOCAL_TIME_KEY.

Definition at line 366 of file GlobalMetadataStore.cc.

◆ initialize() [2/2]

void BESFileLockingCache::initialize	(	const std::string &	cache_dir,
		const std::string &	prefix,
		unsigned long long	size )

inherited

Initialize an instance of FileLockingCache.

Initialize and instance of FileLockingCache using the passed values for the cache directory, item prefix and max cache size. This will ignore the value of enable_cache() (but will correctly (re)set it based on the directory, ..., values). This provides a way for clients to re-initialize caches on the fly.

Parameters

cache_dir	The directory into which the cache files will be written.
prefix	The prefix that will be added to each cache file.
size	The size of the cache in MBytes

Exceptions

BESInternalError	If the cache_dir does not exist or is not writable. size is 0, or if cache dir does not exist.
BESError	If the parameters (directory, ...) are invalid.

Definition at line 259 of file BESFileLockingCache.cc.

◆ insert_xml_base()

void GlobalMetadataStore::insert_xml_base	(	int	fd,
		std::ostream &	os,
		const std::string &	xml_base )

staticprotected

like transfer_bytes(), but adds the xml:base attribute to the DMR/++

Note: This is a static method so the function will be scoped with this class.

Parameters

fd	Open file descriptor to read from; assumed open and positioned at the start of the file.
os	Write to this C++ stream
xml_base	Value of the xml:base attribute.

Exceptions

BESInternalError Thrown if there's a problem reading or writing.

Definition at line 157 of file GlobalMetadataStore.cc.

◆ is_available_helper()

bool GlobalMetadataStore::is_available_helper	(	const std::string &	realName,
		const std::string &	relativeName,
		const std::string &	fileType,
		const std::string &	suffix )

virtual

helper function that checks if last modified time is greater than cached file

Parameters

realName	- complete path to file used to find actual file
relativeName	- relative filename used to find cached file
fileType	- used to retrieve correct BESRequestHandler from BESRequestHandlerList
suffix	- One of 'dmr_r', 'dds_r', 'das_r' or 'dmrpp_r'

Returns: true if actual file has been modified since cached file has been created, false otherwiseS

Definition at line 983 of file GlobalMetadataStore.cc.

◆ is_das_available() [1/2]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_das_available ( const BESContainer & container )

virtual

Definition at line 899 of file GlobalMetadataStore.cc.

◆ is_das_available() [2/2]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_das_available ( const std::string & name )

virtual

Is the DAS response for.

@Deprecated - 6.25.19 SBL

name in the MDS?
Parameters

name Find the DAS response for
name.
Returns
A MDSReadLock object.

See also
is_dmr_available() for more information.

Definition at line 893 of file GlobalMetadataStore.cc.

◆ is_dds_available() [1/2]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_dds_available ( const BESContainer & container )

virtual

Is the DDS response for.

name in the MDS?

This is the preferred method since it checks the LMT of the underlying dataset.

Parameters

container

Find the DDS response for the dataset in

container.

Returns: A MDSReadLock object.

See also: is_dmr_available() for more information.

Definition at line 862 of file GlobalMetadataStore.cc.

◆ is_dds_available() [2/2]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_dds_available ( const std::string & name )

virtual

Is the DDS response for.

@Deprecated - 6.25.19 SBL

name in the MDS?
Parameters

name Find the DDS response for
name.
Returns
A MDSReadLock object.

See also
is_dmr_available() for more information.

Definition at line 846 of file GlobalMetadataStore.cc.

◆ is_dmr_available() [1/3]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_dmr_available ( const BESContainer & container )

virtual

Definition at line 791 of file GlobalMetadataStore.cc.

◆ is_dmr_available() [2/3]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_dmr_available ( const std::string & name )

virtual

Is the DMR response for.

@Deprecated - 6.25.19 SBL

name in the MDS?

Look in the MDS to see if the DMR response has been stored/cached for

name.

Note: This method and the matching methods for the DDS and DAS use LOG() to record cache hits and misses. Other methods also record information about cache hits, but only using VERBOSE(), so that output will not show up in a normal log.

Parameters

name	Find the DMR response for name.

Returns: A MDSReadLock object. This object is true if the item was found (and a read lock was obtained), false if either of those things are not true. When the MDSReadLock object goes out of scope, the read lock is released.

Definition at line 785 of file GlobalMetadataStore.cc.

◆ is_dmr_available() [3/3]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_dmr_available	(	const std::string &	realName,
		const std::string &	relativeName,
		const std::string &	fileType )

virtual

Definition at line 815 of file GlobalMetadataStore.cc.

◆ is_dmrpp_available() [1/2]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_dmrpp_available ( const BESContainer & container )

virtual

Definition at line 948 of file GlobalMetadataStore.cc.

◆ is_dmrpp_available() [2/2]

GlobalMetadataStore::MDSReadLock GlobalMetadataStore::is_dmrpp_available ( const std::string & name )

virtual

Is the DMR++ response for.

@Deprecated - 6.25.19 SBL

name in the MDS?

Look in the MDS to see if the DMR++ response has been stored/cached for

name.

Note: This method uses LOG() to record cache hits and misses. Other methods also record information about cache hits, but only using VERBOSE(), so that output will not show up in a normal log.

Parameters

name	Find the DMR++ response for name.

Returns: A MDSReadLock object. This object is true if the item was found (and a read lock was obtained), false if either of those things are not true. When the MDSReadLock object goes out of scope, the read lock is released.

Definition at line 942 of file GlobalMetadataStore.cc.

◆ is_unlimited()

bool BESFileLockingCache::is_unlimited ( ) const

inlineinherited

Is this cache allowed to store as much as it wants?

If the size of the cache is zero bytes, then it is allowed to grow with out bounds.

Returns: True if the cache is unlimited in size, false if values will be purged after a preset size is exceeded.

Definition at line 181 of file BESFileLockingCache.h.

◆ parse_das_from_mds()

void GlobalMetadataStore::parse_das_from_mds	(	libdap::DAS *	das,
		const std::string &	name )

virtual

Definition at line 1305 of file GlobalMetadataStore.cc.

◆ purge_file()

void BESFileLockingCache::purge_file ( const std::string & file )

virtualinherited

Purge a single file from the cache.

Purge a single file from the cache. The file might be old, etc., and need to be removed. Don't use this to shrink the cache when it gets too big, use update_and_purge() instead since that file optimizes accesses to the cache control file for several changes in a row.

Parameters

file	The name of the file to purge.

Definition at line 1125 of file BESFileLockingCache.cc.

◆ remove_response_helper()

bool GlobalMetadataStore::remove_response_helper	(	const std::string &	name,
		const std::string &	suffix,
		const std::string &	object_name )

protected

Common code to remove a stored response.

Parameters

name	Granule name
suffix	One of 'dds_r', 'das_r' or 'dmr_r'
object_name	One of DDS, DAS or DMR

Definition at line 1167 of file GlobalMetadataStore.cc.

◆ remove_responses()

bool GlobalMetadataStore::remove_responses ( const std::string & name )

virtual

Remove all cached responses and objects for a granule.

Parameters

name

Returns

Definition at line 1190 of file GlobalMetadataStore.cc.

◆ store_dap_response()

bool GlobalMetadataStore::store_dap_response	(	StreamDAP &	writer,
		const std::string &	key,
		const std::string &	name,
		const std::string &	response_name )

protected

Store the DAP metadata responses

Parameters

writer	A child instance of StreamDAP, instantiated using a DDS or DMR. An instance of StreamDDS will write a DDS response, StreamDAS writes a DAS response,and StreamDMR writes a DMR response.
key	Unique Id for this response; used to store the response in the MDS.
name	The granule/file name or pathname
response_name	The name of the particular response (DDS, DAS, DMR). Used for log messages.

Returns: True if the operation succeeded, False if the key is in use.

Exceptions

BESInternalError If ...

Definition at line 571 of file GlobalMetadataStore.cc.

◆ transfer_bytes()

void GlobalMetadataStore::transfer_bytes	(	int	fd,
		std::ostream &	os )

staticprotected

Hacked from GNU wc (in coreutils). This was found to be faster than a memory mapped file read.

https://stackoverflow.com/questions/17925051/fast-textfile-reading-in-c

Note: This is a static method so the function will be scoped with this class.

Parameters

fd	Open file descriptor to read from; assumed open and positioned at the start of the file.
os	Write to this C++ stream

Exceptions

BESInternalError Thrown if there's a problem reading or writing.

Definition at line 121 of file GlobalMetadataStore.cc.

◆ unlock_and_close()

void BESFileLockingCache::unlock_and_close ( const std::string & target )

virtualinherited

Get an exclusive lock on the 'cache info' file. The 'cache info' file is used to control certain cache actions, ensuring that they are atomic. These include making sure that the create_and_lock() and read_and_lock() operations are atomic as well as the purge and related operations.

Note: This is intended to be used internally only but might be useful in some settings. Unlock the named file.

This does not do any name mangling; it just closes and unlocks whatever is named (or throws BESBESInternalError if the file cannot be closed). If the file was opened more than once, all descriptors are closed. If you need to close a specific descriptor, use the other version of unlock_and_close().

Note: This method assumes that the file was opened/locked using one of read_and_lock() or create_and_lock(). Those methods record the name/file- descriptor pairs so that the files can be properly closed and locks released.

Parameters

file_name The name of the file to unlock.

Exceptions

BESBESInternalError

Definition at line 745 of file BESFileLockingCache.cc.

◆ update_and_purge()

void BESFileLockingCache::update_and_purge ( const std::string & new_file )

virtualinherited

Purge files from the cache.

Purge files, oldest to newest, if the current size of the cache exceeds the size of the cache specified in the constructor. This method uses an exclusive lock on the cache for the duration of the purge process.

Note: If the cache size in bytes is zero, calling this method has no affect (the cache is unlimited in size). Other public methods like update_cache_info() and get_cache_size() still work, however.

Parameters

new_file Do not delete this file. The name of a file this process just added to the cache. Using fcntl(2) locking there is no way this process can detect its own lock, so the shared read lock on the new file won't keep this process from deleting it (but will keep other processes from deleting it).

Definition at line 977 of file BESFileLockingCache.cc.

◆ update_cache_info()

unsigned long long BESFileLockingCache::update_cache_info ( const std::string & target )

virtualinherited

Update the cache info file to include 'target'.

Add the size of the named file to the total cache size recorded in the cache info file. The cache info file is exclusively locked by this method for its duration. This updates the cache info file and returns the new size.

Parameters

target The name of the file

Returns: The new size of the cache

Definition at line 769 of file BESFileLockingCache.cc.

◆ write_das_response()

void GlobalMetadataStore::write_das_response	(	const std::string &	name,
		std::ostream &	os )

virtual

Write the stored DAS response to a stream.

Parameters

name	The (path)name of the granule
os	Write to this stream

Definition at line 1108 of file GlobalMetadataStore.cc.

◆ write_dds_response()

void GlobalMetadataStore::write_dds_response	(	const std::string &	name,
		std::ostream &	os )

virtual

Write the stored DDS response to a stream.

Parameters

name	The (path)name of the granule
os	Write to this stream

Definition at line 1096 of file GlobalMetadataStore.cc.

◆ write_dmr_response()

void GlobalMetadataStore::write_dmr_response	(	const std::string &	name,
		std::ostream &	os )

virtual

Write the stored DMR response to a stream.

Parameters

name	The (path)name of the granule
os	Write to this stream

Definition at line 1120 of file GlobalMetadataStore.cc.

◆ write_dmrpp_response()

void GlobalMetadataStore::write_dmrpp_response	(	const std::string &	name,
		std::ostream &	os )

virtual

Write the stored DMR++ response to a stream.

Parameters

name	The (path)name of the granule
os	Write to this stream

Definition at line 1143 of file GlobalMetadataStore.cc.

◆ write_ledger()

void GlobalMetadataStore::write_ledger ( )

protected

Write the current text of d_ledger_entry to the metadata store ledger

Definition at line 450 of file GlobalMetadataStore.cc.

◆ write_response_helper() [1/2]

void GlobalMetadataStore::write_response_helper	(	const std::string &	name,
		std::ostream &	os,
		const std::string &	suffix,
		const std::string &	object_name )

protected

Common code to copy a response to an output stream.

Parameters

name	Granule name
os	Write the response to this stream
suffix	One of 'dds_r', 'das_r' or 'dmr_r'
object_name	One of DDS, DAS or DMR; used for error reporting.

Definition at line 1034 of file GlobalMetadataStore.cc.

◆ write_response_helper() [2/2]

void GlobalMetadataStore::write_response_helper	(	const std::string &	name,
		std::ostream &	os,
		const std::string &	suffix,
		const std::string &	xml_base,
		const std::string &	object_name )

protected

This version looks at the first few bytes and substitutes a new value for xml:base.

Parameters

name	Granule name
os	Write the response to this stream
suffix	One of 'dds_r', 'das_r' or 'dmr_r'
xml_base	Value of the xml:base attribute in the <Dataset...> element
object_name	One of DDS, DAS or DMR; used for error reporting.

Definition at line 1064 of file GlobalMetadataStore.cc.

Friends And Related Symbol Documentation

◆ DmrppMetadataStore

friend class DmrppMetadataStore

friend

Definition at line 106 of file GlobalMetadataStore.h.

◆ DmrppMetadataStoreTest

friend class DmrppMetadataStoreTest

friend

Definition at line 107 of file GlobalMetadataStore.h.

◆ GlobalMetadataStoreTest

friend class GlobalMetadataStoreTest

friend

Definition at line 108 of file GlobalMetadataStore.h.

Member Data Documentation

◆ d_ledger_entry

std::string bes::GlobalMetadataStore::d_ledger_entry

protected

Definition at line 111 of file GlobalMetadataStore.h.

The documentation for this class was generated from the following files:

Classes

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Friends

Detailed Description

Member Typedef Documentation

◆ MDSReadLock

Constructor & Destructor Documentation

◆ GlobalMetadataStore() [1/2]

◆ GlobalMetadataStore() [2/2]

◆ ~GlobalMetadataStore()

Member Function Documentation

◆ add_responses() [1/2]

◆ add_responses() [2/2]

◆ cache_enabled()

◆ cache_too_big()

◆ create_and_lock()

◆ dir_exists()

◆ disable()

◆ dump()

◆ enable()

◆ exclusive_to_shared_lock()

◆ get_cache_dir_from_config()

◆ get_cache_directory()

◆ get_cache_file_name()

◆ get_cache_file_prefix()

◆ get_cache_lmt()

◆ get_cache_prefix_from_config()

◆ get_cache_size()

◆ get_cache_size_from_config()

◆ get_dds_object()

◆ get_dmr_object()

◆ get_exclusive_lock()

◆ get_exclusive_lock_nb()

◆ get_hash()

◆ get_instance() [1/2]

◆ get_instance() [2/2]

◆ get_read_lock()

◆ get_read_lock_helper()

◆ initialize() [1/2]

◆ initialize() [2/2]

◆ insert_xml_base()

◆ is_available_helper()

◆ is_das_available() [1/2]

◆ is_das_available() [2/2]

◆ is_dds_available() [1/2]

◆ is_dds_available() [2/2]

◆ is_dmr_available() [1/3]

◆ is_dmr_available() [2/3]

◆ is_dmr_available() [3/3]

◆ is_dmrpp_available() [1/2]

◆ is_dmrpp_available() [2/2]

◆ is_unlimited()

◆ parse_das_from_mds()

◆ purge_file()

◆ remove_response_helper()

◆ remove_responses()

◆ store_dap_response()

◆ transfer_bytes()

◆ unlock_and_close()

◆ update_and_purge()

◆ update_cache_info()

◆ write_das_response()

◆ write_dds_response()

◆ write_dmr_response()

◆ write_dmrpp_response()

◆ write_ledger()

◆ write_response_helper() [1/2]

◆ write_response_helper() [2/2]

Friends And Related Symbol Documentation

◆ DmrppMetadataStore

◆ DmrppMetadataStoreTest

◆ GlobalMetadataStoreTest

Member Data Documentation

◆ d_ledger_entry