Extend libdap::Array so that a handler can read data using a DMR++ file. More...

#include <DmrppArray.h>

Inheritance diagram for dmrpp::DmrppArray:

Collaboration diagram for dmrpp::DmrppArray:

Public Member Functions
void	accumlate_storage_size (unsigned long long chunk_storage_size)

virtual unsigned long	add_chunk (const std::string &byte_order, const std::string &fill_value, libdap::Type fv_type, unsigned long long chunk_size, const std::vector< unsigned long long > &position_in_array)

virtual unsigned long	add_chunk (const std::string &byte_order, const std::string &fill_value, libdap::Type fv_type, unsigned long long chunk_size, const std::vector< unsigned long long > &position_in_array, const std::vector< std::pair< libdap::Type, int > > &structure_type_element)

virtual unsigned long	add_chunk (const std::string &byte_order, const std::string &fill_value, libdap::Type fv_type, unsigned long long chunk_size, const std::vector< unsigned long long > &position_in_array, unsigned int struct_size)

virtual unsigned long	add_chunk (const std::string &byte_order, const std::string &position_in_array, const std::vector< std::pair< unsigned long long, unsigned long long > > &lb_offset_length)

virtual unsigned long	add_chunk (const std::string &byte_order, unsigned long long size, unsigned long long offset, bool linked_block, unsigned int linked_block_index)

virtual unsigned long	add_chunk (const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
	Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

virtual unsigned long	add_chunk (const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &position_in_array)
	Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

virtual unsigned long	add_chunk (const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &position_in_array, bool multi_linked_blocks, unsigned int linked_block_index)

virtual unsigned long	add_chunk (const std::string &byte_order, unsigned long long size, unsigned long long offset, unsigned int filter_mask, const std::vector< unsigned long long > &position_in_array)

virtual unsigned long	add_chunk (std::shared_ptr< http::url > d_data_url, const std::string &byte_order, const std::string &position_in_array, const std::vector< std::pair< unsigned long long, unsigned long long > > &lb_offset_length)

virtual unsigned long	add_chunk (std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, bool linked_block, unsigned int linked_block_index)

virtual unsigned long	add_chunk (std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
	Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

virtual unsigned long	add_chunk (std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &position_in_array)
	Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

virtual unsigned long	add_chunk (std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &position_in_array, bool multi_linked_blocks, unsigned int multi_linked_block_index_in_dmrpp_file)

virtual unsigned long	add_chunk (std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, unsigned int filter_mask, const std::string &position_in_array)

virtual unsigned long	add_chunk (std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, unsigned int filter_mask, const std::vector< unsigned long long > &position_in_array)

unsigned int	buf2val (void **val) override

vector< u_int8_t > &	compact_str_buffer ()

	DmrppArray (const DmrppArray &)=default

	DmrppArray (const std::string &n, const std::string &d, libdap::BaseType *v)

	DmrppArray (const std::string &n, libdap::BaseType *v)

	DmrppArray (const string &n, BaseType *v, shared_ptr< DMZ > dmz)

	DmrppArray (const string &n, const string &d, BaseType *v, shared_ptr< DMZ > dmz)

void	dump (ostream &strm) const override

virtual void	dump (std::ostream &strm) const

virtual bool	get_attributes_loaded () const
	Have the attributes been loaded?

virtual std::string	get_byte_order () const

unsigned long long	get_bytes_per_element () const

virtual const std::vector< unsigned long long > &	get_chunk_dimension_sizes () const
	The chunk dimension sizes held in a const vector.

virtual unsigned long long	get_chunk_size_in_elements () const
	Get the number of elements in this chunk.

std::vector< std::shared_ptr< Chunk > >	get_chunks () const

virtual bool	get_chunks_loaded () const
	Have the chunks been loaded?

virtual size_t	get_chunks_size () const
	Use this when the number of chunks is needed.

const std::vector< unsigned int > &	get_deflate_levels () const

virtual std::string	get_fill_value () const

virtual libdap::Type	get_fill_value_type () const

virtual std::string	get_filters () const
	Return the names of all the filters in the order they were applied.

string_pad_type	get_fixed_length_string_pad () const

std::string	get_fixed_length_string_pad_str () const

unsigned long long	get_fixed_string_length () const

virtual const std::vector< std::shared_ptr< Chunk > > &	get_immutable_chunks () const
	A const reference to the vector of chunks.

virtual bool	get_one_chunk_fill_value () const

void	get_ons_objs (vector< ons > &ons_list)

std::string	get_ons_string ()

bool	get_processing_fv_chunks () const

virtual std::vector< unsigned long long >	get_shape (bool constrained)
	Get the array shape.

virtual unsigned long long	get_size (bool constrained=false)
	Return the total number of elements in this Array.

bool	get_special_structure_flag ()

const std::vector< unsigned int > &	get_struct_offsets () const

char *	get_structure_array_buf_ptr ()

vector< char > &	get_structure_array_str_buffer ()

unsigned int	get_total_linked_blocks () const

virtual bool	get_uses_fill_value () const

bool	get_using_linked_block () const

unsigned long long	get_var_chunks_storage_size () const

virtual const pugi::xml_node &	get_xml_node () const

virtual void	ingest_byte_order (const std::string &byte_order_string)
	Parses the text content of the XML element chunks:byteOrder.

virtual void	ingest_compression_type (const std::string &compression_type_string)
	Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned int> representation.

virtual bool	is_compact_layout () const
	Returns true if this object utilizes COMPACT layout.

virtual bool	is_disable_dio () const
	Returns true if this object describes the missing data.

virtual bool	is_filters_empty () const

bool	is_flsa () const

virtual bool	is_missing_data () const
	Returns true if this object describes the missing data.

virtual bool	is_multi_linked_blocks_chunk () const
	Returns true if this object contains a chunk that have multiple linked blocks .

bool	is_projected ()
	Is this Array subset?

bool	is_vlsa () const

virtual void	load_attributes (libdap::BaseType *btp)
	Load the attribute information for this variable.

virtual void	load_chunks (libdap::BaseType *btp)
	Load chunk information for this variable.

DmrppArray &	operator= (const DmrppArray &rhs)

virtual void	parse_chunk_dimension_sizes (const std::string &chunk_dim_sizes_string)
	Set the dimension sizes for a chunk.

void	print_chunks_element (libdap::XMLWriter &xml, const std::string &name_space="")
	Print the Chunk information.

void	print_compact_element (libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="") const
	Print the Compact base64-encoded information.

void	print_dap4 (libdap::XMLWriter &writer, bool constrained=false) override
	Shadow libdap::Array::print_dap4() - optionally prints DMR++ chunk information.

void	print_dmrpp (libdap::XMLWriter &writer, bool constrained=false)
	Print the DMR++ response for the Scalar types.

void	print_missing_data_element (const libdap::XMLWriter &xml, const std::string &name_space, const char *data, int length) const

void	print_missing_data_element (const libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="") const

void	print_special_structure_element (const libdap::XMLWriter &xml, const std::string &name_space="", const std::string &encoded="") const

libdap::BaseType *	ptr_duplicate () override

bool	read () override
	Read data for the array.

virtual void	set_attributes_loaded (bool state)

void	set_bytes_per_element (unsigned long long bpe)

void	set_chunk_dimension_sizes (const std::vector< unsigned long long > &chunk_dims)
	Set the value of the chunk dimension sizes given a vector of HDF5 hsize_t.

virtual void	set_chunks_loaded (bool state)

void	set_compact (bool value)
	Set the value of the compact property.

void	set_deflate_levels (const std::vector< unsigned int > &def_levels)

void	set_disable_dio (bool value)
	Set the value of the compact property.

virtual void	set_fill_value_string (const std::string &fv)
	Set the fill value (using a string)

virtual void	set_fill_value_type (libdap::Type t)
	Set the libdap data type to use with the fill value.

void	set_filter (const std::string &value)
	Set the value of the filters property.

void	set_fixed_length_string_pad (const string_pad_type pad)

string_pad_type	set_fixed_length_string_pad_type (const std::string &pad_str)

unsigned long long	set_fixed_string_length (const string &length_str)

void	set_fixed_string_length (const unsigned long long length)

void	set_is_flsa (bool state)
	Marks the array as a Fixed length string array, or not, depending on state.

void	set_is_vlsa (bool state)

void	set_missing_data (bool value)
	Set the value of the missing data.

void	set_multi_linked_blocks_chunk (bool value)
	Set the value of the boolean variable that indicates this variable contains multiple linked blocks in a chunk.

virtual void	set_one_chunk_fill_value (bool ufv)
	Set the one_chunk_fill_value property.

void	set_ons_string (const std::string &ons_str)

void	set_ons_string (const vector< ons > &ons_pairs)

void	set_processing_fv_chunks ()

void	set_send_p (bool state) override

void	set_special_structure_flag (bool is_special_struct)

void	set_struct_offsets (const std::vector< unsigned int > &s_offs)

void	set_total_linked_blocks (unsigned tlbs)

virtual void	set_uses_fill_value (bool ufv)
	Set the uses_fill_value property.

void	set_using_linked_block ()

virtual void	set_xml_node (pugi::xml_node node)

virtual bool	twiddle_bytes () const
	Returns true if this object utilizes shuffle compression.

Static Public Member Functions
static string	ingest_fixed_length_string (const char *buf, unsigned long long fixed_str_len, string_pad_type pad_type)

static std::string	pad_type_to_str (string_pad_type pad_type)

Static Public Attributes
static std::string	d_dmrpp_ns = "http://xml.opendap.org/dap/dmrpp/1.0.0#"
	The DMR++ XML namespace.

static std::string	d_ns_prefix = "dmrpp"
	The XML namespace prefix to use.

static bool	d_print_chunks = false
	if true, print_dap4() prints chunk elements

Protected Member Functions
virtual char *	read_atomic (const std::string &name)
	read method for the atomic types

virtual char *	read_atomic (const std::string &name, size_t &buf_size)

Friends
class	DmrppArrayTest

void	process_one_chunk (std::shared_ptr< Chunk > chunk, DmrppArray *array, const vector< unsigned long long > &constrained_array_shape)

void	process_one_chunk_unconstrained (std::shared_ptr< Chunk > chunk, const vector< unsigned long long > &chunk_shape, DmrppArray *array, const vector< unsigned long long > &array_shape)

void	process_one_chunk_unconstrained_dio (std::shared_ptr< Chunk > chunk, const vector< unsigned long long > &chunk_shape, DmrppArray *array, const vector< unsigned long long > &array_shape)

Detailed Description

Extend libdap::Array so that a handler can read data using a DMR++ file.

Note: A key feature of HDF5 is that is can 'chunk' data, breaking up an array into a number of smaller pieces, each of which can be compressed. This code will read both array data that are chunked (and possibly compressed) as well as data that are not (essentially the entire array is written in a single 'chunk'). Because the two cases are different and susceptible to different kinds of optimizations, we have implemented two different read() methods, one for the 'no chunks' case and one for arrays 'with chunks.'

Definition at line 77 of file DmrppArray.h.

Constructor & Destructor Documentation

◆ DmrppArray() [1/4]

dmrpp::DmrppArray::DmrppArray	(	const std::string &	n,
		libdap::BaseType *	v )

inline

Definition at line 176 of file DmrppArray.h.

◆ DmrppArray() [2/4]

dmrpp::DmrppArray::DmrppArray	(	const std::string &	n,
		const std::string &	d,
		libdap::BaseType *	v )

inline

Definition at line 180 of file DmrppArray.h.

◆ DmrppArray() [3/4]

dmrpp::DmrppArray::DmrppArray	(	const string &	n,
		BaseType *	v,
		shared_ptr< DMZ >	dmz )

inline

Definition at line 184 of file DmrppArray.h.

◆ DmrppArray() [4/4]

dmrpp::DmrppArray::DmrppArray	(	const string &	n,
		const string &	d,
		BaseType *	v,
		shared_ptr< DMZ >	dmz )

inline

Definition at line 188 of file DmrppArray.h.

Member Function Documentation

◆ accumlate_storage_size()

void dmrpp::DmrppCommon::accumlate_storage_size ( unsigned long long chunk_storage_size )

inlineinherited

Definition at line 471 of file DmrppCommon.h.

◆ add_chunk() [1/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		const std::string &	fill_value,
		libdap::Type	fv_type,
		unsigned long long	chunk_size,
		const std::vector< unsigned long long > &	position_in_array )

virtualinherited

Definition at line 420 of file DmrppCommon.cc.

◆ add_chunk() [2/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		const std::string &	fill_value,
		libdap::Type	fv_type,
		unsigned long long	chunk_size,
		const std::vector< unsigned long long > &	position_in_array,
		unsigned int	struct_size )

virtualinherited

Definition at line 446 of file DmrppCommon.cc.

◆ add_chunk() [3/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		const std::string &	position_in_array,
		const std::vector< std::pair< unsigned long long, unsigned long long > > &	lb_offset_length )

virtualinherited

Definition at line 345 of file DmrppCommon.cc.

◆ add_chunk() [4/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		bool	linked_block,
		unsigned int	linked_block_index )

virtualinherited

Definition at line 289 of file DmrppCommon.cc.

◆ add_chunk() [5/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		const std::string &	position_in_array )

virtualinherited

Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

NB: This method is used by build_dmrpp during the production of the dmr++ file. The fact that the URL is not set is fine in that circumstance because when the DMZ parser reads the dmr++ the chunks without the explicit URL (dmrpp:href attribute) will inherit the primary URL (dmrpp:href) value from the top level Dataset element.

Parameters

byte_order	The stored byte order of the chunk
size	The sie of the chunk
offset	Chunk offset in the target dataset binary object.
position_in_array	A string description of the chunks position in the array with a nominal format of [x,y,...,z] where x, ..., are unsigned integers

Returns: The number of chunk refs (byteStreams) held in the internal chunks vector..

Definition at line 371 of file DmrppCommon.cc.

◆ add_chunk() [6/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		const std::vector< unsigned long long > &	position_in_array )

virtualinherited

Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

NB: This method is used by build_dmrpp during the production of the dmr++ file. The fact that the URL is not set is fine in that circumstance because when the DMZ parser reads the dmr++ the chunks without the explicit URL (dmrpp:href attributes) will inherit the primary URL (dmrpp:href) value from the top level Dataset element.

Parameters

byte_order	The stored byte order of the chunk
size	The sie of the chunk
offset	Chunk offset in the target dataset binary object.
position_in_array	Described as a vector of unsigned long long values

Returns: The number of chunk refs (byteStreams) held in the internal chunks vector..

Definition at line 396 of file DmrppCommon.cc.

◆ add_chunk() [7/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		const std::vector< unsigned long long > &	position_in_array,
		bool	multi_linked_blocks,
		unsigned int	linked_block_index )

virtualinherited

Definition at line 318 of file DmrppCommon.cc.

◆ add_chunk() [8/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		unsigned int	filter_mask,
		const std::vector< unsigned long long > &	position_in_array )

virtualinherited

Definition at line 408 of file DmrppCommon.cc.

◆ add_chunk() [9/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	std::shared_ptr< http::url >	d_data_url,
		const std::string &	byte_order,
		const std::string &	position_in_array,
		const std::vector< std::pair< unsigned long long, unsigned long long > > &	lb_offset_length )

virtualinherited

Definition at line 333 of file DmrppCommon.cc.

◆ add_chunk() [10/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	std::shared_ptr< http::url >	d_data_url,
		const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		bool	linked_block,
		unsigned int	linked_block_index )

virtualinherited

Definition at line 275 of file DmrppCommon.cc.

◆ add_chunk() [11/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	std::shared_ptr< http::url >	d_data_url,
		const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		const std::string &	position_in_array )

virtualinherited

Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

Parameters

data_url	The URL, file or http(s) protocol, that identifies the location of the binary object that holds the referenced chunk
byte_order	The stored byte order of the chunk
size	The sie of the chunk
offset	Chunk offset in the target dataset binary object.
position_in_array	A string description of the chunks position in the array with a nominal format of [x,y,...,z] where x, ..., are unsigned integers

Returns: The number of chunks in the internal chunks vector for this variable.

Definition at line 212 of file DmrppCommon.cc.

◆ add_chunk() [12/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	std::shared_ptr< http::url >	d_data_url,
		const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		const std::vector< unsigned long long > &	position_in_array )

virtualinherited

Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal vector.

Parameters

data_url	The URL, file or http(s) protocol, that identifies the location of the binary object that holds the referenced chunk
byte_order	The stored byte order of the chunk
size	The sie of the chunk
offset	Chunk offset in the target dataset binary object.
position_in_array	Described as a vector of unsigned long long values

Returns: The number of chunk refs (byteStreams) held in the internal chunks vector..

Definition at line 248 of file DmrppCommon.cc.

◆ add_chunk() [13/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	std::shared_ptr< http::url >	d_data_url,
		const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		const std::vector< unsigned long long > &	position_in_array,
		bool	multi_linked_blocks,
		unsigned int	multi_linked_block_index_in_dmrpp_file )

virtualinherited

Definition at line 303 of file DmrppCommon.cc.

◆ add_chunk() [14/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	std::shared_ptr< http::url >	d_data_url,
		const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		unsigned int	filter_mask,
		const std::string &	position_in_array )

virtualinherited

Definition at line 224 of file DmrppCommon.cc.

◆ add_chunk() [15/15]

unsigned long dmrpp::DmrppCommon::add_chunk	(	std::shared_ptr< http::url >	d_data_url,
		const std::string &	byte_order,
		unsigned long long	size,
		unsigned long long	offset,
		unsigned int	filter_mask,
		const std::vector< unsigned long long > &	position_in_array )

virtualinherited

Definition at line 261 of file DmrppCommon.cc.

◆ buf2val()

unsigned int dmrpp::DmrppArray::buf2val ( void ** val )

override

Definition at line 3292 of file DmrppArray.cc.

◆ compact_str_buffer()

vector< u_int8_t > & dmrpp::DmrppArray::compact_str_buffer ( )

inline

Definition at line 244 of file DmrppArray.h.

◆ dump() [1/2]

void dmrpp::DmrppArray::dump ( ostream & strm ) const

override

Definition at line 3282 of file DmrppArray.cc.

◆ dump() [2/2]

void dmrpp::DmrppCommon::dump ( std::ostream & strm ) const

virtualinherited

Definition at line 846 of file DmrppCommon.cc.

◆ get_attributes_loaded()

virtual bool dmrpp::DmrppCommon::get_attributes_loaded ( ) const

inlinevirtualinherited

Have the attributes been loaded?

Definition at line 255 of file DmrppCommon.h.

◆ get_byte_order()

virtual std::string dmrpp::DmrppCommon::get_byte_order ( ) const

inlinevirtualinherited

Definition at line 341 of file DmrppCommon.h.

◆ get_bytes_per_element()

unsigned long long dmrpp::DmrppArray::get_bytes_per_element ( ) const

inline

Definition at line 249 of file DmrppArray.h.

◆ get_chunk_dimension_sizes()

virtual const std::vector< unsigned long long > & dmrpp::DmrppCommon::get_chunk_dimension_sizes ( ) const

inlinevirtualinherited

The chunk dimension sizes held in a const vector.

Returns: A reference to a const vector of chunk dimension sizes

Definition at line 274 of file DmrppCommon.h.

◆ get_chunk_size_in_elements()

virtual unsigned long long dmrpp::DmrppCommon::get_chunk_size_in_elements ( ) const

inlinevirtualinherited

Get the number of elements in this chunk.

Returns: The number of elements; multiply by element size to get the number of bytes.

Definition at line 280 of file DmrppCommon.h.

◆ get_chunks()

std::vector< std::shared_ptr< Chunk > > dmrpp::DmrppCommon::get_chunks ( ) const

inlineinherited

Definition at line 266 of file DmrppCommon.h.

◆ get_chunks_loaded()

virtual bool dmrpp::DmrppCommon::get_chunks_loaded ( ) const

inlinevirtualinherited

Have the chunks been loaded?

Definition at line 251 of file DmrppCommon.h.

◆ get_chunks_size()

virtual size_t dmrpp::DmrppCommon::get_chunks_size ( ) const

inlinevirtualinherited

Use this when the number of chunks is needed.

Returns: the number of Chunk objects for this variable

Definition at line 270 of file DmrppCommon.h.

◆ get_deflate_levels()

const std::vector< unsigned int > & dmrpp::DmrppCommon::get_deflate_levels ( ) const

inlineinherited

Definition at line 179 of file DmrppCommon.h.

◆ get_fill_value()

virtual std::string dmrpp::DmrppCommon::get_fill_value ( ) const

inlinevirtualinherited

Returns: Return the fill value as a string or "" if get_fill_value() is false

Definition at line 302 of file DmrppCommon.h.

◆ get_fill_value_type()

virtual libdap::Type dmrpp::DmrppCommon::get_fill_value_type ( ) const

inlinevirtualinherited

Returns: Return the fill value as a string or "" if get_fill_value() is false

Definition at line 305 of file DmrppCommon.h.

◆ get_filters()

virtual std::string dmrpp::DmrppCommon::get_filters ( ) const

inlinevirtualinherited

Return the names of all the filters in the order they were applied.

Definition at line 173 of file DmrppCommon.h.

◆ get_fixed_length_string_pad()

string_pad_type dmrpp::DmrppArray::get_fixed_length_string_pad ( ) const

inline

Definition at line 231 of file DmrppArray.h.

◆ get_fixed_length_string_pad_str()

std::string dmrpp::DmrppArray::get_fixed_length_string_pad_str ( ) const

inline

Definition at line 232 of file DmrppArray.h.

◆ get_fixed_string_length()

unsigned long long dmrpp::DmrppArray::get_fixed_string_length ( ) const

inline

Definition at line 227 of file DmrppArray.h.

◆ get_immutable_chunks()

virtual const std::vector< std::shared_ptr< Chunk > > & dmrpp::DmrppCommon::get_immutable_chunks ( ) const

inlinevirtualinherited

A const reference to the vector of chunks.

Definition at line 262 of file DmrppCommon.h.

◆ get_one_chunk_fill_value()

virtual bool dmrpp::DmrppCommon::get_one_chunk_fill_value ( ) const

inlinevirtualinherited

Returns: Return true if this variable contains one chunk and the data are all 'fill value.'

Definition at line 311 of file DmrppCommon.h.

◆ get_ons_objs()

void dmrpp::DmrppArray::get_ons_objs ( vector< ons > & ons_list )

Ingests the (possibly long) ons (offset and size) string that itemizes every offset and size for the members of variable length string array and creates from the string offset:size pairs

Parameters

ons_str
vlen_str_addrs

Definition at line 2960 of file DmrppArray.cc.

◆ get_ons_string()

std::string dmrpp::DmrppArray::get_ons_string ( )

inline

Definition at line 236 of file DmrppArray.h.

◆ get_processing_fv_chunks()

bool dmrpp::DmrppCommon::get_processing_fv_chunks ( ) const

inlineinherited

Definition at line 191 of file DmrppCommon.h.

◆ get_shape()

vector< unsigned long long > dmrpp::DmrppArray::get_shape ( bool constrained )

virtual

Get the array shape.

Parameters

constrained If true, return the shape of the constrained array.

Returns: A vector<int> that describes the shape of the array.

Definition at line 706 of file DmrppArray.cc.

◆ get_size()

unsigned long long dmrpp::DmrppArray::get_size ( bool constrained = false )

virtual

Return the total number of elements in this Array.

Parameters

constrained If true, use the constrained size of the array, otherwise use the full size.

Returns: The number of elements in this Array

Definition at line 689 of file DmrppArray.cc.

◆ get_special_structure_flag()

bool dmrpp::DmrppArray::get_special_structure_flag ( )

inline

Definition at line 252 of file DmrppArray.h.

◆ get_struct_offsets()

const std::vector< unsigned int > & dmrpp::DmrppCommon::get_struct_offsets ( ) const

inlineinherited

Definition at line 188 of file DmrppCommon.h.

◆ get_structure_array_buf_ptr()

char * dmrpp::DmrppArray::get_structure_array_buf_ptr ( )

inline

Definition at line 247 of file DmrppArray.h.

◆ get_structure_array_str_buffer()

vector< char > & dmrpp::DmrppArray::get_structure_array_str_buffer ( )

inline

Definition at line 246 of file DmrppArray.h.

◆ get_total_linked_blocks()

unsigned int dmrpp::DmrppCommon::get_total_linked_blocks ( ) const

inlineinherited

Definition at line 197 of file DmrppCommon.h.

◆ get_uses_fill_value()

virtual bool dmrpp::DmrppCommon::get_uses_fill_value ( ) const

inlinevirtualinherited

Returns: Return true if the the chunk uses 'fill value.'

Definition at line 299 of file DmrppCommon.h.

◆ get_using_linked_block()

bool dmrpp::DmrppCommon::get_using_linked_block ( ) const

inlineinherited

Definition at line 194 of file DmrppCommon.h.

◆ get_var_chunks_storage_size()

unsigned long long dmrpp::DmrppCommon::get_var_chunks_storage_size ( ) const

inlineinherited

Definition at line 472 of file DmrppCommon.h.

◆ get_xml_node()

virtual const pugi::xml_node & dmrpp::DmrppCommon::get_xml_node ( ) const

inlinevirtualinherited

Definition at line 258 of file DmrppCommon.h.

◆ ingest_byte_order()

void dmrpp::DmrppCommon::ingest_byte_order ( const std::string & byte_order_string )

virtualinherited

Parses the text content of the XML element chunks:byteOrder.

Parameters

byte_order_string One of "LE", "BE"

Definition at line 181 of file DmrppCommon.cc.

◆ ingest_compression_type()

void dmrpp::DmrppCommon::ingest_compression_type ( const std::string & compression_type_string )

virtualinherited

Parses the text content of the XML element h4:chunkDimensionSizes into the internal vector<unsigned int> representation.

Parameters

compression_type_string

Definition at line 170 of file DmrppCommon.cc.

◆ ingest_fixed_length_string()

string dmrpp::DmrppArray::ingest_fixed_length_string	(	const char *	buf,
		unsigned long long	fixed_str_len,
		string_pad_type	pad_type )

static

Definition at line 2334 of file DmrppArray.cc.

◆ is_compact_layout()

virtual bool dmrpp::DmrppCommon::is_compact_layout ( ) const

inlinevirtualinherited

Returns true if this object utilizes COMPACT layout.

Definition at line 204 of file DmrppCommon.h.

◆ is_disable_dio()

virtual bool dmrpp::DmrppCommon::is_disable_dio ( ) const

inlinevirtualinherited

Returns true if this object describes the missing data.

Definition at line 235 of file DmrppCommon.h.

◆ is_filters_empty()

virtual bool dmrpp::DmrppCommon::is_filters_empty ( ) const

inlinevirtualinherited

Definition at line 199 of file DmrppCommon.h.

◆ is_flsa()

bool dmrpp::DmrppArray::is_flsa ( ) const

inline

Definition at line 218 of file DmrppArray.h.

◆ is_missing_data()

virtual bool dmrpp::DmrppCommon::is_missing_data ( ) const

inlinevirtualinherited

Returns true if this object describes the missing data.

Definition at line 214 of file DmrppCommon.h.

◆ is_multi_linked_blocks_chunk()

virtual bool dmrpp::DmrppCommon::is_multi_linked_blocks_chunk ( ) const

inlinevirtualinherited

Returns true if this object contains a chunk that have multiple linked blocks .

Definition at line 224 of file DmrppCommon.h.

◆ is_projected()

bool dmrpp::DmrppArray::is_projected ( )

Is this Array subset?

Returns: True if the array has a projection expression, false otherwise

Definition at line 675 of file DmrppArray.cc.

◆ is_vlsa()

bool dmrpp::DmrppArray::is_vlsa ( ) const

inline

Definition at line 223 of file DmrppArray.h.

◆ load_attributes()

void dmrpp::DmrppCommon::load_attributes ( libdap::BaseType * btp )

virtualinherited

Load the attribute information for this variable.

Parameters

btp

Definition at line 883 of file DmrppCommon.cc.

◆ load_chunks()

void dmrpp::DmrppCommon::load_chunks ( libdap::BaseType * btp )

virtualinherited

Load chunk information for this variable.

Parameters

btp	Load the chunk information for/into this variable

Definition at line 874 of file DmrppCommon.cc.

◆ operator=()

DmrppArray & dmrpp::DmrppArray::operator= ( const DmrppArray & rhs )

Definition at line 659 of file DmrppArray.cc.

◆ pad_type_to_str()

std::string dmrpp::DmrppArray::pad_type_to_str ( string_pad_type pad_type )

static

Definition at line 2887 of file DmrppArray.cc.

◆ parse_chunk_dimension_sizes()

void dmrpp::DmrppCommon::parse_chunk_dimension_sizes ( const std::string & chunk_dim_sizes_string )

virtualinherited

Set the dimension sizes for a chunk.

The string argument holds a space-separated list of integers that represent the dimensions of a chunk. Parse that string and store the integers in this instance.

Parameters

chunk_dims The sizes as a list of integers separated by spaces, e.g., '50 50'

Definition at line 132 of file DmrppCommon.cc.

◆ print_chunks_element()

void dmrpp::DmrppCommon::print_chunks_element	(	libdap::XMLWriter &	xml,
		const std::string &	name_space = "" )

inherited

Print the Chunk information.

Note

Should not be called when the d_chunks vector has no elements because it will write out a <chunks> element that is going to be empty when it might just be the case that the chunks have not been read.

Added support for chunks that use the HDF5 Fill Value system - Those chunks have no data to read and thus no offset or length. They do have a Chunk Position in Array and Fill Value, however. Here's an example:

  <dmrpp:chunk  fillValue="-32678" chunkPositionInArray="[...]"/>

Definition at line 514 of file DmrppCommon.cc.

◆ print_compact_element()

void dmrpp::DmrppCommon::print_compact_element	(	libdap::XMLWriter &	xml,
		const std::string &	name_space = "",
		const std::string &	encoded = "" ) const

inherited

Print the Compact base64-encoded information.

See also: https://gnome.pages.gitlab.gnome.org/libxml2/devhelp/libxml2-xmlwriter.html#xmlTextWriterWriteElementNS

Definition at line 690 of file DmrppCommon.cc.

◆ print_dap4()

void dmrpp::DmrppArray::print_dap4	(	libdap::XMLWriter &	writer,
		bool	constrained = false )

override

Shadow libdap::Array::print_dap4() - optionally prints DMR++ chunk information.

This version of libdap::BaseType::print_dap4() will print information about HDF5 chunks when the value of the static class filed dmrpp::DmrppCommon::d_print_chunks is true. The method DMRpp::print_dmrpp() will set the d_pprint_chunks field to true causing this method to include the chunks elements in its output. When the field's value is false, this method prints the same output as libdap::Array.

Note: There are, no doubt, better ways to do this than using what is essentially a global flag; one way is to synchronize access to a DMR C++ object and a DOM tree for the same DMR document. The chunk information can be read from the DMR and inserted into the DOM tree, which then printed. If the approach I took here becomes an issue (i.e., if we have to fix problems in libdap and here because of code duplication), we should probably recode this and the related methods to use the 'DOM tree approach.'

Parameters

xml	Write the XML to this instance of XMLWriter
constrained	True if the response should be constrained. False by default

See also: DmrppCommon::print_dmrpp(); DMRpp::print_dmrpp()

Definition at line 3198 of file DmrppArray.cc.

◆ print_dmrpp()

void dmrpp::DmrppCommon::print_dmrpp	(	libdap::XMLWriter &	writer,
		bool	constrained = false )

inherited

Print the DMR++ response for the Scalar types.

Note: See DmrppArray::print_dap4() for a discussion about the design of this, and related, method.

Parameters

xml	Write the XML to this instance of XMLWriter
constrained	If true, print the constrained DMR. False by default.

See also: DmrppArray::print_dap4()

Definition at line 752 of file DmrppCommon.cc.

◆ print_special_structure_element()

void dmrpp::DmrppCommon::print_special_structure_element	(	const libdap::XMLWriter &	xml,
		const std::string &	name_space = "",
		const std::string &	encoded = "" ) const

inherited

Definition at line 709 of file DmrppCommon.cc.

◆ ptr_duplicate()

libdap::BaseType * dmrpp::DmrppArray::ptr_duplicate ( )

inlineoverride

Definition at line 198 of file DmrppArray.h.

◆ read()

bool dmrpp::DmrppArray::read ( )

override

Read data for the array.

This reads data for a variable and loads it into memory. The software is specialized for reading data using HTTP for either arrays stored in one contiguous piece of memory or in a series of chunks.

Returns: Always returns true

Exceptions

BESError Thrown when the data cannot be read, for a number of reasons, including various network I/O issues.

Definition at line 2586 of file DmrppArray.cc.

◆ read_atomic() [1/2]

char * dmrpp::DmrppCommon::read_atomic ( const std::string & name )

protectedvirtualinherited

read method for the atomic types

This method is used by the specializations of BaseType::read() in the 'atomic' type classes (libdap::Byte, libdap::In32, ...) to read data when those data are contained in a single chunk (i.e., using HDF5 contiguous storage).

Note: It is assumed that these data are never compressed. However, it is possible to call Chunk::inflate_chunk(...) after calling this method and then call Chunk::get_rbuf() to access the decompressed data.

Parameters

name	The name of the variable, used for error messages

Returns: Pointer to a char buffer holding the data.

Exceptions

BESInternalError on error.

Definition at line 476 of file DmrppCommon.cc.

◆ read_atomic() [2/2]

char * dmrpp::DmrppCommon::read_atomic	(	const std::string &	name,
		size_t &	buf_size )

protectedvirtualinherited

Definition at line 489 of file DmrppCommon.cc.

◆ set_attributes_loaded()

virtual void dmrpp::DmrppCommon::set_attributes_loaded ( bool state )

inlinevirtualinherited

Definition at line 256 of file DmrppCommon.h.

◆ set_bytes_per_element()

void dmrpp::DmrppArray::set_bytes_per_element ( unsigned long long bpe )

inline

Definition at line 250 of file DmrppArray.h.

◆ set_chunk_dimension_sizes()

void dmrpp::DmrppCommon::set_chunk_dimension_sizes ( const std::vector< unsigned long long > & chunk_dims )

inlineinherited

Set the value of the chunk dimension sizes given a vector of HDF5 hsize_t.

Definition at line 323 of file DmrppCommon.h.

◆ set_chunks_loaded()

virtual void dmrpp::DmrppCommon::set_chunks_loaded ( bool state )

inlinevirtualinherited

Definition at line 252 of file DmrppCommon.h.

◆ set_compact()

void dmrpp::DmrppCommon::set_compact ( bool value )

inlineinherited

Set the value of the compact property.

Definition at line 209 of file DmrppCommon.h.

◆ set_deflate_levels()

void dmrpp::DmrppCommon::set_deflate_levels ( const std::vector< unsigned int > & def_levels )

inlineinherited

Definition at line 180 of file DmrppCommon.h.

◆ set_disable_dio()

void dmrpp::DmrppCommon::set_disable_dio ( bool value )

inlineinherited

Set the value of the compact property.

Definition at line 240 of file DmrppCommon.h.

◆ set_fill_value_string()

virtual void dmrpp::DmrppCommon::set_fill_value_string ( const std::string & fv )

inlinevirtualinherited

Set the fill value (using a string)

Definition at line 293 of file DmrppCommon.h.

◆ set_fill_value_type()

virtual void dmrpp::DmrppCommon::set_fill_value_type ( libdap::Type t )

inlinevirtualinherited

Set the libdap data type to use with the fill value.

Definition at line 296 of file DmrppCommon.h.

◆ set_filter()

void dmrpp::DmrppCommon::set_filter ( const std::string & value )

inherited

Set the value of the filters property.

Definition at line 106 of file DmrppCommon.cc.

◆ set_fixed_length_string_pad()

void dmrpp::DmrppArray::set_fixed_length_string_pad ( const string_pad_type pad )

inline

Definition at line 229 of file DmrppArray.h.

◆ set_fixed_length_string_pad_type()

string_pad_type dmrpp::DmrppArray::set_fixed_length_string_pad_type ( const std::string & pad_str )

Definition at line 2915 of file DmrppArray.cc.

◆ set_fixed_string_length() [1/2]

unsigned long long dmrpp::DmrppArray::set_fixed_string_length ( const string & length_str )

Definition at line 2849 of file DmrppArray.cc.

◆ set_fixed_string_length() [2/2]

void dmrpp::DmrppArray::set_fixed_string_length ( const unsigned long long length )

inline

Definition at line 225 of file DmrppArray.h.

◆ set_is_flsa()

void dmrpp::DmrppArray::set_is_flsa ( bool state )

inline

Marks the array as a Fixed length string array, or not, depending on state.

Parameters

state

Definition at line 215 of file DmrppArray.h.

◆ set_is_vlsa()

void dmrpp::DmrppArray::set_is_vlsa ( bool state )

inline

Definition at line 220 of file DmrppArray.h.

◆ set_missing_data()

void dmrpp::DmrppCommon::set_missing_data ( bool value )

inlineinherited

Set the value of the missing data.

Definition at line 219 of file DmrppCommon.h.

◆ set_multi_linked_blocks_chunk()

void dmrpp::DmrppCommon::set_multi_linked_blocks_chunk ( bool value )

inlineinherited

Set the value of the boolean variable that indicates this variable contains multiple linked blocks in a chunk.

Definition at line 229 of file DmrppCommon.h.

◆ set_one_chunk_fill_value()

virtual void dmrpp::DmrppCommon::set_one_chunk_fill_value ( bool ufv )

inlinevirtualinherited

Set the one_chunk_fill_value property.

Definition at line 308 of file DmrppCommon.h.

◆ set_ons_string() [1/2]

void dmrpp::DmrppArray::set_ons_string ( const std::string & ons_str )

Definition at line 2934 of file DmrppArray.cc.

◆ set_ons_string() [2/2]

void dmrpp::DmrppArray::set_ons_string ( const vector< ons > & ons_pairs )

Definition at line 2939 of file DmrppArray.cc.

◆ set_processing_fv_chunks()

void dmrpp::DmrppCommon::set_processing_fv_chunks ( )

inlineinherited

Definition at line 190 of file DmrppCommon.h.

◆ set_send_p()

void dmrpp::DmrppArray::set_send_p ( bool state )

override

Definition at line 2294 of file DmrppArray.cc.

◆ set_special_structure_flag()

void dmrpp::DmrppArray::set_special_structure_flag ( bool is_special_struct )

inline

Definition at line 251 of file DmrppArray.h.

◆ set_struct_offsets()

void dmrpp::DmrppCommon::set_struct_offsets ( const std::vector< unsigned int > & s_offs )

inlineinherited

Definition at line 184 of file DmrppCommon.h.

◆ set_total_linked_blocks()

void dmrpp::DmrppCommon::set_total_linked_blocks ( unsigned tlbs )

inlineinherited

Definition at line 196 of file DmrppCommon.h.

◆ set_uses_fill_value()

virtual void dmrpp::DmrppCommon::set_uses_fill_value ( bool ufv )

inlinevirtualinherited

Set the uses_fill_value property.

Definition at line 290 of file DmrppCommon.h.

◆ set_using_linked_block()

void dmrpp::DmrppCommon::set_using_linked_block ( )

inlineinherited

Definition at line 193 of file DmrppCommon.h.

◆ set_xml_node()

virtual void dmrpp::DmrppCommon::set_xml_node ( pugi::xml_node node )

inlinevirtualinherited

Definition at line 259 of file DmrppCommon.h.

◆ twiddle_bytes()

virtual bool dmrpp::DmrppCommon::twiddle_bytes ( ) const

inlinevirtualinherited

Returns true if this object utilizes shuffle compression.

Definition at line 245 of file DmrppCommon.h.

Friends And Related Symbol Documentation

◆ DmrppArrayTest

friend class DmrppArrayTest

friend

Definition at line 119 of file DmrppArray.h.

Member Data Documentation

◆ d_dmrpp_ns

string dmrpp::DmrppCommon::d_dmrpp_ns = "http://xml.opendap.org/dap/dmrpp/1.0.0#"

staticinherited

The DMR++ XML namespace.

Definition at line 161 of file DmrppCommon.h.

◆ d_ns_prefix

string dmrpp::DmrppCommon::d_ns_prefix = "dmrpp"

staticinherited

The XML namespace prefix to use.

Definition at line 162 of file DmrppCommon.h.

◆ d_print_chunks

bool dmrpp::DmrppCommon::d_print_chunks = false

staticinherited

if true, print_dap4() prints chunk elements

Definition at line 160 of file DmrppCommon.h.

The documentation for this class was generated from the following files:

Public Member Functions

Static Public Member Functions

Static Public Attributes

Protected Member Functions

Friends

Detailed Description

Constructor & Destructor Documentation

◆ DmrppArray() [1/4]

◆ DmrppArray() [2/4]

◆ DmrppArray() [3/4]

◆ DmrppArray() [4/4]

Member Function Documentation

◆ accumlate_storage_size()

◆ add_chunk() [1/15]

◆ add_chunk() [2/15]

◆ add_chunk() [3/15]

◆ add_chunk() [4/15]

◆ add_chunk() [5/15]

◆ add_chunk() [6/15]

◆ add_chunk() [7/15]

◆ add_chunk() [8/15]

◆ add_chunk() [9/15]

◆ add_chunk() [10/15]

◆ add_chunk() [11/15]

◆ add_chunk() [12/15]

◆ add_chunk() [13/15]

◆ add_chunk() [14/15]

◆ add_chunk() [15/15]

◆ buf2val()

◆ compact_str_buffer()

◆ dump() [1/2]

◆ dump() [2/2]

◆ get_attributes_loaded()

◆ get_byte_order()

◆ get_bytes_per_element()

◆ get_chunk_dimension_sizes()

◆ get_chunk_size_in_elements()

◆ get_chunks()

◆ get_chunks_loaded()

◆ get_chunks_size()

◆ get_deflate_levels()

◆ get_fill_value()

◆ get_fill_value_type()

◆ get_filters()

◆ get_fixed_length_string_pad()

◆ get_fixed_length_string_pad_str()

◆ get_fixed_string_length()

◆ get_immutable_chunks()

◆ get_one_chunk_fill_value()

◆ get_ons_objs()

◆ get_ons_string()

◆ get_processing_fv_chunks()

◆ get_shape()

◆ get_size()

◆ get_special_structure_flag()

◆ get_struct_offsets()

◆ get_structure_array_buf_ptr()

◆ get_structure_array_str_buffer()

◆ get_total_linked_blocks()

◆ get_uses_fill_value()

◆ get_using_linked_block()

◆ get_var_chunks_storage_size()

◆ get_xml_node()

◆ ingest_byte_order()

◆ ingest_compression_type()

◆ ingest_fixed_length_string()

◆ is_compact_layout()

◆ is_disable_dio()

◆ is_filters_empty()

◆ is_flsa()

◆ is_missing_data()

◆ is_multi_linked_blocks_chunk()

◆ is_projected()

◆ is_vlsa()

◆ load_attributes()

◆ load_chunks()

◆ operator=()

◆ pad_type_to_str()

◆ parse_chunk_dimension_sizes()

◆ print_chunks_element()