bes Updated for version 3.21.1
The Backend Server (BES) is the lower two tiers of the Hyrax data server
build_dmrpp_util.cc
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of the Hyrax data server.
4
5// Copyright (c) 2022 OPeNDAP, Inc.
6// Author: James Gallagher <jgallagher@opendap.org>
7//
8// This library is free software; you can redistribute it and/or
9// modify it under the terms of the GNU Lesser General Public
10// License as published by the Free Software Foundation; either
11// version 2.1 of the License, or (at your option) any later version.
12//
13// This library is distributed in the hope that it will be useful,
14// but WITHOUT ANY WARRANTY; without even the implied warranty of
15// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16// Lesser General Public License for more details.
17//
18// You should have received a copy of the GNU Lesser General Public
19// License along with this library; if not, write to the Free Software
20// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21//
22// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23
24#include "config.h"
25
26#include <iostream>
27#include <sstream>
28#include <memory>
29#include <iterator>
30#include <unordered_set>
31#include <iomanip> // std::put_time()
32#include <ctime> // std::gmtime_r()
33
34#include <H5Ppublic.h>
35#include <H5Dpublic.h>
36#include <H5Epublic.h>
37#include <H5Zpublic.h> // Constants for compression filters
38#include <H5Spublic.h>
39#include <H5Tpublic.h>
40
41#include "h5common.h" // This is in the hdf5 handler
42
43#include <libdap/Str.h>
44#include <libdap/util.h>
45#include <libdap/D4Attributes.h>
46#include <libdap/Array.h>
47
48#include <Base64.h>
49
50#include <BESNotFoundError.h>
51#include <BESInternalError.h>
52#include <BESInternalFatalError.h>
53
54#include <TheBESKeys.h>
55#include <BESContextManager.h>
56
57#include "DMRpp.h"
58#include "DmrppTypeFactory.h"
59#include "DmrppD4Group.h"
60#include "DmrppArray.h"
61#include "DmrppStructure.h"
62#include "D4ParserSax2.h"
63
64#include "UnsupportedTypeException.h"
65
66using namespace std;
67using namespace libdap;
68using namespace dmrpp;
69
70namespace build_dmrpp_util {
71
72bool verbose = false; // Optionally set by build_dmrpp's main().
73
74#define VERBOSE(x) do { if (verbose) (x); } while(false)
75#define prolog std::string("# build_dmrpp::").append(__func__).append("() - ")
76
77#define INVOCATION_CONTEXT "invocation"
78
79// FYI: Filter IDs
80// H5Z_FILTER_ERROR (-1) no filter
81// H5Z_FILTER_NONE 0 reserved indefinitely
82// H5Z_FILTER_DEFLATE 1 deflation like gzip
83// H5Z_FILTER_SHUFFLE 2 shuffle the data
84// H5Z_FILTER_FLETCHER32 3 fletcher32 checksum of EDC
85// H5Z_FILTER_SZIP 4 szip compression
86// H5Z_FILTER_NBIT 5 nbit compression
87// H5Z_FILTER_SCALEOFFSET 6 scale+offset compression
88// H5Z_FILTER_RESERVED 256 filter ids below this value are reserved for library use
89// FYI: Filter IDs
90// H5Z_FILTER_ERROR (-1) no filter
91// H5Z_FILTER_NONE 0 reserved indefinitely
92// H5Z_FILTER_DEFLATE 1 deflation like gzip
93// H5Z_FILTER_SHUFFLE 2 shuffle the data
94// H5Z_FILTER_FLETCHER32 3 fletcher32 checksum of EDC
95// H5Z_FILTER_SZIP 4 szip compression
96// H5Z_FILTER_NBIT 5 nbit compression
97// H5Z_FILTER_SCALEOFFSET 6 scale+offset compression
98// H5Z_FILTER_RESERVED 256 filter ids below this value are reserved for library use
99
105string h5_filter_name(H5Z_filter_t filter_type) {
106 string name;
107 switch(filter_type) {
108 case H5Z_FILTER_NONE:
109 name = "H5Z_FILTER_NONE";
110 break;
111 case H5Z_FILTER_DEFLATE:
112 name = "H5Z_FILTER_DEFLATE";
113 break;
114 case H5Z_FILTER_SHUFFLE:
115 name = "H5Z_FILTER_SHUFFLE";
116 break;
117 case H5Z_FILTER_FLETCHER32:
118 name = "H5Z_FILTER_FLETCHER32";
119 break;
120 case H5Z_FILTER_SZIP:
121 name = "H5Z_FILTER_SZIP";
122 break;
123 case H5Z_FILTER_NBIT:
124 name = "H5Z_FILTER_NBIT";
125 break;
126 case H5Z_FILTER_SCALEOFFSET:
127 name = "H5Z_FILTER_SCALEOFFSET";
128 break;
129 default:
130 {
131 ostringstream oss("ERROR! Unknown HDF5 FILTER (H5Z_filter_t) type: ", std::ios::ate);
132 oss << filter_type;
133 throw BESInternalError(oss.str(),__FILE__,__LINE__);
134 }
135 }
136 return name;
137}
138
144hid_t create_h5plist(hid_t dataset){
145 hid_t plist_id;
146 // Get creation properties list
147 plist_id = H5Dget_create_plist(dataset);
148 if ( plist_id < 0 )
149 throw BESInternalError("Unable to open HDF5 dataset id.", __FILE__, __LINE__);
150 return plist_id;
151}
152
158DmrppCommon *toDC(BaseType *btp){
159 auto *dc = dynamic_cast<DmrppCommon *>(btp);
160 if (!dc) {
161 stringstream msg;
162 msg << "ERROR: Expected a BaseType that was also a DmrppCommon instance.";
163 msg << "(variable_name: "<< ((btp)?btp->name():"unknown") << ").";
164 throw BESInternalError(msg.str(), __FILE__, __LINE__);
165 }
166 return dc;
167}
168
174DmrppArray *toDA(BaseType *btp){
175 auto *da = dynamic_cast<DmrppArray *>(btp);
176 if (!da) {
177 stringstream msg;
178 msg << "ERROR: Expected a BaseType that was also a DmrppArray instance.";
179 msg << "(variable_name: "<< ((btp)?btp->name():"unknown") << ").";
180 throw BESInternalError(msg.str(), __FILE__, __LINE__);
181 }
182 return da;
183}
184
185
192static void set_filter_information(hid_t dataset_id, DmrppCommon *dc, bool disable_dio) {
193
194 hid_t plist_id = create_h5plist(dataset_id);
195
196 try {
197 int numfilt = H5Pget_nfilters(plist_id);
198 VERBOSE(cerr << prolog << "Number of filters associated with dataset: " << numfilt << endl);
199 string filters;
200 size_t nelmts = 20;
201 unsigned int cd_values[20];
202 vector<unsigned int> deflate_levels;
203
204 for (int filter = 0; filter < numfilt; filter++) {
205 unsigned int flags;
206 H5Z_filter_t filter_type = H5Pget_filter2(plist_id, filter, &flags, &nelmts,
207 cd_values, 0, nullptr, nullptr);
208 VERBOSE(cerr << prolog << "Found H5 Filter Type: " << h5_filter_name(filter_type) << " (" << filter_type << ")" << endl);
209 switch (filter_type) {
210 case H5Z_FILTER_DEFLATE:
211 filters.append("deflate ");
212 VERBOSE(cerr << prolog << "Deflate compression level: " << cd_values[0] << endl);
213 deflate_levels.push_back(cd_values[0]);
214 break;
215 case H5Z_FILTER_SHUFFLE:
216 filters.append("shuffle ");
217 break;
218 case H5Z_FILTER_FLETCHER32:
219 filters.append("fletcher32 ");
220 break;
221 default:
222 ostringstream oss("Unsupported HDF5 filter: ", std::ios::ate);
223 oss << filter_type;
224 oss << " (" << h5_filter_name(filter_type) << ")";
225 throw BESInternalError(oss.str(), __FILE__, __LINE__);
226 }
227 }
228 H5Pclose(plist_id);
229
230 //trimming trailing space from compression (aka filter) string
231 filters = filters.substr(0, filters.size() - 1);
232 dc->set_filter(filters);
233 dc->set_deflate_levels(deflate_levels);
234 if (!filters.empty())
235 dc->set_disable_dio(disable_dio);
236 }
237 catch (...) {
238 H5Pclose(plist_id);
239 throw;
240 }
241}
242
243
249short
250is_hdf5_fill_value_defined(hid_t dataset_id)
251{
252
253 // Suppress errors to stderr.
254 H5Eset_auto2(H5E_DEFAULT, nullptr, nullptr);
255
256 auto plist_id = create_h5plist(dataset_id);
257
258 // ret_value 0: UNDEFINED
259 // ret_value 1: DEFAULT
260 // ret_value 2: USER_DEFINED
261 short ret_value = -1;
262
263 // How the fill value is defined?
264 H5D_fill_value_t status;
265 if ((H5Pfill_value_defined(plist_id, &status)) < 0) {
266 H5Pclose(plist_id);
267 throw BESInternalError("Unable to access HDF5 Fillvalue information.", __FILE__, __LINE__);
268 }
269 if (status == H5D_FILL_VALUE_DEFAULT)
270 ret_value = 1;
271 else if (status == H5D_FILL_VALUE_USER_DEFINED)
272 ret_value = 2;
273 else if (status == H5D_FILL_VALUE_UNDEFINED)
274 ret_value = 0;
275
276 return ret_value;
277
278}
279
290string
291get_value_as_string(hid_t h5_type_id, vector<char> &value)
292{
293 H5T_class_t class_type = H5Tget_class(h5_type_id);
294 switch (class_type) {
295 case H5T_INTEGER: {
296 int sign;
297 sign = H5Tget_sign(h5_type_id);
298 switch (H5Tget_size(h5_type_id)) {
299 case 1:
300 if (sign == H5T_SGN_2)
301 return to_string(*(int8_t *) (value.data()));
302 else
303 return to_string(*(uint8_t *) (value.data()));
304
305 case 2:
306 if (sign == H5T_SGN_2)
307 return to_string(*(int16_t *) (value.data()));
308 else
309 return to_string(*(uint16_t *) (value.data()));
310
311 case 4:
312 if (sign == H5T_SGN_2)
313 return to_string(*(int32_t *) (value.data()));
314 else
315 return to_string(*(uint32_t *) (value.data()));
316
317 case 8:
318 if (sign == H5T_SGN_2)
319 return to_string(*(int64_t *) (value.data()));
320 else
321 return to_string(*(uint64_t *) (value.data()));
322
323 default:
324 throw BESInternalError("Unable to extract integer fill value.", __FILE__, __LINE__);
325 }
326 break;
327 }
328
329 case H5T_FLOAT: {
330 ostringstream oss;
331 switch (H5Tget_size(h5_type_id)) {
332 case 4:
333 oss << *(float *) (value.data());
334 return oss.str();
335
336 case 8:
337 oss << *(double *) (value.data());
338 return oss.str();
339
340 default:
341 throw BESInternalError("Unable to extract float fill value.", __FILE__, __LINE__);
342 }
343 break;
344 }
345
346 case H5T_STRING: {
347 if (H5Tis_variable_str(h5_type_id)) {
348
349 // Reading the value like this doesn't work for squat.
350 string fv_str(value.begin(),value.end());
351 // Now fv_str is garbage,.
352
353 stringstream msg(prolog);
354 msg << "UnsupportedTypeException: Your data granule contains a variable length H5T_STRING ";
355 msg << "as a fillValue type. This is not yet supported by the dmr++ creation machinery. ";
356 msg << "The variable/dataset type screening code should have intercepted this prior. ";
357 msg << "fillValue(" + to_string(fv_str.length()) +" chars): 0x";
358 for(auto c : fv_str){
359 msg << std::hex << +c ;
360 }
361 throw UnsupportedTypeException(msg.str());
362 }
363 else {
364 string str_fv(value.begin(),value.end());
365 return str_fv;
366 }
367 break;
368 }
369 case H5T_ARRAY: {
370 string msg(prolog + "UnsupportedTypeException: Your data granule contains an H5T_ARRAY as a fillValue type. "
371 "This is not yet supported by the dmr++ creation machinery."
372 "The variable/dataset type screening code should intercepted this prior.");
373 throw UnsupportedTypeException(msg);
374 }
375 case H5T_COMPOUND: {
376 // The fill value of compound datatype is obtained by calling get_compound_fv_as_string() else where.
377 // An error should generate and be thrown.
378 string msg(prolog + "UnsupportedTypeException: The fill value of a compound datatype should not be obtained in this function. "
379 "get_compound_fv_as_string() is the right function to get the value.");
380 throw UnsupportedTypeException(msg);
381 }
382
383 case H5T_REFERENCE:
384 default: {
385 throw BESInternalError("Unable to extract fill value from HDF5 file.", __FILE__, __LINE__);
386 }
387 }
388}
389
390string
391get_compound_base_fill_value_as_string(hid_t h5_type_id, char* value_ptr)
392{
393 H5T_class_t class_type = H5Tget_class(h5_type_id);
394 switch (class_type) {
395 case H5T_INTEGER: {
396 int sign;
397 sign = H5Tget_sign(h5_type_id);
398 switch (H5Tget_size(h5_type_id)) {
399 case 1:
400 if (sign == H5T_SGN_2)
401 return to_string(*(int8_t *) value_ptr);
402 else
403 return to_string(*(uint8_t *) value_ptr);
404
405 case 2:
406 if (sign == H5T_SGN_2)
407 return to_string(*(int16_t *) value_ptr);
408 else
409 return to_string(*(uint16_t *) value_ptr);
410
411 case 4:
412 if (sign == H5T_SGN_2)
413 return to_string(*(int32_t *) value_ptr);
414 else
415 return to_string(*(uint32_t *) value_ptr);
416
417 case 8:
418 if (sign == H5T_SGN_2)
419 return to_string(*(int64_t *) value_ptr);
420 else
421 return to_string(*(uint64_t *) value_ptr);
422
423 default:
424 throw BESInternalError("Unable to extract integer fill value.", __FILE__, __LINE__);
425 }
426 }
427
428 case H5T_FLOAT: {
429 ostringstream oss;
430 switch (H5Tget_size(h5_type_id)) {
431 case 4:
432 oss << *(float *) value_ptr;
433 return oss.str();
434
435 case 8:
436 oss << *(double *) value_ptr;
437 return oss.str();
438
439 default:
440 throw BESInternalError("Unable to extract float fill value.", __FILE__, __LINE__);
441 }
442 }
443 default:
444 throw BESInternalError("The member of compound datatype that has user-defined datatype has to be either integer or float..", __FILE__, __LINE__);
445 }
446
447}
448
449string obtain_compound_user_defined_fvalues(hid_t dtype_id, hid_t h5_plist_id, vector<char> &value) {
450
451 string ret_value;
452 hid_t memtype = -1;
453
454 if ((memtype = H5Tget_native_type(dtype_id, H5T_DIR_ASCEND))<0) {
455 H5Pclose(h5_plist_id);
456 throw BESInternalError ("Fail to obtain memory datatype.", __FILE__, __LINE__);
457 }
458
459 int nmembs = 0;
460 if ((nmembs = H5Tget_nmembers(memtype)) < 0) {
461 H5Tclose(memtype);
462 H5Pclose(h5_plist_id);
463 string err_msg = "Fail to obtain number of HDF5 compound datatype.";
464 throw BESInternalError (err_msg, __FILE__, __LINE__);
465 }
466
467 // We only need to retrieve the values and save them as strings.
468 // We will only have the one-layer int/float structure to handle.
469 // We do need to know the memb type to put the special handling of a string.
470 for (unsigned int u = 0; u < (unsigned)nmembs; u++) {
471
472 hid_t memb_id = -1;
473 H5T_class_t memb_cls = H5T_NO_CLASS;
474 size_t memb_offset = 0;
475
476 // Get member type ID
477 if((memb_id = H5Tget_member_type(memtype, u)) < 0) {
478 H5Tclose(memtype);
479 H5Pclose(h5_plist_id);
480 string err_msg = "Fail to obtain the datatype of an HDF5 compound datatype member.";
481 throw BESInternalError (err_msg, __FILE__, __LINE__);
482 }
483
484 // Get member type class
485 if((memb_cls = H5Tget_member_class (memtype, u)) < 0) {
486 H5Pclose(h5_plist_id);
487 H5Tclose(memtype);
488 H5Tclose(memb_id);
489 string err_msg = "Fail to obtain the datatype class of an HDF5 compound datatype member.";
490 throw BESInternalError (err_msg, __FILE__, __LINE__);
491 }
492
493 // Get member offset,H5Tget_member_offset only fails
494 // when H5Tget_memeber_class fails. Sinc H5Tget_member_class
495 // is checked above. So no need to check the return value.
496 memb_offset= H5Tget_member_offset(memtype,u);
497
498 if (memb_cls == H5T_ARRAY) {
499
500 hid_t at_base_type = H5Tget_super(memb_id);
501 size_t at_base_type_size = H5Tget_size(at_base_type);
502 H5T_class_t array_cls = H5Tget_class(at_base_type);
503
504 if (array_cls != H5T_INTEGER && array_cls !=H5T_FLOAT) {
505 H5Tclose(memtype);
506 H5Tclose(memb_id);
507 string err_msg = "The base class of an HDF5 compound datatype member must be integer or float.";
508 throw BESInternalError (err_msg, __FILE__, __LINE__);
509 }
510
511 // Need to retrieve the number of elements of the array
512 int at_ndims = H5Tget_array_ndims(memb_id);
513 if (at_ndims <= 0) {
514 H5Pclose(h5_plist_id);
515 H5Tclose(memtype);
516 H5Tclose(at_base_type);
517 H5Tclose(memb_id);
518 string err_msg = "Fail to obtain number of dimensions of the array datatype.";
519 throw BESInternalError (err_msg, __FILE__, __LINE__);
520 }
521
522 vector<hsize_t>at_dims_h(at_ndims,0);
523
524 // Obtain the number of elements for each dims
525 if (H5Tget_array_dims(memb_id,at_dims_h.data())<0) {
526 H5Pclose(h5_plist_id);
527 H5Tclose(memtype);
528 H5Tclose(at_base_type);
529 H5Tclose(memb_id);
530 string err_msg = "Fail to obtain each imension size of the array datatype.";
531 throw BESInternalError (err_msg, __FILE__, __LINE__);
532 }
533
534 vector<hsize_t>at_dims_offset(at_ndims,0);
535 size_t total_array_nums = 1;
536 for (const auto & ad:at_dims_h)
537 total_array_nums *=ad;
538
539 // We need to convert each value to a string and save that string as one value of a string.
540 for (unsigned ar_index = 0; ar_index <total_array_nums; ar_index++) {
541 char *value_ptr = value.data() + memb_offset + ar_index *at_base_type_size;
542 string tmp_value = get_compound_base_fill_value_as_string(at_base_type,value_ptr);
543 if (u == 0 && ar_index== 0)
544 ret_value = tmp_value;
545 else
546 ret_value = ret_value + ' '+ tmp_value;
547 }
548
549 H5Tclose(at_base_type);
550
551 }
552 else {// Scalar int/float
553
554 //We need to figure out the datatype and convert each data value to a string.
555 char *value_ptr = value.data() + memb_offset;
556 string tmp_value = get_compound_base_fill_value_as_string(memb_id,value_ptr);
557 if ( u == 0)
558 ret_value = tmp_value;
559 else
560 ret_value = ret_value + ' '+ tmp_value;
561 }
562 H5Tclose(memb_id);
563 }
564
565 H5Tclose(memtype);
566
567 return ret_value;
568}
569
570unsigned short is_supported_compound_type(hid_t h5_type) {
571
572 unsigned short ret_value = 1;
573 bool has_string_memb_type = false;
574 hid_t memtype = -1;
575 if ((memtype = H5Tget_native_type(h5_type, H5T_DIR_ASCEND)) < 0) {
576 throw InternalErr(__FILE__, __LINE__, "Fail to obtain memory datatype.");
577 }
578
579 hid_t memb_id = -1;
580 H5T_class_t memb_cls = H5T_NO_CLASS;
581 int nmembs = 0;
582 char *memb_name = nullptr;
583
584 if ((nmembs = H5Tget_nmembers(memtype)) < 0) {
585 throw InternalErr(__FILE__, __LINE__, "Fail to obtain number of HDF5 compound datatype.");
586 }
587
588 for (unsigned int u = 0; u < (unsigned) nmembs; u++) {
589
590 if ((memb_id = H5Tget_member_type(memtype, u)) < 0)
591 throw InternalErr(__FILE__, __LINE__,
592 "Fail to obtain the datatype of an HDF5 compound datatype member.");
593
594 // Get member type class
595 memb_cls = H5Tget_member_class(memtype, u);
596
597 // Get member name
598 memb_name = H5Tget_member_name(memtype, u);
599 if (memb_name == nullptr)
600 throw InternalErr(__FILE__, __LINE__, "Fail to obtain the name of an HDF5 compound datatype member.");
601
602 if (memb_cls == H5T_COMPOUND)
603 ret_value = 0;
604 else if (memb_cls == H5T_ARRAY) {
605
606 hid_t at_base_type = H5Tget_super(memb_id);
607 H5T_class_t array_cls = H5Tget_class(at_base_type);
608 if (array_cls != H5T_INTEGER && array_cls != H5T_FLOAT && array_cls != H5T_STRING)
609 ret_value = 0;
610 else if (array_cls == H5T_STRING && has_string_memb_type == false)
611 has_string_memb_type = true;
612 H5Tclose(at_base_type);
613
614
615 } else if (memb_cls != H5T_INTEGER && memb_cls != H5T_FLOAT) {
616 if (memb_cls == H5T_STRING) {
617 if (has_string_memb_type == false)
618 has_string_memb_type = true;
619 }
620 else
621 ret_value = 0;
622 }
623
624 // Close member type ID
625 H5Tclose(memb_id);
626 free(memb_name);
627 if (ret_value == 0)
628 break;
629 } // end for
630
631 if (has_string_memb_type)
632 ret_value = 2;
633 return ret_value;
634
635}
636
637
638string
639get_compound_fv_as_string(hid_t dtype_id, hid_t h5_plist_id, vector<char> &value)
640{
641 H5D_fill_value_t fill_value_status;
642 if (H5Pfill_value_defined(h5_plist_id, &fill_value_status)<0) {
643 H5Pclose(h5_plist_id);
644 throw BESInternalError("H5Pfill_value_defined failed.", __FILE__, __LINE__);
645 }
646
647 string ret_str;
648 string H5_Default_fvalue = "0";
649 if (fill_value_status == H5D_FILL_VALUE_DEFAULT)
650 ret_str = H5_Default_fvalue;
651 else if (fill_value_status == H5D_FILL_VALUE_USER_DEFINED) {
652
653 // We don't support when a compound datatype has a string member and the fill value is user-defined.
654 if (is_supported_compound_type(dtype_id) == 2) {
655
656 string msg(prolog + "UnsupportedTypeException: Your data granule contains an H5T_COMPOUND as user-defined fillValue type"
657 "and one member of H5T_COMPOUND is a string. "
658 "This is not yet supported by the dmr++ creation machinery. ");
659 string str_fv(value.begin(),value.end());
660 throw UnsupportedTypeException(msg);
661 }
662
663 ret_str = obtain_compound_user_defined_fvalues(dtype_id, h5_plist_id, value);
664 }
665 else if (fill_value_status == H5D_FILL_VALUE_UNDEFINED) {
666 H5Pclose(h5_plist_id);
667 throw BESInternalError("The fill value is undefined, the dmrpp module cannot handle this case now.", __FILE__, __LINE__);
668 }
669
670 return ret_str;
671}
672
673bool is_supported_vlen_type(hid_t dataset_id, hid_t h5_type) {
674
675 bool ret_value = false;
676 hid_t base_type = H5Tget_super(h5_type);
677 hid_t dspace = H5Dget_space(dataset_id);
678 if (H5S_SIMPLE == H5Sget_simple_extent_type(dspace) &&
679 (H5Tget_class(base_type) == H5T_INTEGER || H5Tget_class(base_type) == H5T_FLOAT))
680 ret_value = true;
681 H5Tclose(base_type);
682 H5Sclose(dspace);
683 return ret_value;
684
685}
686
687
693string get_hdf5_fill_value_str(hid_t dataset_id)
694{
695 // Suppress errors to stderr.
696 H5Eset_auto2(H5E_DEFAULT, nullptr, nullptr);
697
698 // Get creation properties list
699 hid_t plist_id = create_h5plist(dataset_id);
700 if (plist_id < 0 )
701 throw BESInternalError("Unable to open HDF5 dataset id.", __FILE__, __LINE__);
702
703 try {
704 hid_t dtype_id = H5Dget_type(dataset_id);
705 if (dtype_id < 0)
706 throw BESInternalError("Unable to get HDF5 dataset type id.", __FILE__, __LINE__);
707
708 vector<char> value(H5Tget_size(dtype_id), 0);
709 if (H5Pget_fill_value(plist_id, dtype_id, value.data()) < 0)
710 throw BESInternalError("Unable to access HDF5 Fill Value.", __FILE__, __LINE__);
711
712 string fvalue_str;
713 // The fill value of the compound datatype needs to be handled separately.
714 if (H5Tget_class(dtype_id) == H5T_COMPOUND)
715 fvalue_str = get_compound_fv_as_string(dtype_id,plist_id,value);
716 else
717 fvalue_str = get_value_as_string(dtype_id, value);
718
719 H5Pclose(plist_id);
720
721 return fvalue_str;
722 }
723 catch (...) {
724 H5Pclose(plist_id);
725 throw;
726 }
727}
728
729
736string_pad_type convert_h5_str_pad_type(const H5T_str_t str_pad){
737 string_pad_type pad_type;
738 switch(str_pad){
739 case H5T_STR_SPACEPAD:
740 pad_type = dmrpp::space_pad;
741 break;
742
743 case H5T_STR_NULLTERM:
744 pad_type = dmrpp::null_term;
745 break;
746
747 case H5T_STR_NULLPAD:
748 pad_type = dmrpp::null_pad;
749 break;
750
751 default:
752 {
753 stringstream msg;
754 msg << "ERROR: Received unrecognized value for H5T_str_t: " << str_pad << endl;
755 throw BESInternalError(msg.str(),__FILE__,__LINE__);
756 }
757 }
758 return pad_type;
759}
760
768string_pad_type get_pad_type(const hid_t dataset) {
769 hid_t h5_type = H5Dget_type(dataset);
770 if(h5_type < 0){
771 stringstream msg;
772 msg << "ERROR: H5Dget_type() failed. returned: " << h5_type;
773 throw BESInternalError(msg.str(),__FILE__, __LINE__);
774 }
775 H5T_str_t str_pad = H5Tget_strpad(h5_type);
776 if(str_pad < 0) {
777 stringstream msg;
778 msg << "ERROR: H5Tget_strpad() failed. returned: " << str_pad;
779 throw BESInternalError(msg.str(),__FILE__, __LINE__);
780 }
781 return convert_h5_str_pad_type(str_pad);
782}
783
792void add_fixed_length_string_array_state(const hid_t dataset_id, DmrppArray *array_var){
793
794 hid_t h5_type = H5Dget_type(dataset_id);
795 if (H5Tis_variable_str(h5_type) > 0 ){
796 cout << "# The dataset '" << array_var->name() << "' is a variable length string array, skipping..." << endl;
797 return;
798 }
799
800 VERBOSE( cerr << prolog << "Processing the array dariable: " << array_var->name() << endl);
801 auto data_type = array_var->var()->type();
802
803 if(data_type == libdap::dods_str_c){
804 VERBOSE( cerr << prolog << "The array template variable has type libdap::dods_str_c" << endl);
805
806 array_var->set_is_flsa(true);
807
808 auto pad_type = get_pad_type(dataset_id);
809 VERBOSE( cerr << prolog << "pad_type: " << pad_type << endl);
810 array_var->set_fixed_length_string_pad(pad_type);
811
812 auto type_size = H5Tget_size(h5_type);
813 VERBOSE( cerr << prolog << "type_size: " << type_size << endl);
814 array_var->set_fixed_string_length(type_size);
815 }
816}
817
818
819
826static void add_string_array_info(const hid_t dataset, BaseType *btp){
827
828 Type dap_type = btp->type();
829 if(dap_type != dods_array_c){
830 // NOT AN ARRAY SKIPPING...
831 VERBOSE( cerr << prolog << "Variable " << btp->name() << " is not a DAP Array. Skipping..." << endl);
832 return;
833 }
834 auto dap_array = toDA(btp);
835 if (dap_array->var()->type() != dods_str_c) {
836 // NOT A STRING ARRAY SKIPPING...
837 VERBOSE( cerr << prolog << "Variable " << dap_array->name() << " is an Array of " << dap_array->var()->type_name() << " not String. Skipping..." << endl);
838 return;
839 }
840
841 auto h5_dataset_type = H5Dget_type(dataset);
842 if(h5_dataset_type == H5I_INVALID_HID){
843 throw BESInternalError("ERROR: H5Dget_type() failed for variable '" + dap_array->name() + "'",
844 __FILE__, __LINE__);
845 }
846
847 auto h5_type_class = H5Tget_class(h5_dataset_type);
848 if(h5_type_class != H5T_STRING){
849 VERBOSE( cerr << prolog << "H5Dataset " << dap_array->name() << " is not a String type (type: " << h5_type_class << "). Skipping..." << endl);
850 return;
851 }
852
853 hid_t dspace = H5Dget_space(dataset);
854 if (H5S_SCALAR == H5Sget_simple_extent_type(dspace)){
855 VERBOSE( cerr << prolog << "H5Dataset " << dap_array->name() << " is a scalar type. Skipping..." << endl);
856 return;
857 }
858
859 if (H5Tis_variable_str(h5_dataset_type) > 0) {
860 VERBOSE( cerr << prolog << "Found variable length string array: " << dap_array->name() << endl);
861 dap_array->set_is_vlsa(true);
862 }
863 else {
864 VERBOSE( cerr << prolog << "Found fixed length string array: " << dap_array->name() << endl);
865 add_fixed_length_string_array_state( dataset, dap_array);
866 }
867}
868
874string byte_order_str(hid_t dataset){
875 string byte_order_string;
876 hid_t dtypeid = H5Dget_type(dataset);
877 auto b_order = H5Tget_order(dtypeid);
878 switch (b_order) {
879 case H5T_ORDER_LE:
880 byte_order_string = "LE";
881 break;
882 case H5T_ORDER_BE:
883 byte_order_string = "BE";
884 break;
885 case H5T_ORDER_NONE:
886 break;
887 default:
888 // unsupported enumerations: H5T_ORDER_[ERROR,VAX,MIXED]
889 ostringstream oss("Unsupported HDF5 dataset byteOrder: ", std::ios::ate);
890 oss << b_order << ".";
891 throw BESInternalError(oss.str(), __FILE__, __LINE__);
892 }
893 return byte_order_string;
894}
895
896void obtain_structure_offset(hid_t dataset, vector<unsigned int>& struct_offsets) {
897
898 hid_t dtypeid = H5Dget_type(dataset);
899
900 hid_t memb_id;
901 size_t memb_offset = 0;
902
903 int nmembs = H5Tget_nmembers(dtypeid);
904 if (nmembs <0) {
905 H5Tclose(dtypeid);
906 throw BESInternalError("Cannot get the number of base datatypes in a compound datatype.", __FILE__, __LINE__);
907 }
908
909 for (unsigned int u = 0; u < (unsigned) nmembs; u++) {
910
911 if ((memb_id = H5Tget_member_type(dtypeid, u)) < 0) {
912 H5Tclose(dtypeid);
913 throw BESInternalError("Cannot get the number of base datatypes in a compound datatype.", __FILE__, __LINE__);
914 }
915
916 // Get member offset
917 memb_offset = H5Tget_member_offset(dtypeid, u);
918 if (u !=0)
919 struct_offsets.push_back(memb_offset);
920
921 H5Tclose(memb_id);
922
923 }
924
925 // We need to add the size of the datatype to correctly retrieve the value of the next element.
926 size_t type_size = H5Tget_size(dtypeid);
927 if (type_size == 0) {
928 H5Tclose(dtypeid);
929 throw BESInternalError("Cannot get the correct data type size.", __FILE__, __LINE__);
930 }
931 struct_offsets.push_back(type_size);
932 H5Tclose(dtypeid);
933
934}
935
936
937
943void process_contiguous_layout_dariable(hid_t dataset, BaseType *btp){
944 VERBOSE(cerr << prolog << " Storage: contiguous" << endl);
945
946 haddr_t cont_addr = H5Dget_offset(dataset);
947 hsize_t cont_size = H5Dget_storage_size(dataset);
948 string byte_order = byte_order_str(dataset);
949
950 VERBOSE(cerr << prolog << " Addr: " << cont_addr << endl);
951 VERBOSE(cerr << prolog << " Size: " << cont_size << endl);
952 VERBOSE(cerr << prolog << "byteOrder: " << byte_order << endl);
953
954 if (cont_size > 0) {
955 auto dc = toDC(btp);
956 VERBOSE(cerr << prolog << " Before add_chunk: " <<btp->name() << endl);
957 dc->add_chunk(byte_order, cont_size, cont_addr, "");
958 }
959}
960
966void process_chunked_layout_dariable(hid_t dataset, BaseType *btp, bool disable_dio) {
967
968 DmrppCommon *dc = toDC(btp);
969 hid_t fspace_id = H5Dget_space(dataset);
970 int dataset_rank = H5Sget_simple_extent_ndims(fspace_id);
971 string byte_order = byte_order_str(dataset);
972
973 hsize_t num_chunks = 0;
974 herr_t status = H5Dget_num_chunks(dataset, fspace_id, &num_chunks);
975 if (status < 0) {
976 throw BESInternalError("Could not get the number of chunks for variable "+ btp->name(), __FILE__, __LINE__);
977 }
978
979 VERBOSE(cerr << prolog << "Storage: chunked." << endl);
980 VERBOSE(cerr << prolog << "Number of chunks is: " << num_chunks << endl);
981
982 set_filter_information(dataset, dc, disable_dio);
983
984 // Get chunking information: rank and dimensions
985 vector<hsize_t> chunk_dims(dataset_rank, 0);
986
987 unsigned int chunk_rank = 0;
988 hid_t plist_id = create_h5plist(dataset);
989 try {
990 chunk_rank = H5Pget_chunk(plist_id, dataset_rank, chunk_dims.data());
991 H5Pclose(plist_id);
992 }
993 catch (...) {
994 H5Pclose(plist_id);
995 throw;
996 }
997
998 if (chunk_rank != dataset_rank)
999 throw BESNotFoundError(
1000 "Found a chunk with rank different than the dataset's (aka variables') rank", __FILE__,
1001 __LINE__);
1002
1003 dc->set_chunk_dimension_sizes(chunk_dims);
1004
1005 for (unsigned int i = 0; i < num_chunks; ++i) {
1006 vector<hsize_t> chunk_coords(dataset_rank, 0);
1007 haddr_t addr = 0;
1008 hsize_t size = 0;
1009
1010 unsigned filter_mask = 0;
1011
1012 status = H5Dget_chunk_info(dataset, fspace_id, i, chunk_coords.data(),
1013 &filter_mask, &addr, &size);
1014 if (status < 0) {
1015 VERBOSE(cerr << "ERROR" << endl);
1016 throw BESInternalError("Cannot get HDF5 dataset storage info.", __FILE__, __LINE__);
1017 }
1018
1019 VERBOSE(cerr << prolog << "chk_idk: " << i << ", addr: " << addr << ", size: " << size << endl);
1020 dc->add_chunk(byte_order, size, addr, filter_mask, chunk_coords);
1021 }
1022}
1023
1024H5D_layout_t get_h5_storage_layout(hid_t dataset){
1025 H5D_layout_t layout_type;
1026 hid_t plist_id = create_h5plist(dataset);
1027 try {
1028 layout_type = H5Pget_layout(plist_id);
1029 }
1030 catch(...){
1031 H5Pclose(plist_id);
1032 throw;
1033 }
1034 return layout_type;
1035}
1036
1037void process_compact_layout_scalar(hid_t dataset, BaseType *btp)
1038{
1039
1040 // The variable is a scalar, not an array
1041
1042 VERBOSE(cerr << prolog << "Processing scalar dariable. Storage: compact" << endl);
1043
1044 hid_t dtypeid = H5Dget_type(dataset);
1045 VERBOSE(cerr << prolog << " H5Dget_type(): " << dtypeid << endl);
1046
1047 auto type_size = H5Tget_size(dtypeid);
1048 VERBOSE(cerr << prolog << " H5Tget_size(): " << type_size << " (The size of the datatype in bytes)" << endl);
1049
1050 size_t compact_storage_size = H5Dget_storage_size(dataset);
1051 VERBOSE(cerr << prolog << " H5Dget_storage_size(): " << compact_storage_size << " (The amount of storage space, in bytes, or 0.)" << endl);
1052 if (compact_storage_size == 0) {
1053 throw BESInternalError("Cannot obtain the compact storage size.", __FILE__, __LINE__);
1054 }
1055
1056 Type dap_type = btp->type();
1057 unsigned long long memRequired = 0;
1058 if (dap_type == dods_str_c)
1059 // The string length is 0 if no string has been stored in the internal buffer. So we cannot use length()
1060 // We know the length is 1 for a scalar string.
1061 memRequired = type_size;
1062 else
1063 memRequired = btp->length() * type_size;
1064
1065 // For variable length string, the storage size and the datatype size is not the same.
1066 // And we don't need to know then since it is a variable length string.
1067 if (H5Tis_variable_str(dtypeid) == 0) {
1068 if (compact_storage_size != memRequired)
1069 throw BESInternalError("Compact storage size does not match D4Array or scalar.", __FILE__,
1070 __LINE__);
1071 }
1072
1073 switch (dap_type)
1074 {
1075 case dods_byte_c:
1076 case dods_char_c:
1077 case dods_int8_c:
1078 case dods_uint8_c:
1079 case dods_int16_c:
1080 case dods_uint16_c:
1081 case dods_int32_c:
1082 case dods_uint32_c:
1083 case dods_float32_c:
1084 case dods_float64_c:
1085 case dods_int64_c:
1086 case dods_uint64_c:
1087 {
1088 vector<uint8_t> values(memRequired, 0);
1089 get_data(dataset, reinterpret_cast<void *>(values.data()));
1090 btp->set_read_p(true);
1091 btp->val2buf(reinterpret_cast<void *>(values.data()));
1092 break;
1093 }
1094
1095 case dods_url_c:
1096 case dods_str_c:
1097 {
1098 auto str = dynamic_cast<libdap::Str *>(btp);
1099 if (H5Tis_variable_str(dtypeid) > 0) {
1100 vector<string> finstrval; // passed by reference to read_vlen_string
1101 // @TODO Why push an empty string into the first array position? WHY?
1102 finstrval.emplace_back("");
1103 read_vlen_string(dataset, 1, nullptr, nullptr, nullptr, finstrval);
1104 string vlstr = finstrval[0];
1105 str->set_value(vlstr);
1106 str->set_read_p(true);
1107 }
1108 else {
1109 // A single string for scalar.
1110 vector<uint8_t> values(memRequired, 0);
1111 get_data(dataset, reinterpret_cast<void *>(values.data()));
1112 string fstr(values.begin(), values.end());
1113 str->set_value(fstr);
1114 str->set_read_p(true);
1115 }
1116 break;
1117 }
1118
1119 default:
1120 throw BESInternalError("Unsupported compact storage variable type.", __FILE__, __LINE__);
1121 }
1122}
1123
1124
1125void process_compact_flsa(hid_t dataset, BaseType *btp){
1126
1127 add_string_array_info(dataset, btp);
1128
1129 auto pad_type = get_pad_type(dataset);
1130 VERBOSE( cerr << prolog << "pad_type: " << pad_type << endl);
1131
1132 auto h5_type = H5Dget_type(dataset);
1133 VERBOSE( cerr << prolog << "H5Dget_type(): " << h5_type << endl);
1134
1135 // Since this is a fixed length string, the H5Tget_size() returns the
1136 // length in characters (i.e. bytes) of the fixed length string
1137 auto fls_length = H5Tget_size(h5_type);
1138 VERBOSE( cerr << prolog << "fls_length: " << fls_length << endl);
1139
1140 auto memRequired = btp->length_ll() * fls_length;
1141
1142 auto array = toDA(btp);
1143 auto &string_buf = array->compact_str_buffer();
1144 string_buf.resize(memRequired);
1145 get_data(dataset, reinterpret_cast<void *>(string_buf.data()));
1146 array->set_read_p(true);
1147}
1148
1149void process_compact_layout_array(hid_t dataset, BaseType *btp) {
1150
1151 VERBOSE(cerr << prolog << "BEGIN (" << btp->type_name() << " " << btp->name() << ")" << endl);
1152
1153 hid_t dtypeid = H5Dget_type(dataset);
1154 VERBOSE(cerr << prolog << " H5Dget_type(): " << dtypeid << endl);
1155
1156 auto type_size = H5Tget_size(dtypeid);
1157 VERBOSE(cerr << prolog << " H5Tget_size(): " << type_size << " (The size of the datatype in bytes)" << endl);
1158
1159 size_t compact_storage_size = H5Dget_storage_size(dataset);
1160 VERBOSE(cerr << prolog << " H5Dget_storage_size(): " << compact_storage_size << " (The amount of storage space, in bytes, or 0.)" << endl);
1161 if (compact_storage_size == 0) {
1162 throw BESInternalError("Cannot obtain the compact storage size.", __FILE__, __LINE__);
1163 }
1164
1165 Type dap_type = btp->type();
1166 unsigned long long memRequired = 0;
1167 if (dap_type == dods_str_c)
1168 // The string length is 0 if no string has been stored in the internal buffer. So we cannot use length()
1169 // We know the length is 1 for a scalar string.
1170 memRequired = type_size;
1171 else
1172 memRequired = btp->length() * type_size;
1173
1174 // For variable length string, the storage size and the datatype size is not the same.
1175 // And we don't need to know then since it is a variable length string.
1176 if (H5Tis_variable_str(dtypeid) == 0) {
1177 if (compact_storage_size != memRequired)
1178 throw BESInternalError("Compact storage size does not match D4Array or scalar.", __FILE__,
1179 __LINE__);
1180 }
1181
1182 auto array = toDA(btp);
1183 switch (array->var()->type()) {
1184 case dods_byte_c:
1185 case dods_char_c:
1186 case dods_int8_c:
1187 case dods_uint8_c:
1188 case dods_int16_c:
1189 case dods_uint16_c:
1190 case dods_int32_c:
1191 case dods_uint32_c:
1192 case dods_float32_c:
1193 case dods_float64_c:
1194 case dods_int64_c:
1195 case dods_uint64_c:
1196 {
1197 vector<uint8_t> values(memRequired, 0);
1198 get_data(dataset, reinterpret_cast<void *>(values.data()));
1199 array->set_read_p(true);
1200 array->val2buf(reinterpret_cast<void *>(values.data()));
1201 break;
1202
1203 }
1204 case dods_url_c:
1205 case dods_str_c:
1206 {
1207 if (H5Tis_variable_str(dtypeid) > 0) {
1208 // Variable length string case.
1209 vector<string> finstrval; // passed by reference to read_vlen_string
1210 // @TODO Why push an empty string into the first array position? WHY?
1211 finstrval.emplace_back("");
1212 read_vlen_string(dataset, 1, nullptr, nullptr, nullptr, finstrval);
1213 array->set_value(finstrval, (int) finstrval.size());
1214 array->set_read_p(true);
1215 }
1216 else {
1217 // Fixed length string case.
1218 process_compact_flsa(dataset, btp);
1219 }
1220 break;
1221 }
1222
1223 default:
1224 throw BESInternalError("Unsupported compact storage variable type.", __FILE__, __LINE__);
1225 }
1226}
1227
1228
1234void process_compact_layout_dariable(hid_t dataset, BaseType *btp){
1235
1236 VERBOSE(cerr << prolog << "Processing Compact Storage Layout Dariable" << endl);
1237 // - - - - - - - - - -:- - - - - - - - - -
1238 // This next block is all the QC stuff "sanitize your inputs"
1239 //
1240 auto dc = toDC(btp); // throws if no match
1241
1242 Type dap_type = btp->type();
1243 if ( dap_type == dods_structure_c
1244 || dap_type == dods_sequence_c
1245 || dap_type == dods_grid_c) {
1246 stringstream msg;
1247 msg << "The variable " << btp->FQN() << " is an instance of " << btp->type_name() << ", and utilizes ";
1248 msg << "the hdf5 compact storage layout (H5D_COMPACT). ";
1249 msg << "Only arrays of string and numeric data types are supported for the compact storage layout.";
1250 throw BESInternalError(msg.str(), __FILE__, __LINE__);
1251 }
1252
1253 auto layout_type = get_h5_storage_layout(dataset);
1254 if (layout_type != H5D_COMPACT)
1255 throw BESInternalError(string("ERROR: The dataset is not stored with compact layout."), __FILE__, __LINE__);
1256
1257 // - - - - - - - - - -:- - - - - - - - - -
1258 // Now the QC stuff is finished, so we go to work on the compact layout variable.
1259 //
1260
1261 dc->set_compact(true);
1262
1263 if (dap_type == dods_array_c) {
1264 process_compact_layout_array(dataset, btp);
1265 }
1266 else {
1267 process_compact_layout_scalar(dataset, btp);
1268 }
1269}
1270
1271
1277void set_fill_value(hid_t dataset, BaseType *btp){
1278 short fill_value_defined = is_hdf5_fill_value_defined(dataset);
1279 if (fill_value_defined >0) {
1280 string fill_value = get_hdf5_fill_value_str(dataset);
1281 auto dc = toDC(btp);
1282 dc->set_uses_fill_value(fill_value_defined);
1283 dc->set_fill_value_string(fill_value);
1284 }
1285}
1286
1287
1288bool obtain_structure_string_value(hid_t memtype, size_t ty_size, hssize_t num_elms, vector<char>& encoded_struct_value,const vector<char>& struct_value, string & err_msg) {
1289
1290 bool ret_value = true;
1291 size_t values_offset = 0;
1292
1293 // Loop through all the elements in this compound datatype variable.
1294 for (int64_t element = 0; element < num_elms; ++element) {
1295
1296 int nmembs = 0;
1297 size_t struct_elem_offset = ty_size*element;
1298
1299 if ((nmembs = H5Tget_nmembers(memtype)) < 0) {
1300 err_msg = "Fail to obtain number of HDF5 compound datatype.";
1301 ret_value = false;
1302 break;
1303 }
1304
1305 // We only need to retrieve the values and re-assemble them.
1306 // We will only have the one-layer string-contained structure to handle.
1307 // We do need to know the memb type to put the special handling of a string.
1308 for (unsigned int u = 0; u < (unsigned)nmembs; u++) {
1309
1310 hid_t memb_id = -1;
1311 H5T_class_t memb_cls = H5T_NO_CLASS;
1312 size_t memb_offset = 0;
1313
1314 // Get member type ID
1315 if((memb_id = H5Tget_member_type(memtype, u)) < 0) {
1316 err_msg = "Fail to obtain the datatype of an HDF5 compound datatype member.";
1317 ret_value = false;
1318 break;
1319 }
1320
1321 // Get member type class
1322 if((memb_cls = H5Tget_member_class (memtype, u)) < 0) {
1323 H5Tclose(memb_id);
1324 err_msg = "Fail to obtain the datatype class of an HDF5 compound datatype member.";
1325 ret_value = false;
1326 break;
1327 }
1328
1329 size_t memb_size = H5Tget_size(memb_id);
1330
1331 // Get member offset,H5Tget_member_offset only fails
1332 // when H5Tget_memeber_class fails. Sinc H5Tget_member_class
1333 // is checked above. So no need to check the return value.
1334 memb_offset= H5Tget_member_offset(memtype,u);
1335
1336 // Here we have the offset from the original structure variable.
1337 values_offset = struct_elem_offset + memb_offset;
1338 if (memb_cls == H5T_ARRAY) {
1339
1340 hid_t at_base_type = H5Tget_super(memb_id);
1341 size_t at_base_type_size = H5Tget_size(at_base_type);
1342 H5T_class_t array_cls = H5Tget_class(at_base_type);
1343
1344 // Need to retrieve the number of elements of the array
1345 // and encode each string with base64 and then separate them
1346 // with ";".
1347 // memb_id, obtain the number of dimensions
1348 int at_ndims = H5Tget_array_ndims(memb_id);
1349 if (at_ndims <= 0) {
1350 H5Tclose(at_base_type);
1351 H5Tclose(memb_id);
1352 err_msg = "Fail to obtain number of dimensions of the array datatype.";
1353 ret_value = false;
1354 break;
1355 }
1356
1357 vector<hsize_t>at_dims_h(at_ndims,0);
1358
1359 // Obtain the number of elements for each dims
1360 if (H5Tget_array_dims(memb_id,at_dims_h.data())<0) {
1361 H5Tclose(at_base_type);
1362 H5Tclose(memb_id);
1363 err_msg = "Fail to obtain each imension size of the array datatype.";
1364 ret_value = false;
1365 break;
1366 }
1367
1368 vector<hsize_t>at_dims_offset(at_ndims,0);
1369 size_t total_array_nums = 1;
1370 for (const auto & ad:at_dims_h)
1371 total_array_nums *=ad;
1372
1373 if (array_cls == H5T_STRING) {
1374
1375 vector<string> str_val;
1376 str_val.resize(total_array_nums);
1377
1378 if (H5Tis_variable_str(at_base_type) >0){
1379 auto src = (void*)(struct_value.data()+values_offset);
1380 auto temp_bp =(char*)src;
1381 for (int64_t i = 0;i <total_array_nums; i++){
1382 string tempstrval;
1383 get_vlen_str_data(temp_bp,tempstrval);
1384 str_val[i] = tempstrval;
1385 temp_bp += at_base_type_size;
1386 }
1387 }
1388 else {
1389 auto src = (void*)(struct_value.data()+values_offset);
1390 vector<char> fix_str_val;
1391 fix_str_val.resize(total_array_nums*at_base_type_size);
1392 memcpy((void*)fix_str_val.data(),src,total_array_nums*at_base_type_size);
1393 string total_in_one_string(fix_str_val.begin(),fix_str_val.end());
1394 for (int64_t i = 0; i<total_array_nums;i++)
1395 str_val[i] = total_in_one_string.substr(i*at_base_type_size,at_base_type_size);
1396 }
1397 vector<string> encoded_str_val;
1398 encoded_str_val.resize(str_val.size());
1399
1400 // "Matthew John;James Peter" becomes "base64(Matthew);base64(John;James);base64(Peter);"
1401 for (int i = 0; i < str_val.size(); i++) {
1402 string temp_str = str_val[i];
1403 vector<u_int8_t>temp_val(temp_str.begin(),temp_str.end());
1404 encoded_str_val[i] = base64::Base64::encode(temp_val.data(), temp_str.size()) + ";";
1405
1406 }
1407 // TODO: use memcpy or other more efficient method later. We expect the size is not big.
1408 for (const auto &es_val:encoded_str_val) {
1409 string temp_str = es_val;
1410 for(const auto &ts:temp_str)
1411 encoded_struct_value.push_back(ts);
1412 }
1413
1414 }
1415 else { // integer or float array, just obtain the whole value.
1416 vector<char> int_float_array;
1417 int_float_array.resize(total_array_nums*at_base_type_size);
1418 memcpy((void*)int_float_array.data(),struct_value.data()+values_offset,total_array_nums*at_base_type_size);
1419 for (const auto &int_float:int_float_array)
1420 encoded_struct_value.push_back(int_float);
1421 }
1422 H5Tclose(at_base_type);
1423
1424 }
1425 else if (memb_cls == H5T_STRING) {// Scalar string
1426
1427 string encoded_str;
1428
1429 if (H5Tis_variable_str(memb_id) >0){
1430 auto src = (void*)(struct_value.data()+values_offset);
1431 auto temp_bp =(char*)src;
1432 string tempstrval;
1433 get_vlen_str_data(temp_bp,tempstrval);
1434 vector<u_int8_t>temp_val(tempstrval.begin(),tempstrval.end());
1435 encoded_str = base64::Base64::encode(temp_val.data(), tempstrval.size()) + ";";
1436
1437 }
1438 else {
1439 auto src = (void*)(struct_value.data()+values_offset);
1440 vector<char> fix_str_val;
1441 fix_str_val.resize(memb_size);
1442 memcpy((void*)fix_str_val.data(),src,memb_size);
1443 string fix_str_value(fix_str_val.begin(),fix_str_val.end());
1444 vector<u_int8_t>temp_val(fix_str_value.begin(),fix_str_value.end());
1445 encoded_str = base64::Base64::encode(temp_val.data(), fix_str_value.size()) + ";";
1446 }
1447 for (const auto &es:encoded_str)
1448 encoded_struct_value.push_back(es);
1449
1450 }
1451 else {// Scalar int/float
1452 vector<char> int_float;
1453 int_float.resize(memb_size);
1454 memcpy((void*)int_float.data(),struct_value.data()+values_offset,memb_size);
1455 int_float.resize(memb_size);
1456 memcpy((void*)int_float.data(),struct_value.data()+values_offset,memb_size);
1457 for (const auto &int_f:int_float)
1458 encoded_struct_value.push_back(int_f);
1459 }
1460
1461 } // end "for(unsigned u = 0)"
1462
1463 if (ret_value == false)
1464 break;
1465 } // end "for (int element=0"
1466
1467 return ret_value;
1468
1469}
1470
1471void process_string_in_structure(hid_t dataset, hid_t type_id, BaseType *btp) {
1472
1473 hid_t memtype = -1;
1474 size_t ty_size = -1;
1475
1476 bool is_scalar = false;
1477
1478 if ((memtype = H5Tget_native_type(type_id, H5T_DIR_ASCEND))<0)
1479 throw InternalErr (__FILE__, __LINE__, "Fail to obtain memory datatype.");
1480
1481 ty_size = H5Tget_size(memtype);
1482
1483 hid_t dspace = -1;
1484 if ((dspace = H5Dget_space(dataset))<0) {
1485 H5Tclose(memtype);
1486 throw InternalErr (__FILE__, __LINE__, "Cannot obtain data space.");
1487 }
1488
1489 hssize_t num_elms = H5Sget_simple_extent_npoints(dspace);
1490 if (num_elms < 0) {
1491 H5Tclose(memtype);
1492 H5Sclose(dspace);
1493 throw InternalErr (__FILE__, __LINE__, "Cannot obtain the number of elements of the data space.");
1494 }
1495
1496 vector<char> struct_value;
1497 struct_value.resize(num_elms*ty_size);
1498 if (H5Dread(dataset,memtype, H5S_ALL,H5S_ALL,H5P_DEFAULT,(void*)struct_value.data())<0) {
1499 H5Tclose(memtype);
1500 H5Sclose(dspace);
1501 throw InternalErr (__FILE__, __LINE__, "Cannot read the dataset.");
1502 }
1503
1504 if (H5S_SCALAR == H5Sget_simple_extent_type(dspace))
1505 is_scalar = true;
1506
1507 H5Sclose(dspace);
1508
1509 bool ret_value = false;
1510 string err_msg;
1511 if (is_scalar) {
1512 auto ds = dynamic_cast<DmrppStructure *>(btp);
1513 vector<char> & ds_buffer = ds->get_structure_str_buffer();
1514 ret_value = obtain_structure_string_value(memtype,ty_size,num_elms,ds_buffer,struct_value,err_msg);
1515 ds->set_special_structure_flag(true);
1516 ds->set_read_p(true);
1517 }
1518 else {
1519 auto da = dynamic_cast<DmrppArray *>(btp);
1520 vector<char> &da_buffer = da->get_structure_array_str_buffer();
1521 ret_value = obtain_structure_string_value(memtype,ty_size,num_elms,da_buffer,struct_value,err_msg);
1522 da->set_special_structure_flag(true);
1523 da->set_read_p(true);
1524 }
1525
1526 H5Tclose(memtype);
1527 if (ret_value == false)
1528 throw InternalErr (__FILE__, __LINE__, err_msg);
1529
1530}
1531
1532//Note: the error handling part may be improved in the future.
1533bool handle_vlen_float_int_internal(hid_t dset_id, BaseType *btp) {
1534
1535 hid_t vlen_type = H5Dget_type(dset_id);
1536 hid_t vlen_basetype = H5Tget_super(vlen_type);
1537 if (H5Tget_class(vlen_basetype) != H5T_INTEGER && H5Tget_class(vlen_basetype) != H5T_FLOAT) {
1538 H5Dclose(dset_id);
1539 throw InternalErr(__FILE__, __LINE__,"Only support float or intger variable-length datatype.");
1540 }
1541
1542 hid_t vlen_base_memtype = H5Tget_native_type(vlen_basetype, H5T_DIR_ASCEND);
1543 hid_t vlen_memtype = H5Tvlen_create(vlen_base_memtype);
1544
1545 // Will not support the scalar type.
1546 hid_t vlen_space = H5Dget_space(dset_id);
1547 if (H5Sget_simple_extent_type(vlen_space) != H5S_SIMPLE) {
1548 H5Dclose(dset_id);
1549 throw InternalErr(__FILE__, __LINE__,"Only support array of float or intger variable-length datatype.");
1550 }
1551
1552 hssize_t vlen_number_elements = H5Sget_simple_extent_npoints(vlen_space);
1553 vector<hvl_t> vlen_data(vlen_number_elements);
1554 if (H5Dread(dset_id, vlen_memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, vlen_data.data()) <0) {
1555 H5Dclose(dset_id);
1556 throw InternalErr(__FILE__, __LINE__,"Cannot read variable-length datatype data.");
1557 }
1558
1559 auto da = dynamic_cast<DmrppArray *>(btp);
1560 if (!da) {
1561 string err_msg = "Expected to find a DmrppArray instance but did not in handle_vlen_float_int_internal().";
1562 throw BESInternalError(err_msg, __FILE__, __LINE__);
1563 }
1564 switch (da->var()->type()) {
1565 case dods_byte_c:
1566 case dods_uint8_c:
1567 case dods_char_c:
1568 case dods_int8_c:
1569 case dods_int16_c:
1570 case dods_uint16_c:
1571 case dods_int32_c:
1572 case dods_uint32_c:
1573 case dods_int64_c:
1574 case dods_uint64_c:
1575 case dods_float32_c:
1576 case dods_float64_c: {
1577 // Retrieve the last dimension size.
1578 libdap::Array::Dim_iter last_dim_iter = da->dim_end()-1;
1579 int64_t last_dim_size = da->dimension_size(last_dim_iter);
1580 size_t bytes_per_element = da->var()->width_ll();
1581 size_t total_data_buf_size = da->get_size(false)*bytes_per_element;
1582 vector<char> data_buf(total_data_buf_size,0);
1583 char *temp_data_buf_ptr = data_buf.data();
1584
1585 for (ssize_t i = 0; i < vlen_number_elements; i++) {
1586
1587 size_t vlen_element_size = vlen_data[i].len * bytes_per_element;
1588 vector<char> temp_buf(vlen_element_size);
1589
1590 // Copy the vlen data to the data buffer.
1591 memcpy(temp_data_buf_ptr,vlen_data[i].p,vlen_element_size);
1592
1593 // Move the data buffer pointer to the next element.
1594 // In this regular array, the rest data will be filled with zero.
1595 temp_data_buf_ptr += last_dim_size*bytes_per_element;
1596
1597 }
1598 da->val2buf(data_buf.data());
1599 da->set_missing_data(true);
1600 da->set_read_p(true);
1601
1602 break;
1603 }
1604 default:
1605 throw InternalErr(__FILE__, __LINE__, "Vector::val2buf: bad type");
1606 }
1607
1608 H5Dvlen_reclaim(vlen_memtype, vlen_space, H5P_DEFAULT, (void*)(vlen_data.data()));
1609 H5Sclose(vlen_space);
1610 H5Tclose(vlen_base_memtype);
1611 H5Tclose(vlen_basetype);
1612 H5Tclose(vlen_type);
1613 H5Tclose(vlen_memtype);
1614
1615 return true;
1616
1617}
1618
1619bool handle_vlen_float_int(hid_t dataset, BaseType *btp) {
1620
1621 bool ret_value = false;
1622 hid_t type_id = H5Dget_type(dataset);
1623 if (H5Tget_class(type_id) == H5T_VLEN)
1624 ret_value = handle_vlen_float_int_internal(dataset,btp);
1625 H5Tclose(type_id);
1626 return ret_value;
1627}
1628
1629void handle_vlen_float_int_index(hid_t file, BaseType *btp) {
1630
1631 string vlen_index_name = btp->FQN();
1632 size_t vlen_name_pos = vlen_index_name.rfind("_vlen_index");
1633 if (vlen_name_pos == string::npos) {
1634 string err_msg = vlen_index_name + " is not a variable length index variable name.";
1635 H5Fclose(file);
1636 throw BESInternalError(err_msg, __FILE__, __LINE__);
1637 }
1638
1639 string vlen_name = vlen_index_name.substr(0,vlen_name_pos);
1640
1641 H5Eset_auto2(H5E_DEFAULT, nullptr, nullptr);
1642 hid_t dset_id = H5Dopen2(file, vlen_name.c_str(), H5P_DEFAULT);
1643 if (dset_id < 0)
1644 throw BESInternalError("HDF5 vlen dataset '" + vlen_name + "' cannot be opened.", __FILE__, __LINE__);
1645
1646 hid_t vlen_type = H5Dget_type(dset_id);
1647 hid_t vlen_basetype = H5Tget_super(vlen_type);
1648 if (H5Tget_class(vlen_basetype) != H5T_INTEGER && H5Tget_class(vlen_basetype) != H5T_FLOAT) {
1649 H5Dclose(dset_id);
1650 throw InternalErr(__FILE__, __LINE__,"Only support float or intger variable-length datatype.");
1651 }
1652
1653 hid_t vlen_base_memtype = H5Tget_native_type(vlen_basetype, H5T_DIR_ASCEND);
1654 hid_t vlen_memtype = H5Tvlen_create(vlen_base_memtype);
1655
1656 // Will not support the scalar type.
1657 hid_t vlen_space = H5Dget_space(dset_id);
1658 if (H5Sget_simple_extent_type(vlen_space) != H5S_SIMPLE) {
1659 H5Dclose(dset_id);
1660 throw InternalErr(__FILE__, __LINE__,"Only support array of float or intger variable-length datatype.");
1661 }
1662
1663 hssize_t vlen_number_elements = H5Sget_simple_extent_npoints(vlen_space);
1664 vector<hvl_t> vlen_data(vlen_number_elements);
1665 if (H5Dread(dset_id, vlen_memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, vlen_data.data()) <0) {
1666 H5Dclose(dset_id);
1667 throw InternalErr(__FILE__, __LINE__,"Cannot read variable-length datatype data.");
1668 }
1669
1670 auto da = dynamic_cast<DmrppArray *>(btp);
1671 if (!da) {
1672 H5Dclose(dset_id);
1673 string err_msg = "Expected to find a DmrppArray instance but did not in handle_vlen_float_int_internal().";
1674 throw BESInternalError(err_msg, __FILE__, __LINE__);
1675 }
1676 if (da->var()->type() != dods_int32_c) {
1677 H5Dclose(dset_id);
1678 string err_msg = "vlen_index datatype must be 32-bit integer.";
1679 throw BESInternalError(err_msg, __FILE__, __LINE__);
1680 }
1681 vector<int> vlen_index_data;
1682 for (ssize_t i = 0; i<vlen_number_elements; i++)
1683 vlen_index_data.push_back(vlen_data[i].len);
1684 da->set_value_ll(vlen_index_data.data(),vlen_number_elements);
1685 da->set_missing_data(true);
1686 da->set_read_p(true);
1687
1688 H5Dvlen_reclaim(vlen_memtype, vlen_space, H5P_DEFAULT, (void*)(vlen_data.data()));
1689 H5Sclose(vlen_space);
1690 H5Tclose(vlen_base_memtype);
1691 H5Tclose(vlen_basetype);
1692 H5Tclose(vlen_type);
1693 H5Tclose(vlen_memtype);
1694 H5Dclose(dset_id);
1695
1696}
1707static void get_variable_chunk_info(hid_t dataset, BaseType *btp, bool disable_dio) {
1708
1709 if(verbose) {
1710 string type_name = btp->type_name();
1711 if (btp->type() == dods_array_c) {
1712 auto array = toDA(btp);
1713 type_name = array->var()->type_name();
1714 }
1715 cerr << prolog << "Processing dataset/variable: " << type_name << " " << btp->name() << endl;
1716 }
1717
1718 if (true == handle_vlen_float_int(dataset,btp))
1719 return;
1720
1721 // Added support for HDF5 Fill Value. jhrg 4/22/22
1722 set_fill_value(dataset, btp);
1723
1724 // Here we want to check if this dataset is a compound datatype
1725 hid_t type_id = H5Dget_type(dataset);
1726 if (type_id <0) {
1727 string err_msg = "Cannot obtain the HDF5 data type of the dataset: " + btp->name() ;
1728 throw BESInternalError(err_msg, __FILE__, __LINE__);
1729 }
1730 if (H5T_COMPOUND == H5Tget_class(type_id)) {
1731
1732 unsigned short supported_compound_type = is_supported_compound_type(type_id);
1733 if (supported_compound_type ==2) {
1734 // When the compound datatype contains string, we need to process this dataset differently.
1735 process_string_in_structure(dataset,type_id, btp);
1736 H5Tclose(type_id);
1737 return;
1738 }
1739 else if (supported_compound_type ==1) {
1740
1741 auto layout_type = get_h5_storage_layout(dataset);
1742
1743 // For contiguous or chunk storage layouts, the compound member offset and size need to be saved.
1744 if (layout_type != H5D_COMPACT) {
1745
1746 vector<unsigned int> struct_offsets;
1747 obtain_structure_offset(dataset,struct_offsets);
1748 VERBOSE(cerr << prolog << "struct_offsets[0]: " << struct_offsets[0]<< endl);
1749 // Add struct offset
1750 auto dc = toDC(btp);
1751 dc->set_struct_offsets(struct_offsets);
1752 }
1753 }
1754 }
1755 else
1756 H5Tclose(type_id);
1757
1758 auto layout_type = get_h5_storage_layout(dataset);
1759
1760 switch (layout_type) {
1761 case H5D_CONTIGUOUS: { /* Contiguous Storage Layout */
1762 process_contiguous_layout_dariable(dataset, btp);
1763 break;
1764 }
1765 case H5D_CHUNKED: { /* Chunked Storage Layout */
1766 process_chunked_layout_dariable(dataset, btp, disable_dio);
1767 break;
1768 }
1769 case H5D_COMPACT: { /* Compact Storage Layout */
1770 process_compact_layout_dariable(dataset,btp);
1771 break;
1772 }
1773 default:
1774 ostringstream oss("Unsupported HDF5 dataset layout type: ", std::ios::ate);
1775 oss << layout_type << ".";
1776 throw BESInternalError(oss.str(), __FILE__, __LINE__);
1777 }
1778}
1779
1780
1786string get_type_decl(BaseType *btp){
1787 stringstream type_decl;
1788 if(btp->type() == libdap::dods_array_c){
1789 auto array = toDA(btp);
1790 type_decl << array->var()->type_name() << " " << btp->FQN();
1791 for(auto dim_itr = array->dim_begin(); dim_itr!=array->dim_end(); dim_itr++){
1792 auto dim = *dim_itr;
1793 type_decl << "[";
1794 if(!dim.name.empty()){
1795 type_decl << dim.name << "=";
1796 }
1797 type_decl << dim.size << "]";
1798 }
1799 }
1800 else {
1801 type_decl << btp->type_name() << " " << btp->FQN();
1802 }
1803 return type_decl.str();
1804}
1805
1806
1807bool is_unsupported_type(hid_t dataset_id, BaseType *btp, string &msg){
1808 VERBOSE(cerr << prolog << "BEGIN " << get_type_decl(btp) << endl);
1809
1810 bool is_unsupported = false;
1811 hid_t h5_type_id = H5Dget_type(dataset_id);
1812 H5T_class_t class_type = H5Tget_class(h5_type_id);
1813
1814 bool isArray = btp->type() == dods_array_c;
1815
1816 switch (class_type) {
1817 case H5T_STRING: {
1818 if (H5Tis_variable_str(h5_type_id) && isArray) {
1819 stringstream msgs;
1820 msgs << "UnsupportedTypeException: Your data contains the dataset/variable: ";
1821 msgs << get_type_decl(btp) << " ";
1822 msgs << "which the underlying HDF5/NetCDF-4 file has stored as a";
1823 msgs << (isArray?"n array of ":" ");
1824 msgs << "variable length string";
1825 msgs << (isArray?"s (AVLS). ":". ");
1826 msgs << "This data architecture is not currently supported by ";
1827 msgs << "the dmr++ creation machinery. One solution available to you is to rewrite the granule ";
1828 msgs << "so that these arrays are represented as arrays of fixed length strings (AFLS). While ";
1829 msgs << "these may not be as 'elegant' as AVLS, the ragged ends of the AFLS compress well, so ";
1830 msgs << "the storage penalty is minimal.";
1831 msg = msgs.str();
1832 is_unsupported = false;
1833 }
1834 break;
1835 }
1836 case H5T_ARRAY: {
1837 stringstream msgs;
1838 msgs << "UnsupportedTypeException: Your data contains the dataset/variable: ";
1839 msgs << get_type_decl(btp) << " ";
1840 msgs << "which the underlying HDF5/NetCDF-4 file has stored as an array of H5T_ARRAY. ";
1841 msgs << "This is not yet supported by the dmr++ creation machinery.";
1842 msg = msgs.str();
1843 is_unsupported = true;
1844 break;
1845 }
1846 case H5T_COMPOUND: {
1847 unsigned short supported_compound_type = is_supported_compound_type(h5_type_id);
1848 if (supported_compound_type == 0) {
1849 stringstream msgs;
1850 msgs << "UnsupportedTypeException: Your data contains the dataset/variable: ";
1851 msgs << get_type_decl(btp) << " ";
1852 msgs << "which the underlying HDF5/NetCDF-4 file has stored as an HDF5 compound datatype and ";
1853 msgs << "the basetype of the compound datatype is not integer or float. ";
1854 msgs << "This is not yet supported by the dmr++ creation machinery.";
1855 msg = msgs.str();
1856 is_unsupported = true;
1857 }
1858
1859 break;
1860 }
1861 case H5T_VLEN: {
1862 bool supported_vlen_type = is_supported_vlen_type(dataset_id,h5_type_id);
1863 if (supported_vlen_type == false) {
1864 stringstream msgs;
1865 msgs << "UnsupportedTypeException: Your data contains the dataset/variable: ";
1866 msgs << get_type_decl(btp) << " ";
1867 msgs << "which the underlying HDF5/NetCDF-4 file has stored as an HDF5 vlen datatype and ";
1868 msgs << "the basetype of the vlen datatype is not integer or float. ";
1869 msgs << "This is not yet supported by the dmr++ creation machinery.";
1870 msg = msgs.str();
1871 is_unsupported = true;
1872 }
1873
1874 break;
1875
1876 }
1877
1878 default:
1879 break;
1880 }
1881 VERBOSE(cerr << prolog << "END is_unsupported: " << (is_unsupported?"true":"false") << endl);
1882 return is_unsupported;
1883}
1884
1885
1893bool process_variable_length_string_scalar(const hid_t dataset, BaseType *btp){
1894
1895 // btp->type() == dods_str_c means a scalar string, if it was an
1896 // array of strings then btp->type() == dods_array_c would be true
1897 if(btp->type() != dods_str_c) {
1898 return false;
1899 }
1900
1901 auto h5_type_id = H5Dget_type(dataset);
1902 if(H5Tis_variable_str(h5_type_id) <= 0) {
1903 return false; // Not a variable length string, so, again, not our problem.
1904 }
1905
1906 VERBOSE(cerr << prolog << "Processing VLSS: " << btp->FQN() << "\n");
1907
1908 vector<string> vls_values; // passed by reference to read_vlen_string
1909 vls_values.emplace_back(""); // initialize array for it's trip to Cville
1910
1911 // Read the scalar string.
1912 read_vlen_string(dataset, 1, nullptr, nullptr, nullptr, vls_values);
1913 string vlss = vls_values[0];
1914 VERBOSE(cerr << prolog << " read_vlen_string(): " << vlss << endl);
1915
1916 // Convert variable to a compact representation
1917 // so that its value can be stored in the dmr++
1918 auto dc = toDC(btp);
1919 dc->set_compact(true);
1920
1921 // And then set the value.
1922 auto str = dynamic_cast<libdap::Str *>(btp);
1923 str->set_value(vlss);
1924 str->set_read_p(true);
1925
1926 return true;
1927
1928
1929}
1930
1938bool process_variable_length_string_array(const hid_t dataset, BaseType *btp){
1939
1940 if(btp->type() != dods_array_c) {
1941 return false; // Not an array, not our problem...
1942 }
1943 auto dap_array = toDA(btp);
1944 if(!dap_array){
1945 throw BESInternalError("Malformed DAP object " + btp->FQN() +
1946 " Identifies as dods_array_c but cast to DmrppArray fails!", __FILE__, __LINE__);
1947 }
1948
1949 if(dap_array->prototype()->type() != dods_str_c){
1950 return false; // Not a string, not our problem...
1951 }
1952
1953 hid_t h5_type_id = H5Dget_type(dataset);
1954 if(H5Tis_variable_str(h5_type_id) <= 0) {
1955 return false; // Not a variable length string, so, again, not our problem.
1956 }
1957 VERBOSE(cerr << prolog << "h5_type_id: " << h5_type_id << "\n");
1958
1959 dap_array->set_is_vlsa(true);
1960 VERBOSE(cerr << prolog << "Processing VLSA: " << dap_array->FQN() << "\n");
1961
1962 auto dspace = H5Dget_space(dataset);
1963
1964 int ndims = H5Sget_simple_extent_ndims(dspace);
1965 VERBOSE(cerr << prolog << "ndims: " << ndims << "\n");
1966
1967 vector<hsize_t>count(ndims,0);
1968 if(H5Sget_simple_extent_dims(dspace, count.data(), nullptr) < 0){
1969 H5Sclose(dspace);
1970 H5Tclose(h5_type_id);
1971 H5Dclose(dataset);
1972 throw BESInternalError("Failed to get hdf5 count for variable: " + btp->FQN(), __FILE__, __LINE__);
1973 }
1974
1975 stringstream msg;
1976 msg << "count[]: ";
1977 for(int i=0; i<ndims; i++) {
1978 if(i) msg << ",";
1979 msg << count[i];
1980 }
1981 VERBOSE(cerr << prolog << msg.str() << "\n");
1982
1983 vector<hsize_t> offset(ndims,0);
1984 for(int i=0; i<ndims; i++)
1985 offset.emplace_back(0);
1986
1987
1988 // The following line causes an issue on a 1-element VLSA. The num_elements becomes 0.
1989 // See https://bugs.earthdata.nasa.gov/browse/HYRAX-1538
1990#if 0
1991 //uint64_t num_elements = dap_array->get_size(false);
1992#endif
1993 hssize_t num_elements = H5Sget_simple_extent_npoints(dspace);
1994 if (num_elements < 0) {
1995 H5Sclose(dspace);
1996 H5Tclose(h5_type_id);
1997 H5Dclose(dataset);
1998 throw BESInternalError("Failed to obtain the number of elements for the variable : " + btp->FQN(), __FILE__, __LINE__);
1999 }
2000
2001 VERBOSE(cerr << prolog << "num_elements: " << num_elements << "\n");
2002
2003 vector<string> vls_values(num_elements,"");
2004 read_vlen_string(dataset,
2005 num_elements,
2006 offset.data(),
2007 nullptr,
2008 count.data(),
2009 vls_values);
2010
2011#ifndef NDEBUG
2012 VERBOSE(cerr << prolog << " vls_values.size(): " << vls_values.size() << "\n");
2013 uint64_t indx = 0;
2014 for (const auto &sval: vls_values) {
2015 VERBOSE(cerr << prolog << " vls_values[" << to_string(indx++) << "]: '" << sval << "'\n");
2016 }
2017#endif
2018
2019 dap_array->set_value(vls_values,(int) vls_values.size());
2020 dap_array->set_read_p(true);
2021
2022 return true;
2023}
2024
2025bool check_enable_cf_fake_cv(BaseType *btp, const string& FQN) {
2026
2027 bool ret_value = false;
2028 if (FQN.find_last_of('/')==0) {
2029 if (btp->type() == dods_array_c) {
2030 auto da = dynamic_cast<DmrppArray *>(btp);
2031 if (!da) {
2032 string err_msg = "Expected to find a DmrppArray instance but did not in check_enable_cf_fake_cv().";
2033 throw BESInternalError(err_msg, __FILE__, __LINE__);
2034 }
2035 // Must be 1-D floating data.
2036 if (btp->var()->type() == dods_float32_c && da->dimensions() == 1) {
2037 // Must not have attributes and the dimension name is the same as the variable name(FQN).
2038 const D4Attributes *d4_attrs = btp->attributes();
2039 if (d4_attrs) {
2040 if (d4_attrs->empty()) {
2041 // Now we can check dimension name.
2042 Array::Dim_iter da_dim = da->dim_begin();
2043 if (da->dimension_name(da_dim) == btp->name())
2044 ret_value = true;
2045 }
2046 }
2047 }
2048 }
2049 }
2050 return ret_value;
2051}
2052
2060hid_t get_h5_dataset_id(hid_t file, BaseType *btp, const unordered_set<string> &nc4_non_coord_candidate) {
2061 D4Attributes *d4_attrs = btp->attributes();
2062 if (!d4_attrs)
2063 throw BESInternalError("Expected to find an attribute table for " + btp->name() + " but did not.",
2064 __FILE__, __LINE__);
2065
2066 // Look for the full name path for this variable
2067 // If one was not given via an attribute, use BaseType::FQN() which
2068 // relies on the variable's position in the DAP dataset hierarchy.
2069 const D4Attribute *attr = d4_attrs->get("fullnamepath");
2070 // I believe the logic is more clear in this way:
2071 // If fullnamepath exists and the H5Dopen2 fails to open, it should throw an error.
2072 // If fullnamepath doesn't exist, we should ignore the error as the reason described below:
2073 // (However, we should suppress the HDF5 dataset open error message.) KY 2019-12-02
2074 // It's not an error if a DAP variable in a DMR from the hdf5 handler
2075 // doesn't exist in the file _if_ there's no 'fullnamepath' because
2076 // that variable was synthesized (likely for CF compliance)
2077 hid_t dataset = -1;
2078 if (attr) {
2079 string FQN;
2080 if (attr->num_values() == 1)
2081 FQN = attr->value(0);
2082 else
2083 FQN = btp->FQN();
2084
2085 VERBOSE(cerr << prolog << "Working on: " << FQN << endl);
2086 // Here we have a case to handle the netCDF-4 file coming from the fileout netCDF-4 module.
2087 // The fullnamepath is kept to remember the original HDF5 file, but for the netCDF-4 file generated
2088 // from the fileout netCDF-4 module, this is no longer the case. The CF option makes everything flattened.
2089 // So if the H5Dopen2 fails with the name obtained from the fullnamepath attribute,
2090 // we should directly open the variable with the name.
2091
2092 H5Eset_auto2(H5E_DEFAULT, nullptr, nullptr);
2093 dataset = H5Dopen2(file, FQN.c_str(), H5P_DEFAULT);
2094 if (dataset < 0) {
2095 // We have one more try with the variable name before throwing an error.
2096 dataset = H5Dopen2(file,btp->name().c_str(),H5P_DEFAULT);
2097 if (dataset <0)
2098 throw BESInternalError("HDF5 dataset '" + FQN + "' cannot be opened.", __FILE__, __LINE__);
2099 }
2100 }
2101 else {
2102 // The current design seems to still prefer to open the dataset when the fullnamepath doesn't exist
2103 // So go ahead to open the dataset. Continue even if the dataset cannot be open. KY 2019-12-02
2104 //
2105 // A comment from an older version of the code:
2106 // It's not an error if a DAP variable in a DMR from the hdf5 handler
2107 // doesn't exist in the file _if_ there's no 'fullnamepath' because
2108 // that variable was synthesized (likely for CF compliance)
2109 H5Eset_auto2(H5E_DEFAULT, nullptr, nullptr);
2110 string FQN = btp->FQN();
2111 if (nc4_non_coord_candidate.find(btp->name()) != nc4_non_coord_candidate.end()) {
2112 string real_name_candidate = "_nc4_non_coord_" + btp->name();
2113 size_t fqn_last_fslash_pos = btp->FQN().find_last_of('/');
2114 string real_path_candidate = btp->FQN().substr(0, fqn_last_fslash_pos + 1) + real_name_candidate;
2115 dataset = H5Dopen2(file, real_path_candidate.c_str(), H5P_DEFAULT);
2116 }
2117
2118 // Here we need to handle a special case for a dmr file generated by the EnableCF option in the HDF5 handler.
2119 // The netCDF-4's pure dimension is mapped to a fake coordinate by the EnableCF option and is ignored by the default option.
2120 // However, netCDF-4 still stores this dimension as an HDF5 variable with 0 values. The EnableCF option replaces
2121 // those 0 values with 0,1,2... as the fake coordinate. So here we need to ignore this kind of variables and let the
2122 // code after this call to handle it as the EnableCF required.
2123
2124 VERBOSE(cerr << prolog << "Working on: " << FQN << endl);
2125 if (dataset < 0) {
2126 dataset = H5Dopen2(file, FQN.c_str(), H5P_DEFAULT);
2127 if (dataset < 0) {
2128 VERBOSE(cerr << prolog << "WARNING: HDF5 dataset '" << FQN << "' cannot be opened." << endl);
2129 }
2130 else if(check_enable_cf_fake_cv(btp, FQN) == true)
2131 dataset = -1;
2132 }
2133 }
2134 return dataset;
2135}
2136
2150void mk_nc4_non_coord_candidates(D4Group *group, unordered_set<string> &nc4_non_coord_candidate){
2151
2152 // First obtain dimension names.
2153 unordered_set<string> dimname_list;
2154 D4Dimensions *grp_dims = group->dims();
2155
2156 if (grp_dims) {
2157 for (auto di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di)
2158 dimname_list.insert((*di)->name());
2159 }
2160
2161 if (!dimname_list.empty()) {
2162 // Then find the nc4_non_coord candidate variables,
2163 for (auto btp = group->var_begin(), ve = group->var_end(); btp != ve; ++btp) {
2164 if (dimname_list.find((*btp)->name())!=dimname_list.end())
2165 nc4_non_coord_candidate.insert((*btp)->name());
2166 }
2167 }
2168
2169}
2170
2171
2179void get_chunks_for_all_variables(hid_t file, D4Group *group, bool disable_dio) {
2180
2181 unordered_set<string> nc4_non_coord_candidate;
2182 mk_nc4_non_coord_candidates(group,nc4_non_coord_candidate);
2183
2184 // variables in the group
2185
2186 for(auto btp : group->variables()) {
2187 VERBOSE(cerr << prolog << "-------------------------------------------------------" << endl);
2188 VERBOSE(cerr << prolog);
2189 VERBOSE(btp->print_decl(cerr,"",false,false,false) );
2190 VERBOSE(cerr << endl);
2191
2192 auto dataset = get_h5_dataset_id(file, btp, nc4_non_coord_candidate);
2193 if(dataset > 0) {
2194 // If we have a valid dataset then we have a variable with data. I think.
2195 // If it's not valid we skip it, I think because the associated BaseType,
2196 // btp, may be a semantic object, like a dimension, with no data
2197 // associated with it.
2198 try {
2199 string msg;
2200 if (is_unsupported_type(dataset, btp, msg)) {
2201 throw UnsupportedTypeException(msg);
2202 }
2203
2204 if (!process_variable_length_string_scalar(dataset, btp) && !process_variable_length_string_array(dataset,btp)) {
2205
2206 VERBOSE(cerr << prolog << "Building chunks for: " << get_type_decl(btp) << endl);
2207 get_variable_chunk_info(dataset, btp, disable_dio);
2208
2209 VERBOSE(cerr << prolog << "Annotating String Arrays as needed for: " << get_type_decl(btp) << endl);
2210 add_string_array_info(dataset, btp);
2211 }
2212 H5Dclose(dataset);
2213 }
2214 catch (...) {
2215 H5Dclose(dataset);
2216 throw;
2217 }
2218 }
2219 else {
2220 VERBOSE(cerr << prolog << "Unable to open " << btp->FQN()
2221 << " with the hdf5 api. Skipping chunk production. "
2222 << "Need to check if we need to embed the data to the dmrpp file." << endl);
2223
2224 // Currently we only check if this is the artificial coordinate added by the HDF5 handler.
2225 D4Attributes *d4_attrs = btp->attributes();
2226 if (d4_attrs==nullptr)
2227 return;
2228
2229 if (d4_attrs->empty() == false) {
2230
2231 D4Attribute *attr = d4_attrs->find("units");
2232 if (attr) {
2233 string attr_value = attr->value(0);
2234 if (attr_value == "level") {
2235 auto dc = dynamic_cast<DmrppCommon *>(btp);
2236 if (!dc) {
2237 string err_msg = "Expected to find a DmrppCommon instance but did not in get_chunks_for_all_variables().";
2238 throw BESInternalError(err_msg, __FILE__, __LINE__);
2239 }
2240 auto da = dynamic_cast<DmrppArray *>(btp);
2241 if (!da) {
2242 string err_msg = "Expected to find a DmrppArray instance but did not in get_chunks_for_all_variables().";
2243 throw BESInternalError(err_msg, __FILE__, __LINE__);
2244 }
2245
2246 if (da->dimensions() ==1 && btp->var()->type() == dods_int32_c){
2247
2248 vector<int> level_value;
2249 level_value.resize((size_t)(da->length()));
2250 for (int32_t i = 0; i <da->length(); i++)
2251 level_value[i] = i;
2252
2253 da->set_value(level_value.data(),da->length());
2254 da->set_missing_data(true);
2255 da->set_read_p(true);
2256 }
2257 }
2258 }
2259 attr = d4_attrs->find("orig_datatype");
2260 if (attr) {
2261 string attr_value = attr->value(0);
2262 if (attr_value == "VLEN_INDEX") {
2263 // This is a vlen index variable. We need to find the corresponding vlen variable.
2264 handle_vlen_float_int_index(file,btp);
2265 }
2266 }
2267 }
2268 else {// The coordinate of the netCDF-4 pure dimension added by HDF5 handler's CF option doesn't have any attribute.
2269 // Check if this variable is 1-D floating-point array.
2270 if (btp->type() == dods_array_c) {
2271
2272 auto da = dynamic_cast<DmrppArray *>(btp);
2273 if (!da) {
2274 string err_msg = "Expected to find a DmrppArray instance but did not in get_chunks_for_all_variables().";
2275 throw BESInternalError(err_msg, __FILE__, __LINE__);
2276 }
2277
2278 if (da->dimensions() ==1 && btp->var()->type() == dods_float32_c){
2279
2280 da->set_missing_data(true);
2281
2282 vector<float> level_value;
2283 level_value.resize((size_t)(da->length()));
2284 for (int32_t i = 0; i <da->length(); i++)
2285 level_value[i] = i;
2286
2287 da->set_value(level_value.data(),da->length());
2288 da->set_missing_data(true);
2289 da->set_read_p(true);
2290 }
2291
2292 }
2293 }
2294 }
2295 }
2296
2297 // all groups in the group
2298 for(auto g:group->groups()) {
2299 get_chunks_for_all_variables(file, g,disable_dio);
2300 }
2301
2302}
2303
2309void add_chunk_information(const string &h5_file_name, DMRpp *dmrpp, bool disable_dio)
2310{
2311 // Open the hdf5 file
2312 hid_t file = H5Fopen(h5_file_name.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2313 if (file < 0) {
2314 stringstream msg;
2315 msg << "Error: HDF5 file '" << h5_file_name << "' cannot be opened." << endl;
2316 throw BESNotFoundError(msg.str(), __FILE__, __LINE__);
2317 }
2318
2319 // iterate over all the variables in the DMR
2320 try {
2321 get_chunks_for_all_variables(file, dmrpp->root(), disable_dio);
2322 H5Fclose(file);
2323 }
2324 catch (...) {
2325 H5Fclose(file);
2326 throw;
2327 }
2328}
2329
2330
2346void qc_input_file(const string &file_fqn)
2347{
2348 //Use an ifstream file to run a check on the provided file's signature
2349 // to see if it is an HDF5 file. - kln 5/18/23
2350
2351 if (file_fqn.empty()) {
2352 stringstream msg;
2353 msg << "HDF5 input file name must be provided (-f <input>) and be a fully qualified path name." << endl;
2354 throw BESInternalFatalError(msg.str(), __FILE__, __LINE__);
2355 }
2356
2357 std::ifstream file(file_fqn, ios::binary);
2358 auto errnum = errno;
2359 if (!file) // This is same as if(file.fail()){...}
2360 {
2361 stringstream msg;
2362 msg << "Encountered a Read/writing error when attempting to open the file: " << file_fqn << endl;
2363 msg << "* strerror(errno): " << strerror(errnum) << endl;
2364 msg << "* failbit: " << (((file.rdstate() & std::ifstream::failbit) != 0) ? "true" : "false") << endl;
2365 msg << "* badbit: " << (((file.rdstate() & std::ifstream::badbit) != 0) ? "true" : "false") << endl;
2366 msg << "Things to check:" << endl;
2367 msg << "* Does the file exist at expected location?" << endl;
2368 msg << "* Does your user have permission to read the file?" << endl;
2369 throw BESInternalFatalError(msg.str(), __FILE__, __LINE__);
2370 }
2371
2372 //NetCDF3 signatures:
2373 const char netcdf3Signature[] = {'C', 'D', 'F'};
2374
2375 //Read the first 8 bytes (file signature) from the file
2376 string signature;
2377 signature.resize(8);
2378 file.read(&signature[0], signature.size());
2379
2380 htri_t temp_is_hdf5 = H5Fis_hdf5(file_fqn.c_str());
2381
2382 bool isHDF5 = (temp_is_hdf5>0)?true:false;
2383 //First check if file is NOT an HDF5 file, then, if it is not, check if it is netcdf3
2384 if (!isHDF5) {
2385 //Reset the file stream to read from the beginning
2386 file.clear();
2387 file.seekg(0);
2388
2389 char newSignature[3];
2390 file.read(&signature[0], signature.size());
2391
2392 bool isNetCDF3 = memcmp(newSignature, netcdf3Signature, sizeof(netcdf3Signature)) == 0;
2393 if (isNetCDF3) {
2394 stringstream msg;
2395 msg << "The file submitted, " << file_fqn << ", ";
2396 msg << "is a NetCDF-3 classic file and is not compatible with dmr++ production at this time." << endl;
2397 throw BESInternalFatalError(msg.str(), __FILE__, __LINE__);
2398 }
2399 else {
2400 stringstream msg;
2401 msg << "The provided file: " << file_fqn << " - ";
2402 msg << "is neither an HDF5 or a NetCDF-4 file, currently only HDF5 and NetCDF-4 files ";
2403 msg << "are supported for dmr++ production" << endl;
2404 throw BESInternalFatalError(msg.str(), __FILE__, __LINE__);
2405 }
2406 }
2407}
2408
2409
2417static string recreate_cmdln_from_args(int argc, char *argv[])
2418{
2419 stringstream ss;
2420 for(int i=0; i<argc; i++) {
2421 if (i > 0)
2422 ss << " ";
2423 ss << argv[i];
2424 }
2425 return ss.str();
2426}
2427
2433std::string what_time_is_it(){
2434 // Get current time as a time_point
2435 auto now = std::chrono::system_clock::now();
2436
2437 // Convert to system time (time_t)
2438 auto time_t_now = std::chrono::system_clock::to_time_t(now);
2439
2440 // Convert to tm structure (GMT time)
2441 struct tm tbuf{};
2442 const std::tm* gmt_time = gmtime_r(&time_t_now, &tbuf);
2443
2444 // Format the time using a stringstream
2445 std::stringstream ss;
2446 ss << std::put_time(gmt_time, "%Y-%m-%dT%H:%M:%SZ");
2447
2448 return ss.str();
2449}
2450
2459void inject_build_dmrpp_metadata_worker( DMRpp *dmrpp, const string &bes_conf_doc, const string &invocation)
2460{
2461 dmrpp->set_version(CVER);
2462
2463 // Build the version attributes for the DMR++
2464 auto version = new D4Attribute("build_dmrpp_metadata", StringToD4AttributeType("container"));
2465
2466 auto creation_date = new D4Attribute("created", StringToD4AttributeType("string"));
2467 creation_date->add_value(what_time_is_it());
2468 version->attributes()->add_attribute_nocopy(creation_date);
2469
2470 auto build_dmrpp_version = new D4Attribute("build_dmrpp", StringToD4AttributeType("string"));
2471 build_dmrpp_version->add_value(CVER);
2472 version->attributes()->add_attribute_nocopy(build_dmrpp_version);
2473
2474 auto bes_version = new D4Attribute("bes", StringToD4AttributeType("string"));
2475 bes_version->add_value(CVER);
2476 version->attributes()->add_attribute_nocopy(bes_version);
2477
2478 stringstream ldv;
2479 ldv << libdap_name() << "-" << libdap_version();
2480 auto libdap4_version = new D4Attribute("libdap", StringToD4AttributeType("string"));
2481 libdap4_version->add_value(ldv.str());
2482 version->attributes()->add_attribute_nocopy(libdap4_version);
2483
2484 if(!bes_conf_doc.empty()) {
2485 stringstream ss(bes_conf_doc);
2486 string line;
2487 string new_bes_conf;
2488
2489 // Iterate through each line and remove the bes module library path
2490 while (getline(ss, line)) {
2491 // Check if the line contains "BES.module."
2492 if (line.find("BES.module.") == string::npos) {
2493 // If the line doesn't contain "BES.module.", add it to the result
2494 new_bes_conf += line + "\n";
2495 }
2496 }
2497
2498 // Add the BES configuration used to create the base DMR
2499 auto config = new D4Attribute("configuration", StringToD4AttributeType("string"));
2500 config->add_value(new_bes_conf);
2501 version->attributes()->add_attribute_nocopy(config);
2502 }
2503
2504 if(!invocation.empty()) {
2505 // How was build_dmrpp invoked?
2506 auto invoke = new D4Attribute("invocation", StringToD4AttributeType("string"));
2507 invoke->add_value(invocation);
2508 version->attributes()->add_attribute_nocopy(invoke);
2509 }
2510 // Inject version and configuration attributes into DMR here.
2511 dmrpp->root()->attributes()->add_attribute_nocopy(version);
2512}
2513
2514
2528 void inject_build_dmrpp_metadata(int argc, char **argv, const string &bes_conf_file_used_to_create_dmr, DMRpp *dmrpp)
2529{
2530 string bes_configuration;
2531 string invocation;
2532 if(!bes_conf_file_used_to_create_dmr.empty()) {
2533 // Add the BES configuration used to create the base DMR
2534 TheBESKeys::ConfigFile = bes_conf_file_used_to_create_dmr;
2535 bes_configuration = TheBESKeys::TheKeys()->get_as_config();
2536 }
2537
2538 invocation = recreate_cmdln_from_args(argc, argv);
2539
2540 inject_build_dmrpp_metadata_worker(dmrpp, bes_configuration, invocation);
2541
2542}
2543
2554void inject_build_dmrpp_metadata(DMRpp *dmrpp)
2555{
2556 bool found;
2557
2558 string bes_configuration;
2559 string invocation;
2560 if(!TheBESKeys::ConfigFile.empty()) {
2561 // Add the BES configuration used to create the base DMR
2562 bes_configuration = TheBESKeys::TheKeys()->get_as_config();
2563 }
2564
2565 // How was build_dmrpp invoked?
2566 invocation = BESContextManager::TheManager()->get_context(INVOCATION_CONTEXT, found);
2567
2568 // Do the work now...
2569 inject_build_dmrpp_metadata_worker(dmrpp, bes_configuration, invocation);
2570}
2571
2572
2585void build_dmrpp_from_dmr_file(const string &dmrpp_href_value, const string &dmr_filename, const string &h5_file_fqn,
2586 bool add_production_metadata, const string &bes_conf_file_used_to_create_dmr, bool disable_dio, int argc, char *argv[])
2587{
2588 // Get dmr:
2589 DMRpp dmrpp;
2590 DmrppTypeFactory dtf;
2591 dmrpp.set_factory(&dtf);
2592
2593 ifstream in(dmr_filename.c_str());
2594 D4ParserSax2 parser;
2595 parser.intern(in, &dmrpp, false);
2596
2597 add_chunk_information(h5_file_fqn, &dmrpp, disable_dio);
2598
2599 if (add_production_metadata) {
2600 inject_build_dmrpp_metadata(argc, argv, bes_conf_file_used_to_create_dmr, &dmrpp);
2601 }
2602
2603 XMLWriter writer;
2604 dmrpp.print_dmrpp(writer, dmrpp_href_value);
2605 cout << writer.get_doc();
2606}
2607
2608
2609} // namespace build_dmrpp_util
BESContextManager::get_context
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
Definition BESContextManager.cc:93
TheBESKeys::get_as_config
std::string get_as_config() const
Definition TheBESKeys.cc:488
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Access to the singleton.
Definition TheBESKeys.cc:85
TheBESKeys::ConfigFile
static std::string ConfigFile
Definition TheBESKeys.h:117
dmrpp::DMRpp::print_dmrpp
virtual void print_dmrpp(libdap::XMLWriter &xml, const std::string &href="", bool constrained=false, bool print_chunks=true)
Print the DMR++ response.
Definition DMRpp.cc:75
dmrpp::DmrppArray::set_is_flsa
void set_is_flsa(bool state)
Marks the array as a Fixed length string array, or not, depending on state.
Definition DmrppArray.h:215
dmrpp::DmrppCommon::set_fill_value_string
virtual void set_fill_value_string(const std::string &fv)
Set the fill value (using a string)
Definition DmrppCommon.h:293
dmrpp::DmrppCommon::set_disable_dio
void set_disable_dio(bool value)
Set the value of the compact property.
Definition DmrppCommon.h:240
dmrpp::DmrppCommon::set_uses_fill_value
virtual void set_uses_fill_value(bool ufv)
Set the uses_fill_value property.
Definition DmrppCommon.h:290
dmrpp::DmrppCommon::set_chunk_dimension_sizes
void set_chunk_dimension_sizes(const std::vector< unsigned long long > &chunk_dims)
Set the value of the chunk dimension sizes given a vector of HDF5 hsize_t.
Definition DmrppCommon.h:323
dmrpp::DmrppCommon::add_chunk
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Adds a chunk to the vector of chunk refs (byteStreams) and returns the size of the chunks internal ve...
Definition DmrppCommon.cc:212
dmrpp::DmrppCommon::set_filter
void set_filter(const std::string &value)
Set the value of the filters property.
Definition DmrppCommon.cc:106
dmrpp::DmrppCommon::set_compact
void set_compact(bool value)
Set the value of the compact property.
Definition DmrppCommon.h:209
get_data
void get_data(hid_t dset, void *buf)
Definition dmrpp_module/h5common.cc:50
h5common.h
Type
Type
Type of JSON value.
Definition rapidjson.h:664