bes Updated for version 3.21.1
The Backend Server (BES) is the lower two tiers of the Hyrax data server
HDF5Array.cc
Go to the documentation of this file.
1// This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2// data server.
3
4// Authors: Hyo-Kyung Lee <hyoklee@hdfgroup.org> and Kent Yang
5// <myang6@hdfgroup.org>
6
7// Copyright (c) 2009-2023 The HDF Group, Inc. and OPeNDAP, Inc.
8//
9// This is H5free_memory software; you can redistribute it and/or modify it under the
10// terms of the GNU Lesser General Public License as published by the Free
11// Software Foundation; either version 2.1 of the License, or (at your
12// option) any later version.
13//
14// This software is distributed in the hope that it will be useful, but
15// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
16// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
17// License for more details.
18//
19// You should have received a copy of the GNU Lesser General Public
20// License along with this library; if not, write to the Free Software
21// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22//
23// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24// You can contact The HDF Group, Inc. at 410 E University Ave,
25// Suite 200, Champaign, IL 61820
33
34#include <iostream>
35#include <memory>
36
37#include <BESDebug.h>
38#include <libdap/Error.h>
39#include <libdap/InternalErr.h>
40
41#include "HDF5Array.h"
42#include "HDF5Structure.h"
43#include "HDF5Str.h"
44
45using namespace std;
46using namespace libdap;
47
48BaseType *HDF5Array::ptr_duplicate() {
49 auto HDF5Array_unique = make_unique<HDF5Array>(*this);
50 return HDF5Array_unique.release();
51}
52
53HDF5Array::HDF5Array(const string & n, const string &d, BaseType * v) :
54 Array(n, d, v) {
55}
56
57int64_t HDF5Array::format_constraint(int64_t *offset, int64_t *step, int64_t *count) {
58
59 // For the 0-length array case, just return 0.
60 if(length() == 0)
61 return 0;
62
63 int64_t nels = 1;
64 int id = 0;
65
66 Dim_iter p = dim_begin();
67
68 while (p != dim_end()) {
69
70 int64_t start = dimension_start_ll(p, true);
71 int64_t stride = dimension_stride_ll(p, true);
72 int64_t stop = dimension_stop_ll(p, true);
73
74 // Check for empty constraint
75 if (start > stop) {
76 ostringstream oss;
77
78 oss << "Array/Grid hyperslab start point "<< start <<
79 " is greater than stop point " << stop <<".";
80 throw Error(malformed_expr, oss.str());
81 }
82
83 offset[id] = start;
84 step[id] = stride;
85 count[id] = ((stop - start) / stride) + 1; // count of elements
86 nels *= count[id]; // total number of values for variable
87
88 BESDEBUG("h5",
89 "=format_constraint():"
90 << "id=" << id << " offset=" << offset[id]
91 << " step=" << step[id]
92 << " count=" << count[id]
93 << endl);
94
95 id++;
96 p++;
97 }
98
99 return nels;
100}
101
102
103bool HDF5Array::read()
104{
105 BESDEBUG("h5",
106 ">read() dataset=" << dataset()
107 << " dimension=" << d_num_dim
108 << " data_size=" << d_memneed << " length=" << length()
109 << endl);
110
111 hid_t file_id = H5Fopen(dataset().c_str(),H5F_ACC_RDONLY,H5P_DEFAULT);
112
113 BESDEBUG("h5","variable name is "<<name() <<endl);
114 BESDEBUG("h5","variable path is "<<var_path <<endl);
115
116 hid_t dset_id = -1;
117
118 if (true == is_dap4())
119 dset_id = H5Dopen2(file_id,var_path.c_str(),H5P_DEFAULT);
120 else
121 dset_id = H5Dopen2(file_id,name().c_str(),H5P_DEFAULT);
122
123 BESDEBUG("h5","after H5Dopen2 "<<endl);
124
125 hid_t dtype_id = H5Dget_type(dset_id);
126 if(dtype_id < 0) {
127 H5Dclose(dset_id);
128 H5Fclose(file_id);
129 throw InternalErr(__FILE__,__LINE__, "Fail to obtain the datatype .");
130 }
131
132
133 vector<int64_t> offset(d_num_dim);
134 vector<int64_t> count(d_num_dim);
135 vector<int64_t> step(d_num_dim);
136 int64_t nelms = format_constraint(offset.data(), step.data(), count.data()); // Throws Error.
137 vector<char>values;
138
139 // We only map the reference to URL when the dataset is an array of reference.
140 if (get_dap_type(dtype_id,is_dap4()) == "Url") {
141 bool ret_ref = false;
142 try {
143 ret_ref = m_array_of_reference(dset_id,dtype_id);
144 H5Tclose(dtype_id);
145 H5Dclose(dset_id);
146 H5Fclose(file_id);
147
148 }
149 catch(...) {
150 H5Tclose(dtype_id);
151 H5Dclose(dset_id);
152 H5Fclose(file_id);
153 throw;
154
155 }
156 return ret_ref;
157 }
158
159 try {
160 do_array_read(dset_id,dtype_id,values,nelms,offset.data(),count.data(),step.data());
161 }
162 catch(...) {
163 H5Tclose(dtype_id);
164 H5Dclose(dset_id);
165 H5Fclose(file_id);
166 throw;
167 }
168
169 H5Tclose(dtype_id);
170 H5Dclose(dset_id);
171 H5Fclose(file_id);
172
173 return true;
174}
175
176void HDF5Array::do_array_read(hid_t dset_id,hid_t dtype_id,vector<char>&values,
177 int64_t nelms,const int64_t* offset,const int64_t* count, const int64_t* step)
178{
179
180 H5T_class_t tcls = H5Tget_class(dtype_id);
181 bool has_values = false;
182 size_t values_offset = 0;
183
184 if (H5T_COMPOUND == tcls)
185 m_array_of_structure(dset_id,values,has_values,values_offset,nelms,offset,count,step);
186 else if(H5T_INTEGER == tcls || H5T_FLOAT == tcls || H5T_STRING == tcls)
187 m_array_of_atomic(dset_id,dtype_id,nelms,offset,count,step);
188 else if (H5T_ENUM == tcls) {
189 hid_t basetype = H5Tget_super(dtype_id);
190 m_array_of_atomic(dset_id,basetype,nelms,offset,count,step);
191 H5Tclose(basetype);
192 }
193 else {
194 throw InternalErr(__FILE__,__LINE__,"Fail to read the data for Unsupported datatype.");
195 }
196
197}
198
199void HDF5Array:: m_array_of_atomic(hid_t dset_id, hid_t dtype_id, int64_t nelms,const int64_t* offset,
200 const int64_t* count, const int64_t* step)
201{
202
203 hid_t memtype = -1;
204 if((memtype = H5Tget_native_type(dtype_id, H5T_DIR_ASCEND))<0) {
205 throw InternalErr (__FILE__, __LINE__, "Fail to obtain memory datatype.");
206 }
207
208 // First handle variable-length string
209 if (H5Tis_variable_str(memtype) && H5Tget_class(memtype) == H5T_STRING) {
210
211 vector<hsize_t> hoffset;
212 vector<hsize_t>hcount;
213 vector<hsize_t>hstep;
214 hoffset.resize(d_num_dim);
215 hcount.resize(d_num_dim);
216 hstep.resize(d_num_dim);
217 for (int i = 0; i <d_num_dim; i++) {
218 hoffset[i] = (hsize_t) offset[i];
219 hcount[i] = (hsize_t) count[i];
220 hstep[i] = (hsize_t) step[i];
221 }
222 handle_vlen_string(dset_id, memtype, nelms, hoffset, hcount, hstep);
223 return ;
224 }
225
226 try {
227 if (nelms == (int64_t) d_num_elm)
228 handle_array_read_whole(dset_id, memtype, nelms);
229 else
230 handle_array_read_slab(dset_id, memtype, nelms, offset, step, count);
231
232 H5Tclose(memtype);
233 }
234 catch (...) {
235 H5Tclose(memtype);
236 throw;
237 }
238
239}
240
241void HDF5Array::handle_array_read_whole(hid_t dset_id, hid_t memtype, int64_t nelms) {
242
243 vector<char> convbuf(d_memneed);
244 get_data(dset_id, (void *) convbuf.data());
245
246 // Check if a Signed Byte to Int16 conversion is necessary, this is only valid for DAP2.
247 if (false == is_dap4() && (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype))) {
248 vector<short> convbuf2(nelms);
249 for (int64_t i = 0; i < nelms; i++) {
250 convbuf2[i] = (signed char) (convbuf[i]);
251 BESDEBUG("h5", "convbuf[" << i << "]="
252 << (signed char) convbuf[i] << endl);
253 BESDEBUG("h5", "convbuf2[" << i << "]="
254 << convbuf2[i] << endl);
255 }
256 m_intern_plain_array_data((char *) convbuf2.data(), memtype);
257
258 } else
259 m_intern_plain_array_data(convbuf.data(), memtype);
260
261}
262
263void HDF5Array::handle_array_read_slab(hid_t dset_id, hid_t memtype, int64_t nelms,
264 const int64_t *offset, const int64_t *step, const int64_t *count)
265{
266 size_t data_size = nelms * H5Tget_size(memtype);
267 if (data_size == 0)
268 throw InternalErr(__FILE__, __LINE__, "get_size failed");
269
270 vector<char> convbuf(data_size);
271 get_slabdata(dset_id, offset, step, count, d_num_dim, convbuf.data());
272
273 // Check if a Signed Byte to Int16 conversion is necessary.
274 if (false == is_dap4() && (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype))) {
275 vector<short> convbuf2(data_size);
276 for (int64_t i = 0; i < (int)data_size; i++)
277 convbuf2[i] = static_cast<signed char> (convbuf[i]);
278 m_intern_plain_array_data((char*) convbuf2.data(),memtype);
279 }
280 else
281 m_intern_plain_array_data(convbuf.data(),memtype);
282}
283
284void HDF5Array::handle_vlen_string(hid_t dset_id, hid_t memtype, int64_t nelms, const vector<hsize_t>& hoffset,
285 const vector<hsize_t>& hcount, const vector<hsize_t>& hstep){
286
287 vector<string>finstrval;
288 finstrval.resize(nelms);
289 try {
290 read_vlen_string(dset_id, nelms, hoffset.data(), hstep.data(), hcount.data(),finstrval);
291 }
292 catch(...) {
293 H5Tclose(memtype);
294 throw InternalErr(__FILE__,__LINE__,"Fail to read variable-length string.");
295 }
296 set_value_ll(finstrval,nelms);
297 H5Tclose(memtype);
298
299}
300
301bool HDF5Array::m_array_of_structure(hid_t dsetid, vector<char>&values,bool has_values, size_t values_offset,
302 int64_t nelms,const int64_t* offset,const int64_t* count, const int64_t* step) {
303
304 BESDEBUG("h5", "=read() Array of Structure length=" << length() << endl);
305 hid_t mspace = -1;
306 hid_t memtype = -1;
307 hid_t dtypeid = -1;
308 size_t ty_size = -1;
309
310 if ((dtypeid = H5Dget_type(dsetid)) < 0)
311 throw InternalErr (__FILE__, __LINE__, "Cannot obtain the datatype.");
312
313 if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
314 H5Tclose(dtypeid);
315 throw InternalErr (__FILE__, __LINE__, "Fail to obtain memory datatype.");
316 }
317
318 ty_size = H5Tget_size(memtype);
319
320 if (false == has_values) {
321
322 hid_t dspace = -1;
323
324 if ((dspace = H5Dget_space(dsetid))<0) {
325 H5Tclose(memtype);
326 H5Tclose(dtypeid);
327 throw InternalErr (__FILE__, __LINE__, "Cannot obtain data space.");
328 }
329
330 d_num_dim = H5Sget_simple_extent_ndims(dspace);
331 if (d_num_dim < 0) {
332 H5Tclose(memtype);
333 H5Tclose(dtypeid);
334 H5Sclose(dspace);
335 throw InternalErr (__FILE__, __LINE__, "Cannot obtain the number of dimensions of the data space.");
336 }
337
338 vector<hsize_t> hoffset;
339 vector<hsize_t>hcount;
340 vector<hsize_t>hstep;
341 hoffset.resize(d_num_dim);
342 hcount.resize(d_num_dim);
343 hstep.resize(d_num_dim);
344 for (int i = 0; i <d_num_dim; i++) {
345 hoffset[i] = (hsize_t) offset[i];
346 hcount[i] = (hsize_t) count[i];
347 hstep[i] = (hsize_t) step[i];
348 }
349
350 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
351 hoffset.data(), hstep.data(),
352 hcount.data(), nullptr) < 0) {
353 H5Tclose(memtype);
354 H5Tclose(dtypeid);
355 H5Sclose(dspace);
356 throw InternalErr (__FILE__, __LINE__, "Cannot generate the hyperslab of the HDF5 dataset.");
357 }
358
359 mspace = H5Screate_simple(d_num_dim, hcount.data(),nullptr);
360 if (mspace < 0) {
361 H5Sclose(dspace);
362 H5Tclose(memtype);
363 H5Tclose(dtypeid);
364 throw InternalErr (__FILE__, __LINE__, "Cannot create the memory space.");
365 }
366
367 values.resize(nelms*ty_size);
368 hid_t read_ret = -1;
369 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,
370 (void*)values.data());
371 if (read_ret < 0) {
372 H5Tclose(memtype);
373 H5Tclose(dtypeid);
374 H5Sclose(dspace);
375 throw InternalErr (__FILE__, __LINE__, "Fail to read the HDF5 compound datatype dataset.");
376 }
377
378 H5Sclose(dspace);
379 has_values = true;
380 } // end of "if(false == has_values)" block
381
382 HDF5Structure *h5s = nullptr;
383 hid_t memb_id = -1;
384 char* memb_name = nullptr;
385
386 try {
387
388 // Loop through all the elements in this compound datatype array.
389 for (int64_t element = 0; element < nelms; ++element) {
390
391 h5s = dynamic_cast<HDF5Structure*>(var()->ptr_duplicate());
392 int nmembs = 0;
393 size_t struct_elem_offset = values_offset + ty_size*element;
394
395 if ((nmembs = H5Tget_nmembers(memtype)) < 0)
396 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of HDF5 compound datatype.");
397
398 for (unsigned int u = 0; u < (unsigned)nmembs; u++) {
399 // Get member name
400 memb_name = H5Tget_member_name(memtype,u);
401 if (memb_name == nullptr)
402 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the name of an HDF5 compound datatype member.");
403
404 BaseType *field = h5s->var(memb_name);
405 m_array_of_structure_member(field, memtype, u,dsetid, values, has_values, struct_elem_offset );
406 H5free_memory(memb_name);
407
408 } // end "for(unsigned u = 0)"
409 h5s->set_read_p(true);
410 set_vec_ll((uint64_t)element,h5s);
411 delete h5s;
412 } // end "for (int element=0"
413
414 // Close HDF5-related resources
415 m_array_of_structure_close_hdf5_ids(values, has_values, mspace, dtypeid, memtype);
416 }
417 catch(...) {
418
419 m_array_of_structure_catch_close_hdf5_ids(memb_id, memb_name, values, has_values, mspace, dtypeid, memtype);
420 delete h5s;
421 throw;
422 }
423
424 set_read_p(true);
425
426 return false;
427}
428
429
430void HDF5Array:: m_array_of_structure_member(BaseType *field, hid_t memtype, unsigned int u, hid_t dsetid,
431 vector<char>&values,bool has_values, size_t struct_elem_offset ) const
432{
433
434 hid_t memb_id = -1;
435 H5T_class_t memb_cls = H5T_NO_CLASS;
436 size_t memb_offset = 0;
437 // Get member type ID
438 if((memb_id = H5Tget_member_type(memtype, u)) < 0)
439 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype of an HDF5 compound datatype member.");
440
441 // Get member type class
442 if((memb_cls = H5Tget_member_class (memtype, u)) < 0)
443 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype class of an HDF5 compound datatype member.");
444
445 // Get member offset,H5Tget_member_offset only fails
446 // when H5Tget_memeber_class fails. Sinc H5Tget_member_class
447 // is checked above. So no need to check the return value.
448 memb_offset= H5Tget_member_offset(memtype,u);
449
450 if (memb_cls == H5T_COMPOUND) {
451 HDF5Structure &memb_h5s = dynamic_cast<HDF5Structure&>(*field);
452 memb_h5s.do_structure_read(dsetid, memb_id,values,has_values,(int)(memb_offset+struct_elem_offset));
453 }
454 else if(memb_cls == H5T_ARRAY) {
455
456 // memb_id, obtain the number of dimensions
457 int at_ndims = H5Tget_array_ndims(memb_id);
458 if(at_ndims <= 0)
459 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
460
461 HDF5Array &h5_array_type = dynamic_cast<HDF5Array&>(*field);
462 vector<int64_t> at_offset(at_ndims,0);
463 vector<int64_t> at_count(at_ndims,0);
464 vector<int64_t> at_step(at_ndims,0);
465
466 int64_t at_nelms = h5_array_type.format_constraint(at_offset.data(),at_step.data(),at_count.data());
467
468 // Read the array data
469 h5_array_type.do_h5_array_type_read(dsetid,memb_id,values,has_values,(int)(memb_offset+struct_elem_offset),
470 at_nelms,at_offset.data(),at_count.data(),at_step.data());
471 }
472 else if(memb_cls == H5T_INTEGER || memb_cls == H5T_FLOAT) {
473
474
475 if (true == promote_char_to_short(memb_cls,memb_id) && (field->is_dap4() == false)) {
476
477 auto src = (const void*)(values.data() + struct_elem_offset +memb_offset);
478 char val_int8;
479 memcpy(&val_int8,src,1);
480 auto val_short=(short)val_int8;
481 field->val2buf(&val_short);
482 }
483 else {
484 field->val2buf(values.data() + struct_elem_offset +memb_offset);
485 }
486 }
487 else if(memb_cls == H5T_STRING) {
488
489 // distinguish between variable length and fixed length
490 if (true == H5Tis_variable_str(memb_id)) {
491 auto src = (void*)(values.data()+struct_elem_offset + memb_offset);
492 string final_str;
493 get_vlen_str_data((char*)src,final_str);
494 field->val2buf(&final_str);
495 }
496 else {// Obtain fixed-size string value
497 auto src = (const void*)(values.data()+struct_elem_offset + memb_offset);
498 vector<char> str_val;
499 size_t memb_size = H5Tget_size(memb_id);
500 if (memb_size == 0) {
501 H5Tclose(memb_id);
502 throw InternalErr (__FILE__, __LINE__,"Fail to obtain the size of HDF5 compound datatype.");
503 }
504 str_val.resize(memb_size);
505 memcpy(str_val.data(),src,memb_size);
506 string temp_string(str_val.begin(),str_val.end());
507 field->val2buf(&temp_string);
508 }
509 }
510 else {
511 H5Tclose(memb_id);
512 throw InternalErr (__FILE__, __LINE__,
513 "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
514 }
515
516 // Close member type ID
517 H5Tclose(memb_id);
518 field->set_read_p(true);
519
520}
521
522void HDF5Array::m_array_of_structure_close_hdf5_ids(vector<char> &values, bool has_values, hid_t mspace,
523 hid_t dtypeid, hid_t memtype) const
524{
525 if (true == has_values) {
526 if (-1 == mspace)
527 throw InternalErr(__FILE__, __LINE__,
528 "memory type and memory space for this compound datatype should be valid.");
529
530 if (H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)values.data())<0)
531 throw InternalErr(__FILE__, __LINE__, "Unable to reclaim the compound datatype array.");
532 H5Sclose(mspace);
533 }
534
535 H5Tclose(dtypeid);
536 H5Tclose(memtype);
537
538}
539
540void HDF5Array:: m_array_of_structure_catch_close_hdf5_ids(hid_t memb_id, char * memb_name, vector<char> &values,
541 bool has_values, hid_t mspace, hid_t dtypeid, hid_t memtype) const {
542
543 if (memb_id != -1)
544 H5Tclose(memb_id);
545 if (memb_name != nullptr)
546 H5free_memory(memb_name);
547 if (true == has_values) {
548 if(H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)(values.data()))<0) {
549 H5Tclose(memtype);
550 H5Sclose(mspace);
551 }
552 H5Sclose(mspace);
553 }
554 H5Tclose(memtype);
555 H5Tclose(dtypeid);
556}
557
558// Haven't checked the codes and comments
559// Haven't added the close handles routines for error handlings yet. KY 2011-11-18
560bool HDF5Array::m_array_of_reference(hid_t dset_id,hid_t dtype_id)
561{
562
563#if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
564 hid_t d_dset_id = dset_id;
565
566 vector<int64_t> offset(d_num_dim);
567 vector<int64_t> count(d_num_dim);
568 vector<int64_t> step(d_num_dim);
569
570 int64_t nelms = format_constraint(offset.data(), step.data(), count.data());
571 vector<string> v_str(nelms);
572
573 BESDEBUG("h5", "=read() URL type is detected. "
574 << "nelms=" << nelms << " full_size=" << d_num_elm << endl);
575
576 // Handle regional reference.
577 if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) < 0) {
578 throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed");
579 }
580
581 if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) > 0)
582 m_array_of_region_reference(d_dset_id,v_str, nelms, offset, step);
583
584 // Handle object reference.
585 if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) < 0)
586 throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed.");
587
588 if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) > 0)
589 m_array_of_object_reference( d_dset_id, v_str, nelms,offset,step);
590
591 set_value_ll(v_str.data(), nelms);
592 return false;
593
594#else
595 return m_array_of_reference_new_h5_apis(dset_id,dtype_id);
596
597#endif
598}
599
600void HDF5Array:: m_array_of_region_reference(hid_t d_dset_id, vector<string>& v_str,
601 int64_t nelms, const vector<int64_t>& offset,
602 const vector<int64_t> &step) {
603
604 hdset_reg_ref_t *rbuf = nullptr;
605 BESDEBUG("h5", "=read() Got regional reference. " << endl);
606
607 // Vector doesn't work for this case. somehow it doesn't support the type.
608 rbuf = new hdset_reg_ref_t[d_num_elm];
609 if (rbuf == nullptr){
610 throw InternalErr(__FILE__, __LINE__, "new() failed.");
611 }
612 if (H5Dread(d_dset_id, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0)
613 throw InternalErr(__FILE__, __LINE__, "H5Dread() failed.");
614
615 for (int64_t i = 0; i < nelms; i++) {
616
617 hdset_reg_ref_t *temp_rbuf = rbuf + offset[0]+i*step[0];
618
619 // Let's assume that URL array is always 1 dimension.
620 if (temp_rbuf!= nullptr) {
621
622 char r_name[DODS_NAMELEN];
623
624 hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_DATASET_REGION, (const void*)(temp_rbuf));
625 if (did_r < 0)
626 throw InternalErr(__FILE__, __LINE__, "H5RDEREFERENCE() failed.");
627
628 if (H5Iget_name(did_r, (char *) r_name, DODS_NAMELEN) < 0)
629 throw InternalErr(__FILE__, __LINE__, "H5Iget_name() failed.");
630
631 BESDEBUG("h5", "=read() dereferenced name is " << r_name << endl);
632
633 string varname(r_name);
634 hid_t space_id = H5Rget_region(did_r, H5R_DATASET_REGION, (const void*)(temp_rbuf));
635 if (space_id < 0)
636 throw InternalErr(__FILE__, __LINE__, "H5Rget_region() failed.");
637
638 int ndim = H5Sget_simple_extent_ndims(space_id);
639 if (ndim < 0) {
640 throw InternalErr(__FILE__, __LINE__, "H5Sget_simple_extent_ndims() failed.");
641 }
642
643 BESDEBUG("h5", "=read() dim is " << ndim << endl);
644
645 string expression;
646 switch (H5Sget_select_type(space_id)) {
647
648 case H5S_SEL_NONE:
649 BESDEBUG("h5", "=read() None selected." << endl);
650 break;
651
652 case H5S_SEL_POINTS: {
653 m_array_of_region_reference_point_selection(space_id, ndim,varname,v_str, i);
654 break;
655 }
656 case H5S_SEL_HYPERSLABS: {
657 m_array_of_region_reference_hyperslab_selection(space_id, ndim, varname, v_str, i);
658 break;
659 }
660 case H5S_SEL_ALL:
661 BESDEBUG("h5", "=read() All selected." << endl);
662 break;
663
664 default:
665 BESDEBUG("h5", "Unknown space type." << endl);
666 break;
667 }
668
669 } else {
670 v_str[i] = "";
671 }
672 }
673 delete[] rbuf;
674}
675
676void HDF5Array:: m_array_of_region_reference_point_selection(hid_t space_id, int ndim, const string &varname,
677 vector<string> &v_str,int64_t i) const {
678
679 string expression;
680
681 BESDEBUG("h5", "=read() Points selected." << endl);
682 hssize_t npoints = H5Sget_select_npoints(space_id);
683 if (npoints < 0) {
684 throw InternalErr(__FILE__, __LINE__,
685 "Cannot determine number of elements in the dataspace selection");
686 }
687
688 BESDEBUG("h5", "=read() npoints are " << npoints << endl);
689
690 vector<hsize_t> buf(npoints * ndim);
691 if (H5Sget_select_elem_pointlist(space_id, 0, npoints, buf.data()) < 0)
692 throw InternalErr(__FILE__, __LINE__, "H5Sget_select_elem_pointlist() failed.");
693
694 for (int64_t j = 0; j < (int) npoints; j++) {
695 // Name of the dataset.
696 expression.append(varname);
697 for (int k = 0; k < ndim; k++) {
698 ostringstream oss;
699 oss << "[" << (int) buf[j * ndim + k] << "]";
700 expression.append(oss.str());
701 }
702 if (j != (int64_t) (npoints - 1)) {
703 expression.append(",");
704 }
705 }
706 v_str[i].append(expression);
707
708}
709
710void HDF5Array:: m_array_of_region_reference_hyperslab_selection(hid_t space_id, int ndim, const string &varname,
711 vector<string> &v_str,int64_t i) const
712{
713 string expression;
714
715 vector<hsize_t> start(ndim);
716 vector<hsize_t> end(ndim);
717 vector<hsize_t> stride(ndim);
718 vector<hsize_t> s_count(ndim);
719 vector<hsize_t> block(ndim);
720
721 BESDEBUG("h5", "=read() Slabs selected." << endl);
722 BESDEBUG("h5", "=read() nblock is " << H5Sget_select_hyper_nblocks(space_id) << endl);
723
724#if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 8)
725 if (H5Sget_select_bounds(space_id, start.data(), end.data()) < 0) {
726 throw InternalErr(__FILE__, __LINE__, "H5Sget_select_bounds() failed.");
727 }
728#else
729 if (H5Sget_regular_hyperslab(space_id, start.data(), stride.data(), s_count.data(),
730 block.data()) < 0) {
731 throw InternalErr(__FILE__, __LINE__, "H5Sget_regular_hyperslab() failed.");
732 }
733#endif
734
735 for (int j = 0; j < ndim; j++) {
736 ostringstream oss;
737 BESDEBUG("h5", "start " << start[j]
738 << "stride " << stride[j]
739 << "count " << s_count[j]
740 << "block " << block[j]
741 << endl);
742
743 // Map from HDF5's start,stride,count,block to DAP's start,stride,end.
744 end[j] = start[j] + stride[j] * (s_count[j] - 1) + (block[j] - 1);
745 BESDEBUG("h5", "=read() start is " << start[j]
746 << "=read() end is " << end[j] << endl);
747 oss << "[" << start[j] << ":" << stride[j] << ":" << end[j] << "]";
748 expression.append(oss.str());
749 BESDEBUG("h5", "=read() expression is " << expression << endl);
750 }
751 v_str[i] = varname;
752 if (!expression.empty())
753 v_str[i].append(expression);
754
755}
756void HDF5Array:: m_array_of_object_reference(hid_t d_dset_id, vector<string>& v_str,
757 int64_t nelms, const vector<int64_t>& offset,
758 const vector<int64_t> &step) const
759{
760
761 BESDEBUG("h5", "=read() Got object reference. " << endl);
762 vector<hobj_ref_t> orbuf;
763 orbuf.resize(d_num_elm);
764 if (H5Dread(d_dset_id, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL,
765 H5P_DEFAULT, orbuf.data()) < 0) {
766 throw InternalErr(__FILE__, __LINE__, "H5Dread failed()");
767 }
768
769 for (int64_t i = 0; i < nelms; i++) {
770
771 // Let's assume that URL array is always 1 dimension.
772 hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_OBJECT, &orbuf[offset[0] + i * step[0]]);
773 if (did_r < 0)
774 throw InternalErr(__FILE__, __LINE__, "H5RDEREFERENCE() failed.");
775
776 char r_name[DODS_NAMELEN];
777 if (H5Iget_name(did_r, (char *) r_name, DODS_NAMELEN) < 0)
778 throw InternalErr(__FILE__, __LINE__, "H5Iget_name() failed.");
779
780 // Shorten the dataset name
781 string varname(r_name);
782
783 BESDEBUG("h5", "=read() dereferenced name is " << r_name <<endl);
784 v_str[i] = varname;
785 }
786}
787
788bool HDF5Array::m_array_of_reference_new_h5_apis(hid_t dset_id,hid_t dtype_id) {
789
790#if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
791 throw InternalErr(__FILE__, __LINE__,
792 "The HDF5 handler compiled with earlier version (<=110)of the HDF5 library should not call method that uses new reference APIs");
793 return false;
794#else
795
796 H5R_ref_t *rbuf = nullptr;
797 hid_t mem_space_id = 0;
798 hid_t file_space_id;
799
800 try {
801
802 // First we need to read the reference data from DAP's hyperslab selection.
803 vector<int64_t> offset(d_num_dim);
804 vector<int64_t> count(d_num_dim);
805 vector<int64_t> step(d_num_dim);
806 vector<hsize_t> hoffset(d_num_dim);
807 vector<hsize_t>hcount(d_num_dim);
808 vector<hsize_t>hstep(d_num_dim);
809
810 int64_t nelms = format_constraint(offset.data(), step.data(), count.data());
811 for (int i = 0; i <d_num_dim; i++) {
812 hoffset[i] = (hsize_t) offset[i];
813 hcount[i] = (hsize_t) count[i];
814 hstep[i] = (hsize_t) step[i];
815 }
816
817 BESDEBUG("h5", "=read() URL type is detected. "
818 << "nelms=" << nelms << endl);
819
820 rbuf = new H5R_ref_t[nelms];
821
822 file_space_id = H5Dget_space(dset_id);
823 if(file_space_id < 0)
824 throw InternalErr(__FILE__, __LINE__, "Fail to obtain reference dataset file space.");
825
826 if (H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET,
827 hoffset.data(), hstep.data(),
828 hcount.data(), nullptr) < 0)
829 throw InternalErr (__FILE__, __LINE__, "Fail to select the hyperslab for reference dataset.");
830
831
832 mem_space_id = H5Screate_simple(d_num_dim,hcount.data(),nullptr);
833 if(mem_space_id < 0)
834 throw InternalErr(__FILE__, __LINE__, "Fail to obtain reference dataset memory space.");
835
836 if(H5Dread(dset_id,H5T_STD_REF,mem_space_id,file_space_id,H5P_DEFAULT,&rbuf[0])<0)
837 throw InternalErr(__FILE__, __LINE__, "Fail to read hyperslab reference dataset.");
838
839 H5Sclose(mem_space_id);
840 H5Sclose(file_space_id);
841
842 // Now we need to retrieve the reference info. fot the nelms elements.
843 vector<string> v_str;
844
845 H5R_type_t ref_type = H5Rget_type((const H5R_ref_t *)&rbuf[0]);
846
847 // The referenced objects can only be either objects or dataset regions.
848 if(ref_type != H5R_OBJECT2 && ref_type !=H5R_DATASET_REGION2)
849 throw InternalErr(__FILE__, __LINE__, "Unsupported reference: neither object nor region references");
850
851 for (int64_t i = 0; i < nelms; i++) {
852
853 hid_t obj_id = H5Ropen_object((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, H5P_DEFAULT);
854 if(obj_id < 0)
855 throw InternalErr(__FILE__, __LINE__, "Cannot open the object the reference points to");
856
857 vector<char> objname;
858 ssize_t objnamelen = -1;
859 if ((objnamelen= H5Iget_name(obj_id,nullptr,0))<=0) {
860 H5Oclose(obj_id);
861 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the name length of the object the reference points to");
862 }
863 objname.resize(objnamelen+1);
864 if ((objnamelen= H5Iget_name(obj_id,objname.data(),objnamelen+1))<=0) {
865 H5Oclose(obj_id);
866 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the name length of the object the reference points to");
867 }
868
869 string objname_str = string(objname.begin(),objname.end());
870 string trim_objname = objname_str.substr(0,objnamelen);
871
872 // For object references, we just need to save the object full path.
873 if(ref_type == H5R_OBJECT2)
874 v_str.push_back(trim_objname);
875 else {// Must be region reference.
876 H5O_type_t obj_type;
877 if(H5Rget_obj_type3((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, &obj_type) < 0){
878 H5Oclose(obj_id);
879 throw InternalErr(__FILE__, __LINE__, "H5Rget_obj_type3() failed.");
880 }
881 if(obj_type != H5O_TYPE_DATASET) {
882 H5Oclose(obj_id);
883 throw InternalErr(__FILE__, __LINE__, "Region reference must point to a dataset.");
884 }
885 hid_t region_space_id = H5Ropen_region(&rbuf[i],H5P_DEFAULT,H5P_DEFAULT);
886 if (region_space_id < 0) {
887 H5Oclose(obj_id);
888 throw InternalErr(__FILE__, __LINE__, "Cannot obtain the space ID the reference points to");
889 }
890
891 int ndim = H5Sget_simple_extent_ndims(region_space_id);
892 if (ndim < 0) {
893 H5Sclose(region_space_id);
894 H5Oclose(obj_id);
895 throw InternalErr(__FILE__, __LINE__, "H5Sget_simple_extent_ndims() failed.");
896 }
897
898 string expression;
899 switch (H5Sget_select_type(region_space_id)) {
900
901 case H5S_SEL_NONE:
902 BESDEBUG("h5", "=read() None selected." << endl);
903 break;
904
905 case H5S_SEL_POINTS: {
906 BESDEBUG("h5", "=read() Points selected." << endl);
907 hssize_t npoints = H5Sget_select_npoints(region_space_id);
908 if (npoints < 0) {
909 H5Sclose(region_space_id);
910 H5Oclose(obj_id);
911 throw InternalErr(__FILE__, __LINE__,
912 "Cannot determine number of elements in the dataspace selection");
913 }
914
915 BESDEBUG("h5", "=read() npoints are " << npoints
916 << endl);
917 vector<hsize_t> buf(npoints * ndim);
918 if (H5Sget_select_elem_pointlist(region_space_id, 0, npoints, buf.data()) < 0) {
919 H5Sclose(region_space_id);
920 H5Oclose(obj_id);
921 throw InternalErr(__FILE__, __LINE__, "H5Sget_select_elem_pointlist() failed.");
922 }
923
924#if 0
925 for (int j = 0; j < npoints * ndim; j++) {
926 "h5", "=read() npoints buf[0] =" << buf[j] <<endl;
927 }
928#endif
929
930 for (int64_t j = 0; j < (int) npoints; j++) {
931 // Name of the dataset.
932 expression.append(trim_objname);
933 for (int k = 0; k < ndim; k++) {
934 ostringstream oss;
935 oss << "[" << (int) buf[j * ndim + k] << "]";
936 expression.append(oss.str());
937 }
938 if (j != (int64_t) (npoints - 1)) {
939 expression.append(",");
940 }
941 }
942 v_str.push_back(expression);
943
944 break;
945 }
946 case H5S_SEL_HYPERSLABS: {
947 vector<hsize_t> start(ndim);
948 vector<hsize_t> end(ndim);
949 vector<hsize_t>stride(ndim);
950 vector<hsize_t>s_count(ndim);
951 vector<hsize_t>block(ndim);
952
953 BESDEBUG("h5", "=read() Slabs selected." << endl);
954 BESDEBUG("h5", "=read() nblock is " <<
955 H5Sget_select_hyper_nblocks(region_space_id) << endl);
956
957 if (H5Sget_regular_hyperslab(region_space_id, start.data(), stride.data(), s_count.data(), block.data()) < 0) {
958 H5Sclose(region_space_id);
959 H5Oclose(obj_id);
960 throw InternalErr(__FILE__, __LINE__, "H5Sget_regular_hyperslab() failed.");
961 }
962
963 expression.append(trim_objname);
964 for (int j = 0; j < ndim; j++) {
965 ostringstream oss;
966 BESDEBUG("h5", "start " << start[j]
967 << "stride "<<stride[j]
968 << "count "<< s_count[j]
969 << "block "<< block[j]
970 <<endl);
971
972 // Map from HDF5's start,stride,count,block to DAP's start,stride,end.
973 end[j] = start[j] + stride[j]*(s_count[j]-1)+(block[j]-1);
974 BESDEBUG("h5", "=read() start is " << start[j]
975 << "=read() end is " << end[j] << endl);
976 oss << "[" << start[j] << ":" << stride[j] << ":" << end[j] << "]";
977 expression.append(oss.str());
978 BESDEBUG("h5", "=read() expression is "
979 << expression << endl)
980 ;
981 }
982 v_str.push_back(expression);
983 // Constraint expression. [start:stride:end]
984 break;
985 }
986 case H5S_SEL_ALL:
987 BESDEBUG("h5", "=read() All selected." << endl);
988 break;
989
990 default:
991 BESDEBUG("h5", "Unknown space type." << endl);
992 break;
993 }
994 H5Sclose(region_space_id);
995 }
996 H5Oclose(obj_id);
997 }
998 for (int64_t i = 0; i<nelms; i++)
999 H5Rdestroy(&rbuf[i]);
1000 delete[] rbuf;
1001 H5Sclose(mem_space_id);
1002 H5Sclose(file_space_id);
1003 set_value_ll(v_str.data(), nelms);
1004 return false;
1005 }
1006 catch (...) {
1007 if(rbuf!= nullptr)
1008 delete[] rbuf;
1009 H5Sclose(mem_space_id);
1010 H5Sclose(file_space_id);
1011 throw;
1012 }
1013#endif
1014}
1015
1016
1017void HDF5Array::m_intern_plain_array_data(char *convbuf,hid_t memtype)
1018{
1019 if (check_h5str(memtype)) {
1020
1021 vector<string> v_str(d_num_elm);
1022 size_t elesize = H5Tget_size(memtype);
1023 if (elesize == 0) {
1024 throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1025 }
1026 vector<char> strbuf(elesize + 1);
1027 BESDEBUG("h5", "=read()<check_h5str() element size=" << elesize
1028 << " d_num_elm=" << d_num_elm << endl);
1029
1030 for (int64_t strindex = 0; strindex < (int64_t)d_num_elm; strindex++) {
1031 get_strdata(strindex, convbuf, strbuf.data(), (int)elesize);
1032 BESDEBUG("h5", "=read()<get_strdata() strbuf=" << strbuf.data() << endl);
1033 v_str[strindex] = strbuf.data();
1034 }
1035 set_read_p(true);
1036 val2buf((void *) v_str.data());
1037 }
1038 else {
1039 set_read_p(true);
1040 val2buf((void *) convbuf);
1041 }
1042}
1043
1044bool HDF5Array::do_h5_array_type_read(hid_t dsetid, hid_t memb_id,vector<char>&values,bool has_values,size_t values_offset,
1045 int64_t at_nelms,int64_t* at_offset,int64_t* at_count, int64_t* at_step){
1046
1047 //1. Call do array first(datatype must be derived) and the value must be set. We don't support Array datatype
1048 // unless it is inside a compound datatype
1049 if(has_values != true)
1050 throw InternalErr (__FILE__, __LINE__,
1051 "Only support the retrieval of HDF5 Array datatype values from the parent compound datatype read.");
1052
1053 hid_t at_base_type = H5Tget_super(memb_id);
1054 if (at_base_type < 0)
1055 throw InternalErr (__FILE__, __LINE__, "Fail to obtain the basetype of the array datatype.");
1056
1057 // memb_id, obtain the number of dimensions
1058 int at_ndims = H5Tget_array_ndims(memb_id);
1059 if (at_ndims <= 0) {
1060 H5Tclose(at_base_type);
1061 throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
1062 }
1063
1064 vector<hsize_t>at_dims_h(at_ndims,0);
1065
1066 // Obtain the number of elements for each dims
1067 if (H5Tget_array_dims(memb_id,at_dims_h.data())<0) {
1068 H5Tclose(at_base_type);
1069 throw InternalErr (__FILE__, __LINE__, "Fail to obtain dimensions of the array datatype.");
1070 }
1071 vector<int64_t>at_dims(at_ndims,0);
1072 for (int64_t i = 0;i<at_ndims;i++)
1073 at_dims[i] = (int64_t)at_dims_h[i];
1074
1075 int64_t at_total_nelms = 1;
1076 for (int i = 0; i <at_ndims; i++)
1077 at_total_nelms = at_total_nelms*at_dims[i];
1078
1079 H5T_class_t array_cls = H5Tget_class(at_base_type);
1080 size_t at_base_type_size = H5Tget_size(at_base_type);
1081
1082 // H5 Array type, the basetype is COMPOUND.
1083 if(H5T_COMPOUND == array_cls) {
1084
1085 // These vectors are used to handle subset of array datatype
1086 vector<int64_t> at_end(at_ndims, 0);
1087 vector<int64_t> at_pos(at_ndims, 0);
1088 for (int i = 0; i < at_ndims; i++) {
1089 at_pos[i] = at_offset[i];
1090 at_end[i] = at_offset[i] + (at_count[i] - 1) * at_step[i];
1091 }
1092
1093 int64_t at_orig_index = INDEX_nD_TO_1D(at_dims, at_pos);
1094
1095 // To read the array of compound (structure) in DAP, one must read one element each. set_vec is used afterwards.
1096 for (int64_t array_index = 0; array_index < at_nelms; array_index++) {
1097
1098 // The basetype of the array datatype is compound,-- check if the following line is valid.
1099 auto h5s = dynamic_cast<HDF5Structure *>(var()->ptr_duplicate());
1100 hid_t child_memb_id;
1101 H5T_class_t child_memb_cls;
1102 int child_nmembs;
1103 size_t child_memb_offset;
1104
1105 if ((child_nmembs = H5Tget_nmembers(at_base_type)) < 0) {
1106 H5Tclose(at_base_type);
1107 delete h5s;
1108 throw InternalErr(__FILE__, __LINE__, "Fail to obtain number of HDF5 compound datatype.");
1109 }
1110
1111 for (unsigned child_u = 0; child_u < (unsigned) child_nmembs; child_u++) {
1112
1113 // Get member type ID
1114 if ((child_memb_id = H5Tget_member_type(at_base_type, child_u)) < 0) {
1115 H5Tclose(at_base_type);
1116 delete h5s;
1117 throw InternalErr(__FILE__, __LINE__,
1118 "Fail to obtain the datatype of an HDF5 compound datatype member.");
1119 }
1120
1121 // Get member type class
1122 if ((child_memb_cls = H5Tget_member_class(at_base_type, child_u)) < 0) {
1123 H5Tclose(child_memb_id);
1124 H5Tclose(at_base_type);
1125 delete h5s;
1126 throw InternalErr(__FILE__, __LINE__,
1127 "Fail to obtain the datatype class of an HDF5 compound datatype member.");
1128 }
1129
1130 // Get member offset
1131 child_memb_offset = H5Tget_member_offset(at_base_type, child_u);
1132
1133 // Get member name
1134 char *child_memb_name = H5Tget_member_name(at_base_type, child_u);
1135 if (child_memb_name == nullptr) {
1136 H5Tclose(child_memb_id);
1137 H5Tclose(at_base_type);
1138 delete h5s;
1139 throw InternalErr(__FILE__, __LINE__,
1140 "Fail to obtain the name of an HDF5 compound datatype member.");
1141 }
1142
1143 BaseType *field = h5s->var(child_memb_name);
1144 H5free_memory(child_memb_name);
1145 try {
1146 do_h5_array_type_read_base_compound_member(dsetid, field, child_memb_id, child_memb_cls, values,
1147 has_values, values_offset, at_nelms, at_total_nelms, at_base_type_size,
1148 array_index, at_orig_index, child_memb_offset);
1149 }
1150 catch(...){
1151 delete h5s;
1152 }
1153
1154 } // end "for ( child_u = 0)"
1155 h5s->set_read_p(true);
1156
1157 // Save the value of this element to DAP structure.
1158 set_vec_ll((uint64_t) array_index, h5s);
1159 delete h5s;
1160
1161 vector<int64_t> at_offsetv(at_pos.size(), 0);
1162 vector<int64_t> at_stepv(at_pos.size(), 0);
1163 for (int64_t at_index = 0; at_index < (int64_t) (at_pos.size()); at_index++) {
1164 at_offsetv[at_index] = at_offset[at_index];
1165 at_stepv[at_index] = at_step[at_index];
1166 }
1167 //obtain the next position of the selected point based on the offset,end and step.
1168 obtain_next_pos(at_pos, at_offsetv, at_end, at_stepv, (int) (at_pos.size()));
1169 at_orig_index = INDEX_nD_TO_1D(at_dims, at_pos);
1170 }// end for "(array_index = 0) for array (compound)datatype"
1171
1172 }
1173 else if(H5T_INTEGER == array_cls|| H5T_FLOAT == array_cls) {
1174 do_h5_array_type_read_base_atomic(array_cls, at_base_type, at_base_type_size, values, values_offset, at_nelms,
1175 at_total_nelms,at_ndims, at_dims, at_offset, at_step, at_count);
1176 }
1177 else if(H5T_STRING == array_cls) {
1178
1179 // set the original position to the starting point
1180 vector<int64_t>at_pos(at_ndims,0);
1181 for (int i = 0; i< at_ndims; i++)
1182 at_pos[i] = at_offset[i];
1183
1184 vector<string>total_strval;
1185 total_strval.resize(at_total_nelms);
1186
1187 if (true == H5Tis_variable_str(at_base_type)) {
1188 auto src = (void*)(values.data()+values_offset);
1189 auto temp_bp =(char*)src;
1190 for (int64_t i = 0;i <at_total_nelms; i++){
1191 string tempstrval;
1192 get_vlen_str_data(temp_bp,tempstrval);
1193 total_strval[i] = tempstrval;
1194 temp_bp += at_base_type_size;
1195 }
1196 if (at_total_nelms == at_nelms)
1197 set_value_ll(total_strval,at_total_nelms);
1198 else {// obtain subset for variable-length string.
1199 vector<string>final_val;
1200 subset<string>(
1201 total_strval.data(),
1202 at_ndims,
1203 at_dims,
1204 at_offset,
1205 at_step,
1206 at_count,
1207 &final_val,
1208 at_pos,
1209 0
1210 );
1211
1212 set_value_ll(final_val,at_nelms);
1213
1214 }
1215
1216 }
1217 else {// For fixed-size string.
1218 auto src = (void*)(values.data()+values_offset);
1219 for(int64_t i = 0; i <at_total_nelms; i++)
1220 total_strval[i].resize(at_base_type_size);
1221
1222 vector<char> str_val;
1223 str_val.resize(at_total_nelms*at_base_type_size);
1224 memcpy((void*)str_val.data(),src,at_total_nelms*at_base_type_size);
1225 string total_in_one_string(str_val.begin(),str_val.end());
1226 for (int64_t i = 0; i<at_total_nelms;i++)
1227 total_strval[i] = total_in_one_string.substr(i*at_base_type_size,at_base_type_size);
1228
1229 if (at_total_nelms == at_nelms)
1230 set_value_ll(total_strval,at_total_nelms);
1231 else {
1232 vector<string>final_val;
1233 subset<string>(
1234 total_strval.data(),
1235 at_ndims,
1236 at_dims,
1237 at_offset,
1238 at_step,
1239 at_count,
1240 &final_val,
1241 at_pos,
1242 0
1243 );
1244 set_value_ll(final_val,at_nelms);
1245
1246 }
1247 }
1248 }
1249 else {
1250 H5Tclose(at_base_type);
1251 throw InternalErr (__FILE__, __LINE__,
1252 "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
1253 }
1254
1255 H5Tclose(at_base_type);
1256
1257 return true;
1258}
1259
1260void HDF5Array:: do_h5_array_type_read_base_compound_member(hid_t dsetid, BaseType *field, hid_t child_memb_id,
1261 H5T_class_t child_memb_cls, vector<char>&values,
1262 bool has_values, size_t values_offset, int64_t at_nelms,
1263 int64_t at_total_nelms, size_t at_base_type_size,
1264 int64_t array_index, int64_t at_orig_index,
1265 size_t child_memb_offset) const {
1266 if (child_memb_cls == H5T_COMPOUND) {
1267 HDF5Structure &memb_h5s = dynamic_cast<HDF5Structure &>(*field);
1268 //
1269 // Call structure read when reading the whole array. sa1{sa2[100]}
1270 // sa2[100] is an array datatype.
1271 // If reading the whole buffer, just add the buffer.
1272 if (at_total_nelms == at_nelms) {
1273 memb_h5s.do_structure_read(dsetid, child_memb_id, values, has_values,
1274 values_offset + at_base_type_size * array_index + child_memb_offset);
1275 }
1276 // Subset of sa2, sa2[10:100:2]; sa2[100] is an array datatype.
1277 // The whole array sa2[100] is to be read into somewhere in buffer values.
1278 else {
1279 // The subset should be considered. adjust memb_offset+values_offset+???,make sure only the subset is selected.
1280 // at_total_nelms is 100 but at_nelms is (100-10)/2+1=46.
1281 // The starting point of the whole array is values+memb_offset_values_offset
1282 // When the datatype is structure, we have to obtain the index one by one.
1283
1284 memb_h5s.do_structure_read(dsetid, child_memb_id, values, has_values,
1285 values_offset + at_base_type_size * at_orig_index +
1286 child_memb_offset);
1287 }
1288 }
1289 else if (child_memb_cls == H5T_ARRAY) {
1290
1291 // memb_id, obtain the number of dimensions
1292 int child_at_ndims = H5Tget_array_ndims(child_memb_id);
1293 if (child_at_ndims <= 0) {
1294 H5Tclose(child_memb_id);
1295 throw InternalErr(__FILE__, __LINE__,
1296 "Fail to obtain number of dimensions of the array datatype.");
1297 }
1298
1299 HDF5Array &h5_array_type = dynamic_cast<HDF5Array &>(*field);
1300 vector<int64_t> child_at_offset(child_at_ndims, 0);
1301 vector<int64_t> child_at_count(child_at_ndims, 0);
1302 vector<int64_t> child_at_step(child_at_ndims, 0);
1303
1304 int64_t child_at_nelms = h5_array_type.format_constraint(child_at_offset.data(),
1305 child_at_step.data(),
1306 child_at_count.data());
1307 if (at_total_nelms == at_nelms) {
1308 h5_array_type.do_h5_array_type_read(dsetid, child_memb_id, values, has_values,
1309 child_memb_offset + values_offset +
1310 at_base_type_size * array_index,
1311 child_at_nelms, child_at_offset.data(),
1312 child_at_count.data(), child_at_step.data());
1313 } else {// Adjust memb_offset+values_offset, basically change at_base_type_size*array_index
1314 h5_array_type.do_h5_array_type_read(dsetid, child_memb_id, values, has_values,
1315 child_memb_offset + values_offset +
1316 at_base_type_size * at_orig_index,
1317 child_at_nelms, child_at_offset.data(),
1318 child_at_count.data(), child_at_step.data());
1319
1320 }
1321 }
1322 else if (H5T_INTEGER == child_memb_cls || H5T_FLOAT == child_memb_cls) {
1323
1324 int64_t number_index = ((at_total_nelms == at_nelms) ? array_index : at_orig_index);
1325 if (true == promote_char_to_short(child_memb_cls, child_memb_id) && (field->is_dap4() == false)) {
1326 auto src = (const void *) (values.data() + (number_index * at_base_type_size) + values_offset +
1327 child_memb_offset);
1328 char val_int8;
1329 memcpy(&val_int8, src, 1);
1330 auto val_short = (short) val_int8;
1331 field->val2buf(&val_short);
1332 }
1333 else
1334 field->val2buf(
1335 values.data() + (number_index * at_base_type_size) + values_offset + child_memb_offset);
1336 }
1337 else if (H5T_STRING == child_memb_cls) {
1338
1339 int64_t string_index = ((at_total_nelms == at_nelms) ? array_index : at_orig_index);
1340 size_t data_offset = (size_t)string_index * at_base_type_size + values_offset +
1341 child_memb_offset;
1342 do_h5_array_type_read_base_compound_member_string(field, child_memb_id, values, data_offset);
1343
1344 }
1345 else {
1346 H5Tclose(child_memb_id);
1347 throw InternalErr(__FILE__, __LINE__, "Unsupported datatype class for the array base type.");
1348 }
1349 field->set_read_p(true);
1350 H5Tclose(child_memb_id);
1351}
1352
1353void HDF5Array:: do_h5_array_type_read_base_compound_member_string(BaseType *field, hid_t child_memb_id,
1354 const vector<char> &values, size_t data_offset) const
1355{
1356 // distinguish between variable length and fixed length
1357 auto src = (void *) (values.data() + data_offset);
1358 if (true == H5Tis_variable_str(child_memb_id)) {
1359
1360 string final_str;
1361 auto temp_bp = (char *) src;
1362 get_vlen_str_data(temp_bp, final_str);
1363 field->val2buf(&final_str[0]);
1364 }
1365 else {// Obtain string
1366 vector<char> str_val;
1367 size_t memb_size = H5Tget_size(child_memb_id);
1368 if (memb_size == 0) {
1369 H5Tclose(child_memb_id);
1370 throw InternalErr(__FILE__, __LINE__, "Fail to obtain the size of HDF5 compound datatype.");
1371 }
1372 str_val.resize(memb_size);
1373 memcpy(str_val.data(), src, memb_size);
1374 field->val2buf(str_val.data());
1375 }
1376}
1377
1378void HDF5Array::do_h5_array_type_read_base_atomic(H5T_class_t array_cls, hid_t at_base_type, size_t at_base_type_size,
1379 vector<char>&values,
1380 size_t values_offset, int64_t at_nelms,int64_t at_total_nelms,
1381 int at_ndims, vector<int64_t> &at_dims, int64_t* at_offset,
1382 int64_t* at_step, int64_t *at_count) {
1383
1384 // If no subset for the array datatype, just read the whole buffer.
1385 if (at_total_nelms == at_nelms)
1386 do_h5_array_type_read_base_atomic_whole_data(array_cls, at_base_type, at_nelms, values, values_offset);
1387 else {
1388 // Adjust the value for the subset of the array datatype
1389 // Obtain the corresponding DAP type of the HDF5 data type
1390 string dap_type = get_dap_type(at_base_type, is_dap4());
1391
1392 // The total array type data is read.
1393 auto src = (void *) (values.data() + values_offset);
1394
1395 // set the original position to the starting point
1396 vector<int64_t> at_pos(at_ndims, 0);
1397 for (int64_t i = 0; i < at_ndims; i++)
1398 at_pos[i] = at_offset[i];
1399
1400 if (BYTE == dap_type) {
1401
1402 vector<unsigned char> total_val;
1403 total_val.resize(at_total_nelms);
1404 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1405
1406 vector<unsigned char> final_val;
1407 subset<unsigned char>(
1408 total_val.data(),
1409 at_ndims,
1410 at_dims,
1411 at_offset,
1412 at_step,
1413 at_count,
1414 &final_val,
1415 at_pos,
1416 0
1417 );
1418
1419 set_value_ll(final_val.data(), at_nelms);
1420
1421 } else if (INT16 == dap_type) {
1422
1423 // promote char to short,DAP2 doesn't have "char" type
1424 if (true == promote_char_to_short(array_cls, at_base_type) && (is_dap4() == false)) {
1425 vector<char> total_val;
1426 total_val.resize(at_total_nelms);
1427 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1428
1429 vector<char> final_val;
1430 subset<char>(
1431 total_val.data(),
1432 at_ndims,
1433 at_dims,
1434 at_offset,
1435 at_step,
1436 at_count,
1437 &final_val,
1438 at_pos,
1439 0
1440 );
1441
1442 vector<short> final_val_short;
1443 final_val_short.resize(at_nelms);
1444 for (int64_t i = 0; i < at_nelms; i++)
1445 final_val_short[i] = final_val[i];
1446
1447 val2buf(final_val_short.data());
1448
1449 } else {// short
1450
1451 vector<short> total_val;
1452 total_val.resize(at_total_nelms);
1453 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1454
1455 vector<short> final_val;
1456 subset<short>(
1457 total_val.data(),
1458 at_ndims,
1459 at_dims,
1460 at_offset,
1461 at_step,
1462 at_count,
1463 &final_val,
1464 at_pos,
1465 0
1466 );
1467
1468 val2buf(final_val.data());
1469 }
1470 } else if (UINT16 == dap_type) {
1471 vector<unsigned short> total_val;
1472 total_val.resize(at_total_nelms);
1473 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1474
1475 vector<unsigned short> final_val;
1476 subset<unsigned short>(
1477 total_val.data(),
1478 at_ndims,
1479 at_dims,
1480 at_offset,
1481 at_step,
1482 at_count,
1483 &final_val,
1484 at_pos,
1485 0
1486 );
1487
1488 val2buf(final_val.data());
1489
1490 } else if (UINT32 == dap_type) {
1491 vector<unsigned int> total_val;
1492 total_val.resize(at_total_nelms);
1493 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1494
1495 vector<unsigned int> final_val;
1496 subset<unsigned int>(
1497 total_val.data(),
1498 at_ndims,
1499 at_dims,
1500 at_offset,
1501 at_step,
1502 at_count,
1503 &final_val,
1504 at_pos,
1505 0
1506 );
1507 val2buf(final_val.data());
1508
1509
1510 } else if (INT32 == dap_type) {
1511 vector<int> total_val;
1512 total_val.resize(at_total_nelms);
1513 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1514
1515 vector<int> final_val;
1516 subset<int>(
1517 total_val.data(),
1518 at_ndims,
1519 at_dims,
1520 at_offset,
1521 at_step,
1522 at_count,
1523 &final_val,
1524 at_pos,
1525 0
1526 );
1527
1528 val2buf(final_val.data());
1529
1530 } else if (FLOAT32 == dap_type) {
1531 vector<float> total_val;
1532 total_val.resize(at_total_nelms);
1533 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1534
1535 vector<float> final_val;
1536 subset<float>(
1537 total_val.data(),
1538 at_ndims,
1539 at_dims,
1540 at_offset,
1541 at_step,
1542 at_count,
1543 &final_val,
1544 at_pos,
1545 0
1546 );
1547
1548 val2buf(final_val.data());
1549
1550 } else if (FLOAT64 == dap_type) {
1551 vector<double> total_val;
1552 total_val.resize(at_total_nelms);
1553 memcpy(total_val.data(), src, at_total_nelms * at_base_type_size);
1554
1555 vector<double> final_val;
1556 subset<double>(
1557 total_val.data(),
1558 at_ndims,
1559 at_dims,
1560 at_offset,
1561 at_step,
1562 at_count,
1563 &final_val,
1564 at_pos,
1565 0
1566 );
1567 val2buf(final_val.data());
1568
1569 } else {
1570 H5Tclose(at_base_type);
1571 throw InternalErr(__FILE__, __LINE__,
1572 "Non-supported integer or float datatypes");
1573 }
1574 }
1575}
1576
1577void HDF5Array::do_h5_array_type_read_base_atomic_whole_data(H5T_class_t array_cls, hid_t at_base_type,int64_t at_nelms,
1578 vector<char> &values, size_t values_offset)
1579{
1580 // For DAP2 char should be mapped to short
1581 if (true == promote_char_to_short(array_cls, at_base_type) && is_dap4() == false) {
1582
1583 vector<char> val_int8;
1584 val_int8.resize(at_nelms);
1585 auto src = (void *) (values.data() + values_offset);
1586 memcpy(val_int8.data(), src, at_nelms);
1587
1588 vector<short> val_short;
1589 for (int64_t i = 0; i < at_nelms; i++)
1590 val_short[i] = (short) val_int8[i];
1591
1592 val2buf(val_short.data());
1593
1594 }
1595 else // shortcut for others
1596 val2buf(values.data() + values_offset);
1597}
1599inline int64_t
1600HDF5Array::INDEX_nD_TO_1D (const std::vector < int64_t > &dims,
1601 const std::vector < int64_t > &pos) const
1602{
1603 //
1604 // "int a[10][20][30] // & a[1][2][3] == a + (20*30+1 + 30*2 + 1 *3)"
1605 // "int b[10][2]; // &b[1][2] == b + (20*1 + 2);"
1606 //
1607 assert (dims.size () == pos.size ());
1608 int64_t sum = 0;
1609 int start = 1;
1610
1611 for (const auto &apos:pos) {
1612 int64_t m = 1;
1613 for (unsigned int j = start; j < dims.size (); j++)
1614 m *= dims[j];
1615 sum += m * apos;
1616 start++;
1617 }
1618 return sum;
1619}
1620
1621// Obtain the dimension index of the next pos. of the point based on the offset, step and end
1622bool HDF5Array::obtain_next_pos(vector<int64_t>& pos, vector<int64_t>&start,vector<int64_t>&end,vector<int64_t>&step,
1623 int rank_change) {
1624
1625 if((pos[rank_change-1] + step[rank_change-1])<=end[rank_change-1]) {
1626 pos[rank_change-1] = pos[rank_change-1] + step[rank_change-1];
1627 return true;
1628 }
1629 else {
1630 if( 1 == rank_change)
1631 return false;
1632 pos[rank_change-1] = start[rank_change-1];
1633 obtain_next_pos(pos,start,end,step,rank_change-1);
1634 }
1635 return true;
1636}
1637
1639//
1640// \param input Input variable
1641// \param dim dimension info of the input
1642// \param start start indexes of each dim
1643// \param stride stride of each dim
1644// \param edge count of each dim
1645// \param poutput output variable
1646// \parrm index dimension index
1647// \return 0 if successful. -1 otherwise.
1648//
1649template<typename T>
1650int HDF5Array::subset(
1651 const T input[],
1652 int rank,
1653 vector<int64_t> & dim,
1654 int64_t start[],
1655 int64_t stride[],
1656 int64_t edge[],
1657 std::vector<T> *poutput,
1658 vector<int64_t>& pos,
1659 int index)
1660{
1661 for(int64_t k=0; k<edge[index]; k++)
1662 {
1663 pos[index] = start[index] + k*stride[index];
1664 if(index+1<rank)
1665 subset(input, rank, dim, start, stride, edge, poutput,pos,index+1);
1666 if(index==rank-1)
1667 {
1668 poutput->push_back(input[INDEX_nD_TO_1D( dim, pos)]);
1669 }
1670 } // end of for
1671 return 0;
1672} // end of template<typename T> static int subset
1673
1674
1675// public functions to set all parameters needed in read function.
1676
1677
1678void HDF5Array::set_memneed(size_t need) {
1679 d_memneed = need;
1680}
1681
1682void HDF5Array::set_numdim(int ndims) {
1683 d_num_dim = ndims;
1684}
1685
1686void HDF5Array::set_numelm(hsize_t nelms) {
1687 d_num_elm = nelms;
1688}
1689
1690// We don't inherit libdap Array Class's transform_to_dap4 method since CF option is still using it.
1691BaseType* HDF5Array::h5dims_transform_to_dap4(D4Group *grp,const vector<string> &dimpath) {
1692
1693 BESDEBUG("h5", "<h5dims_transform_to_dap4" << endl);
1694
1695 if (grp == nullptr)
1696 return nullptr;
1697
1698 Array *dest = dynamic_cast<HDF5Array*>(ptr_duplicate());
1699
1700 // If there is just a size, don't make
1701 // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1702 // a name). jhrg 3/18/14
1703
1704 int k = 0;
1705 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1706
1707 if (false == (*d).name.empty()) {
1708
1709 BESDEBUG("h5", "<coming to the dimension loop, has name " << (*d).name<<endl);
1710 BESDEBUG("h5", "<coming to the dimension loop, has dimpath " << dimpath[k] <<endl);
1711 BESDEBUG("h5", "<coming to the dimension loop, has dimpath group "
1712 << dimpath[k].substr(0,dimpath[k].find_last_of("/")+1) <<endl);
1713
1714 D4Group *temp_grp = grp;
1715 D4Dimension *d4_dim = nullptr;
1716 bool is_dim_nonc4_grp = handle_one_dim(d,temp_grp, d4_dim, dimpath, k);
1717
1718 // Not find this dimension in any of the ancestor groups, add it to this group.
1719 // The following block is fine, but to avoid the complaint from sonarcloud.
1720 // Use a bool.
1721 if (true == is_dim_nonc4_grp) {
1722 string err= "The variable " + var_path +" has dimension ";
1723 err += dimpath[k] + ". This dimension is not under its ancestor or the current group.";
1724 err += " This is not supported.";
1725 delete dest;
1726 throw InternalErr(__FILE__,__LINE__,err);
1727 }
1728
1729 bool d4_dim_null = ((d4_dim==nullptr)?true:false);
1730 if (d4_dim_null == true) {
1731 auto d4_dim_unique = make_unique<D4Dimension>((*d).name, (*d).size);
1732 D4Dimensions * dims = grp->dims();
1733 BESDEBUG("h5", "<Just before adding D4 dimension to group" << grp->FQN() <<endl);
1734 d4_dim = d4_dim_unique.release();
1735 dims->add_dim_nocopy(d4_dim);
1736 (*d).dim = d4_dim;
1737 }
1738 }
1739 k++;
1740 }
1741
1742 dest->set_is_dap4(true);
1743
1744 return dest;
1745
1746}
1747
1748bool HDF5Array::handle_one_dim(Array::Dim_iter d, D4Group *temp_grp, D4Dimension * &d4_dim,
1749 const vector<string> &dimpath, int k) const
1750{
1751 bool is_dim_nonc4_grp = false;
1752 while (temp_grp) {
1753
1754 BESDEBUG("h5", "<coming to the group has name " << temp_grp->name()<<endl);
1755 BESDEBUG("h5", "<coming to the group has fullpath " << temp_grp->FQN()<<endl);
1756
1757 //Obtain all the dimensions of this group.
1758 D4Dimensions *temp_dims = temp_grp->dims();
1759
1760 // Check if this dimension is defined in this group
1761 d4_dim = temp_dims->find_dim((*d).name);
1762
1763 // Need the full path of the dimension name
1764 string d4_dim_path = dimpath[k].substr(0,dimpath[k].find_last_of("/")+1);
1765 BESDEBUG("h5", "d4_dim_path is " << d4_dim_path<<endl);
1766
1767 bool ancestor_grp = false;
1768
1769 // If the dim_path is within this group or its ancestor, this is valid.
1770 if(d4_dim_path.find(temp_grp->FQN())==0 || temp_grp->FQN().find(d4_dim_path)==0)
1771 ancestor_grp = true;
1772
1773 // If we find this dimension and the dimension is on the ancestral path,
1774 // this follows the netCDF-4/DAP4 dimension model, break.
1775 if(d4_dim && (temp_grp->FQN() == d4_dim_path)) {
1776 BESDEBUG("h5", "<FInd dimension name " << (*d).name<<endl);
1777 (*d).dim = d4_dim;
1778 is_dim_nonc4_grp = false;
1779 break;
1780 }
1781 // If the dimension name is not on the ancestral path, this
1782 // dimension must be on another path, mark it.
1783 else if( ancestor_grp == false) {
1784 is_dim_nonc4_grp = true;
1785 break;
1786 }
1787 else
1788 d4_dim = nullptr;
1789
1790 if(temp_grp->get_parent())
1791 temp_grp = static_cast<D4Group*>(temp_grp->get_parent());
1792 else
1793 temp_grp = nullptr;
1794
1795 }
1796 return is_dim_nonc4_grp;
1797}
HDF5Array.h
A class for handling all types of array in HDF5 for the default option.
HDF5Str.h
This class that translates HDF5 string into DAP string for the default option.
HDF5Structure.h
This class converts HDF5 compound type into DAP structure for the default option.
HDF5Array::set_numelm
void set_numelm(hsize_t nelms)
remembers number of elements in this array.
Definition HDF5Array.cc:1686
HDF5Array::ptr_duplicate
libdap::BaseType * ptr_duplicate() override
Definition HDF5Array.cc:48
HDF5Array::read
bool read() override
Reads HDF5 array data into local buffer.
Definition HDF5Array.cc:103
HDF5Array::set_numdim
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition HDF5Array.cc:1682
HDF5Array::HDF5Array
HDF5Array(const std::string &n, const std::string &d, libdap::BaseType *v)
Constructor.
Definition HDF5Array.cc:53
HDF5Array::set_memneed
void set_memneed(size_t need)
remembers memory size needed.
Definition HDF5Array.cc:1678
get_data
void get_data(hid_t dset, void *buf)
Definition dmrpp_module/h5common.cc:50
DODS_NAMELEN
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition hdf5_handler.h:65