bes Updated for version 3.21.1
The Backend Server (BES) is the lower two tiers of the Hyrax data server
HDF5RequestHandler.cc
Go to the documentation of this file.
1// This file is part of hdf5_handler, a data handler for the OPeNDAP data
2// server.
3
4// Copyright (c) 2002,2003 OPeNDAP, Inc.
5// Copyright (c) 2007-2023 The HDF Group, Inc. and OPeNDAP, Inc.
6// Authors:
7// Kent Yang <myang6@hdfgroup.org>
8// James Gallagher <jgallagher@opendap.org>
9//
10// This is free software; you can redistribute it and/or modify it under the
11// terms of the GNU Lesser General Public License as published by the Free
12// Software Foundation; either version 2.1 of the License, or (at your
13// option) any later version.
14//
15// This software is distributed in the hope that it will be useful, but
16// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18// License for more details.
19//
20// You should have received a copy of the GNU Lesser General Public
21// License along with this library; if not, write to the Free Software
22// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23//
24// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25// You can contact The HDF Group, Inc. at 410 E University Ave,
26// Suite 200, Champaign, IL 61820
27
28// HDF5RequestHandler.cc
34
35#include <iostream>
36#include <string>
37#include <unordered_map>
38
39#include <sys/stat.h>
40#include <fcntl.h>
41#include <unistd.h>
42
43#include <libdap/DMR.h>
44#include <libdap/D4BaseTypeFactory.h>
45#include <ObjMemCache.h>
46#include "HDF5_DMR.h"
47
48#include <libdap/mime_util.h>
49#include "hdf5_handler.h"
50#include "HDF5RequestHandler.h"
51#include "HDF5CFModule.h"
52#include "HDF5_DDS.h"
53
54#include <BESDASResponse.h>
55#include <libdap/Ancillary.h>
56#include <BESInfo.h>
57#include <BESDapNames.h>
58#include <BESResponseNames.h>
59#include <BESVersionInfo.h>
60#include <BESServiceRegistry.h>
61#include <BESUtil.h>
62#include <BESDapError.h>
63#include <BESInternalFatalError.h>
64#include <BESSyntaxUserError.h>
65#include <TheBESKeys.h>
66#include <BESDebug.h>
67#include <BESStopWatch.h>
68#include "h5get.h"
69
70#define HDF5_NAME "h5"
71#include "h5cfdaputil.h"
72
73using namespace std;
74using namespace libdap;
75
76
77#define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
78
79// The debug function to dump all the contents of a DAS table.
80void get_attr_contents(AttrTable* temp_table);
81
82// Five functions to generate a DAS cache file.
83// 1. The wrapper function to call write_das_table_to_file to generate the cache.
84void write_das_to_file(DAS*das_ptr,FILE* das_file);
85
86// 2. The main function to generate the DAS cache.
87void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
88
89// 3. The function to write the DAS container/table name to the cache
90void write_container_name_to_file(const string&,FILE* das_file);
91
92// 4. The function to write the DAS attribute name,type and values to the cache
93void write_das_attr_info(AttrTable* dtp,const string&,const string&,FILE * das_file);
94
95// 5. The function to copy a string to a memory buffer.
96char* copy_str(char *temp_ptr,const string & str);
97
98// These two functions are for reading the DAS from the DAS cache file
99// 1. Obtain the string from a memory buffer.
100char* obtain_str(char*temp_ptr,string & str);
101
102// 2. The main function to obtain DAS info from the cache.
103char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at);
104
105
106// Obtain the BES key as an integer
107static unsigned int get_uint_key(const string &key,unsigned int def_val);
108static unsigned long get_ulong_key(const string &key,unsigned long def_val);
109
110// Obtain the BES key as a floating-pointer number.
111static float get_float_key(const string &key, float def_val);
112
113// Obtain the BES key as a string.
114static string get_beskeys(const string&);
115
116// Obtain the BES key info.
117bool obtain_beskeys_info(const string &, bool &);
118
119// For the CF option to read DAS, DDS and DMR.
120extern void read_cfdas(DAS &das, const string & filename,hid_t fileid);
121extern void read_cfdds(DDS &dds, const string & filename,hid_t fileid);
122extern void read_cfdmr(DMR *dmr, const string & filename,hid_t fileid);
123
124// Check the description of cache_entries and cache_purge_level at h5.conf.in.
125unsigned int HDF5RequestHandler::_mdcache_entries = 500;
126unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
127unsigned int HDF5RequestHandler::_srdcache_entries = 0;
128float HDF5RequestHandler::_cache_purge_level = 0.2F;
129
130// Metadata object cache at DAS,DDS and DMR.
131ObjMemCache *HDF5RequestHandler::das_cache = nullptr;
132ObjMemCache *HDF5RequestHandler::dds_cache = nullptr;
133ObjMemCache *HDF5RequestHandler::datadds_cache = nullptr;
134ObjMemCache *HDF5RequestHandler::dmr_cache = nullptr;
135
136ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = nullptr;
137ObjMemCache *HDF5RequestHandler::srdata_mem_cache = nullptr;
138
139// Set default values of all BES keys according to h5.conf.in.
140// This will help the dmrpp module. No need to
141// set multiple keys if the user's setting is the same as
142// the h5.conf.in. KY 2021-08-23
143
144// Note the CF option is still true by default. This may change if more users move to DAP4. KY 2023-07-11
145bool HDF5RequestHandler::_usecf = true;
146bool HDF5RequestHandler::_pass_fileid = false;
147bool HDF5RequestHandler::_disable_structmeta = true;
148bool HDF5RequestHandler::_disable_ecsmeta = false;
149bool HDF5RequestHandler::_keep_var_leading_underscore = false;
150bool HDF5RequestHandler::_check_name_clashing = false;
151bool HDF5RequestHandler::_add_path_attrs = true;
152bool HDF5RequestHandler::_drop_long_string = true;
153bool HDF5RequestHandler::_fillvalue_check = true;
154bool HDF5RequestHandler::_check_ignore_obj = false;
155bool HDF5RequestHandler::_flatten_coor_attr = true;
156bool HDF5RequestHandler::_default_handle_dimension = true; //Ignored when _usecf=true.
157bool HDF5RequestHandler::_eos5_rm_convention_attr_path = true;
158bool HDF5RequestHandler::_dmr_long_int = true;
159bool HDF5RequestHandler::_no_zero_size_fullnameattr = false;
160bool HDF5RequestHandler::_enable_coord_attr_add_path = true;
161
162bool HDF5RequestHandler::_usecfdmr = true;
163bool HDF5RequestHandler::_add_dap4_coverage = true;
164
165bool HDF5RequestHandler::_common_cache_dirs = false;
166
167bool HDF5RequestHandler::_use_disk_cache = false;
168bool HDF5RequestHandler::_use_disk_dds_cache = false;
169string HDF5RequestHandler::_disk_cache_dir;
170string HDF5RequestHandler::_disk_cachefile_prefix;
171unsigned long long HDF5RequestHandler::_disk_cache_size = 0;
172
173bool HDF5RequestHandler::_disk_cache_comp_data = false;
174bool HDF5RequestHandler::_disk_cache_float_only_comp_data = false;
175float HDF5RequestHandler::_disk_cache_comp_threshold = 1.0;
176unsigned long HDF5RequestHandler::_disk_cache_var_size = 0;
177
178bool HDF5RequestHandler::_use_disk_meta_cache = false;
179string HDF5RequestHandler::_disk_meta_cache_path;
180
181bool HDF5RequestHandler::_use_latlon_disk_cache = false;
182long HDF5RequestHandler::_latlon_disk_cache_size = 0;
183string HDF5RequestHandler::_latlon_disk_cache_dir;
184string HDF5RequestHandler::_latlon_disk_cachefile_prefix;
185
186bool HDF5RequestHandler::_escape_utf8_attr = true;
187
188DMR* HDF5RequestHandler::dmr_int64 = nullptr;
189
190string HDF5RequestHandler::_stp_east_filename;
191string HDF5RequestHandler::_stp_north_filename;
192vector<string> HDF5RequestHandler::lrd_cache_dir_list;
193vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
194vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
195
196
197HDF5RequestHandler::HDF5RequestHandler(const string & name)
198 :BESRequestHandler(name)
199{
200 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
201
202 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
203 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
204 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
205 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
206 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
207
208 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
209 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
210
211 // DYNAMIC_CONFIG_ENABLED is defined by TheBESKeys.h. As of 3/11/22, this
212 // is 0 so the dynamic keys feature is not used. jhrg 3/8/22
213#if !(DYNAMIC_CONFIG_ENABLED)
214 load_config();
215#endif
216
217 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
218}
219
220HDF5RequestHandler::~HDF5RequestHandler()
221{
222 // delete the cache.
223 delete das_cache;
224 delete dds_cache;
225 delete datadds_cache;
226 delete dmr_cache;
227 delete lrdata_mem_cache;
228 delete srdata_mem_cache;
229
230}
231
235void HDF5RequestHandler::load_config()
236{
237 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
238 BES_STOPWATCH_START(HDF5_NAME, prolog + "ClockTheBESKeys");
239
240 // Obtain the metadata cache entries and purge level.
241 HDF5RequestHandler::_mdcache_entries = get_uint_key("H5.MetaDataMemCacheEntries", 0);
242 HDF5RequestHandler::_lrdcache_entries = get_uint_key("H5.LargeDataMemCacheEntries", 0);
243 HDF5RequestHandler::_srdcache_entries = get_uint_key("H5.SmallDataMemCacheEntries", 0);
244 HDF5RequestHandler::_cache_purge_level = get_float_key("H5.CachePurgeLevel", 0.2F);
245
246 if (get_mdcache_entries()) { // else it stays at its default of null
247 das_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
248 dds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
249 datadds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
250 dmr_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
251 }
252
253 // Starting from hyrax 1.16.5, users don't need to explicitly set the BES keys if
254 // they are happy with the default settings in the h5.conf.in.
255 // In the previous releases, we required users to set BES key values. Otherwise,
256 // the BES key values of true/false will always be false if the key is not found.
257 // Hopefully this change will make it convenient for the general users.
258 // In the meantime, the explicit BES key settings will not be affected.
259 // KY 2021-10-13
260 //
261 // Check if the EnableCF key is set.
262 bool has_key = false;
263 bool key_value = obtain_beskeys_info("H5.EnableCF",has_key);
264 if (has_key)
265 _usecf = key_value;
266 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCF: " << (_usecf?"true":"false") << endl);
267
268 // The DefaultHandleDimension is effective only when EnableCF=false
269 key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
270 if (has_key)
271 _default_handle_dimension = key_value;
272 BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
273
274 // The following keys are only effective when EnableCF is true or unset(EnableCF is true if users don't set the key).
275 key_value = obtain_beskeys_info("H5.EnablePassFileID",has_key);
276 if (has_key)
277 _pass_fileid = key_value;
278 BESDEBUG(HDF5_NAME, prolog << "H5.EnablePassFileID: " << (_pass_fileid?"true":"false") << endl);
279
280 key_value = obtain_beskeys_info("H5.DisableStructMetaAttr",has_key);
281 if (has_key)
282 _disable_structmeta = key_value;
283 BESDEBUG(HDF5_NAME, prolog << "H5.DisableStructMetaAttr: " << (_disable_structmeta?"true":"false") << endl);
284
285 key_value = obtain_beskeys_info("H5.DisableECSMetaAttr",has_key);
286 if (has_key)
287 _disable_ecsmeta = key_value;
288 BESDEBUG(HDF5_NAME, prolog << "H5.DisableECSMetaAttr: " << (_disable_ecsmeta?"true":"false") << endl);
289
290 key_value = obtain_beskeys_info("H5.KeepVarLeadingUnderscore",has_key);
291 if (has_key)
292 _keep_var_leading_underscore = key_value;
293 BESDEBUG(HDF5_NAME, prolog << "H5.KeepVarLeadingUnderscore: " << (_keep_var_leading_underscore?"true":"false") << endl);
294
295 key_value = obtain_beskeys_info("H5.EnableCheckNameClashing",has_key);
296 if (has_key)
297 _check_name_clashing = key_value;
298 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCheckNameClashing: " << (_check_name_clashing?"true":"false") << endl);
299
300 key_value = obtain_beskeys_info("H5.EnableAddPathAttrs",has_key);
301 if (has_key)
302 _add_path_attrs = key_value;
303 BESDEBUG(HDF5_NAME, prolog << "H5.EnableAddPathAttrs: " << (_add_path_attrs?"true":"false") << endl);
304
305 key_value = obtain_beskeys_info("H5.EnableDropLongString",has_key);
306 if (has_key)
307 _drop_long_string = key_value;
308 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDropLongString: " << (_drop_long_string?"true":"false") << endl);
309
310 key_value = obtain_beskeys_info("H5.EnableFillValueCheck",has_key);
311 if (has_key)
312 _fillvalue_check = key_value;
313 BESDEBUG(HDF5_NAME, prolog << "H5.EnableFillValueCheck: " << (_fillvalue_check?"true":"false") << endl);
314
315
316 key_value = obtain_beskeys_info("H5.CheckIgnoreObj",has_key);
317 if (has_key)
318 _check_ignore_obj = key_value;
319 BESDEBUG(HDF5_NAME, prolog << "H5.CheckIgnoreObj: " << (_check_ignore_obj?"true":"false") << endl);
320
321 key_value = obtain_beskeys_info("H5.ForceFlattenNDCoorAttr",has_key);
322 if (has_key)
323 _flatten_coor_attr = key_value;
324 BESDEBUG(HDF5_NAME, prolog << "H5.ForceFlattenNDCoorAttr: " << (_flatten_coor_attr?"true":"false") << endl);
325
326 key_value = obtain_beskeys_info("H5.RmConventionAttrPath",has_key);
327 if (has_key)
328 _eos5_rm_convention_attr_path = key_value;
329 BESDEBUG(HDF5_NAME, prolog << "H5.RmConventionAttrPath: " << (_eos5_rm_convention_attr_path?"true":"false") << endl);
330
331 key_value = obtain_beskeys_info("H5.EnableDMR64bitInt",has_key);
332 if (has_key)
333 _dmr_long_int = key_value;
334 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDMR64bitInt: " << (_dmr_long_int?"true":"false") << endl);
335
336 key_value = obtain_beskeys_info("H5.NoZeroSizeFullnameAttr",has_key);
337 if (has_key)
338 _no_zero_size_fullnameattr = key_value;
339 BESDEBUG(HDF5_NAME, prolog << "H5.NoZeroSizeFullnameAttr: " << (_no_zero_size_fullnameattr?"true":"false") << endl);
340
341 key_value = obtain_beskeys_info("H5.EnableCoorattrAddPath",has_key);
342 if (has_key)
343 _enable_coord_attr_add_path = key_value;
344 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCoorattrAddPath: " << (_enable_coord_attr_add_path?"true":"false") << endl);
345
346 // This is a critical key to ask the handler to generate the DMR directly for the CF option.
347 // If this key is not true, the DMR will be generated via DDS and DAS. 64-bit integer mapping may be ignored or
348 // incomplete. KY 2023-07-11
349 key_value = obtain_beskeys_info("H5.EnableCFDMR",has_key);
350 if (has_key)
351 _usecfdmr = key_value;
352 BESDEBUG(HDF5_NAME, prolog << "H5.EnableCFDMR: " << (_usecfdmr?"true":"false") << endl);
353
354 key_value = obtain_beskeys_info("H5.EnableDAP4Coverage",has_key);
355 if (has_key)
356 _add_dap4_coverage = key_value;
357 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDAP4Coverage: " << (_add_dap4_coverage?"true":"false") << endl);
358
359 // Load a bunch of keys to set the disk cache. Currently it only applies to CF but in the future it may
360 // apply to the default option. KY 2023-07-11
361 load_config_disk_cache();
362
363 // The UTF8 escaping key applies to both CF and default options.
364 key_value = obtain_beskeys_info("H5.EscapeUTF8Attr", has_key);
365 if (has_key)
366 _escape_utf8_attr = key_value;
367
368 // load the cache keys when CF option is set.
369 if (get_usecf())
370 load_config_cf_cache();
371
372 _stp_east_filename = get_beskeys("H5.STPEastFileName");
373 _stp_north_filename = get_beskeys("H5.STPNorthFileName");
374 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
375}
376
377void HDF5RequestHandler::load_config_disk_cache() {
378
379 bool has_key = false;
380 bool key_value = obtain_beskeys_info("H5.EnableDiskDataCache",has_key);
381 if (has_key)
382 _use_disk_cache = key_value;
383 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDataCache: " << (_use_disk_cache?"true":"false") << endl);
384
385 _disk_cache_dir = get_beskeys("H5.DiskCacheDataPath");
386 _disk_cachefile_prefix = get_beskeys("H5.DiskCacheFilePrefix");
387 _disk_cache_size = get_ulong_key("H5.DiskCacheSize",0);
388
389 key_value = obtain_beskeys_info("H5.DiskCacheComp",has_key);
390 if (has_key)
391 _disk_cache_comp_data = key_value;
392 BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheComp: " << (_disk_cache_comp_data?"true":"false") << endl);
393
394 key_value = obtain_beskeys_info("H5.DiskCacheFloatOnlyComp",has_key);
395 if (has_key)
396 _disk_cache_float_only_comp_data = key_value;
397 BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheFloatOnlyComp: " << (_disk_cache_float_only_comp_data?"true":"false") << endl);
398 _disk_cache_comp_threshold = get_float_key("H5.DiskCacheCompThreshold",1.0);
399 _disk_cache_var_size = 1024*get_uint_key("H5.DiskCacheCompVarSize",0);
400
401 key_value = obtain_beskeys_info("H5.EnableDiskMetaDataCache",has_key);
402 if (has_key)
403 _use_disk_meta_cache = key_value;
404 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskMetaDataCache: " << (_use_disk_meta_cache?"true":"false") << endl);
405
406 key_value = obtain_beskeys_info("H5.EnableDiskDDSCache",has_key);
407 if (has_key)
408 _use_disk_dds_cache = key_value;
409 BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDDSCache: " << (_use_disk_dds_cache?"true":"false") << endl);
410 _disk_meta_cache_path = get_beskeys("H5.DiskMetaDataCachePath");
411
412 key_value = obtain_beskeys_info("H5.EnableEOSGeoCacheFile",has_key);
413 if (has_key)
414 _use_latlon_disk_cache = key_value;
415 BESDEBUG(HDF5_NAME, prolog << "H5.EnableEOSGeoCacheFile: " << (_use_latlon_disk_cache?"true":"false") << endl);
416 _latlon_disk_cache_size = get_uint_key("H5.Cache.latlon.size",0);
417 _latlon_disk_cache_dir = get_beskeys("H5.Cache.latlon.path");
418 _latlon_disk_cachefile_prefix= get_beskeys("H5.Cache.latlon.prefix");
419
420 if (_disk_cache_comp_data == true && _use_disk_cache == true) {
421 if (_disk_cache_comp_threshold < 1.0) {
422 ostringstream ss;
423 ss<< _disk_cache_comp_threshold;
424 string _comp_threshold_str(ss.str());
425 string invalid_comp_threshold ="The Compression Threshold is the total size of the variable array";
426 invalid_comp_threshold+=" divided by the storage size of compressed array. It should always be >1";
427 invalid_comp_threshold+=" The current threshold set at h5.conf is ";
428 invalid_comp_threshold+=_comp_threshold_str;
429 invalid_comp_threshold+=" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
430 throw BESInternalError(invalid_comp_threshold,__FILE__,__LINE__);
431 }
432 }
433}
434
435void HDF5RequestHandler::load_config_cf_cache() {
436
437 bool has_key = false;
438 if (get_lrdcache_entries()) {
439
440 lrdata_mem_cache = new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
441 bool has_LFMC_config = false;
442 bool key_value = obtain_beskeys_info("H5.LargeDataMemCacheConfig",has_key);
443 if (has_key)
444 has_LFMC_config = key_value;
445 BESDEBUG(HDF5_NAME, prolog << "H5.LargeDataMemCacheConfig: " << (has_LFMC_config?"true":"false") << endl);
446 if (true == has_LFMC_config)
447 _common_cache_dirs =obtain_lrd_common_cache_dirs();
448 }
449 if (get_srdcache_entries()) {
450
451 BESDEBUG(HDF5_NAME, prolog << "Generate memory cache for smaller coordinate variables" << endl);
452 srdata_mem_cache = new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
453
454 }
455
456
457}
458
459// Build DAS
460bool HDF5RequestHandler::hdf5_build_das(BESDataHandlerInterface & dhi)
461{
462 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
463#if DYNAMIC_CONFIG_ENABLED
464 load_config();
465#endif
466
467 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
468 hid_t cf_fileid = -1;
469
470 // Obtain the HDF5 file name.
471 string filename = dhi.container->access();
472
473 // Obtain the BES object from the client
474 BESResponseObject *response = dhi.response_handler->get_response_object() ;
475
476 // Convert to the BES DAS response
477 auto bdas = dynamic_cast < BESDASResponse * >(response) ;
478 if( !bdas )
479 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
480
481 try {
482 bdas->set_container( dhi.container->get_symbolic_name() ) ;
483 DAS *das = bdas->get_das();
484
485 // Look inside the memory cache to see if it's initialized
486 DAS *cached_das_ptr = nullptr;
487 bool use_das_cache = false;
488 if (das_cache)
489 cached_das_ptr = dynamic_cast<DAS*>(das_cache->get(filename));
490 if (cached_das_ptr)
491 use_das_cache = true;
492
493 if (true == use_das_cache) {
494
495 // copy the cached DAS into the BES response object
496 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
497 *das = *cached_das_ptr;
498 }
499 else
500 hdf5_build_das_internal_no_mem_cache(filename, das, cf_fileid);
501 bdas->clear_container() ;
502 }
503 catch(const BESSyntaxUserError & e) {
504 if (cf_fileid !=-1)
505 H5Fclose(cf_fileid);
506 BESDEBUG(HDF5_NAME, prolog << "Caught BESSyntaxUserError! Message: " << e.get_message() << endl);
507 throw;
508 }
509
510 catch(const BESError & e) {
511 if (cf_fileid !=-1)
512 H5Fclose(cf_fileid);
513 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
514 throw;
515 }
516 catch(const InternalErr & e) {
517 if (cf_fileid !=-1)
518 H5Fclose(cf_fileid);
519 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
520 __FILE__, __LINE__);
521 }
522 catch(const Error & e) {
523 if (cf_fileid !=-1)
524 H5Fclose(cf_fileid);
525 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
526 __FILE__, __LINE__);
527 }
528 catch(...) {
529 if (cf_fileid !=-1)
530 H5Fclose(cf_fileid);
531 string s = "unknown exception caught building HDF5 DAS";
532 throw BESInternalFatalError(s, __FILE__, __LINE__);
533 }
534
535 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
536 return true;
537}
538
539void HDF5RequestHandler::hdf5_build_das_internal_no_mem_cache(const string& filename, DAS *das, hid_t &cf_fileid)
540{
541 bool das_from_dc = false;
542 string das_cache_fname;
543
544 // If the use_disk_meta_cache is set, check if the cache file exists and sets the flag.
545 if (_use_disk_meta_cache == true) {
546
547 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
548 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
549
550 if (access(das_cache_fname.c_str(),F_OK) !=-1)
551 das_from_dc = true;
552
553 }
554
555 // If reading DAS from the disk cache, call the corresponding function
556 if (true == das_from_dc) {
557 read_das_from_disk_cache(das_cache_fname,das);
558
559 // If the memory cache is set, adding the DAS copy to the memory cache
560 if (das_cache) {
561 // add a copy
562 BESDEBUG(HDF5_NAME, prolog << "HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
563 das_cache->add(new DAS(*das), filename);
564 }
565 }
566
567 else {// Need to build from the HDF5 file
568 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
569 if (true == _usecf) {//CF option
570
571 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
572 if (cf_fileid < 0){
573 string invalid_file_msg="Could not open this HDF5 file ";
574 invalid_file_msg +=filename;
575 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
576 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
577 invalid_file_msg +=" distributor.";
578 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
579 }
580 // Need to check if DAP4 DMR CF 64-bit integer mapping is on.
581 if (HDF5RequestHandler::get_dmr_64bit_int()!=nullptr)
582 HDF5RequestHandler::set_dmr_64bit_int(nullptr);
583 read_cfdas( *das,filename,cf_fileid);
584 H5Fclose(cf_fileid);
585 }
586 else {// Default option
587 hid_t fileid = get_fileid(filename.c_str());
588 if (fileid < 0) {
589 string invalid_file_msg="Could not open this HDF5 file ";
590 invalid_file_msg +=filename;
591 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
592 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
593 invalid_file_msg +=" distributor.";
594 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
595 }
596
597 find_gloattr(fileid, *das);
598 depth_first(fileid, "/", *das);
599 close_fileid(fileid);
600 }
601
602 Ancillary::read_ancillary_das( *das, filename ) ;
603
604#if 0
605// Dump all attribute contents
606AttrTable* top_table = das->get_top_level_attributes();
607get_attr_contents(top_table);
608
609// Dump all variable contents
610AttrTable::Attr_iter start_aiter=das->var_begin();
611AttrTable::Attr_iter it = start_aiter;
612AttrTable::Attr_iter end_aiter = das->var_end();
613while(it != end_aiter) {
614AttrTable* temp_table = das->get_table(it);
615if(temp_table!=0){
616cerr<<"var_begin"<<endl;
617temp_table->print(cerr);
618}
619++it;
620}
621#endif
622 // If the memory cache is turned on
623 if (das_cache) {
624 // add a copy
625 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
626 das_cache->add(new DAS(*das), filename);
627 }
628
629 // DAS disk cache fname will be set only when the metadata disk cache is turned on
630 // So if it comes here, the das cache should be generated.
631 if (das_cache_fname.empty() == false) {
632 BESDEBUG(HDF5_NAME, prolog << "HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
633 write_das_to_disk_cache(das_cache_fname,das);
634 }
635 }
636}
637// Convenient function that helps build DDS and Data
638// Since this function will be used by both the DDS and Data services, we need to pass both BESDDSResponse
639// and BESDataDDSResponse. These two parameters are necessary for the future DDS disk cache feature.
640void HDF5RequestHandler::get_dds_with_attributes( BESDDSResponse*bdds,BESDataDDSResponse*data_bdds,
641 const string &container_name, const string& filename,
642 const string &dds_cache_fname, const string &das_cache_fname,
643 bool dds_from_dc,bool das_from_dc, bool build_data)
644{
645 DDS *dds;
646 if (true == build_data)
647 dds = data_bdds->get_dds();
648 else
649 dds = bdds->get_dds();
650
651 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
652 hid_t fileid = -1;
653 hid_t cf_fileid = -1;
654
655 try {
656
657 // Look in memory cache to see if it's initialized
658 const DDS* cached_dds_ptr = nullptr;
659 bool use_dds_cache = false;
660 if (dds_cache)
661 cached_dds_ptr = dynamic_cast<DDS*>(dds_cache->get(filename));
662 if (cached_dds_ptr)
663 use_dds_cache = true;
664 if (true == use_dds_cache) {
665 // copy the cached DDS into the BES response object. Assume that any cached DDS
666 // includes the DAS information.
667 BESDEBUG(HDF5_NAME, prolog << "DDS Metadata Cached hit for : " << filename << endl);
668 *dds = *cached_dds_ptr; // Copy the referenced object
669 }
670 else if (true ==dds_from_dc) {//Currently the dds_from_ds is always false by default.
671 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,
672 das_cache_fname,-1,das_from_dc);
673 }
674 else {
675 read_dds_from_file(dds, filename, cf_fileid, fileid, dds_cache_fname, dds_from_dc);
676
677 // Add attributes
678 add_das_to_dds_wrapper(dds,filename,cf_fileid,fileid,container_name,das_cache_fname,das_from_dc);
679
680 // Add memory cache if possible
681 if (dds_cache) {
682 // add a copy
683 BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
684 dds_cache->add(new DDS(*dds), filename);
685 }
686
687 H5Fclose(cf_fileid);
688 H5Fclose(fileid);
689
690 }
691
692 }
693 catch(const InternalErr & e) {
694 close_h5_files(cf_fileid, fileid);
695 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
696 __FILE__, __LINE__);
697 }
698 catch(const Error & e) {
699 close_h5_files(cf_fileid, fileid);
700 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
701 __FILE__, __LINE__);
702 }
703 catch(const BESSyntaxUserError & e) {
704 close_h5_files(cf_fileid, fileid);
705 BESDEBUG(HDF5_NAME, prolog << "Caught BESSyntaxUserError! Message: " << e.get_message() << endl);
706 throw;
707 }
708 catch(...) {
709 close_h5_files(cf_fileid, fileid);
710 string s = "unknown exception caught building HDF5 DDS";
711 throw BESInternalFatalError(s, __FILE__, __LINE__);
712 }
713}
714
715void HDF5RequestHandler::read_dds_from_file(DDS *dds, const string &filename, hid_t &cf_fileid, hid_t &fileid,
716 const string &dds_cache_fname, bool dds_from_dc)
717{
718 BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
719 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
720 dds->filename(filename);
721
722 // For the time being, not mess up CF's fileID with Default's fileID
723 if (true == _usecf) {
724
725 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
726 if (cf_fileid < 0){
727 string invalid_file_msg="Could not open this HDF5 file ";
728 invalid_file_msg +=filename;
729 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
730 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
731 invalid_file_msg +=" distributor.";
732 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
733 }
734
735 // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
736 // to let the handler map the 64-bit integer.
737 if(HDF5RequestHandler::get_dmr_64bit_int() != nullptr)
738 HDF5RequestHandler::set_dmr_64bit_int(nullptr);
739 read_cfdds(*dds,filename,cf_fileid);
740 }
741 else {
742
743 fileid = get_fileid(filename.c_str());
744 if (fileid < 0) {
745 string invalid_file_msg="Could not open this HDF5 file ";
746 invalid_file_msg +=filename;
747 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
748 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
749 invalid_file_msg +=" distributor.";
750 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
751 }
752
753 depth_first(fileid, (char*)"/", *dds, filename.c_str());
754
755 }
756 // Check semantics
757 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
758 dds->print(cerr);
759 throw InternalErr(__FILE__, __LINE__,
760 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
761 }
762
763 Ancillary::read_ancillary_dds( *dds, filename ) ;
764
765 // Generate the DDS cached file if needed,currently this is always false by default
766 if(dds_cache_fname.empty() == false && dds_from_dc == false)
767 write_dds_to_disk_cache(dds_cache_fname,dds);
768
769}
770
771void HDF5RequestHandler::add_das_to_dds_wrapper(DDS *dds, const string &filename, hid_t cf_fileid, hid_t fileid,
772 const string &container_name, const string &das_cache_fname,
773 bool das_from_dc) {
774 hid_t h5_fd = -1;
775 if (_usecf == true)
776 h5_fd = cf_fileid;
777 else
778 h5_fd = fileid;
779
780 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
781}
782
783void HDF5RequestHandler::get_dds_without_attributes_datadds(BESDataDDSResponse*data_bdds, const string& filename)
784{
785 DDS *dds = data_bdds->get_dds();
786
787 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
788 hid_t fileid = -1;
789 hid_t cf_fileid = -1;
790
791 try {
792
793 // Look in memory cache to see if it's initialized
794 const DDS* cached_dds_ptr = nullptr;
795 bool use_datadds_cache = false;
796
797 if (datadds_cache)
798 cached_dds_ptr = dynamic_cast<DDS*>(datadds_cache->get(filename));
799
800 if (cached_dds_ptr)
801 use_datadds_cache = true;
802 if (true == use_datadds_cache) {
803 // copy the cached DDS into the BES response object.
804 // The DAS information is not included.
805 BESDEBUG(HDF5_NAME, prolog << "DataDDS Metadata Cached hit for : " << filename << endl);
806 *dds = *cached_dds_ptr; // Copy the referenced object
807 }
808 else
809 read_datadds_from_file(dds, filename, cf_fileid, fileid);
810
811 BESDEBUG(HDF5_NAME, prolog << "Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
812 data_bdds->set_ia_flag(false);
813
814 }
815 catch(const BESSyntaxUserError & e) {
816
817 BESDEBUG(HDF5_NAME, prolog << "Caught BESSyntaxUserError! Message: " << e.get_message() << endl);
818 close_h5_files(cf_fileid, fileid);
819 throw;
820 }
821
822 catch(const InternalErr & e) {
823
824 close_h5_files(cf_fileid, fileid);
825 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
826 __FILE__, __LINE__);
827 }
828 catch(const Error & e) {
829
830 close_h5_files(cf_fileid, fileid);
831 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
832 __FILE__, __LINE__);
833 }
834 catch(...) {
835
836 close_h5_files(cf_fileid, fileid);
837 string s = "unknown exception caught building HDF5 DDS";
838 throw BESInternalFatalError(s, __FILE__, __LINE__);
839 }
840}
841
842void HDF5RequestHandler::read_datadds_from_file(DDS *dds, const string &filename, hid_t &cf_fileid, hid_t &fileid)
843{
844 BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
845 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
846 dds->filename(filename);
847
848 // For the time being, not mess up CF's fileID with Default's fileID
849 if (true == _usecf) {
850
851 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
852 if (cf_fileid < 0){
853 string invalid_file_msg="Could not open this HDF5 file ";
854 invalid_file_msg +=filename;
855 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
856 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
857 invalid_file_msg +=" distributor.";
858 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
859 }
860 // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
861 // to let the handler map the 64-bit integer.
862 if (HDF5RequestHandler::get_dmr_64bit_int() != nullptr)
863 HDF5RequestHandler::set_dmr_64bit_int(nullptr);
864 read_cfdds(*dds,filename,cf_fileid);
865 }
866 else {
867 fileid = get_fileid(filename.c_str());
868 if (fileid < 0) {
869 string invalid_file_msg="Could not open this HDF5 file ";
870 invalid_file_msg +=filename;
871 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
872 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
873 invalid_file_msg +=" distributor.";
874 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
875 }
876
877 depth_first(fileid, (char*)"/", *dds, filename.c_str());
878 }
879 // Check semantics
880 if (!dds->check_semantics()) { // DDS didn't comply with the semantics
881 dds->print(cerr);
882 throw InternalErr(__FILE__, __LINE__,
883 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
884 }
885 Ancillary::read_ancillary_dds( *dds, filename ) ;
886
887 // Add memory cache if possible
888 if (datadds_cache) {
889 // add a copy
890 BESDEBUG(HDF5_NAME, prolog << "DataDDS added to the cache for : " << filename << endl);
891 datadds_cache->add(new DDS(*dds), filename);
892 }
893
894 close_h5_files(cf_fileid, fileid);
895
896}
897
898// Build DDS
899bool HDF5RequestHandler::hdf5_build_dds(BESDataHandlerInterface & dhi)
900{
901 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
902#if DYNAMIC_CONFIG_ENABLED
903 load_config();
904#endif
905
906 // Obtain the HDF5 file name.
907 string filename = dhi.container->access();
908
909 string container_name = dhi.container->get_symbolic_name();
910 BESResponseObject *response = dhi.response_handler->get_response_object();
911 auto bdds = dynamic_cast < BESDDSResponse * >(response);
912 if( !bdds )
913 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
914 bdds->set_container(container_name);
915
916 try {
917
918 bool dds_from_dc = false;
919 bool das_from_dc = false;
920 bool build_data = false;
921 string dds_cache_fname;
922 string das_cache_fname;
923
924 if (_use_disk_meta_cache) {
925
926 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
927
928 // The _use_disk_dds_cache is always set to false by default
929 if (_use_disk_dds_cache) {
930 dds_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_dds";
931 if (access(dds_cache_fname.c_str(),F_OK) !=-1)
932 dds_from_dc = true;
933 }
934
935 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
936 // Check if das files exist
937 if (access(das_cache_fname.c_str(),F_OK) !=-1)
938 das_from_dc = true;
939 }
940
941 get_dds_with_attributes(bdds, nullptr,container_name,filename, dds_cache_fname,das_cache_fname,
942 dds_from_dc,das_from_dc,build_data);
943
944 bdds->set_constraint( dhi ) ;
945 bdds->clear_container() ;
946
947 }
948
949 catch(const BESSyntaxUserError & e) {
950 BESDEBUG(HDF5_NAME, prolog << "Caught BESSyntaxUserError! Message: " << e.get_message() << endl);
951 throw;
952 }
953 catch(const BESError & e) {
954 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
955 throw;
956 }
957 catch(const InternalErr & e) {
958 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
959 __FILE__, __LINE__);
960 }
961 catch(const Error & e) {
962 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
963 __FILE__, __LINE__);
964 }
965 catch(...) {
966 string s = "unknown exception caught building HDF5 DDS";
967 throw BESInternalFatalError(s, __FILE__, __LINE__);
968 }
969
970 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
971 return true;
972}
973
974bool HDF5RequestHandler::hdf5_build_data(BESDataHandlerInterface & dhi)
975{
976 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
977#if DYNAMIC_CONFIG_ENABLED
978 load_config();
979#endif
980
981 if (_usecf && _pass_fileid)
982 return hdf5_build_data_with_IDs(dhi);
983
984 string filename = dhi.container->access();
985
986 string container_name = dhi.container->get_symbolic_name();
987 BESResponseObject *response = dhi.response_handler->get_response_object();
988 auto bdds = dynamic_cast < BESDataDDSResponse * >(response);
989 if( !bdds )
990 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
991 bdds->set_container(container_name);
992
993 try {
994
995 // DDS from disk cache is not currently supported. It may be supported in the future.
996 // We will leave the code as commented.
997#if 0
998 bool dds_from_dc = false;
999 bool build_data = true;
1000#endif
1001
1002 // The code inside the following #if 0 #endif is not used anymore. So comment out. KY 2022-12-11
1003#if 0
1004 bool das_from_dc = false;
1005 string dds_cache_fname;
1006 string das_cache_fname;
1007
1008
1009 // Only DAS is read from the cache. dds_from_dc is always false.
1010 if(_use_disk_meta_cache == true) {
1011
1012 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1013 das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1014
1015 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1016 das_from_dc = true;
1017
1018 }
1019#endif
1020
1021 get_dds_without_attributes_datadds(bdds, filename);
1022
1023 bdds->set_constraint( dhi ) ;
1024 bdds->clear_container() ;
1025 }
1026
1027 catch(const BESSyntaxUserError & e) {
1028 BESDEBUG(HDF5_NAME, prolog << "Caught BESSyntaxUserError! Message: " << e.get_message() << endl);
1029 throw;
1030 }
1031 catch(const BESError & e) {
1032 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1033 throw;
1034 }
1035 catch(const InternalErr & e) {
1036 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1037 __FILE__, __LINE__);
1038 }
1039 catch(const Error & e) {
1040 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1041 __FILE__, __LINE__);
1042 }
1043 catch(...) {
1044 string s = "unknown exception caught building HDF5 DDS";
1045 throw BESInternalFatalError(s, __FILE__, __LINE__);
1046 }
1047
1048 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1049 return true;
1050}
1051
1052// Obtain data when turning on the pass fileID key.The memory cache is not used.
1053bool HDF5RequestHandler::hdf5_build_data_with_IDs(BESDataHandlerInterface & dhi)
1054{
1055 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1056#if DYNAMIC_CONFIG_ENABLED
1057 load_config();
1058#endif
1059
1060 BESDEBUG(HDF5_NAME,prolog << "Building DataDDS by passing file IDs. "<<endl);
1061 hid_t cf_fileid = -1;
1062
1063 string filename = dhi.container->access();
1064
1065 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1066 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1067 if (cf_fileid < 0){
1068 string invalid_file_msg="Could not open this HDF5 file ";
1069 invalid_file_msg +=filename;
1070 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1071 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1072 invalid_file_msg +=" distributor.";
1073 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1074 }
1075
1076 BESResponseObject *response = dhi.response_handler->get_response_object();
1077 auto bdds = dynamic_cast < BESDataDDSResponse * >(response);
1078 if( !bdds )
1079 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1080
1081 try {
1082
1083 bdds->set_container( dhi.container->get_symbolic_name() ) ;
1084
1085 auto hdds_unique = make_unique<HDF5DDS>(bdds->get_dds());
1086 delete bdds->get_dds();
1087
1088 auto hdds = hdds_unique.release();
1089 bdds->set_dds(hdds);
1090 hdds->setHDF5Dataset(cf_fileid);
1091
1092 read_cfdds( *hdds,filename,cf_fileid);
1093
1094 if (!hdds->check_semantics()) { // DDS didn't comply with the DAP semantics
1095 hdds->print(cerr);
1096 throw InternalErr(__FILE__, __LINE__,
1097 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1098 }
1099
1100 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1101
1102 auto das_unique = make_unique<DAS>();
1103 auto das = das_unique.release();
1104 BESDASResponse bdas( das ) ;
1105 bdas.set_container( dhi.container->get_symbolic_name() ) ;
1106 read_cfdas( *das,filename,cf_fileid);
1107 Ancillary::read_ancillary_das( *das, filename ) ;
1108
1109 hdds->transfer_attributes(das);
1110 bdds->set_constraint( dhi ) ;
1111 bdds->clear_container() ;
1112
1113 }
1114
1115 catch(const BESSyntaxUserError & e) {
1116 BESDEBUG(HDF5_NAME, prolog << "Caught BESSyntaxUserError! Message: " << e.get_message() << endl);
1117
1118 if(cf_fileid !=-1)
1119 H5Fclose(cf_fileid);
1120 throw;
1121 }
1122 catch(const BESError & e) {
1123 if(cf_fileid !=-1)
1124 H5Fclose(cf_fileid);
1125 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1126 throw;
1127 }
1128 catch(const InternalErr & e) {
1129 if(cf_fileid !=-1)
1130 H5Fclose(cf_fileid);
1131 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1132 __FILE__, __LINE__);
1133 }
1134 catch(const Error & e) {
1135 if(cf_fileid !=-1)
1136 H5Fclose(cf_fileid);
1137 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1138 __FILE__, __LINE__);
1139 }
1140 catch(...) {
1141 if(cf_fileid !=-1)
1142 H5Fclose(cf_fileid);
1143 string s = "unknown exception caught building HDF5 DataDDS";
1144 throw BESInternalFatalError(s, __FILE__, __LINE__);
1145 }
1146
1147 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1148 return true;
1149}
1150
1151bool HDF5RequestHandler::hdf5_build_dmr(BESDataHandlerInterface & dhi)
1152{
1153 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1154#if DYNAMIC_CONFIG_ENABLED
1155 load_config();
1156#endif
1157
1158 // Extract the DMR Response object - this holds the DMR used by the
1159 // other parts of the framework.
1160 BESResponseObject *response = dhi.response_handler->get_response_object();
1161 BESDMRResponse &bes_dmr_response = dynamic_cast<BESDMRResponse &>(*response);
1162
1163 string filename = dhi.container->access();
1164
1165 DMR *dmr = bes_dmr_response.get_dmr();
1166
1167 // For the time being, separate CF file ID from the default file ID(mainly for debugging)
1168 hid_t fileid = -1;
1169 hid_t cf_fileid = -1;
1170
1171 try {
1172
1173 const DMR* cached_dmr_ptr = nullptr;
1174 if (dmr_cache){
1175 BESDEBUG(HDF5_NAME, prolog << "Checking DMR cache for : " << filename << endl);
1176 cached_dmr_ptr = dynamic_cast<DMR*>(dmr_cache->get(filename));
1177 }
1178
1179 if (cached_dmr_ptr) {
1180 // copy the cached DMR into the BES response object
1181 BESDEBUG(HDF5_NAME, prolog << "DMR cache hit for : " << filename << endl);
1182 *dmr = *cached_dmr_ptr; // Copy the referenced object
1183 dmr->set_request_xml_base(bes_dmr_response.get_request_xml_base());
1184 }
1185 else {// No cache
1186 if (true == hdf5_build_dmr_from_file(dhi,bes_dmr_response,dmr, filename, cf_fileid, fileid))
1187 return true;
1188 }// else no cache
1189 }// try
1190 catch(const BESError & e) {
1191 close_h5_files(cf_fileid, fileid);
1192 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1193 throw;
1194 }
1195 catch(const InternalErr & e) {
1196
1197 close_h5_files(cf_fileid, fileid);
1198 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1199 __FILE__, __LINE__);
1200 }
1201 catch(const Error & e) {
1202
1203 close_h5_files(cf_fileid, fileid);
1204 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1205 __FILE__, __LINE__);
1206 }
1207 catch(...) {
1208
1209 close_h5_files(cf_fileid, fileid);
1210 string s = "unknown exception caught building HDF5 DMR";
1211 throw BESInternalFatalError(s, __FILE__, __LINE__);
1212 }
1213
1214 // Instead of fiddling with the internal storage of the DHI object,
1215 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1216 // methods to set the constraints. But, why? Ans: from Patrick is that
1217 // in the 'container' mode of BES each container can have a different
1218 // CE.
1219 bes_dmr_response.set_dap4_constraint(dhi);
1220 bes_dmr_response.set_dap4_function(dhi);
1221 dmr->set_factory(nullptr);
1222
1223 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1224 return true;
1225}
1226
1227bool HDF5RequestHandler::hdf5_build_dmr_from_file(BESDataHandlerInterface & dhi, BESDMRResponse &bes_dmr_response,
1228 DMR *dmr, const string &filename,
1229 hid_t &cf_fileid, hid_t &fileid)
1230{
1231 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1232 D4BaseTypeFactory MyD4TypeFactory;
1233 dmr->set_factory(&MyD4TypeFactory);
1234 if(_escape_utf8_attr == false)
1235 dmr->set_utf8_xml_encoding();
1236
1237 if(true ==_usecf) {// CF option
1238
1239 if(true == _usecfdmr) {
1240
1241 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1242 if (cf_fileid < 0) {
1243 string invalid_file_msg="Could not open this HDF5 file ";
1244 invalid_file_msg +=filename;
1245 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1246 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1247 invalid_file_msg +=" distributor.";
1248 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1249 }
1250 read_cfdmr(dmr,filename,cf_fileid);
1251 H5Fclose(cf_fileid);
1252 bes_dmr_response.set_dap4_constraint(dhi);
1253 bes_dmr_response.set_dap4_function(dhi);
1254 dmr->set_factory(nullptr);
1255
1256 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1257
1258 return true;
1259 }
1260
1261 if(true == _pass_fileid)
1262 return hdf5_build_dmr_with_IDs(dhi);
1263
1264 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1265 if (cf_fileid < 0){
1266 string invalid_file_msg="Could not open this HDF5 file ";
1267 invalid_file_msg +=filename;
1268 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1269 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1270 invalid_file_msg +=" distributor.";
1271 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1272 }
1273
1274 BaseTypeFactory factory;
1275 DDS dds(&factory, name_path(filename), "3.2");
1276 dds.filename(filename);
1277
1278 DAS das;
1279
1280 // For the CF option dmr response, we need to map 64-bit integer separately
1281 // So set the flag to map 64-bit integer.
1282 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1283 read_cfdds( dds,filename,cf_fileid);
1284 if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1285 dds.print(cerr);
1286 throw InternalErr(__FILE__, __LINE__,
1287 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1288 }
1289
1290 read_cfdas(das,filename,cf_fileid);
1291 Ancillary::read_ancillary_das( das, filename ) ;
1292
1293 dds.transfer_attributes(&das);
1294
1296 if(cf_fileid !=-1)
1297 H5Fclose(cf_fileid);
1298
1299 dmr->build_using_dds(dds);
1300
1301
1302 }// "if(true == _usecf)"
1303 else {// default option
1304
1305 // Obtain the HDF5 file ID.
1306 fileid = get_fileid(filename.c_str());
1307 if (fileid < 0) {
1308 string invalid_file_msg="Could not open this HDF5 file ";
1309 invalid_file_msg +=filename;
1310 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1311 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1312 invalid_file_msg +=" distributor.";
1313 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1314 }
1315
1316 // Check if this is an HDF-EOS5 file
1317 bool is_eos5 = check_eos5(fileid);
1318
1319 // Check if dim scale is used.
1320 bool use_dimscale = false;
1321 if (true == _default_handle_dimension)
1322 use_dimscale = check_dimscale(fileid);
1323
1324 // If it is an HDF-EOS5 file but not having dimension scale,
1325 // treat it as an HDF-EOS5 file, retrieve all the HDF-EOS5 info.
1326 eos5_dim_info_t eos5_dim_info;
1327 if (is_eos5 && !use_dimscale)
1328 obtain_eos5_dims(fileid,eos5_dim_info);
1329
1330 dmr->set_name(name_path(filename));
1331 dmr->set_filename(name_path(filename));
1332
1333 // The breadth_first() function builds the variables and attributes and
1334 // loads them into the root group (building child groups as needed).
1335 // jhrg 4/30/20
1336 D4Group* root_grp = dmr->root();
1337 BESDEBUG("h5", "use_dimscale is "<< use_dimscale <<endl);
1338
1339 // It is possible that a dimension variable has hardlinks. To make it
1340 // right for the netCDF-4 data model and the current DAP4 implementation,
1341 // we need to choose the shortest path of all hardlinks as the dimension path.
1342 // So to avoid iterate all HDF5 objects multiple times, save the found
1343 // hardlinks and search them when necessary. Note we have to search hardlinks from the root.
1344 // KY 2021-11-15
1345 vector<link_info_t> hdf5_hls;
1346 vector<string> handled_coord_names;
1347
1348 breadth_first(fileid, fileid,(const char*)"/",root_grp,filename.c_str(),use_dimscale,is_eos5,hdf5_hls,
1349 eos5_dim_info,handled_coord_names);
1350
1351 if (is_eos5 == false)
1352 add_dap4_coverage_default(root_grp,handled_coord_names);
1353
1354 // Leave the following block until the HDF-EOS5 is fully supported.
1355#if 0
1356 BESDEBUG("h5", "build_dmr - before obtain dimensions"<< endl);
1357 D4Dimensions *root_dims = root_grp->dims();
1358for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1359BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1360BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1361BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1362BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1363//cout <<"dim size is: "<<(*di)->size()<<endl;
1364//cout <<"dim fully_qualified_name is: "<<(*di)->fully_qualified_name()<<endl;
1365}
1366 BESDEBUG("h5", "build_dmr - after obtain dimensions"<< endl);
1367#endif
1368
1369 close_fileid(fileid);
1370
1371 }// else (default option)
1372
1373 // If the cache is turned on, add the memory cache.
1374 if (dmr_cache) {
1375 // add a copy
1376 BESDEBUG(HDF5_NAME, prolog << "DMR added to the cache for : " << filename << endl);
1377 dmr_cache->add(new DMR(*dmr), filename);
1378 }
1379 return false;
1380}
1381// This function is only used when EnableCF is true.
1382bool HDF5RequestHandler::hdf5_build_dmr_with_IDs(BESDataHandlerInterface & dhi)
1383{
1384 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1385#if DYNAMIC_CONFIG_ENABLED
1386 load_config();
1387#endif
1388
1389 string filename = dhi.container->access();
1390 hid_t cf_fileid = -1;
1391
1392 H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1393 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1394 if (cf_fileid < 0){
1395 string invalid_file_msg="Could not open this HDF5 file ";
1396 invalid_file_msg +=filename;
1397 invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1398 invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1399 invalid_file_msg +=" distributor.";
1400 throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1401 }
1402
1403 BaseTypeFactory factory;
1404 DDS dds(&factory, name_path(filename), "3.2");
1405 dds.filename(filename);
1406
1407 DAS das;
1408
1409 try {
1410 // This is the CF option
1411 read_cfdds( dds,filename,cf_fileid);
1412
1413 if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1414 dds.print(cerr);
1415 throw InternalErr(__FILE__, __LINE__,
1416 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1417 }
1418
1419 Ancillary::read_ancillary_dds( dds, filename ) ;
1420
1421 read_cfdas(das,filename,cf_fileid);
1422
1423 Ancillary::read_ancillary_das( das, filename ) ;
1424
1425 dds.transfer_attributes(&das);
1426
1428
1429 }
1430 catch(const BESError & e) {
1431 if(cf_fileid !=-1)
1432 H5Fclose(cf_fileid);
1433 BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1434 throw;
1435 }
1436 catch(const InternalErr & e) {
1437
1438 if(cf_fileid !=-1)
1439 H5Fclose(cf_fileid);
1440
1441 throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1442 __FILE__, __LINE__);
1443 }
1444 catch(const Error & e) {
1445
1446 if(cf_fileid !=-1)
1447 H5Fclose(cf_fileid);
1448
1449 throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1450 __FILE__, __LINE__);
1451 }
1452 catch(...) {
1453
1454 if(cf_fileid !=-1)
1455 H5Fclose(cf_fileid);
1456
1457 string s = "unknown exception caught building HDF5 DataDDS";
1458 throw BESInternalFatalError(s, __FILE__, __LINE__);
1459 }
1460
1461 // Extract the DMR Response object - this holds the DMR used by the
1462 // other parts of the framework.
1463 BESResponseObject *response = dhi.response_handler->get_response_object();
1464 BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1465
1466 // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1467 // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1468 // the BES built and add this one. The HDF4DMR object will close the open dataset when
1469 // the BES runs the DMR's destructor.
1470
1471 DMR *dmr = bes_dmr.get_dmr();
1472 D4BaseTypeFactory MyD4TypeFactory;
1473 dmr->set_factory(&MyD4TypeFactory);
1474 dmr->build_using_dds(dds);
1475
1476 auto hdf5_dmr_unique = make_unique<HDF5DMR>(dmr);
1477 auto hdf5_dmr = hdf5_dmr_unique.release();
1478 hdf5_dmr->setHDF5Dataset(cf_fileid);
1479 delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1480 bes_dmr.set_dmr(hdf5_dmr); // BESDMRResponse will delete hdf5_dmr
1481
1482 // Instead of fiddling with the internal storage of the DHI object,
1483 // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1484 // methods to set the constraints. But, why? Ans: from Patrick is that
1485 // in the 'container' mode of BES each container can have a different
1486 // CE.
1487 bes_dmr.set_dap4_constraint(dhi);
1488 bes_dmr.set_dap4_function(dhi);
1489 hdf5_dmr->set_factory(nullptr);
1490
1491 BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1492 return true;
1493}
1494
1495bool HDF5RequestHandler::hdf5_build_help(BESDataHandlerInterface & dhi)
1496{
1497 BESResponseObject *response = dhi.response_handler->get_response_object();
1498 auto info = dynamic_cast<BESInfo *>(response);
1499 if( !info )
1500 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1501
1502 string add_info="Just for Test";
1503
1504 map<string, string, std::less<>> attrs ;
1505 attrs["name"] = MODULE_NAME ;
1506 attrs["version"] = MODULE_VERSION ;
1507 list<string> services ;
1508 BESServiceRegistry::TheRegistry()->services_handled( HDF5_NAME, services );
1509 if ( services.empty()==false ) {
1510 string handles = BESUtil::implode( services, ',' ) ;
1511 attrs["handles"] = handles ;
1512 }
1513 info->begin_tag( "module", &attrs ) ;
1514 info->end_tag( "module" ) ;
1515 info->add_data(add_info);
1516
1517 return true;
1518}
1519
1520bool HDF5RequestHandler::hdf5_build_version(BESDataHandlerInterface & dhi)
1521{
1522 BESResponseObject *response = dhi.response_handler->get_response_object();
1523 auto info = dynamic_cast < BESVersionInfo * >(response);
1524 if( !info )
1525 throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1526
1527 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1528
1529 return true;
1530}
1531
1532
1533bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1534{
1535 string lrd_config_fpath;
1536 string lrd_config_fname;
1537
1538 // Obtain DataCache path
1539 lrd_config_fpath = get_beskeys("H5.DataCachePath");
1540
1541 // Obtain the configure file name that specifics the large file configuration
1542 lrd_config_fname = get_beskeys("H5.LargeDataMemCacheFileName");
1543
1544 // If either the configure file path or fname is missing, won't add specific mem. cache dirs.
1545 if (lrd_config_fpath.empty() || lrd_config_fname.empty())
1546 return false;
1547
1548 // temp_line for storing info of one line in the config. file
1549 string temp_line;
1550
1551 // The full path of the configure file
1552 string mcache_config_fname = lrd_config_fpath+"/"+lrd_config_fname;
1553
1554 // Open the configure file
1555 ifstream mcache_config_file(mcache_config_fname.c_str());
1556
1557 // If the configuration file is not open, return false.
1558 if (mcache_config_file.is_open()==false) {
1559 BESDEBUG(HDF5_NAME, prolog << "The large data memory cache configure file "<<mcache_config_fname );
1560 BESDEBUG(HDF5_NAME, prolog << " cannot be opened."<<endl);
1561 return false;
1562 }
1563
1564 // Read the configuration file line by line
1565 while(getline(mcache_config_file,temp_line)) {
1566
1567 // Only consider lines that is no less than 2 characters and the 2nd character is space.
1568 if (temp_line.size() >1 && temp_line.at(1)==' ') {
1569 char sep=' ';
1570 string subline = temp_line.substr(2);
1571 vector<string> temp_name_list;
1572
1573 // Include directories to store common latitude and longitude values
1574 if(temp_line.at(0)=='1') {
1575 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1576 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1577 }
1578 // Include directories not to store common latitude and longitude values
1579 else if(temp_line.at(0)=='0'){
1580 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1581 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1582 }
1583 // Include variable names that the server would like to store in the memory cache
1584 else if(temp_line.at(0)=='2')
1585 obtain_lrd_common_cache_dirs_data_vars(temp_name_list, subline, sep);
1586 }
1587 }
1588
1589
1590 mcache_config_file.close();
1591 if(lrd_cache_dir_list.empty() && lrd_non_cache_dir_list.empty() && lrd_var_cache_file_list.empty())
1592 return false;
1593 else
1594 return true;
1595}
1596
1597void HDF5RequestHandler::obtain_lrd_common_cache_dirs_data_vars(vector<string> &temp_name_list, const string &subline,
1598 char sep)
1599{
1600 // We need to handle the space case inside a variable path
1601 // either "" or '' needs to be used to identify a var path
1602 vector<unsigned int>dq_pos;
1603 vector<unsigned int>sq_pos;
1604 for (unsigned int i = 0; i<subline.size();i++) {
1605 if (subline[i]=='"') {
1606 dq_pos.push_back(i);
1607 }
1608 else if(subline[i]=='\'')
1609 sq_pos.push_back(i);
1610 }
1611 if (dq_pos.empty() && sq_pos.empty())
1612 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1613 else if((dq_pos.empty()==false) && (dq_pos.size()%2==0) && sq_pos.empty()==true) {
1614 unsigned int dq_index= 0;
1615 while(dq_index < dq_pos.size()){
1616 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1617 temp_name_list.push_back
1618 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1619 }
1620 dq_index = dq_index + 2;
1621 }
1622 }
1623 else if((sq_pos.empty()==false) &&(sq_pos.size()%2==0) && dq_pos.empty()==true) {
1624 unsigned int sq_index= 0;
1625 while(sq_index < sq_pos.size()){
1626 if (sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1627 temp_name_list.push_back
1628 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1629 }
1630 sq_index = sq_index+2;
1631 }
1632 }
1633
1634 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1635
1636}
1637bool HDF5RequestHandler::read_das_from_disk_cache(const string & cache_filename,DAS *das_ptr) {
1638
1639 BESDEBUG(HDF5_NAME, prolog << "Coming to read_das_from_disk_cache() " << cache_filename << endl);
1640 bool ret_value = true;
1641 FILE *md_file = nullptr;
1642 md_file = fopen(cache_filename.c_str(),"rb");
1643
1644 if(nullptr == md_file) {
1645 string bes_error = "An error occurred trying to open a metadata cache file " + cache_filename;
1646 throw BESInternalError( bes_error, __FILE__, __LINE__);
1647 }
1648 else {
1649
1650 int fd_md = fileno(md_file);
1651 struct flock *l_md;
1652 l_md = lock(F_RDLCK);
1653
1654 // hold a read(shared) lock to read metadata from a file.
1655 if (fcntl(fd_md,F_SETLKW,l_md) == -1) {
1656 fclose(md_file);
1657 ostringstream oss;
1658 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1659 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1660 }
1661
1662 try {
1663
1664 struct stat sb{};
1665 if (stat(cache_filename.c_str(),&sb) != 0) {
1666 string bes_error = "An error occurred trying to stat a metadata cache file size " + cache_filename;
1667 throw BESInternalError( bes_error, __FILE__, __LINE__);
1668
1669 }
1670
1671 auto bytes_expected_read=(size_t)sb.st_size;
1672 BESDEBUG(HDF5_NAME, prolog << "DAS Disk cache file size is " << bytes_expected_read << endl);
1673
1674 vector<char> buf;
1675 buf.resize(bytes_expected_read);
1676 size_t bytes_to_read =fread((void*)buf.data(),1,bytes_expected_read,md_file);
1677 if (bytes_to_read != bytes_expected_read)
1678 throw InternalErr(__FILE__,__LINE__,"Fail to read the data from the das cache file.");
1679
1680 char* temp_pointer =buf.data();
1681
1682 AttrTable*at = nullptr;
1683 // recursively build DAS, the folloing code is necessary. KY 2022-12-11
1684 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1685 }
1686 catch(...) {
1687 if (fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1688 fclose(md_file);
1689 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1690 }
1691
1692 fclose(md_file);
1693 throw InternalErr(__FILE__,__LINE__,"Fail to parse a das cache file.");
1694 }
1695
1696 // Unlock the cache file
1697 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1698 fclose(md_file);
1699 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1700 }
1701 fclose(md_file);
1702 }
1703 return ret_value;
1704
1705}
1706
1707// This fucntion will NOT be used by default. Leave here for future improvement.
1708bool HDF5RequestHandler::write_dds_to_disk_cache(const string& dds_cache_fname,DDS *dds_ptr) {
1709
1710 BESDEBUG(HDF5_NAME, prolog << "Write DDS to disk cache " << dds_cache_fname << endl);
1711 FILE *dds_file = fopen(dds_cache_fname.c_str(),"w");
1712
1713 if(nullptr == dds_file) {
1714 string bes_error = "An error occurred trying to open a metadata cache file " + dds_cache_fname;
1715 throw BESInternalError( bes_error, __FILE__, __LINE__);
1716 }
1717 else {
1718
1719 int fd_md = fileno(dds_file);
1720 struct flock *l_md;
1721 l_md = lock(F_WRLCK);
1722
1723 // hold a read(shared) lock to read metadata from a file.
1724 if (fcntl(fd_md,F_SETLKW,l_md) == -1) {
1725 fclose(dds_file);
1726 ostringstream oss;
1727 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1728 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1729 }
1730
1731 try {
1732 dds_ptr->print(dds_file);
1733 }
1734 catch(...) {
1735 if (fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1736 fclose(dds_file);
1737 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1738 }
1739
1740 fclose(dds_file);
1741 throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1742 }
1743
1744 if (fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1745 fclose(dds_file);
1746 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1747 }
1748
1749 fclose(dds_file);
1750 }
1751 return true;
1752
1753}
1754
1755// Write DAS to a binary cached file on disk.
1756bool HDF5RequestHandler::write_das_to_disk_cache(const string & das_cache_fname, DAS *das_ptr) {
1757
1758 BESDEBUG(HDF5_NAME, prolog << "Write DAS to disk cache " << das_cache_fname << endl);
1759 FILE *das_file = fopen(das_cache_fname.c_str(),"wb");
1760 if(nullptr == das_file) {
1761 string bes_error = "An error occurred trying to open a metadata cache file " + das_cache_fname;
1762 throw BESInternalError( bes_error, __FILE__, __LINE__);
1763 }
1764 else {
1765 int fd_md = fileno(das_file);
1766 struct flock *l_md;
1767 l_md = lock(F_WRLCK);
1768
1769 // hold a write-lock(exclusive lock) to write metadata to a file.
1770 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1771 fclose(das_file);
1772 ostringstream oss;
1773 oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1774 throw BESInternalError( oss.str(), __FILE__, __LINE__);
1775 }
1776
1777 try {
1778 write_das_to_file(das_ptr,das_file);
1779 }
1780 catch(...) {
1781 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1782 fclose(das_file);
1783 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1784 }
1785
1786 fclose(das_file);
1787 throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1788 }
1789
1790 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1791 fclose(das_file);
1792 throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1793 }
1794
1795 fclose(das_file);
1796
1797 }
1798
1799 return true;
1800
1801}
1802
1803// The wrapper function to call write_das_table_to_file to generate the cache.
1804void write_das_to_file(DAS*das_ptr,FILE* das_file) {
1805
1806 // When category_flag is 2, it marks the end of the file.
1807 uint8_t category_flag = 2;
1808 AttrTable* top_table = das_ptr->get_top_level_attributes();
1809 write_das_table_to_file(top_table,das_file);
1810
1811 // Add the final ending flag for retrieving the info.
1812 fwrite((const void*)&category_flag,1,1,das_file);
1813
1814}
1815
1816// The main function to write DAS to a file
1817void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
1818
1819 if(temp_table !=nullptr) {
1820
1821 // 2 is the end mark of an attribute table
1822 uint8_t category_flag = 2;
1823
1824 // Loop through the whole DAS top table
1825 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
1826 AttrTable::Attr_iter top_endit = temp_table->attr_end();
1827 AttrTable::Attr_iter top_it = top_startit;
1828 while(top_it !=top_endit) {
1829 AttrType atype = temp_table->get_attr_type(top_it);
1830 if (atype == Attr_unknown)
1831 throw InternalErr(__FILE__,__LINE__,"Unsupported DAS Attribute type");
1832 else if (atype!=Attr_container) {
1833 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr name is: "
1834 << temp_table->get_name(top_it) << endl);
1835 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr type is: "
1836 << temp_table->get_type(top_it) << endl);
1837 // For the debugging purpose
1838#if 0
1839 //unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
1840 //cerr<<"Attribute values are "<<endl;
1841 //for (int i = 0; i <num_attrs;i++)
1842 // cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
1843 //cerr<<endl;
1844 //write_das_attr_info(temp_table,top_it,das_file);
1845#endif
1846 // Write DAS attribute info to the file
1847 write_das_attr_info(temp_table,temp_table->get_name(top_it),
1848 temp_table->get_type(top_it),das_file);
1849 }
1850 else {
1851 BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr container name is: "
1852 << (*top_it)->name << endl);
1853 // Write the container and then write the info. in this container
1854 AttrTable* sub_table = temp_table->get_attr_table(top_it);
1855 write_container_name_to_file(sub_table->get_name(),das_file);
1856 write_das_table_to_file(sub_table,das_file);
1857
1858 // Write the end flag
1859 fwrite((const void*)&category_flag,1,1,das_file);
1860
1861 }
1862 ++top_it;
1863 }
1864
1865 }
1866}
1867
1868// Write container name to the disk file
1869void write_container_name_to_file(const string& cont_name,FILE *das_file) {
1870
1871 // 1 marks the starting of a container
1872 uint8_t category_flag = 1;
1873 vector<char> buf;
1874 size_t bytes_to_write = cont_name.size()+sizeof(size_t)+1;
1875 buf.resize(bytes_to_write);
1876 char*temp_pointer =buf.data();
1877 memcpy((void*)temp_pointer,(void*)&category_flag,1);
1878 temp_pointer++;
1879 temp_pointer=copy_str(temp_pointer,cont_name);
1880
1881 size_t bytes_to_be_written = fwrite((const void*)buf.data(),1,bytes_to_write,das_file);
1882 if (bytes_to_be_written != bytes_to_write)
1883 throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS container name to a cache");
1884
1885}
1886
1887
1888// Write DAS attribute info. to the disk cache file
1889void write_das_attr_info(AttrTable* dtp,const string& attr_name, const string & attr_type,FILE * das_file) {
1890
1891 // 0 marks the starting of a DAS attribute
1892 uint8_t category_flag = 0;
1893
1894 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
1895 vector<string> attr_values;
1896 size_t total_attr_values_size = 0;
1897 for (unsigned int i = 0; i <num_attr_elems;i++){
1898 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
1899 total_attr_values_size += attr_values[i].size();
1900 }
1901 // Need to add a flag, value as 0 to indicate the attribute.
1902 // DAS: category flag, sizeof attribute name, attribute name, size of attribute type, attribute type
1903 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2* sizeof(size_t);
1904
1905 // One unsigned int to store the number of element elements i
1906 // + sizeof(size_t) * number of elements to store the number of characters for each attribute
1907 // (in DAP, every attribute is in string format)
1908 // +total size of all attribute values
1909 bytes_to_write_attr += sizeof(unsigned int) + num_attr_elems*sizeof(size_t)+total_attr_values_size;
1910
1911 vector<char>attr_buf;
1912 attr_buf.resize(bytes_to_write_attr);
1913 char* temp_attrp =attr_buf.data();
1914
1915 // The attribute flag
1916 memcpy((void*)temp_attrp,(void*)&category_flag,1);
1917 temp_attrp++;
1918
1919 // The attribute name and type
1920 temp_attrp=copy_str(temp_attrp,attr_name);
1921 temp_attrp=copy_str(temp_attrp,attr_type);
1922
1923 // Number of elements
1924 memcpy((void*)temp_attrp,(void*)&num_attr_elems,sizeof(unsigned int));
1925 temp_attrp+=sizeof(unsigned int);
1926
1927 // All attributes
1928 for (unsigned int i = 0; i <num_attr_elems;i++)
1929 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
1930
1931 size_t bytes_to_be_written = fwrite((const void*)attr_buf.data(),1,bytes_to_write_attr,das_file);
1932 if (bytes_to_be_written != bytes_to_write_attr)
1933 throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS attribute to a cache");
1934
1935}
1936
1937// Read DDS from a disk cache, this function is not used by default.
1938void HDF5RequestHandler::read_dds_from_disk_cache(BESDDSResponse* bdds, BESDataDDSResponse* data_bdds,
1939 bool build_data,const string & container_name,const string & h5_fname,
1940 const string & dds_cache_fname,const string &das_cache_fname, hid_t h5_fd,
1941 bool das_from_dc) {
1942
1943
1944 BESDEBUG(HDF5_NAME, prolog << "BEGIN dds_cache_fname: " << dds_cache_fname << endl);
1945
1946 DDS *dds;
1947 if(true == build_data)
1948 dds = data_bdds->get_dds();
1949 else
1950 dds = bdds->get_dds();
1951
1952 // write a function to pass the following with the lock.
1953 BaseTypeFactory tf;
1954 DDS tdds(&tf,name_path(h5_fname),"3.2");
1955 tdds.filename(h5_fname);
1956
1957 FILE *dds_file = fopen(dds_cache_fname.c_str(),"r");
1958 tdds.parse(dds_file);
1959
1960 auto cache_dds_unique = make_unique<DDS>(tdds);
1961 delete dds;
1962
1963 auto cache_dds = cache_dds_unique.release();
1964 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
1965
1966 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
1967 if(true == build_data)
1968 data_bdds->set_dds(cache_dds);
1969 else
1970 bdds->set_dds(cache_dds);
1971 if(dds_file !=nullptr)
1972 fclose(dds_file);
1973
1974 if (dds_cache) {
1975 // add a copy
1976 BESDEBUG(HDF5_NAME, prolog << "For memory cache, DDS added to the cache for : " << h5_fname << endl);
1977 dds_cache->add(new DDS(*cache_dds), h5_fname);
1978 }
1979
1980}
1981
1982// Add DAS to DDS.
1983void HDF5RequestHandler::add_das_to_dds(DDS *dds, const string &/*container_name*/, const string &filename,
1984 const string &das_cache_fname, hid_t h5_fd, bool das_from_dc) {
1985
1986 BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1987
1988 // Check DAS memory cache
1989 DAS *das = nullptr ;
1990 bool use_das_cache = false;
1991 if (das_cache)
1992 das = dynamic_cast<DAS*>(das_cache->get(filename));
1993 if (das)
1994 use_das_cache = true;
1995
1996 if (true == use_das_cache) {
1997 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
1998 dds->transfer_attributes(das); // no need to copy the cached DAS
1999 }
2000
2001 else {
2002
2003 auto das_unique = make_unique<DAS>();
2004 das = das_unique.get();
2005 // The following block is commented out because the attribute containers in DDX disappear
2006 // when the container_name of DAS is added. Without adding the container_name of DAS,
2007 // the attribute containers show up in DDX. This information is re-discovered while working on
2008 // https://bugs.earthdata.nasa.gov/browse/HYRAX-714 although the following code was commented
2009 // out long time ago. KY 2022-05-27
2010#if 0
2011 if (!container_name.empty())
2012 das->container_name(container_name);
2013#endif
2014 if (das_from_dc == true)
2015 read_das_from_disk_cache(das_cache_fname,das);
2016 else
2017 read_das_from_file(das, filename, das_cache_fname, h5_fd, das_from_dc);
2018
2019 dds->transfer_attributes(das);
2020
2021 if (das_cache) {
2022 // add a copy
2023 BESDEBUG(HDF5_NAME, prolog << "For memory cache, DAS added to the cache for : " << filename << endl);
2024 das_cache->add(new DAS(*das), filename);
2025 }
2026
2027 }
2028
2029}
2030
2031void HDF5RequestHandler::read_das_from_file(DAS *das, const string &filename, const string &das_cache_fname,
2032 hid_t h5_fd, bool das_from_dc)
2033{
2034 // This bool is for the case, when DDS is read from a cache then we need to open the HDF5 file.
2035 bool h5_file_open = true;
2036 if (h5_fd == -1)
2037 h5_file_open = false;
2038 if (true == _usecf) {
2039 // go to the CF option
2040 if (h5_file_open == false)
2041 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2042
2043 read_cfdas( *das,filename,h5_fd);
2044 if (h5_file_open == false)
2045 H5Fclose(h5_fd);
2046 }
2047 else {
2048 if(h5_file_open == false)
2049 h5_fd = get_fileid(filename.c_str());
2050 find_gloattr(h5_fd, *das);
2051 depth_first(h5_fd, "/", *das);
2052 if(h5_file_open == false)
2053 close_fileid(h5_fd);
2054 }
2055
2056 Ancillary::read_ancillary_das( *das, filename ) ;
2057
2058 if (das_cache_fname.empty() == false && das_from_dc == false)
2059 write_das_to_disk_cache(das_cache_fname,das);
2060
2061}
2062bool obtain_beskeys_info(const string& key, bool & has_key) {
2063
2064 bool ret_value = false;
2065 string doset;
2066 TheBESKeys::TheKeys()->get_value( key, doset, has_key ) ;
2067 if (has_key) {
2068 const string dosettrue ="true";
2069 const string dosetyes = "yes";
2070 doset = BESUtil::lowercase(doset) ;
2071 ret_value = (dosettrue == doset || dosetyes == doset);
2072 }
2073 return ret_value;
2074}
2075
2076
2077// get_uint_key and get_float_key are copied from the netCDF handler.
2078static unsigned int get_uint_key(const string &key, unsigned int def_val)
2079{
2080 bool found = false;
2081 string doset;
2082
2083 TheBESKeys::TheKeys()->get_value(key, doset, found);
2084 if (true == found) {
2085 // In C++11, stoi is better.
2086 return stoi(doset);
2087 }
2088 else {
2089 return def_val;
2090 }
2091}
2092
2093static unsigned long get_ulong_key(const string &key, unsigned long def_val)
2094{
2095 bool found = false;
2096 string doset;
2097
2098 TheBESKeys::TheKeys()->get_value(key, doset, found);
2099 if (true == found) {
2100 // In C++11, stoul is better.
2101 return stoul(doset);
2102 }
2103 else {
2104 return def_val;
2105 }
2106}
2107static float get_float_key(const string &key, float def_val)
2108{
2109 bool found = false;
2110 string doset;
2111
2112 TheBESKeys::TheKeys()->get_value(key, doset, found);
2113 if (true == found)
2114 return stof(doset);
2115 else
2116 return def_val;
2117}
2118
2119static string get_beskeys(const string &key) {
2120
2121 bool found = false;
2122 string ret_value;
2123
2124 TheBESKeys::TheKeys()->get_value( key, ret_value, found ) ;
2125 return ret_value;
2126}
2127
2128// The function to copy a string to a memory buffer.
2129char* copy_str(char*temp_ptr,const string & str) {
2130
2131 size_t str_size=str.size();
2132 memcpy((void*)temp_ptr,(void*)&str_size,sizeof(size_t));
2133 temp_ptr+=sizeof(size_t);
2134 vector<char>temp_vc2(str.begin(),str.end());
2135 memcpy((void*)temp_ptr,(void*)temp_vc2.data(),str.size());
2136 temp_ptr+=str.size();
2137 return temp_ptr;
2138
2139}
2140
2141
2142// Obtain the string from a memory buffer.
2143// Note: both char* and string(as a reference) will be returned
2144// The attribute binary first stores the size of the string, then the string itself
2145char* obtain_str(char*temp_ptr,string & str) {
2146
2147 size_t oname_size = *((size_t *)temp_ptr);
2148 temp_ptr = temp_ptr + sizeof(size_t);
2149 string oname;
2150 for(unsigned int i =0; i<oname_size; i++){
2151 oname.push_back(*temp_ptr);
2152 ++temp_ptr;
2153 }
2154 str = oname;
2155 return temp_ptr;
2156
2157}
2158
2159// For our case, there are no global attributes for DAS.
2160// The global attributes are always under HDF_GLOBAL.
2161// The main function to obtain the DAS info. from the cache.
2162char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at_par) {
2163
2164 // 3 is only for the code to come into the loop.
2165 uint8_t flag =3;
2166 while(flag !=2) {
2167 flag = *((uint8_t*)(temp_pointer));
2168 BESDEBUG(HDF5_NAME, prolog << "Build DAS from the disk cache file flag: "
2169 <<" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2170 <<" flag = 3; the initial value to get the attribute retrieval process started."
2171 <<" The flag value is "
2172 << (int)flag <<endl);
2173 temp_pointer++;
2174
2175 if (flag ==1) {
2176 string container_name;
2177 temp_pointer = obtain_str(temp_pointer,container_name);
2178 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, container name is " << container_name << endl);
2179
2180 // Remember the current Attribute table state
2181 AttrTable*temp_at_par = at_par;
2182 if(at_par == nullptr) {
2183 auto new_attr_table_unique = make_unique<libdap::AttrTable>();
2184 auto new_attr_table = new_attr_table_unique.release();
2185 at_par = das->add_table(container_name, new_attr_table);
2186 }
2187 else
2188 at_par = at_par->append_container(container_name);
2189
2190 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2191 // MUST resume the original state
2192 at_par = temp_at_par;
2193
2194 }
2195 else if(flag == 0) {
2196 // The attribute must have a table.
2197 if(at_par ==nullptr)
2198 throw BESInternalError( "The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2199
2200 // Attribute name
2201 string attr_name;
2202 temp_pointer = obtain_str(temp_pointer,attr_name);
2203 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr name is: " << attr_name << endl);
2204
2205 // Attribute type
2206 string attr_type;
2207 temp_pointer = obtain_str(temp_pointer,attr_type);
2208 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr type is: " << attr_type << endl);
2209
2210 // Attribute values
2211 unsigned int num_values = *((unsigned int*)(temp_pointer));
2212 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, number of attribute values is: " << num_values << endl);
2213 temp_pointer+=sizeof(unsigned int);
2214
2215 vector <string> attr_values;
2216
2217 for(unsigned int i = 0; i<num_values; i++) {
2218 string attr_value;
2219 temp_pointer = obtain_str(temp_pointer,attr_value);
2220 attr_values.push_back(attr_value);
2221 BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attribute value is: " << attr_value << endl);
2222 }
2223
2224 at_par->append_attr(attr_name,attr_type,&attr_values);
2225 }
2226
2227 }
2228 return temp_pointer;
2229}
2230
2231// The debugging function to get attribute info.
2232void get_attr_contents(AttrTable*temp_table) {
2233 if(temp_table !=nullptr) {
2234 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2235 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2236 AttrTable::Attr_iter top_it = top_startit;
2237 while(top_it !=top_endit) {
2238 AttrType atype = temp_table->get_attr_type(top_it);
2239 if(atype == Attr_unknown)
2240 cerr<<"unsupported DAS attributes" <<endl;
2241 else if(atype!=Attr_container) {
2242
2243 cerr<<"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2244 cerr<<"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2245 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2246 cerr<<"Attribute values are "<<endl;
2247 for (unsigned int i = 0; i <num_attrs;i++)
2248 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2249 cerr<<endl;
2250 }
2251 else {
2252 cerr<<"Coming to the attribute container. "<<endl;
2253 cerr<<"container name is "<<(*top_it)->name <<endl;
2254 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2255 cerr<<"container table name is "<<sub_table->get_name() <<endl;
2256 get_attr_contents(sub_table);
2257 }
2258 ++top_it;
2259 }
2260
2261 }
2262}
2263
2264
2265void HDF5RequestHandler::add_attributes(BESDataHandlerInterface &dhi) {
2266
2267 BESResponseObject *response = dhi.response_handler->get_response_object();
2268 auto bdds = dynamic_cast<BESDataDDSResponse *>(response);
2269 if (!bdds)
2270 throw BESInternalError("cast error", __FILE__, __LINE__);
2271 DDS *dds = bdds->get_dds();
2272 string container_name = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name(): "";
2273 string filename = dhi.container->access();
2274 DAS* das = nullptr;
2275 bool das_from_mcache = false;
2276 if(das_cache) {
2277 das = dynamic_cast<DAS*>(das_cache->get(filename));
2278 if (das) {
2279 BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2280 dds->transfer_attributes(das); // no need to copy the cached DAS
2281 das_from_mcache = true;
2282 }
2283 }
2284
2285 if(false == das_from_mcache) {
2286 auto das_unique = make_unique<DAS>();
2287 das = das_unique.release();
2288 // This looks at the 'use explicit containers' prop, and if true
2289 // sets the current container for the DAS.
2290 if (!container_name.empty()) das->container_name(container_name);
2291
2292 hid_t h5_fd =-1;
2293 if (true == _usecf) {
2294 // go to the CF option
2295 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2296
2297 read_cfdas( *das,filename,h5_fd);
2298
2299 H5Fclose(h5_fd);
2300 }
2301 else {
2302 h5_fd = get_fileid(filename.c_str());
2303 find_gloattr(h5_fd, *das);
2304 depth_first(h5_fd, "/", *das);
2305 close_fileid(h5_fd);
2306 }
2307
2308 Ancillary::read_ancillary_das(*das, filename);
2309
2310 dds->transfer_attributes(das);
2311
2312 // Only free the DAS if it's not added to the cache
2313 if (das_cache) {
2314 // add a copy
2315 BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
2316 das_cache->add(das, filename);
2317 }
2318 else {
2319 delete das;
2320 }
2321 }
2322 BESDEBUG(HDF5_NAME, prolog << "Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2323 bdds->set_ia_flag(true);
2324
2325}
2326
2327void HDF5RequestHandler::close_h5_files(hid_t cf_fileid, hid_t fileid) {
2328
2329 if (cf_fileid != -1)
2330 H5Fclose(cf_fileid);
2331 if (fileid != -1)
2332 H5Fclose(fileid);
2333
2334}
HDF5CFModule.h
This class describes the different categories of HDF5 products for the CF option.
HDF5RequestHandler.h
include the entry functions to execute the handlers
BESContainer::get_symbolic_name
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition BESContainer.h:222
BESContainer::access
virtual std::string access()=0
returns the true name of this container
BESDDSResponse
Holds a DDS object within the BES.
Definition BESDDSResponse.h:50
BESDDSResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition BESDDSResponse.cc:64
BESDDSResponse::set_dds
void set_dds(libdap::DDS *ddsIn)
Definition BESDDSResponse.h:71
BESDDSResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition BESDDSResponse.cc:73
BESDDSResponse::get_dds
libdap::DDS * get_dds()
Definition BESDDSResponse.h:80
BESDMRResponse
Represents an OPeNDAP DMR DAP4 data object within the BES.
Definition BESDMRResponse.h:49
BESDapResponse::set_dap4_function
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition BESDapResponse.cc:173
BESDapResponse::set_dap4_constraint
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition BESDapResponse.cc:156
BESDapResponse::set_constraint
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition BESDapResponse.cc:134
BESDapResponse::get_explicit_containers
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Definition BESDapResponse.h:70
BESDapResponse::get_request_xml_base
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Definition BESDapResponse.h:76
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition BESDataDDSResponse.h:46
BESDataDDSResponse::set_dds
void set_dds(libdap::DDS *ddsIn)
Definition BESDataDDSResponse.h:73
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition BESDataHandlerInterface.h:56
BESDataHandlerInterface::container
BESContainer * container
pointer to current container in this interface
Definition BESDataHandlerInterface.h:75
BESError::get_message
std::string get_message() const
get the error message for this exception
Definition BESError.h:132
BESInternalError
exception thrown if internal error encountered
Definition BESInternalError.h:43
BESRequestHandler
Represents a specific data type request handler.
Definition BESRequestHandler.h:74
BESResponseHandler::get_response_object
virtual BESResponseObject * get_response_object()
return the current response object
Definition BESResponseHandler.cc:62
BESServiceRegistry::services_handled
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
Definition BESServiceRegistry.cc:347
BESUtil::lowercase
static std::string lowercase(const std::string &s)
Definition BESUtil.cc:257
BESUtil::implode
static std::string implode(const std::list< std::string > &values, char delim)
Definition BESUtil.cc:620
ObjMemCache
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition ObjMemCache.h:83
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition TheBESKeys.cc:288
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Access to the singleton.
Definition TheBESKeys.cc:85
size_t
h5cfdaputil.h
Helper functions for generating DAS attributes and a function to check BES Key.
depth_first
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition h5das.cc:64
find_gloattr
void find_gloattr(hid_t file, DAS &das)
Definition h5das.cc:444
breadth_first
bool breadth_first(hid_t file_id, hid_t pid, const char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, bool is_eos5, vector< link_info_t > &hdf5_hls, eos5_dim_info_t &eos5_dim_info, vector< string > &handled_cv_names)
Definition h5dmr.cc:116
add_dap4_coverage_default
void add_dap4_coverage_default(D4Group *d4_root, const vector< string > &handled_all_cv_names)
Add DAP4 coverage.
Definition h5dmr.cc:1937
hdf5_handler.h
The main header of the HDF5 OPeNDAP handler.