bes Updated for version 3.21.1
The Backend Server (BES) is the lower two tiers of the Hyrax data server
HDF5GMCF.cc
Go to the documentation of this file.
1// This file is part of the hdf5_handler implementing for the CF-compliant
2// Copyright (c) 2011-2023 The HDF Group, Inc. and OPeNDAP, Inc.
3//
4// This is free software; you can redistribute it and/or modify it under the
5// terms of the GNU Lesser General Public License as published by the Free
6// Software Foundation; either version 2.1 of the License, or (at your
7// option) any later version.
8//
9// This software is distributed in the hope that it will be useful, but
10// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12// License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17//
18// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19// You can contact The HDF Group, Inc. at 410 E University Ave,
20// Suite 200, Champaign, IL 61820
21
36
37#include "HDF5CF.h"
38#include "HDF5RequestHandler.h"
39#include "h5apicompatible.h"
40#include <BESDebug.h>
41#include <algorithm>
42#include <memory>
43
44using namespace std;
45using namespace libdap;
46using namespace HDF5CF;
47
48// Copier function.
49GMCVar::GMCVar(const Var*var) {
50
51 BESDEBUG("h5", "Coming to GMCVar()"<<endl);
52 newname = var->newname;
53 name = var->name;
54 fullpath = var->fullpath;
55 rank = var->rank;
56 total_elems = var->total_elems;
57 zero_storage_size = var->zero_storage_size;
58 dtype = var->dtype;
59 unsupported_attr_dtype = var->unsupported_attr_dtype;
60 unsupported_dspace = var->unsupported_dspace;
61 coord_attr_add_path = false;
62
63 for (const auto& vattr:var->attrs) {
64 auto attr_unique = make_unique<Attribute>();
65 auto attr = attr_unique.release();
66 attr->name = vattr->name;
67 attr->newname = vattr->newname;
68 attr->dtype =vattr->dtype;
69 attr->count =vattr->count;
70 attr->strsize = vattr->strsize;
71 attr->fstrsize = vattr->fstrsize;
72 attr->value =vattr->value;
73 attrs.push_back(attr);
74 }
75
76 for (const auto& vdim:var->dims) {
77 auto dim_unique = make_unique<Dimension>(vdim->size);
78 auto dim = dim_unique.release();
79 dim->name = vdim->name;
80 dim->newname = vdim->newname;
81 dim->unlimited_dim = vdim->unlimited_dim;
82 dims.push_back(dim);
83 }
84
85 product_type = General_Product;
86
87}
88
89//Copy function of a special variable.
90GMSPVar::GMSPVar(const Var*var) {
91
92 BESDEBUG("h5", "Coming to GMSPVar()"<<endl);
93 fullpath = var->fullpath;
94 rank = var->rank;
95 total_elems = var->total_elems;
96 zero_storage_size = var->zero_storage_size;
97 unsupported_attr_dtype = var->unsupported_attr_dtype;
98 unsupported_dspace = var->unsupported_dspace;
99 coord_attr_add_path = var->coord_attr_add_path;
100 // The caller of this function should change the following fields.
101 // This is just to make data coverity happy.
102 otype = H5UNSUPTYPE;
103 sdbit = -1;
104 numofdbits = -1;
105
106 for (const auto &vattr:var->attrs) {
107 auto attr_unique = make_unique<Attribute>();
108 auto attr = attr_unique.release();
109 attr->name = vattr->name;
110 attr->newname = vattr->newname;
111 attr->dtype =vattr->dtype;
112 attr->count =vattr->count;
113 attr->strsize = vattr->strsize;
114 attr->fstrsize = vattr->fstrsize;
115 attr->value =vattr->value;
116 attrs.push_back(attr);
117 }
118
119 for (const auto &vdim:var->dims) {
120 auto dim_unique = make_unique<Dimension>(vdim->size);
121 auto dim = dim_unique.release();
122 dim->name = vdim->name;
123 dim->newname = vdim->newname;
124 dim->unlimited_dim = vdim->unlimited_dim;
125 dims.push_back(dim);
126 }
127}
128
129
130GMFile::GMFile(const char*file_fullpath, hid_t file_id, H5GCFProduct product_type, GMPattern gproduct_pattern):
131File(file_fullpath,file_id), product_type(product_type),gproduct_pattern(gproduct_pattern)
132{
133
134}
135
136// destructor
137GMFile::~GMFile()
138{
139
140 if (!this->cvars.empty()){
141 for (auto i= this->cvars.begin(); i!=this->cvars.end(); ++i) {
142 delete *i;
143 }
144 }
145
146 if (!this->spvars.empty()){
147 for (auto i= this->spvars.begin(); i!=this->spvars.end(); ++i) {
148 delete *i;
149 }
150 }
151
152}
153
154// Get CF string
155string GMFile::get_CF_string(string s) {
156
157 // HDF5 group or variable path always starts with '/'. When CF naming rule is applied,
158 // the first '/' is always changes to "_", this is not necessary. However,
159 // to keep the backward compatability, I use a BES key for people to go back with the original name.
160
161 if(s[0] !='/')
162 return File::get_CF_string(s);
163 else if (General_Product == product_type && OTHERGMS == gproduct_pattern) {
164
165 if(true == HDF5RequestHandler::get_keep_var_leading_underscore())
166 return File::get_CF_string(s);
167 else {
168 s.erase(0,1);
169 return File::get_CF_string(s);
170 }
171 }
172 else {
173 // The leading underscore should be removed from all supported products
174 s.erase(0,1);
175 return File::get_CF_string(s);
176 }
177}
178
179// Retrieve all the HDF5 information.
180void GMFile::Retrieve_H5_Info(const char *file_fullpath,
181 hid_t file_id, bool include_attr) {
182
183 BESDEBUG("h5", "Coming to Retrieve_H5_Info()"<<endl);
184 // GPM needs the attribute info. to obtain the lat/lon.
185 // So set the include_attr to be true for these products.
186 if (product_type == Mea_SeaWiFS_L2 || product_type == Mea_SeaWiFS_L3
187 || GPMS_L3 == product_type || GPMM_L3 == product_type || GPM_L1 == product_type || OBPG_L3 == product_type
188 || Mea_Ozone == product_type || General_Product == product_type)
189 File::Retrieve_H5_Info(file_fullpath,file_id,true);
190 else
191 File::Retrieve_H5_Info(file_fullpath,file_id,include_attr);
192
193}
194
195// Update the product type. This is because the file structure may change across different versions of products
196// I need to handle them differently and still support different versions. The goal is to support two versions in a row.
197// Currently, GPM level 3 is changed.
198// This routine should be called right after Retrieve_H5_Info.
200
201 BESDEBUG("h5", "Coming to Update_Product_Type()"<<endl);
202 if(GPMS_L3 == this->product_type || GPMM_L3 == this->product_type) {
203
204 // Check Dimscale attributes
205#if 0
206 //Check_General_Product_Pattern();
207#endif
208
209 Check_Dimscale_General_Product_Pattern();
210 if(GENERAL_DIMSCALE == this->gproduct_pattern){
211 if(GPMS_L3 == this->product_type) {
212 for (auto &var:this->vars)
213 var->newname = var->name;
214 special_gpm_l3 = true;
215 }
216 this->product_type = General_Product;
217 }
218 }
219 else if(General_Product == this->product_type)
220 Check_General_Product_Pattern();
221}
222
224
225 BESDEBUG("h5", "Coming to Remove_Unneeded_Objects()"<<endl);
226 if(General_Product == this->product_type) {
227 string file_path = this->path;
228 if(HDF5CFUtil::obtain_string_after_lastslash(file_path).find("OMPS-NPP")==0)
229 Remove_OMPSNPP_InputPointers();
230 }
231 if((General_Product == this->product_type) && (GENERAL_DIMSCALE == this->gproduct_pattern)) {
232 string nc4_non_coord="_nc4_non_coord_";
233 size_t nc4_non_coord_size= nc4_non_coord.size();
234 for (const auto &var:this->vars) {
235 if(var->name.find(nc4_non_coord)==0) {
236 // Here we need to store the path
237 string temp_vname = HDF5CFUtil::obtain_string_after_lastslash(var->fullpath);
238 string temp_vpath = HDF5CFUtil::obtain_string_before_lastslash(var->fullpath);
239 string temp_vname_removed = temp_vname.substr(nc4_non_coord_size,temp_vname.size()-nc4_non_coord_size);
240 string temp_v_fullpath_removed = temp_vpath + temp_vname_removed;
241 nc4_sdimv_dv_path.insert(temp_v_fullpath_removed);
242 }
243 }
244
245 if(nc4_sdimv_dv_path.empty()==false)
246 this->have_nc4_non_coord = true;
247
248
249 if (true == this->have_nc4_non_coord) {
250 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
251 if(nc4_sdimv_dv_path.find((*irv)->fullpath)!=nc4_sdimv_dv_path.end()){
252 delete(*irv);
253 irv=this->vars.erase(irv);
254 }
255 else
256 ++irv;
257 }
258 }
259
260 }
261 else if(GPM_L3_New == this->product_type) {
262 for (auto irg = this->groups.begin(); irg != this->groups.end(); ) {
263 if((*irg)->attrs.empty()) {
264 delete(*irg);
265 irg = this->groups.erase(irg);
266
267 }
268 else
269 ++irg;
270 }
271 }
272}
273
274void GMFile::Remove_OMPSNPP_InputPointers() {
275 // Here I don't check whether this is a netCDF file by
276 // using Check_Dimscale_General_Product_Pattern() to see if it returns true.
277 // We will see if we need this.
278 for (auto irg = this->groups.begin(); irg != this->groups.end(); ) {
279 if((*irg)->path.find("/InputPointers")==0) {
280 delete(*irg);
281 irg = this->groups.erase(irg);
282
283 }
284 else
285 ++irg;
286 }
287
288 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
289 if((*irv)->fullpath.find("/InputPointers")==0) {
290 delete(*irv);
291 irv = this->vars.erase(irv);
292
293 }
294 else
295 ++irv;
296 }
297}
299
300 for (const auto &cvar:this->cvars) {
301
302 if (cvar->cvartype != CV_NONLATLON_MISS){
303 for (auto & attr:cvar->attrs) {
304 Retrieve_H5_Attr_Value(attr,cvar->fullpath);
305 }
306 }
307 }
308}
309
310// Retrieve HDF5 supported attribute values.
312
313 BESDEBUG("h5", "Coming to Retrieve_H5_Supported_Attr_Values()"<<endl);
314
315 // General attributes
317
318 //Coordinate variable attributes
320
321 // Special variable attributes
322 for (const auto &spv:this->spvars) {
323 for (auto &attr:spv->attrs) {
324 Retrieve_H5_Attr_Value(attr,spv->fullpath);
326 }
327 }
328}
329
330// Adjust attribute values. Currently, this is only for ACOS and OCO2.
331// Reason: DAP2 doesn't support 64-bit integer, and they have 64-bit integer data
332// in these files. Chop them to two 32-bit integers following the data producer's information.
334
335 BESDEBUG("h5", "Coming to Adjust_H5_Attr_Value()"<<endl);
336 if (product_type == ACOS_L2S_OR_OCO2_L1B) {
337 if (("Type" == attr->name) && (H5VSTRING == attr->dtype)) {
338 string orig_attrvalues(attr->value.begin(),attr->value.end());
339 if (orig_attrvalues != "Signed64") return;
340 string new_attrvalues = "Signed32";
341 // Since the new_attrvalues size is the same as the orig_attrvalues size
342 // No need to adjust the strsize and fstrsize. KY 2011-2-1
343 attr->value.clear();
344 attr->value.resize(new_attrvalues.size());
345 copy(new_attrvalues.begin(),new_attrvalues.end(),attr->value.begin());
346 }
347 }
348}
349
350// Unsupported datatype
351void GMFile:: Handle_Unsupported_Dtype(bool include_attr) {
352
353 BESDEBUG("h5", "Coming to Handle_Unsupported_Dtype()"<<endl);
354 if(true == check_ignored) {
355 Gen_Unsupported_Dtype_Info(include_attr);
356 }
357 File::Handle_Unsupported_Dtype(include_attr);
358 Handle_GM_Unsupported_Dtype(include_attr);
359}
360
361// Unsupported datatype for general data products.
362void GMFile:: Handle_GM_Unsupported_Dtype(bool include_attr) {
363
364 BESDEBUG("h5", "Coming to Handle_GM_Unsupported_Dtype()"<<endl);
365 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ) {
366 if (true == include_attr) {
367 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
368 H5DataType temp_dtype = (*ira)->getType();
369 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
370 delete (*ira);
371 ira = (*ircv)->attrs.erase(ira);
372 }
373 else {
374 ++ira;
375 }
376 }
377 }
378 H5DataType temp_dtype = (*ircv)->getType();
379 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
380
381 // This may need to be checked carefully in the future,
382 // My current understanding is that the coordinate variable can
383 // be ignored if the corresponding variable is ignored.
384 // Currently, we don't find any NASA files in this category.
385 // KY 2012-5-21
386 delete (*ircv);
387 ircv = this->cvars.erase(ircv);
388 }
389 else {
390 ++ircv;
391 }
392
393 }
394
395 for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ) {
396 if (true == include_attr) {
397 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
398 H5DataType temp_dtype = (*ira)->getType();
399 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
400 delete (*ira);
401 ira = (*ircv)->attrs.erase(ira);
402 }
403 else {
404 ++ira;
405 }
406 }
407 }
408 H5DataType temp_dtype = (*ircv)->getType();
409 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
410 delete (*ircv);
411 ircv = this->spvars.erase(ircv);
412 }
413 else {
414 ++ircv;
415 }
416
417 }
418}
419
420// Datatype ignore information.
421void GMFile:: Gen_Unsupported_Dtype_Info(bool include_attr) {
422
423 BESDEBUG("h5", "GMFile::Coming to Gen_Unsupported_Dtype_Info()"<<endl);
424 if(true == include_attr) {
425
426 File::Gen_Group_Unsupported_Dtype_Info();
427 File::Gen_Var_Unsupported_Dtype_Info();
428 Gen_VarAttr_Unsupported_Dtype_Info();
429 }
430
431}
432
433// Datatype ignored information for variable attributes
434void GMFile:: Gen_VarAttr_Unsupported_Dtype_Info() {
435
436 BESDEBUG("h5", "GMFile::Coming to Gen_Unsupported_Dtype_Info()"<<endl);
437 // First general variables(non-CV and non-special variable) that use dimension scales.
438 if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
439 || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type) || (Mea_SeaWiFS_L3 == this->product_type)
440 || (OBPG_L3 == this->product_type)) {
441 Gen_DimScale_VarAttr_Unsupported_Dtype_Info();
442 }
443
444 else
445 File::Gen_VarAttr_Unsupported_Dtype_Info();
446
447 // CV and special variables
448 Gen_GM_VarAttr_Unsupported_Dtype_Info();
449
450}
451
452// Generate ignored object,attribute information for the CVs and special variables of general supported products.
453void GMFile:: Gen_GM_VarAttr_Unsupported_Dtype_Info(){
454
455 BESDEBUG("h5", "GMFile::Coming to Gen_GM_VarAttr_Unsupported_Dtype_Info()"<<endl);
456 if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
457 || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type) || (Mea_SeaWiFS_L3 == this->product_type)
458 || (OBPG_L3 == this->product_type)) {
459
460 for (const auto &cvar:this->cvars) {
461 // If the attribute REFERENCE_LIST comes with the attribute CLASS, the
462 // attribute REFERENCE_LIST is okay to ignore. No need to report.
463 bool is_ignored = ignored_dimscale_ref_list(cvar);
464 if (false == cvar->attrs.empty()) {
465 for (const auto &attr:cvar->attrs) {
466 H5DataType temp_dtype = attr->getType();
467 // TODO: check why 64-bit integer is included here.
468 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
469 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
470 // is okay to ignore if the variable has another attribute
471 // CLASS="DIMENSION_SCALE"
472 if (("DIMENSION_LIST" !=attr->name) &&
473 ("REFERENCE_LIST" != attr->name || true == is_ignored))
474 this->add_ignored_info_attrs(false,cvar->fullpath,attr->name);
475 }
476 }
477 }
478 }
479
480 for (const auto&spvar:this->spvars) {
481 // If the attribute REFERENCE_LIST comes with the attribute CLASS, the
482 // attribute REFERENCE_LIST is okay to ignore. No need to report.
483 bool is_ignored = ignored_dimscale_ref_list(spvar);
484 if (false == spvar->attrs.empty()) {
485#if 0
486 //if (true == spvar->unsupported_attr_dtype)
487#endif
488 for (const auto &attr:spvar->attrs) {
489 H5DataType temp_dtype = attr->getType();
490 //TODO; check why 64-bit integer is included here.
491 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
492 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
493 // is okay to ignore if the variable has another attribute
494 // CLASS="DIMENSION_SCALE"
495 if (("DIMENSION_LIST" !=attr->name) &&
496 ("REFERENCE_LIST" != attr->name || true == is_ignored))
497 this->add_ignored_info_attrs(false,spvar->fullpath,attr->name);
498 }
499 }
500 }
501 }
502 }// "if((General_Product == ......)"
503 else {
504 for (const auto &cvar:this->cvars) {
505 if (false == cvar->attrs.empty()) {
506#if 0
507 //if (true == cvar->unsupported_attr_dtype)
508#endif
509 for (const auto &attr:cvar->attrs) {
510 H5DataType temp_dtype = attr->getType();
511 // TODO: check why 64-bit integer is included here.
512 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64)
513 this->add_ignored_info_attrs(false,cvar->fullpath,attr->name);
514 }
515 }
516 }
517
518 for (const auto &spvar:this->spvars) {
519 if (false == spvar->attrs.empty()) {
520#if 0
521 //if (true == spvar->unsupported_attr_dtype)
522#endif
523 for (const auto &attr:spvar->attrs) {
524 H5DataType temp_dtype = attr->getType();
525 //TODO: check why 64-bit integer is included here.
526 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
527 this->add_ignored_info_attrs(false,spvar->fullpath,attr->name);
528 }
529 }
530 }
531 }
532
533 }// else
534
535}
536
537// Unsupported data space
538void GMFile:: Handle_Unsupported_Dspace(bool include_attr) {
539
540 BESDEBUG("h5", "Coming to GMFile:Handle_Unsupported_Dspace()"<<endl);
541 if(true == check_ignored)
542 Gen_Unsupported_Dspace_Info();
543
545 Handle_GM_Unsupported_Dspace(include_attr);
546
547}
548
549// Unsupported data space for coordinate variables and special variables of general products
550void GMFile:: Handle_GM_Unsupported_Dspace(bool include_attr) {
551
552 BESDEBUG("h5", "Coming to GMFile:Handle_GM_Unsupported_Dspace()"<<endl);
553 if(true == this->unsupported_var_dspace) {
554 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ) {
555 if (true == (*ircv)->unsupported_dspace ) {
556
557 // This may need to be checked carefully in the future,
558 // My current understanding is that the coordinate variable can
559 // be ignored if the corresponding variable is ignored.
560 // Currently, we don't find any NASA files in this category.
561 // KY 2012-5-21
562 delete (*ircv);
563 ircv = this->cvars.erase(ircv);
564 }
565 else {
566 ++ircv;
567 }
568 }
569
570 for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ) {
571 if (true == (*ircv)->unsupported_dspace) {
572 delete (*ircv);
573 ircv = this->spvars.erase(ircv);
574 }
575 else {
576 ++ircv;
577 }
578
579 }
580 }// if
581
582 if(true == include_attr) {
583 if(true == this->unsupported_var_attr_dspace) {
584 for (auto &cvar:this->cvars) {
585 if (false == cvar->attrs.empty()) {
586 if (true == cvar->unsupported_attr_dspace) {
587 for (auto ira = cvar->attrs.begin(); ira != cvar->attrs.end(); ) {
588 if (0 == (*ira)->count) {
589 delete (*ira);
590 ira = cvar->attrs.erase(ira);
591 }
592 else {
593 ++ira;
594 }
595 }
596 }
597 }
598 }
599
600 for (auto &spvar:this->spvars) {
601 if (false == spvar->attrs.empty()) {
602 if (true == spvar->unsupported_attr_dspace) {
603 for (auto ira = spvar->attrs.begin(); ira != spvar->attrs.end(); ) {
604 if (0 == (*ira)->count) {
605 delete (*ira);
606 ira = spvar->attrs.erase(ira);
607 }
608 else {
609 ++ira;
610 }
611 }
612 }
613 }
614 } // for
615 }// if
616 }// if
617
618}
619
620// Generate unsupported data space information
621void GMFile:: Gen_Unsupported_Dspace_Info() {
622
623 // Leave like this since we may add more info. in this method later. KY 2022-12-08
624 File::Gen_Unsupported_Dspace_Info();
625
626}
627
628// Handle other unsupported objects
629void GMFile:: Handle_Unsupported_Others(bool include_attr) {
630
631 BESDEBUG("h5", "Coming to GMFile:Handle_Unsupported_Others()"<<endl);
633
634 // Add the removal of CLASS=DIM_SCALE attribute if this is a netCDF-4-like attribute.
635 //
636 if(General_Product != this->product_type
637 || (General_Product == this->product_type && OTHERGMS != this->gproduct_pattern)){
638 //
639#if 0
640 if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
641 || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type)
642 || (Mea_SeaWiFS_L3 == this->product_type)
643 || (OBPG_L3 == this->product_type))
644#endif
645 remove_netCDF_internal_attributes(include_attr);
646 if(include_attr == true) {
647 // We also need to remove the _nc3_strict from the root attributes
648 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
649
650 if((*ira)->name == "_nc3_strict") {
651 delete(*ira);
652 ira =this->root_attrs.erase(ira);
653 //If we have other root attributes to remove, remove the break statement.
654 }
655 else if((*ira)->name == "_NCProperties") {
656 delete(*ira);
657 ira =this->root_attrs.erase(ira);
658 }
659 else if((*ira)->name == "_Netcdf4Coordinates") {
660 delete(*ira);
661 ira =this->root_attrs.erase(ira);
662 }
663
664 else {
665 ++ira;
666 }
667 }
668 for (const auto &cvar:this->cvars) {
669 for(auto ira = cvar->attrs.begin(); ira != cvar->attrs.end();) {
670 if((*ira)->name == "CLASS") {
671 string class_value = Retrieve_Str_Attr_Value(*ira,cvar->fullpath);
672
673 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
674 // "DIMENSION_SCALE", which is 15.
675 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
676 delete(*ira);
677 ira = cvar->attrs.erase(ira);
678 // Add another block to set a key
679 }
680 else {
681 ++ira;
682 }
683 }
684 else if((*ira)->name == "NAME") {// Add a BES Key later
685 delete(*ira);
686 ira =cvar->attrs.erase(ira);
687 //"NAME" attribute causes the file netCDF-4 failed.
688#if 0
689
690 string name_value = Retrieve_Str_Attr_Value(*ira,cvar->fullpath);
691 if( 0 == name_value.compare(0,cvar->name.size(),cvar->name)) {
692 delete(*ira);
693 ira =cvar->attrs.erase(ira);
694 }
695 else {
696 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
697 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
698 delete((*ira));
699 ira =cvar->attrs.erase(ira);
700 }
701 else {
702 ++ira;
703 }
704 }
705#endif
706 }
707 else if((*ira)->name == "_Netcdf4Dimid") {
708 delete(*ira);
709 ira =cvar->attrs.erase(ira);
710 }
711 else if((*ira)->name == "_Netcdf4Coordinates") {
712 delete(*ira);
713 ira =cvar->attrs.erase(ira);
714 }
715
716#if 0
717 else if((*ira)->name == "_nc3_strict") {
718 delete((*ira));
719 ira =cvar->attrs.erase(ira);
720 }
721#endif
722 else {
723 ++ira;
724 }
725 }
726 }
727 }
728 }
729 // netCDF Java lifts the string size limitation. All the string attributes can be
730 // represented by netCDF Java. So comment out the code. KY 2018/08/10
731#if 0
732 if(true == this->check_ignored && true == include_attr) {
733 if(true == HDF5RequestHandler::get_drop_long_string()){
734 for (vector<GMCVar *>::iterator irv = this->cvars.begin();
735 irv != this->cvars.end(); ++irv) {
736 for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
737 ira != (*irv)->attrs.end();++ira) {
738 if(true == Check_DropLongStr((*irv),(*ira))) {
739 this->add_ignored_droplongstr_hdr();
740 this->add_ignored_var_longstr_info((*irv),(*ira));
741 }
742 }
743 }
744
745 for (vector<GMSPVar *>::iterator irv = this->spvars.begin();
746 irv != this->spvars.end(); ++irv) {
747 for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
748 ira != (*irv)->attrs.end();++ira) {
749 if(true == Check_DropLongStr((*irv),(*ira))) {
750 this->add_ignored_droplongstr_hdr();
751 this->add_ignored_var_longstr_info((*irv),(*ira));
752 }
753 }
754
755 }
756 }
757 }
758#endif
759
760 if(false == this->have_ignored)
761 this->add_no_ignored_info();
762
763}
764
765// Add dimension names
767
768 BESDEBUG("h5", "Coming to GMFile:Add_Dim_Name()"<<endl);
769 switch(product_type) {
770 case Mea_SeaWiFS_L2:
771 case Mea_SeaWiFS_L3:
772 Add_Dim_Name_Mea_SeaWiFS();
773 break;
774 case Aqu_L3:
775 Add_Dim_Name_Aqu_L3();
776 break;
777 case OSMAPL2S:
778 Add_Dim_Name_OSMAPL2S();
779 break;
780 case ACOS_L2S_OR_OCO2_L1B:
781 Add_Dim_Name_ACOS_L2S_OCO2_L1B();
782 break;
783 case Mea_Ozone:
784 Add_Dim_Name_Mea_Ozonel3z();
785 break;
786 case GPMS_L3:
787 case GPMM_L3:
788 case GPM_L1:
789 case GPM_L3_New:
790 Add_Dim_Name_GPM();
791 break;
792 case OBPG_L3:
793 Add_Dim_Name_OBPG_L3();
794 break;
795 case General_Product:
796 Add_Dim_Name_General_Product();
797 break;
798 default:
799 throw1("Cannot generate dim. names for unsupported datatype");
800 }
801
802// Just for debugging
803#if 0
804for (vector<Var*>::iterator irv2 = this->vars.begin();
805 irv2 != this->vars.end(); irv2++) {
806 for (vector<Dimension *>::iterator ird = (*irv2)->dims.begin();
807 ird !=(*irv2)->dims.end(); ird++) {
808 cerr<<"Dimension name afet Add_Dim_Name "<<(*ird)->newname <<endl;
809 }
810}
811#endif
812
813}
814
815//Add Dim. Names for OBPG level 3 product
816void GMFile::Add_Dim_Name_OBPG_L3() {
817
818 BESDEBUG("h5", "Coming to Add_Dim_Name_OBPG_L3()"<<endl);
819 // netCDF-4 like structure
820 // Note: We need to change the product type to netCDF-4 like product type and pattern.
821 Check_General_Product_Pattern();
822 Add_Dim_Name_General_Product();
823}
824
825//Add Dim. Names for MeaSures SeaWiFS. Future: May combine with the handling of netCDF-4 products
826void GMFile::Add_Dim_Name_Mea_SeaWiFS() {
827
828 BESDEBUG("h5", "Coming to Add_Dim_Name_Mea_SeaWiFS()"<<endl);
829 pair<set<string>::iterator,bool> setret;
830 if (Mea_SeaWiFS_L3 == product_type)
831 iscoard = true;
832 for (const auto &var:this->vars) {
833 Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(var);
834 for (const auto &dim:var->dims) {
835 setret = dimnamelist.insert(dim->name);
836 if (true == setret.second)
837 Insert_One_NameSizeMap_Element(dim->name,dim->size,dim->unlimited_dim);
838 }
839 }
840
841 if (true == dimnamelist.empty())
842 throw1("This product should have the dimension names, but no dimension names are found");
843}
844
845// Handle Dimension scales for MEasUREs SeaWiFS and OZone.
846void GMFile::Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(Var* var)
847{
848
849 BESDEBUG("h5", "Coming to Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone()"<<endl);
850 Attribute* dimlistattr = nullptr;
851 bool has_dimlist = false;
852 bool has_class = false;
853 bool has_reflist = false;
854
855 for (const auto &attr:var->attrs) {
856 if ("DIMENSION_LIST" == attr->name) {
857 dimlistattr = attr;
858 has_dimlist = true;
859 }
860 if ("CLASS" == attr->name)
861 has_class = true;
862 if ("REFERENCE_LIST" == attr->name)
863 has_reflist = true;
864
865 if (true == has_dimlist)
866 break;
867 if (true == has_class && true == has_reflist)
868 break;
869 }
870
871 if (true == has_dimlist)
872 Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(var,dimlistattr);
873
874 // Dim name is the same as the variable name for dimscale variable
875 else if(true == has_class && true == has_reflist) {
876 if (var->dims.size() !=1)
877 throw2("dimension scale dataset must be 1 dimension, this is not true for variable ",
878 var->name);
879
880 // The var name is the object name, however, we would like the dimension name to be full path.
881 // so that the dim name can be served as the key for future handling.
882 (var->dims)[0]->name = var->fullpath;
883 (var->dims)[0]->newname = var->fullpath;
884 pair<set<string>::iterator,bool> setret;
885 setret = dimnamelist.insert((var->dims)[0]->name);
886 if (true == setret.second)
887 Insert_One_NameSizeMap_Element((var->dims)[0]->name,(var->dims)[0]->size,(var->dims)[0]->unlimited_dim);
888 }
889
890 // No dimension, add fake dim names, this may never happen for MeaSure
891 // but just for coherence and completeness.
892 // For Fake dimension
893 else {
894
895 set<hsize_t> fakedimsize;
896 pair<set<hsize_t>::iterator,bool> setsizeret;
897 for (auto &dim:var->dims) {
898 Add_One_FakeDim_Name(dim);
899 setsizeret = fakedimsize.insert(dim->size);
900 if (false == setsizeret.second)
901 Adjust_Duplicate_FakeDim_Name(dim);
902 }
903// Just for debugging
904#if 0
905 for (int i = 0; i < var->dims.size(); ++i) {
906 Add_One_FakeDim_Name((var->dims)[i]);
907 bool gotoMainLoop = false;
908 for (int j =i-1; j>=0 && !gotoMainLoop; --j) {
909 if (((var->dims)[i])->size == ((var->dims)[j])->size){
910 Adjust_Duplicate_FakeDim_Name((var->dims)[i]);
911 gotoMainLoop = true;
912 }
913 }
914 }
915#endif
916
917 }//end of else
918}
919
920// Helper function to support dimensions of MeaSUrES SeaWiFS and OZone products
921void GMFile::Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(const Var *var,const Attribute*dimlistattr)
922{
923
924 BESDEBUG("h5", "Coming to Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone()"<<endl);
925 ssize_t objnamelen = -1;
926 hobj_ref_t rbuf;
927
928 vector<hvl_t> vlbuf;
929
930 hid_t dset_id = -1;
931 hid_t attr_id = -1;
932 hid_t atype_id = -1;
933 hid_t amemtype_id = -1;
934 hid_t aspace_id = -1;
935 hid_t ref_dset = -1;
936
937
938 if(nullptr == dimlistattr)
939 throw2("Cannot obtain the dimension list attribute for variable ",var->name);
940
941 if (0==var->rank)
942 throw2("The number of dimension should NOT be 0 for the variable ",var->name);
943
944 try {
945
946 vlbuf.resize(var->rank);
947
948 dset_id = H5Dopen(this->fileid,(var->fullpath).c_str(),H5P_DEFAULT);
949 if (dset_id < 0)
950 throw2("Cannot open the dataset ",var->fullpath);
951
952 attr_id = H5Aopen(dset_id,(dimlistattr->name).c_str(),H5P_DEFAULT);
953 if (attr_id <0 )
954 throw4("Cannot open the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
955
956 atype_id = H5Aget_type(attr_id);
957 if (atype_id <0)
958 throw4("Cannot obtain the datatype of the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
959
960 amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
961
962 if (amemtype_id < 0)
963 throw2("Cannot obtain the memory datatype for the attribute ",dimlistattr->name);
964
965
966 if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0)
967 throw2("Cannot obtain the referenced object for the variable ",var->name);
968
969
970 vector<char> objname;
971 int vlbuf_index = 0;
972
973 // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
974 for (auto &vdim:var->dims) {
975
976 if(vlbuf[vlbuf_index].p== nullptr)
977 throw4("The dimension doesn't exist. Var name is ",var->name,"; the dimension index is ",vlbuf_index);
978
979 rbuf =((hobj_ref_t*)vlbuf[vlbuf_index].p)[0];
980 if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0)
981 throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
982 // The above code works with dereferencing references generated with new H5R(H5R_ref_t) APIs with 1.12 and 1.13.
983 // However, in case this h5rdeference2 API stops working with new APIs, the following #if 0 #endif block is a
984 // way to handle this issue.
985#if 0
986
987 rbuf =((hobj_ref_t*)vl_ref[0].p)[0];
988 H5E_BEGIN_TRY {
989 dset1 = H5Rdereference2(attr_id,H5P_DEFAULT,H5R_OBJECT,&ds_ref_buf);
990 } H5E_END_TRY;
991
992 H5R_ref_t new_rbuf =((H5R_ref_t*)vlbuf[vlbuf_index].p)[0];
993 if ((ref_dset = H5Ropen_object((H5R_ref_t *)&new_rbuf, H5P_DEFAULT, H5P_DEFAULT))<0)
994 throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
995 H5Rdestroy(&new_rbuf);
996
997#endif
998 if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0)
999 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
1000 objname.resize(objnamelen+1);
1001 if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0)
1002 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
1003
1004 auto objname_str = string(objname.begin(),objname.end());
1005 string trim_objname = objname_str.substr(0,objnamelen);
1006 vdim->name = string(trim_objname.begin(),trim_objname.end());
1007
1008 pair<set<string>::iterator,bool> setret;
1009 setret = dimnamelist.insert(vdim->name);
1010 if (true == setret.second)
1011 Insert_One_NameSizeMap_Element(vdim->name,vdim->size,vdim->unlimited_dim);
1012 vdim->newname = vdim->name;
1013 H5Dclose(ref_dset);
1014 objname.clear();
1015 vlbuf_index++;
1016 }// end of for
1017
1018 if(vlbuf.empty()==false) {
1019
1020 if ((aspace_id = H5Aget_space(attr_id)) < 0)
1021 throw2("Cannot get hdf5 dataspace id for the attribute ",dimlistattr->name);
1022
1023 if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0)
1024 throw2("Cannot successfully clean up the variable length memory for the variable ",var->name);
1025
1026 H5Sclose(aspace_id);
1027
1028 }
1029
1030 H5Tclose(atype_id);
1031 H5Tclose(amemtype_id);
1032 H5Aclose(attr_id);
1033 H5Dclose(dset_id);
1034
1035 }
1036
1037 catch(...) {
1038
1039 if(atype_id != -1)
1040 H5Tclose(atype_id);
1041
1042 if(amemtype_id != -1)
1043 H5Tclose(amemtype_id);
1044
1045 if(aspace_id != -1)
1046 H5Sclose(aspace_id);
1047
1048 if(attr_id != -1)
1049 H5Aclose(attr_id);
1050
1051 if(dset_id != -1)
1052 H5Dclose(dset_id);
1053
1054 throw;
1055 }
1056
1057}
1058
1059// Add MeaSURES OZone level 3Z dimension names
1060void GMFile::Add_Dim_Name_Mea_Ozonel3z() {
1061
1062 BESDEBUG("h5", "Coming to Add_Dim_Name_Mea_Ozonel3z()"<<endl);
1063 iscoard = true;
1064 bool use_dimscale = false;
1065
1066 for (const auto &grp:this->groups) {
1067 if ("/Dimensions" == grp->path) {
1068 use_dimscale = true;
1069 break;
1070 }
1071 }
1072
1073 if (false == use_dimscale) {
1074
1075 bool has_dimlist = false;
1076 bool has_class = false;
1077 bool has_reflist = false;
1078
1079 for (const auto &var:this->vars) {
1080 for(const auto &attr:var->attrs) {
1081 if ("DIMENSION_LIST" == attr->name)
1082 has_dimlist = true;
1083 }
1084 if (true == has_dimlist)
1085 break;
1086 }
1087
1088 if (true == has_dimlist) {
1089
1090 for (const auto &var:this->vars) {
1091
1092 for (const auto &attr:var->attrs) {
1093 if ("CLASS" == attr->name)
1094 has_class = true;
1095 if ("REFERENCE_LIST" == attr->name)
1096 has_reflist = true;
1097 if (true == has_class && true == has_reflist)
1098 break;
1099 }
1100 if (true == has_class &&
1101 true == has_reflist)
1102 break;
1103
1104 }
1105 if (true == has_class && true == has_reflist)
1106 use_dimscale = true;
1107 } // end of if (true == has_dimlist)
1108 } // end of if (false == use_dimscale)
1109
1110 if (true == use_dimscale) {
1111
1112 pair<set<string>::iterator,bool> setret;
1113 for (const auto &var:this->vars) {
1114 Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(var);
1115 for (const auto &dim:var->dims) {
1116 setret = dimnamelist.insert(dim->name);
1117 if(true == setret.second)
1118 Insert_One_NameSizeMap_Element(dim->name,dim->size,dim->unlimited_dim);
1119 }
1120 }
1121
1122 if (true == dimnamelist.empty())
1123 throw1("This product should have the dimension names, but no dimension names are found");
1124 } // end of if (true == use_dimscale)
1125
1126 else {
1127
1128 // Since the dim. size of each dimension of 2D lat/lon may be the same, so use multimap.
1129 multimap<hsize_t,string> ozonedimsize_to_dimname;
1130 pair<multimap<hsize_t,string>::iterator,multimap<hsize_t,string>::iterator> mm_er_ret;
1131 multimap<hsize_t,string>::iterator irmm;
1132
1133 for (const auto &var:this->vars) {
1134 bool is_cv = check_cv(var->name);
1135 if (true == is_cv) {
1136 if (var->dims.size() != 1)
1137 throw3("The coordinate variable", var->name," must be one dimension for the zonal average product");
1138 ozonedimsize_to_dimname.insert(pair<hsize_t,string>((var->dims)[0]->size,var->fullpath));
1139 }
1140 }// end of for
1141
1142 set<hsize_t> fakedimsize;
1143 pair<set<hsize_t>::iterator,bool> setsizeret;
1144 pair<set<string>::iterator,bool> setret;
1145 pair<set<string>::iterator,bool> tempsetret;
1146 set<string> tempdimnamelist;
1147 bool fakedimflag = false;
1148
1149 for (const auto &var:this->vars) {
1150
1151 for (auto &dim:var->dims) {
1152
1153 fakedimflag = true;
1154 mm_er_ret = ozonedimsize_to_dimname.equal_range(dim->size);
1155 for (irmm = mm_er_ret.first; irmm!=mm_er_ret.second;irmm++) {
1156 setret = tempdimnamelist.insert(irmm->second);
1157 if (true == setret.second) {
1158 dim->name = irmm->second;
1159 dim->newname = dim->name;
1160 setret = dimnamelist.insert(dim->name);
1161 if(setret.second) Insert_One_NameSizeMap_Element(dim->name,dim->size,dim->unlimited_dim);
1162 fakedimflag = false;
1163 break;
1164 }
1165 }
1166
1167 if (true == fakedimflag) {
1168 Add_One_FakeDim_Name(dim);
1169 setsizeret = fakedimsize.insert(dim->size);
1170 if (false == setsizeret.second)
1171 Adjust_Duplicate_FakeDim_Name(dim);
1172 }
1173
1174 } // end of for
1175 tempdimnamelist.clear();
1176 fakedimsize.clear();
1177 } // end of for
1178 } // end of else
1179}
1180
1181// This is a special helper function for MeaSURES ozone products
1182bool GMFile::check_cv(const string & varname) const {
1183
1184 BESDEBUG("h5", "Coming to check_cv()"<<endl);
1185 const string lat_name ="Latitude";
1186 const string time_name ="Time";
1187 const string ratio_pressure_name ="MixingRatioPressureLevels";
1188 const string profile_pressure_name ="ProfilePressureLevels";
1189 const string wave_length_name ="Wavelength";
1190
1191 if (lat_name == varname)
1192 return true;
1193 else if (time_name == varname)
1194 return true;
1195 else if (ratio_pressure_name == varname)
1196 return true;
1197 else if (profile_pressure_name == varname)
1198 return true;
1199 else if (wave_length_name == varname)
1200 return true;
1201 else
1202 return false;
1203}
1204
1205// Add Dimension names for GPM products
1206void GMFile::Add_Dim_Name_GPM()
1207{
1208
1209 BESDEBUG("h5", "Coming to Add_Dim_Name_GPM()"<<endl);
1210 // This is used to create a dimension name set.
1211 pair<set<string>::iterator,bool> setret;
1212
1213 // The commented code is for an old version of GPM products. May remove them later. KY 2015-06-16
1214 // We need to create a fakedim name to fill in. To make the dimension name unique, we use a counter.
1215#if 0
1216 // int dim_count = 0;
1217 // map<string,string> varname_to_fakedim;
1218 // map<int,string> gpm_dimsize_to_fakedimname;
1219#endif
1220
1221 // We find that GPM has an attribute DimensionNames(nlon,nlat) in this case.
1222 // We will use this attribute to specify the dimension names.
1223 for (const auto &var:this->vars) {
1224
1225 for (auto &attr:var->attrs) {
1226
1227 if("DimensionNames" == attr->name) {
1228
1229 Retrieve_H5_Attr_Value(attr,var->fullpath);
1230 string dimname_value(attr->value.begin(),attr->value.end());
1231
1232 vector<string> ind_elems;
1233 char sep=',';
1234 HDF5CFUtil::Split(dimname_value.data(),sep,ind_elems);
1235
1236 if(ind_elems.size() != (size_t)(var->getRank())) {
1237 throw2("The number of dims obtained from the <DimensionNames> attribute is not equal to the rank ",
1238 var->name);
1239 }
1240
1241 for(unsigned int i = 0; i<ind_elems.size(); ++i) {
1242
1243 (var->dims)[i]->name = ind_elems[i];
1244
1245 // Generate a dimension name if the dimension name is missing.
1246 // The routine will ensure that the fakeDim name is unique.
1247 if((var->dims)[i]->name==""){
1248 Add_One_FakeDim_Name((var->dims)[i]);
1249// For debugging
1250#if 0
1251 string fakedim = "FakeDim";
1252 stringstream sdim_count;
1253 sdim_count << dim_count;
1254 fakedim = fakedim + sdim_count.str();
1255 dim_count++;
1256 (var->dims)[i]->name = fakedim;
1257 (var->dims)[i]->newname = fakedim;
1258 ind_elems[i] = fakedim;
1259#endif
1260 }
1261
1262 else {
1263 (var->dims)[i]->newname = ind_elems[i];
1264 setret = dimnamelist.insert((var->dims)[i]->name);
1265
1266 if (true == setret.second) {
1267 Insert_One_NameSizeMap_Element((var->dims)[i]->name,
1268 (var->dims)[i]->size,
1269 (var->dims)[i]->unlimited_dim);
1270 }
1271 else {
1272 if(dimname_to_dimsize[(var->dims)[i]->name] !=(var->dims)[i]->size)
1273 throw5("Dimension ",(var->dims)[i]->name, "has two sizes",
1274 (var->dims)[i]->size,dimname_to_dimsize[(var->dims)[i]->name]);
1275
1276 }
1277 }
1278
1279 }// end of for
1280 break;
1281
1282 } //end of if
1283 } //end of for
1284
1285#if 0
1286 if(false == has_dim_name_attr) {
1287
1288 throw4( "The variable ", var->name, " doesn't have the DimensionNames attribute.",
1289 "We currently don't support this case. Please report to the NASA data center.");
1290 }
1291
1292#endif
1293 } //end of for
1294
1295}
1296
1297// Add Dimension names for Aquarius level 3 products
1298void GMFile::Add_Dim_Name_Aqu_L3() const
1299{
1300 BESDEBUG("h5", "Coming to Add_Dim_Name_Aqu_L3()"<<endl);
1301 for (auto &var:this->vars) {
1302 if ("l3m_data" == var->name) {
1303 (var->dims)[0]->name = "lat";
1304 (var->dims)[0]->newname = "lat";
1305 (var->dims)[1]->name = "lon";
1306 (var->dims)[1]->newname = "lon";
1307 break;
1308 }
1309
1310// For the time being, don't assign dimension names to palette,
1311// we will see if tools can pick up l3m and then make decisions.
1312#if 0
1313 if ("palette" == (*irv)->name) {
1314//"h5","coming to palette" <<endl;
1315 ((*irv)->dims)[0]->name = "paldim0";
1316 ((*irv)->dims)[0]->newname = "paldim0";
1317 ((*irv)->dims)[1]->name = "paldim1";
1318 ((*irv)->dims)[1]->newname = "paldim1";
1319 }
1320#endif
1321
1322 }// end of for
1323}
1324
1325// Add dimension names for OSMAPL2S(note: the SMAP change their structures. The code doesn't apply to them.)
1326void GMFile::Add_Dim_Name_OSMAPL2S(){
1327
1328 BESDEBUG("h5", "Coming to Add_Dim_Name_OSMAPL2S()"<<endl);
1329 string tempvarname;
1330 string key = "_lat";
1331 string osmapl2sdim0 ="YDim";
1332 string osmapl2sdim1 ="XDim";
1333
1334 // Since the dim. size of each dimension of 2D lat/lon may be the same, so use multimap.
1335 multimap<hsize_t,string> osmapl2sdimsize_to_dimname;
1336 pair<multimap<hsize_t,string>::iterator,multimap<hsize_t,string>::iterator> mm_er_ret;
1337 multimap<hsize_t,string>::iterator irmm;
1338
1339 // Generate dimension names based on the size of "???_lat"(one coordinate variable)
1340 for (const auto &var:this->vars) {
1341 tempvarname = var->name;
1342 if ((tempvarname.size() > key.size())&&
1343 (key == tempvarname.substr(tempvarname.size()-key.size(),key.size()))){
1344 if (var->dims.size() !=2)
1345 throw1("Currently only 2D lat/lon is supported for OSMAPL2S");
1346 osmapl2sdimsize_to_dimname.insert(pair<hsize_t,string>((var->dims)[0]->size,osmapl2sdim0));
1347 osmapl2sdimsize_to_dimname.insert(pair<hsize_t,string>((var->dims)[1]->size,osmapl2sdim1));
1348 break;
1349 }
1350 }
1351
1352 set<hsize_t> fakedimsize;
1353 pair<set<hsize_t>::iterator,bool> setsizeret;
1354 pair<set<string>::iterator,bool> setret;
1355 pair<set<string>::iterator,bool> tempsetret;
1356 set<string> tempdimnamelist;
1357 bool fakedimflag = false;
1358
1359
1360 for (const auto &var:this->vars) {
1361 for (auto &dim:var->dims) {
1362
1363 fakedimflag = true;
1364 mm_er_ret = osmapl2sdimsize_to_dimname.equal_range(dim->size);
1365 for (irmm = mm_er_ret.first; irmm!=mm_er_ret.second;irmm++) {
1366 setret = tempdimnamelist.insert(irmm->second);
1367 if (setret.second) {
1368 dim->name = irmm->second;
1369 dim->newname = dim->name;
1370 setret = dimnamelist.insert(dim->name);
1371 if(setret.second) Insert_One_NameSizeMap_Element(dim->name,dim->size,dim->unlimited_dim);
1372 fakedimflag = false;
1373 break;
1374 }
1375 }
1376
1377 if (true == fakedimflag) {
1378 Add_One_FakeDim_Name(dim);
1379 setsizeret = fakedimsize.insert(dim->size);
1380 if (!setsizeret.second)
1381 Adjust_Duplicate_FakeDim_Name(dim);
1382 }
1383 } // end of for dim
1384 tempdimnamelist.clear();
1385 fakedimsize.clear();
1386 } // end of for var
1387}
1388
1389//Add dimension names for ACOS level2S or OCO2 level1B products
1390void GMFile::Add_Dim_Name_ACOS_L2S_OCO2_L1B(){
1391
1392 BESDEBUG("h5", "Coming to Add_Dim_Name_ACOS_L2S_OCO2_L1B()"<<endl);
1393 for (const auto &var:this->vars) {
1394 set<hsize_t> fakedimsize;
1395 pair<set<hsize_t>::iterator,bool> setsizeret;
1396 for (auto dim:var->dims) {
1397 Add_One_FakeDim_Name(dim);
1398 setsizeret = fakedimsize.insert(dim->size);
1399 if (false == setsizeret.second)
1400 Adjust_Duplicate_FakeDim_Name(dim);
1401 }
1402 } // end of for var
1403}
1404
1405// Add dimension names for general products. Read the descrption of Check_General_Product_Pattern() for different patterns we support.
1406void GMFile::Add_Dim_Name_General_Product(){
1407
1408 BESDEBUG("h5", "Coming to Add_Dim_Name_General_Product()"<<endl);
1409
1410 // This general product should follow the HDF5 dimension scale model.
1411 if (GENERAL_DIMSCALE == this->gproduct_pattern){
1412 Add_Dim_Name_Dimscale_General_Product();
1413}
1414 // This general product has 2-D latitude,longitude
1415 else if (GENERAL_LATLON2D == this->gproduct_pattern)
1416 Add_Dim_Name_LatLon2D_General_Product();
1417 // This general product has 1-D latitude,longitude
1418 else if (GENERAL_LATLON1D == this->gproduct_pattern || GENERAL_LATLON_COOR_ATTR == this->gproduct_pattern)
1419 Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product();
1420
1421
1422}
1423
1424// We check four patterns under the General_Product category
1425// 1. General products that uses HDF5 dimension scales following netCDF-4 data model
1426// 2. General products that have 2-D lat/lon variables(lat/lon variable names are used to identify the case) under the root group or
1427// a special geolocation group
1428// 3. General products that have 1-D lat/lon variables(lat/lon variable names are used to identify the case) under the root group or
1429// a special geolocation group
1430// 4. General products that have some variables containing CF "coordinates" attributes. We can support some products if the "coordinates"
1431// attribute contains CF lat/lon units and the variable ranks are 2 or 1.
1432void GMFile::Check_General_Product_Pattern() {
1433
1434 BESDEBUG("h5", "Coming to Check_General_Product_Pattern()"<<endl);
1435 if (false == Check_Dimscale_General_Product_Pattern()) {
1436 //HERE add a check for the GPM. (choose 5 variables equally distance for the attribute)
1437 if (false == Check_And_Update_New_GPM_L3())
1438 if (false == Check_LatLon2D_General_Product_Pattern())
1439 if (false == Check_LatLon1D_General_Product_Pattern())
1440 Check_LatLon_With_Coordinate_Attr_General_Product_Pattern();
1441 }
1442
1443}
1444
1445// Check if this general product is netCDF4-like HDF5 file.
1446// We only need to check "DIMENSION_LIST","CLASS" and CLASS values.
1447bool GMFile::Check_Dimscale_General_Product_Pattern() {
1448
1449 BESDEBUG("h5", "Coming to Check_Dimscale_General_Product_Pattern()"<<endl);
1450 bool ret_value = false;
1451 bool has_dimlist = false;
1452 bool has_dimscalelist = false;
1453
1454 // Check if containing the "DIMENSION_LIST" attribute;
1455 for (const auto &var:this->vars) {
1456 for(const auto &attr:var->attrs) {
1457 if ("DIMENSION_LIST" == attr->name) {
1458 has_dimlist = true;
1459 break;
1460 }
1461 }
1462 if (true == has_dimlist)
1463 break;
1464 }
1465
1466 // Check if containing both the attribute "CLASS" and the attribute "REFERENCE_LIST" for the same variable.
1467 // This is the dimension scale.
1468 // Actually "REFERENCE_LIST" is not necessary for a dimension scale dataset. If a dimension scale doesn't
1469 // have a "REFERENCE_LIST", it is still valid. But no other variables use this dimension scale. We found
1470 // such a case in a matched_airs_aqua product. KY 2012-12-03
1471 for (const auto &var:this->vars) {
1472 for(const auto &attr:var->attrs) {
1473 if ("CLASS" == attr->name) {
1474
1475 Retrieve_H5_Attr_Value(attr,var->fullpath);
1476 string class_value;
1477 class_value.resize(attr->value.size());
1478 copy(attr->value.begin(),attr->value.end(),class_value.begin());
1479
1480 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
1481 // "DIMENSION_SCALE", which is 15.
1482 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
1483 has_dimscalelist = true;
1484 break;
1485 }
1486 }
1487 }
1488 if (true == has_dimscalelist)
1489 break;
1490 }
1491
1492 if (true == has_dimscalelist) {
1493 if (true == has_dimlist ) {
1494 this->gproduct_pattern = GENERAL_DIMSCALE;
1495 ret_value = true;
1496 }
1497 else {
1498 //May fall into the single dimension scale case.
1499 //This is really, really rare,but we do have to check.
1500 // Check if NAME and _Netcdf4Dimid exists for this variable.
1501
1502 bool is_general_dimscale = false;
1503
1504 for (const auto &var:this->vars) {
1505
1506 bool has_class_dscale = false;
1507 bool has_name = false;
1508 bool has_netcdf4_id = false;
1509
1510 for (const auto &attr:var->attrs) {
1511
1512 if ("CLASS" == attr->name) {
1513 Retrieve_H5_Attr_Value(attr,var->fullpath);
1514 string class_value;
1515 class_value.resize(attr->value.size());
1516 copy(attr->value.begin(),attr->value.end(),class_value.begin());
1517
1518 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
1519 // "DIMENSION_SCALE", which is 15.
1520 if (0 == class_value.compare(0,15,"DIMENSION_SCALE"))
1521 has_class_dscale= true;
1522 }
1523 else if ("NAME" == attr->name)
1524 has_name = true;
1525 else if ("_Netcdf4Dimid" == attr->name)
1526 has_netcdf4_id = true;
1527 if(true == has_class_dscale && true == has_name && true == has_netcdf4_id)
1528 is_general_dimscale = true;
1529 }
1530
1531 if(true == is_general_dimscale)
1532 break;
1533 }
1534
1535 if (true == is_general_dimscale) {
1536 this->gproduct_pattern = GENERAL_DIMSCALE;
1537 ret_value = true;
1538 }
1539 }
1540 }
1541
1542 return ret_value;
1543}
1544
1545bool GMFile::Check_And_Update_New_GPM_L3() {
1546
1547 bool is_new_gpm_l3 = false;
1548 unsigned num_vars = this->vars.size();
1549 unsigned sel_steps = num_vars/5;
1550 string dim_name="DimensionNames";
1551 bool has_dim_name = false;
1552 if(sel_steps == 0)
1553 sel_steps = 1;
1554
1555 // Given DimensionNames exists in almost every variable in the new GPM product,
1556 // We will check the existence of this attribute for at most 5 variables.
1557
1558 vector<Var *>::iterator it_var_end;
1559
1560 if(sel_steps ==1)
1561 it_var_end = this->vars.end();
1562 else
1563 it_var_end = this->vars.begin()+5*sel_steps;
1564
1565 for (auto irv = this->vars.begin(); irv != it_var_end; irv+=sel_steps) {
1566 for (const auto &attr:(*irv)->attrs) {
1567
1568 if(H5FSTRING == attr->getType()) {
1569 if(attr->name == dim_name){
1570 has_dim_name = true;
1571 break;
1572 }
1573 }
1574 }
1575 if(true == has_dim_name)
1576 break;
1577 }
1578
1579 // Files that can go to this step should be a small subset, now
1580 // we will check the "??GridHeader" for all the groups.
1581 if(true == has_dim_name) {
1582 string attr_name_subset = "GridHeader";
1583 BESDEBUG("h5", "GMFile::Check_And_Update_New_GPM_L3() has attribute <DimensionNames>. "<<endl);
1584 for (const auto &grp:this->groups) {
1585 for (const auto &attr:grp->attrs) {
1586
1587 string attr_name = attr->name;
1588
1589 // We identify this as a new GPM level 3 product.
1590 if (attr_name.find(attr_name_subset)!=string::npos) {
1591 this->product_type = GPM_L3_New;
1592 is_new_gpm_l3 = true;
1593 break;
1594 }
1595 }
1596 if (true == is_new_gpm_l3)
1597 break;
1598 }
1599 }
1600 return is_new_gpm_l3;
1601}
1602
1603// If having 2-D latitude/longitude,set the general product pattern.
1604// In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" and "cell_lat,cell_lon"names.
1605// The "cell_lat" and "cell_lon" come from SMAP. KY 2015-12-2
1606bool GMFile::Check_LatLon2D_General_Product_Pattern() {
1607
1608 BESDEBUG("h5", "Coming to Check_LatLon2D_General_Product_Pattern()"<<endl);
1609 bool ret_value = false;
1610
1611 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("latitude","longitude");
1612 if(false == ret_value) {
1613 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("Latitude","Longitude");
1614 if(false == ret_value) {
1615 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("lat","lon");
1616 if(false == ret_value)
1617 ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("cell_lat","cell_lon");
1618 }
1619 }
1620
1621 // Make sure set the general product pattern flag for this case.
1622 if(true == ret_value)
1623 this->gproduct_pattern = GENERAL_LATLON2D;
1624 return ret_value;
1625
1626}
1627
1628// Helper function for Check_LatLon2D_General_Product_Pattern,we assume the lat and lon only present either under the root or
1629// a specific group Geolocation.
1630bool GMFile::Check_LatLon2D_General_Product_Pattern_Name_Size(const string & latname,const string & lonname) {
1631
1632 BESDEBUG("h5", "Coming to Check_LatLon2D_General_Product_Pattern_Name_Size()"<<endl);
1633 bool ret_value = false;
1634 bool ll_flag = false;
1635
1636 vector<size_t>lat_size(2,0);
1637 vector<size_t>lon_size(2,0);
1638
1639 const string designed_group1 = "/";
1640 const string designed_group2 = "/Geolocation/";
1641
1642 bool lat_flag_g1 = false;
1643 bool lon_flag_g1 = false;
1644 bool lat_flag_g2 = false;
1645 bool lon_flag_g2 = false;
1646
1647
1648 // This case allows to have both "lat and lon" under either group 1 or group 2 but on not both group 1 and 2.
1649 // This case doesn't allow "lat" and "lon" under separate groups.
1650 // Check if we have lat and lon at the only designated group,group 1 "/"
1651 lat_flag_g1 = is_var_under_group(latname,designed_group1,2,lat_size);
1652 lon_flag_g1 = is_var_under_group(lonname,designed_group1,2,lon_size);
1653 if(lat_flag_g1 == true && lon_flag_g1 == true) {
1654
1655 // Make sure the group 2 "/Geolocation" doesn't have the lat/lon
1656 lat_flag_g2 = is_var_under_group(latname,designed_group2,2,lat_size);
1657 if(lat_flag_g2 == false) {
1658 lon_flag_g2 = is_var_under_group(lonname,designed_group2,2,lon_size);
1659 if(lon_flag_g2 == false)
1660 ll_flag = true;
1661 }
1662 }// If the root doesn't have lat/lon, check the group 2 "/Geolocation".
1663 else if(lat_flag_g1 == false && lon_flag_g1 == false) {
1664 lat_flag_g2 = is_var_under_group(latname,designed_group2,2,lat_size);
1665 if(lat_flag_g2 == true) {
1666 lon_flag_g2 = is_var_under_group(lonname,designed_group2,2,lon_size);
1667 if(lon_flag_g2 == true)
1668 ll_flag = true;
1669 }
1670 }
1671
1672 // We are loose here since this is just to support some NASA products in a customized way.
1673 // If the first two cases don't exist, we allow to check another group "GeolocationData" and
1674 // see if Latitude and Longitude are present. (4 years ? from the first implementation, we got this case.)
1675 // KY 2020-02-27
1676 if(false == ll_flag) {
1677
1678 const string designed_group3 = "/GeolocationData/";
1679 if(is_var_under_group(latname,designed_group3,2,lat_size) &&
1680 is_var_under_group(lonname,designed_group3,2,lon_size))
1681 ll_flag = true;
1682 }
1683
1684#if 0
1685
1686 for (vector<Var *>::iterator irv = this->vars.begin();
1687 irv != this->vars.end(); ++irv) {
1688
1689 if((*irv)->rank == 2) {
1690 if((*irv)->name == latname) {
1691
1692 // Obtain the variable path
1693 string lat_path =HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1694
1695 // Tackle only the root group or the name of the group as "/Geolocation"
1696 // By doing this, we assume that the file has lat/lon either under the root or under the "Geolocation
1697 // but not BOTH. The following code may generate wrong results if the file contains lat/lon under
1698 // both the root and /Geolocation. This is documented in https://jira.hdfgroup.org/browse/HFVHANDLER-175
1699 bool has_right_lat = false;
1700 if("/" == lat_path || "/Geolocation/" == lat_path)
1701 if("/" == lat_path || "/Geolocation/" == lat_path) {
1702 ll_flag++;
1703 lat_size[0] = (*irv)->getDimensions()[0]->size;
1704 lat_size[1] = (*irv)->getDimensions()[1]->size;
1705 }
1706
1707 }
1708 else if((*irv)->name == lonname) {
1709 string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1710 if("/" == lon_path || "/Geolocation/" == lon_path) {
1711 ll_flag++;
1712 lon_size[0] = (*irv)->getDimensions()[0]->size;
1713 lon_size[1] = (*irv)->getDimensions()[1]->size;
1714 }
1715 }
1716 if(2 == ll_flag)
1717 break;
1718 } // if((*irv)->rank == 2)
1719 } // for (vector<Var *>::iterator irv = this->vars.begin();
1720
1721#endif
1722
1723 // Only when both lat/lon are found can we support this case.
1724 // Before that, we also need to check if the lat/lon shares the same dimensions.
1725 if(true == ll_flag) {
1726
1727 bool latlon_size_match = true;
1728 for (unsigned int size_index = 0; size_index <lat_size.size();size_index++) {
1729 if(lat_size[size_index] != lon_size[size_index]){
1730 latlon_size_match = false;
1731 break;
1732 }
1733 }
1734 if (true == latlon_size_match) {
1735 // If we do find the lat/lon pair, save them for later use.
1736 gp_latname = latname;
1737 gp_lonname = lonname;
1738 ret_value = true;
1739 }
1740
1741 }
1742
1743 return ret_value;
1744
1745}
1746
1747// If having 1-D latitude/longitude,set the general product pattern.
1748// In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" and "cell_lat,cell_lon"names.
1749// The "cell_lat" and "cell_lon" come from SMAP. KY 2015-12-2
1750bool GMFile::Check_LatLon1D_General_Product_Pattern() {
1751
1752 BESDEBUG("h5", "Coming to Check_LatLon1D_General_Product_Pattern()"<<endl);
1753 bool ret_value = false;
1754
1755 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("latitude","longitude");
1756 if(false == ret_value) {
1757 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("Latitude","Longitude");
1758 if(false == ret_value) {
1759 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("lat","lon");
1760 if(false == ret_value)
1761 ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("cell_lat","cell_lon");
1762 }
1763 }
1764
1765 if(true == ret_value)
1766 this->gproduct_pattern = GENERAL_LATLON1D;
1767 return ret_value;
1768
1769}
1770
1771// Helper function for Check_LatLon1D_General_Product_Pattern.
1772// We only check if the lat/lon etc. pairs are under "/" or "/Geolocation". Other cases can be easily added.
1773bool GMFile::Check_LatLon1D_General_Product_Pattern_Name_Size(const string & latname,const string & lonname) {
1774
1775 BESDEBUG("h5", "Coming to Check_LatLon1D_General_Product_Pattern_Name_Size()"<<endl);
1776 bool ret_value = false;
1777 short ll_flag = 0;
1778 size_t lat_size = 0;
1779 size_t lon_size = 0;
1780
1781 for (const auto &var:this->vars) {
1782
1783 if(var->rank == 1) {
1784 if(var->name == latname) {
1785
1786 string lat_path =HDF5CFUtil::obtain_string_before_lastslash(var->fullpath);
1787
1788 // Tackle only the root group or the name of the group as "/Geolocation"
1789 // May not generate the correct output. See https://jira.hdfgroup.org/browse/HFVHANDLER-175
1790 if("/" == lat_path || "/Geolocation/" == lat_path) {
1791 ll_flag++;
1792 lat_size = var->getDimensions()[0]->size;
1793 }
1794 }
1795 else if(var->name == lonname) {
1796 string lon_path = HDF5CFUtil::obtain_string_before_lastslash(var->fullpath);
1797 if("/" == lon_path || "/Geolocation/" == lon_path) {
1798 ll_flag++;
1799 lon_size = var->getDimensions()[0]->size;
1800 }
1801 }
1802 if(2 == ll_flag)
1803 break;
1804 }
1805 }
1806
1807 if(2 == ll_flag) {
1808
1809 bool latlon_size_match_grid = true;
1810
1811 // When the size of latitude is equal to the size of longitude for a 1-D lat/lon, it is very possible
1812 // that this is not a regular grid but rather a profile with the lat,lon recorded as the function of time.
1813 // Adding the coordinate/dimension as the normal grid is wrong, so check out this case.
1814 // KY 2015-12-2
1815 if(lat_size == lon_size) {
1816
1817 // It is very unusual that lat_size = lon_size for a grid.
1818 latlon_size_match_grid = false;
1819
1820 // For a normal grid, a >2D variable should exist to have both lat and lon size,
1821 // if such a variable that has the same size exists, we will treat it as a normal grid.
1822 for (const auto &var:this->vars) {
1823 if(var->rank >=2) {
1824 short ll_size_flag = 0;
1825 for (const auto &dim:var->dims) {
1826 if(lat_size == dim->size) {
1827 ll_size_flag++;
1828 if(2 == ll_size_flag){
1829 break;
1830 }
1831 }
1832 }
1833 if(2 == ll_size_flag) {
1834 latlon_size_match_grid = true;
1835 break;
1836 }
1837 }
1838 }
1839 }
1840
1841 // If the sizes of lat and lon match the grid, this is the lat/lon candidate.
1842 // Save the latitude and longitude names for later use.
1843 if (true == latlon_size_match_grid) {
1844 gp_latname = latname;
1845 gp_lonname = lonname;
1846 ret_value = true;
1847 }
1848 }
1849
1850 return ret_value;
1851}
1852
1853// This function checks if this general product contains "coordinates" attributes in some variables
1854// that can be used to handle CF friendly.
1855bool GMFile::Check_LatLon_With_Coordinate_Attr_General_Product_Pattern() {
1856
1857 BESDEBUG("h5", "Coming to Check_LatLon_With_Coordinate_Attr_General_Product_Pattern()"<<endl);
1858 bool ret_value = false;
1859 string co_attrname = "coordinates";
1860 string co_attrvalue;
1861 string unit_attrname = "units";
1862 string lat_unit_attrvalue ="degrees_north";
1863 string lon_unit_attrvalue ="degrees_east";
1864
1865 bool coor_has_lat_flag = false;
1866 bool coor_has_lon_flag = false;
1867
1868 vector<Var*> tempvar_lat;
1869 vector<Var*> tempvar_lon;
1870
1871 // Check if having both lat, lon names stored in the coordinate attribute value by looping through rank >1 variables.
1872 for (const auto &var:this->vars) {
1873
1874 if(var->rank >=2) {
1875 for (const auto &attr:var->attrs) {
1876
1877 // If having attribute "coordinates" for this variable, checking the values and
1878 // see if having lat/lon,latitude/longitude, Latitude/Longitude pairs.
1879 if(attr->name == co_attrname) {
1880 Retrieve_H5_Attr_Value(attr,var->fullpath);
1881 string orig_attr_value(attr->value.begin(),attr->value.end());
1882 vector<string> coord_values;
1883 char sep=' ';
1884 HDF5CFUtil::Split_helper(coord_values,orig_attr_value,sep);
1885
1886 for (const auto &coord_value:coord_values) {
1887 string coord_value_suffix1;
1888 string coord_value_suffix2;
1889 string coord_value_suffix3;
1890
1891 if(coord_value.size() >=3) {
1892
1893 // both "lat" and "lon" have 3 characters.
1894 coord_value_suffix1 = coord_value.substr(coord_value.size()-3,3);
1895
1896 // The word "latitude" has 8 characters and the word "longitude" has 9 characters.
1897 if(coord_value.size() >=8){
1898 coord_value_suffix2 = coord_value.substr(coord_value.size()-8,8);
1899 if(coord_value.size() >=9)
1900 coord_value_suffix3 = coord_value.substr(coord_value.size()-9,9);
1901 }
1902 }
1903
1904 // lat/longitude or latitude/lon pairs in theory are fine.
1905 if(coord_value_suffix1=="lat" || coord_value_suffix2 =="latitude" || coord_value_suffix2 == "Latitude")
1906 coor_has_lat_flag = true;
1907 else if(coord_value_suffix1=="lon" || coord_value_suffix3 =="longitude" || coord_value_suffix3 == "Longitude")
1908 coor_has_lon_flag = true;
1909 }
1910
1911 if(true == coor_has_lat_flag && true == coor_has_lon_flag)
1912 break;
1913 }// end of if(attr->name
1914 }// end of for (ira)
1915
1916 if(true == coor_has_lat_flag && true == coor_has_lon_flag)
1917 break;
1918 else {
1919 coor_has_lat_flag = false;
1920 coor_has_lon_flag = false;
1921 }
1922 } // end of if
1923 }// end of for
1924
1925 // Check the variable names that include latitude and longitude suffixes such as lat,latitude and Latitude.
1926 if(true == coor_has_lat_flag && true == coor_has_lon_flag) {
1927
1928 for (const auto &var:this->vars) {
1929
1930 bool var_is_lat = false;
1931 bool var_is_lon = false;
1932
1933 string varname = var->name;
1934 string ll_ssuffix;
1935 string ll_lsuffix1;
1936 string ll_lsuffix2;
1937 if(varname.size() >=3) {//lat/lon
1938 ll_ssuffix = varname.substr(varname.size()-3,3);
1939 if(varname.size() >=8) {//latitude/Latitude
1940 ll_lsuffix1 = varname.substr(varname.size()-8,8);
1941 if(varname.size() >=9)//Longitude/longitude
1942 ll_lsuffix2 = varname.substr(varname.size()-9,9);
1943 }
1944 }
1945 if(ll_ssuffix=="lat" || ll_lsuffix1 =="latitude" || ll_lsuffix1 == "Latitude")
1946 var_is_lat = true;
1947 else if(ll_ssuffix=="lon" || ll_lsuffix2 =="longitude" || ll_lsuffix2 == "Longitude")
1948 var_is_lon = true;
1949
1950 // Find the lat/lon candidate, save them to temporary vectors
1951 if(true == var_is_lat) {
1952 if(var->rank > 0) {
1953 auto lat_unique = make_unique<Var>(var);
1954 auto lat = lat_unique.release();
1955 tempvar_lat.push_back(lat);
1956 }
1957 }
1958 else if(true == var_is_lon) {
1959 if(var->rank >0) {
1960 auto lon_unique = make_unique<Var>(var);
1961 auto lon = lon_unique.release();
1962 tempvar_lon.push_back(lon);
1963 }
1964 }
1965 }// for of irv
1966
1967 // Build up latloncv_candidate_pairs, Name_Size_2Pairs struct,
1968 // 1) Compare the rank, dimension sizes and the dimension orders of tempvar_lon against tempvar_lat
1969 // rank >=2 the sizes,orders, should be consistent
1970 // rank =1, no check issue.
1971 // 2) If the conditions are fulfilled, save them to the Name_Size struct
1972 for (auto &t_lat:tempvar_lat) {
1973
1974 // Check the rank =1 case
1975 if(t_lat->rank == 1)
1976 Build_lat1D_latlon_candidate(t_lat,tempvar_lon);
1977
1978 // Check the reank>=2 case
1979 else if(t_lat->rank >1)
1980 Build_latg1D_latlon_candidate(t_lat,tempvar_lon);
1981 }
1982
1983#if 0
1984for(vector<struct Name_Size_2Pairs>::iterator ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
1985cerr<<"struct lat lon names are " <<(*ivs).name1 <<" and " << (*ivs).name2 <<endl;
1986}
1987#endif
1988
1989 // Check if there is duplicate latitude variables for one longitude variable in the latloncv_candidate_pairs.
1990 // if yes, remove the ones that have duplicate latitude variables.
1991 // This will assure that the latloncv_candidate_pairs is one-to-one mapping between latitude and longitude.
1992 Build_unique_latlon_candidate();
1993
1994
1995 // Even if we find that there are qualified geo-location coordinate pairs, we still need to check
1996 // the geo-location variable rank.
1997 // If the rank of any one-pair is 2, this case is qualified for the category GENERAL_LATLON_COOR_ATTR.
1998 // If the rank of any one-pair is 1,
1999 // we will check if the sizes of the lat and the lon in a pair are the same.
2000 // If they are not the same, this case is qualified for the category GENERAL_LATLON_COOR_ATTR
2001 // else check if there is any variable that has the "coordinates" attribute and the "coordinates" attribute includes
2002 // the paths of this lat/lon pair. If the dimensions of such a variable have two sizes that are equal to the size of the lat,
2003 // this case is still qualfied for the category GENERAL_LATLON_COOR_ATTR.
2004 // NOTE: here we deliberately ignore the case when the rank of lat/lon is >2. In some recent developments, we find that
2005 // there are 3D lat/lon and some tools like Panoply can visualize those data. So maybe we need to accept some 3D lat/lon in the futurei(KY 2016-07-07).
2006 if(latloncv_candidate_pairs.empty() == false) {
2007 int num_1d_rank = 0;
2008 int num_2d_rank = 0;
2009 int num_g2d_rank = 0;
2010 vector<struct Name_Size_2Pairs> temp_1d_latlon_pairs;
2011 for(const auto &llcv_p:latloncv_candidate_pairs) {
2012 if(1 == llcv_p.rank) {
2013 num_1d_rank++;
2014 temp_1d_latlon_pairs.push_back(llcv_p);
2015 }
2016 else if(2 == llcv_p.rank)
2017 num_2d_rank++;
2018 else if(llcv_p.rank >2)
2019 num_g2d_rank++;
2020 }
2021
2022 // This is the GENERAL_LATLON_COOR_ATTR case.
2023 if (num_2d_rank !=0)
2024 ret_value = true;
2025 else if(num_1d_rank!=0) {
2026
2027 // Check if lat and lon share the same size and the dimension of a variable
2028 // that has the "coordinates" only holds one size.
2029 for (const auto &t_1dll_p:temp_1d_latlon_pairs) {
2030 if(t_1dll_p.size1 != t_1dll_p.size2) {
2031 ret_value = true;
2032 break;
2033 }
2034 else {
2035
2036 // If 1-D lat and lon share the same size,we need to check if there is a variable
2037 // that has both lat and lon as the coordinates but only has one dimension that holds the size.
2038 // If this is true, this is not the GENERAL_LATLON_COOR_ATTR case(SMAP level 2 follows into the category).
2039
2040 ret_value = true;
2041 for (const auto &var:this->vars) {
2042
2043 if(var->rank >=2) {
2044 for (const auto &attr:var->attrs) {
2045
2046 // Check if this variable has the "coordinates" attribute
2047 if(attr->name == co_attrname) {
2048 Retrieve_H5_Attr_Value(attr,var->fullpath);
2049 string orig_attr_value(attr->value.begin(),attr->value.end());
2050 vector<string> coord_values;
2051 char sep=' ';
2052 HDF5CFUtil::Split_helper(coord_values,orig_attr_value,sep);
2053 bool has_lat_flag = false;
2054 bool has_lon_flag = false;
2055 for (const auto &c_value:coord_values) {
2056 if(t_1dll_p.name1 == c_value)
2057 has_lat_flag = true;
2058 else if(t_1dll_p.name2 == c_value)
2059 has_lon_flag = true;
2060 }
2061 // Find both lat and lon, now check the dim. size
2062 if(true == has_lat_flag && true == has_lon_flag) {
2063 short has_same_ll_size = 0;
2064 for(const auto &dim:var->dims){
2065 if(dim->size == t_1dll_p.size1)
2066 has_same_ll_size++;
2067 }
2068 if(has_same_ll_size!=2){
2069 ret_value = false;
2070 break;
2071 }
2072 }
2073 }
2074 }// end of for ira
2075
2076 if(false == ret_value)
2077 break;
2078 }// end of if irv
2079 }// end of for irv
2080
2081 if(true == ret_value)
2082 break;
2083 }// else
2084 }// end of for ivs
2085 } // else of if num_1d_rank
2086 }// end of if of latloncv_candidate_pairs
2087
2088 release_standalone_var_vector(tempvar_lat);
2089 release_standalone_var_vector(tempvar_lon);
2090
2091 }
2092#if 0
2093if(true == ret_value)
2094cerr<<"This product is the coordinate type "<<endl;
2095#endif
2096 // Don't forget to set the flag for this general product pattern.
2097 if(true == ret_value)
2098 this->gproduct_pattern = GENERAL_LATLON_COOR_ATTR;
2099
2100 return ret_value;
2101}
2102
2103// Build 1-D latlon coordinate variables candidate for GENERAL_LATLON_COOR_ATTR.
2104void GMFile::Build_lat1D_latlon_candidate(const Var *lat,const vector<Var*> &lon_vec) {
2105
2106 BESDEBUG("h5", "Coming to Build_lat1D_latlon_candidate()"<<endl);
2107 set<string> lon_candidate_path;
2108 vector< pair<string,hsize_t> > lon_path_size_vec;
2109
2110 // Obtain the path and the size info. from all the potential qualified longitude candidate.
2111 for(const auto &lon:lon_vec) {
2112
2113 if (lat->rank == lon->rank) {
2114 pair<string,hsize_t>lon_path_size;
2115 lon_path_size.first = lon->fullpath;
2116 lon_path_size.second = lon->getDimensions()[0]->size;
2117 lon_path_size_vec.push_back(lon_path_size);
2118 }
2119 }
2120
2121 // If there is only one potential qualified longitude for this latitude, just save this pair.
2122 if(lon_path_size_vec.size() == 1) {
2123
2124 Name_Size_2Pairs latlon_pair;
2125 latlon_pair.name1 = lat->fullpath;
2126 latlon_pair.name2 = lon_path_size_vec[0].first;
2127 latlon_pair.size1 = lat->getDimensions()[0]->size;
2128 latlon_pair.size2 = lon_path_size_vec[0].second;
2129 latlon_pair.rank = lat->rank;
2130 latloncv_candidate_pairs.push_back(latlon_pair);
2131
2132 }
2133 else if(lon_path_size_vec.size() >1) {
2134
2135 // For more than one potential qualified longitude, we can still find a qualified one
2136 // if we find there is only one longitude under the same group of this latitude.
2137 string lat_path = HDF5CFUtil::obtain_string_before_lastslash(lat->fullpath);
2138 pair<string,hsize_t> lon_final_path_size;
2139 short num_lon_match = 0;
2140 for (const auto &lon_size:lon_path_size_vec) {
2141 // Search the longitude path and see if it matches with the latitude.
2142 if(HDF5CFUtil::obtain_string_before_lastslash(lon_size.first)==lat_path) {
2143 num_lon_match++;
2144 if(1 == num_lon_match)
2145 lon_final_path_size = lon_size;
2146 else if(num_lon_match > 1)
2147 break;
2148 }
2149 }
2150 if(num_lon_match ==1) {// insert this lat/lon pair to the struct
2151 Name_Size_2Pairs latlon_pair;
2152 latlon_pair.name1 = lat->fullpath;
2153 latlon_pair.name2 = lon_final_path_size.first;
2154 latlon_pair.size1 = lat->getDimensions()[0]->size;
2155 latlon_pair.size2 = lon_final_path_size.second;
2156 latlon_pair.rank = lat->rank;
2157 latloncv_candidate_pairs.push_back(latlon_pair);
2158 }
2159 }
2160
2161}
2162
2163// Build >1D latlon coordinate variables candidate for GENERAL_LATLON_COOR_ATTR.
2164void GMFile::Build_latg1D_latlon_candidate(Var *lat,const vector<Var*> & lon_vec) {
2165
2166 BESDEBUG("h5", "Coming to Build_latg1D_latlon_candidate()"<<endl);
2167 set<string> lon_candidate_path;
2168
2169 // We will check if the longitude shares the same dimensions of the latitude
2170 for(const auto &lon:lon_vec) {
2171
2172 if (lat->rank == lon->rank) {
2173
2174 // Check the dim order and size.
2175 bool same_dim = true;
2176 for(int dim_index = 0; dim_index <lat->rank; dim_index++) {
2177 if(lat->getDimensions()[dim_index]->size !=
2178 lon->getDimensions()[dim_index]->size){
2179 same_dim = false;
2180 break;
2181 }
2182 }
2183 if(true == same_dim)
2184 lon_candidate_path.insert(lon->fullpath);
2185 }
2186 }
2187
2188 // Check the size of the lon., if the size is not 1, see if we can find the pair under the same group.
2189 if(lon_candidate_path.size() > 1) {
2190
2191 string lat_path = HDF5CFUtil::obtain_string_before_lastslash(lat->fullpath);
2192 vector <string> lon_final_candidate_path_vec;
2193 for(auto islon_path =lon_candidate_path.begin();islon_path!=lon_candidate_path.end();++islon_path) {
2194
2195 // Search the path.
2196 if(HDF5CFUtil::obtain_string_before_lastslash(*islon_path)==lat_path)
2197 lon_final_candidate_path_vec.push_back(*islon_path);
2198 }
2199
2200 if(lon_final_candidate_path_vec.size() == 1) {// insert this lat/lon pair to the struct
2201
2202 Name_Size_2Pairs latlon_pair;
2203
2204 latlon_pair.name1 = lat->fullpath;
2205 latlon_pair.name2 = lon_final_candidate_path_vec[0];
2206 latlon_pair.size1 = lat->getDimensions()[0]->size;
2207 latlon_pair.size2 = lat->getDimensions()[1]->size;
2208 latlon_pair.rank = lat->rank;
2209 latloncv_candidate_pairs.push_back(latlon_pair);
2210 }
2211 else if(lon_final_candidate_path_vec.size() >1) {
2212
2213 // Under the same group, if we have two pairs lat/lon such as foo1_lat,foo1_lon, foo2_lat,foo2_lon, we will
2214 // treat {foo1_lat,foo1_lon} and {foo2_lat,foo2_lon} as two lat,lon coordinate candidates. This is essentially the SMAP L1B case.
2215 // We only compare three potential suffixes, lat/lon, latitude/longitude,Latitude/Longitude. We will treat the pair
2216 // latitude/Longitude and Latitude/longitude as a valid one.
2217
2218 string lat_name = HDF5CFUtil::obtain_string_after_lastslash(lat->fullpath);
2219 string lat_name_prefix1;
2220 string lat_name_prefix2;
2221
2222 // name prefix before the pair lat,note: no need to check if the last 3 characters are lat or lon. We've checked already.
2223 if(lat_name.size() >3) {
2224 lat_name_prefix1 = lat_name.substr(0,lat_name.size()-3);
2225 if(lat_name.size() >8)
2226 lat_name_prefix2 = lat_name.substr(0,lat_name.size()-8);
2227 }
2228 string lon_name_prefix1;
2229 string lon_name_prefix2;
2230
2231 for(auto ilon = lon_final_candidate_path_vec.begin(); ilon!=lon_final_candidate_path_vec.end();++ilon) {
2232 string lon_name = HDF5CFUtil::obtain_string_after_lastslash(*ilon);
2233 if(lon_name.size() >3) {
2234 lon_name_prefix1 = lon_name.substr(0,lon_name.size()-3);
2235 if(lon_name.size() >9)
2236 lon_name_prefix2 = lon_name.substr(0,lon_name.size()-9);
2237 }
2238 if((lat_name_prefix1 !="" && lat_name_prefix1 == lon_name_prefix1) ||
2239 (lat_name_prefix2 !="" && lat_name_prefix2 == lon_name_prefix2)) {// match lat,lon this one is the candidate
2240
2241 Name_Size_2Pairs latlon_pair;
2242 latlon_pair.name1 = lat->fullpath;
2243 latlon_pair.name2 = *ilon;
2244 latlon_pair.size1 = lat->getDimensions()[0]->size;
2245 latlon_pair.size2 = lat->getDimensions()[1]->size;
2246 latlon_pair.rank = lat->rank;
2247 latloncv_candidate_pairs.push_back(latlon_pair);
2248
2249 }
2250 }
2251 }// end else of if lon_final_candidate_path_vec
2252 }// end of if(lon_candidate_path.size() > 1)
2253
2254 else if(lon_candidate_path.size() == 1) {//insert this lat/lon pair to the struct
2255
2256 Name_Size_2Pairs latlon_pair;
2257
2258 latlon_pair.name1 = lat->fullpath;
2259 latlon_pair.name2 = *(lon_candidate_path.begin());
2260 latlon_pair.size1 = lat->getDimensions()[0]->size;
2261 latlon_pair.size2 = lat->getDimensions()[1]->size;
2262 latlon_pair.rank = lat->rank;
2263 latloncv_candidate_pairs.push_back(latlon_pair);
2264
2265 }
2266
2267}
2268
2269// We need to make sure that one lat maps to one lon in the lat/lon pairs.
2270// This routine removes the duplicate ones like (lat1,lon1) and (lat2,lon1).
2271void GMFile::Build_unique_latlon_candidate() {
2272
2273 BESDEBUG("h5", "Coming to Build_unique_latlon_candidate()"<<endl);
2274 set<int> duplicate_index;
2275 for(unsigned int i= 0; i<latloncv_candidate_pairs.size();i++) {
2276 for(unsigned int j=i+1;j<latloncv_candidate_pairs.size();j++) {
2277 if(latloncv_candidate_pairs[i].name2 == latloncv_candidate_pairs[j].name2) {
2278 duplicate_index.insert(i);
2279 duplicate_index.insert(j);
2280 }
2281 }
2282 }
2283
2284 // set is pre-sorted. we used a quick way to remove multiple elements.
2285 for(auto its= duplicate_index.rbegin();its!=duplicate_index.rend();++its) {
2286 latloncv_candidate_pairs[*its] = latloncv_candidate_pairs.back();
2287 latloncv_candidate_pairs.pop_back();
2288 }
2289}
2290// Leave the following code for the time being.
2291#if 0
2292// In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" names.
2293// This routine will check this case.
2294bool GMFile::Check_LatLonName_General_Product(int ll_rank) {
2295
2296 if(ll_rank <1 || ll_rank >2)
2297 throw2("Only support rank = 1 or 2 lat/lon case for the general product. The current rank is ",ll_rank);
2298 bool ret_value = false;
2299 size_t lat2D_dimsize0 = 0;
2300 size_t lat2D_dimsize1 = 0;
2301 size_t lon2D_dimsize0 = 0;
2302 size_t lon2D_dimsize1 = 0;
2303
2304 // The element order is latlon_flag,latilong_flag and LatLon_flag.
2305 vector<short>ll_flag(3,0);
2306
2307 vector<size_t>lat_size;
2308 vector<size_t>lon_size;
2309
2310 // We only need to check 2-D latlon
2311 if(2 == ll_rank) {
2312 //lat/lon is 2-D array, so the size is doubled.
2313 lat_size.assign(6,0);
2314 lon_size.assign(6,0);
2315 }
2316
2317 for (const auto &var:this->vars) {
2318
2319 if(var->rank == ll_rank) {
2320 if(var->name == "lat") {
2321 ll_flag[0]++;
2322 if(ll_rank == 2) {
2323 lat_size[0] = var->getDimensions()[0]->size;
2324 lat_size[1] = var->getDimensions()[1]->size;
2325
2326 }
2327
2328 }
2329 else if(var->name == "lon") {
2330 ll_flag[0]++;
2331 if(ll_rank == 2) {
2332 lon_size[0] = var->getDimensions()[0]->size;
2333 lon_size[1] = var->getDimensions()[1]->size;
2334
2335 }
2336
2337 }
2338 else if(var->name == "latitude"){
2339 ll_flag[1]++;
2340 if(ll_rank == 2) {
2341 lat_size[2] = var->getDimensions()[0]->size;
2342 lat_size[3] = var->getDimensions()[1]->size;
2343
2344 }
2345 }
2346 else if(var->name == "longitude"){
2347 ll_flag[1]++;
2348 if(ll_rank == 2) {
2349 lon_size[2] = var->getDimensions()[0]->size;
2350 lon_size[3] = var->getDimensions()[1]->size;
2351
2352 }
2353
2354 }
2355 else if(var->name == "Latitude"){
2356 ll_flag[2]++;
2357 if(ll_rank == 2) {
2358 lat_size[4] = var->getDimensions()[0]->size;
2359 lat_size[5] = var->getDimensions()[1]->size;
2360
2361 }
2362
2363 }
2364 else if(var->name == "Longitude"){
2365 ll_flag[2]++;
2366 if(ll_rank == 2) {
2367 lon_size[4] = var->getDimensions()[0]->size;
2368 lon_size[5] = var->getDimensions()[1]->size;
2369 }
2370 }
2371 }
2372 }
2373
2374 int total_llflag = 0;
2375 for (int i = 0; i < ll_flag.size();i++)
2376 if(2 == ll_flag[i])
2377 total_llflag ++;
2378
2379 // We only support 1 (L)lat(i)/(L)lon(g) pair.
2380 if(1 == total_llflag) {
2381 bool latlon_size_match = true;
2382 if(2 == ll_rank) {
2383 for (int size_index = 0; size_index <lat_size.size();size_index++) {
2384 if(lat_size[size_index] != lon_size[size_index]){
2385 latlon_size_match = false;
2386 break;
2387 }
2388 }
2389 }
2390
2391 if(true == latlon_size_match) {
2392 ret_value = true;
2393 if(2 == ll_flag[0]) {
2394 gp_latname = "lat";
2395 gp_lonname = "lon";
2396 }
2397 else if ( 2 == ll_flag[1]) {
2398 gp_latname = "latitude";
2399 gp_lonname = "longitude";
2400 }
2401
2402 else if (2 == ll_flag[2]){
2403 gp_latname = "Latitude";
2404 gp_lonname = "Longitude";
2405 }
2406 }
2407 }
2408
2409 return ret_value;
2410}
2411#endif
2412
2413// Add dimension names for the case that has 2-D lat/lon.
2414void GMFile::Add_Dim_Name_LatLon2D_General_Product() {
2415
2416 BESDEBUG("h5", "Coming to Add_Dim_Name_LatLon2D_General_Product()"<<endl);
2417 string latdimname0;
2418 string latdimname1;
2419 size_t latdimsize0 = 0;
2420 size_t latdimsize1 = 0;
2421
2422 // Need to generate fake dimensions.
2423 for (const auto &var:this->vars) {
2424
2425 set<hsize_t> fakedimsize;
2426 pair<set<hsize_t>::iterator,bool> setsizeret;
2427 int num_dup_dim_size = 0;
2428 for (auto &dim:var->dims) {
2429 Add_One_FakeDim_Name(dim);
2430 setsizeret = fakedimsize.insert(dim->size);
2431
2432 // Avoid the same size dimension sharing the same dimension name.
2433 if (false == setsizeret.second){
2434 num_dup_dim_size++;
2435 Adjust_Duplicate_FakeDim_Name2(dim,num_dup_dim_size);
2436 }
2437
2438#if 0
2439 if (false == setsizeret.second)
2440 Adjust_Duplicate_FakeDim_Name(dim);
2441#endif
2442 }
2443
2444 // Find variable name that is latitude or lat or Latitude
2445 // Note that we don't need to check longitude since longitude dim. sizes should be the same as the latitude for this case.
2446 if(var->name == gp_latname) {
2447 if(var->rank != 2) {
2448 throw4("coordinate variables ",gp_latname,
2449 " must have rank 2 for the 2-D latlon case , the current rank is ",
2450 var->rank);
2451 }
2452 latdimname0 = var->getDimensions()[0]->name;
2453 latdimsize0 = var->getDimensions()[0]->size;
2454
2455 latdimname1 = var->getDimensions()[1]->name;
2456 latdimsize1 = var->getDimensions()[1]->size;
2457 }
2458 }
2459
2460
2461 // Now we need to change a dimension of a general variable that shares the same size of lat
2462 // to the dimension name of the lat.
2463 for (auto &var:this->vars) {
2464 int lat_dim0_index = 0;
2465 int lat_dim1_index = 0;
2466 bool has_lat_dims_size = false;
2467
2468 for (unsigned int dim_index = 0; dim_index <var->dims.size(); dim_index++) {
2469
2470 // Find if having the first dimension size of lat
2471 if((var->dims[dim_index])->size == latdimsize0) {
2472
2473 // Find if having the second dimension size of lat
2474 lat_dim0_index = dim_index;
2475 for(unsigned int dim_index2 = dim_index+1;dim_index2 < var->dims.size();dim_index2++) {
2476 if((var->dims[dim_index2])->size == latdimsize1) {
2477 lat_dim1_index = dim_index2;
2478 has_lat_dims_size = true;
2479 break;
2480 }
2481 }
2482 }
2483 if(true == has_lat_dims_size)
2484 break;
2485 }
2486 // Find the lat's dimension sizes, change the (fake) dimension names.
2487 if(true == has_lat_dims_size) {
2488
2489 (var->dims[lat_dim0_index])->name = latdimname0;
2490#if 0
2491 //(var->dims[lat_dim0_index])->newname = latdimname0;
2492#endif
2493 (var->dims[lat_dim1_index])->name = latdimname1;
2494#if 0
2495 //(var->dims[lat_dim1_index])->newname = latdimname1;
2496#endif
2497
2498 }
2499 }
2500
2501 //When we generate Fake dimensions, we may encounter discontiguous Fake dimension names such
2502 // as FakeDim0, FakeDim9 etc. We would like to make Fake dimension names in contiguous order
2503 // FakeDim0,FakeDim1,etc.
2504
2505 // Obtain the tempdimnamelist set.
2506 set<string>tempdimnamelist;
2507
2508 for (const auto &var:this->vars) {
2509 for (const auto &dim:var->dims)
2510 tempdimnamelist.insert(dim->name);
2511 }
2512
2513 // Generate the final dimnamelist,it is a contiguous order: FakeDim0,FakeDim1 etc.
2514 set<string>finaldimnamelist;
2515 string finaldimname_base = "FakeDim";
2516
2517 for(unsigned int i = 0; i<tempdimnamelist.size();i++) {
2518 stringstream sfakedimindex;
2519 sfakedimindex << i;
2520 string finaldimname = finaldimname_base + sfakedimindex.str();
2521 finaldimnamelist.insert(finaldimname);
2522 }
2523
2524 // If the original tempdimnamelist is not the same as the finaldimnamelist,
2525 // we need to generate a map from original name to the final name.
2526 if(finaldimnamelist != tempdimnamelist) {
2527 map<string,string> tempdimname_to_finaldimname;
2528 auto tempit = tempdimnamelist.begin();
2529 auto finalit = finaldimnamelist.begin();
2530 while(tempit != tempdimnamelist.end()) {
2531 tempdimname_to_finaldimname[*tempit] = *finalit;
2532 tempit++;
2533 finalit++;
2534 }
2535
2536 // Change the dimension names of every variable to the final dimension name list.
2537 for (auto &var:this->vars) {
2538 for (auto &dim:var->dims) {
2539 if(tempdimname_to_finaldimname.find(dim->name) !=tempdimname_to_finaldimname.end())
2540 dim->name = tempdimname_to_finaldimname[dim->name];
2541 else
2542 throw3("The dimension names ",dim->name, "cannot be found in the dim. name list.");
2543 }
2544 }
2545 }
2546
2547
2548 dimnamelist.clear();
2549 dimnamelist = finaldimnamelist;
2550
2551 // We need to update dimname_to_dimsize map. This may be used in the future.
2552 dimname_to_dimsize.clear();
2553 for (const auto &var:this->vars) {
2554 for (const auto &dim:var->dims) {
2555 if(finaldimnamelist.find(dim->name)!=finaldimnamelist.end()) {
2556 dimname_to_dimsize[dim->name] = dim->size;
2557 dimname_to_unlimited[dim->name] = dim->unlimited_dim;
2558 finaldimnamelist.erase(dim->name);
2559 }
2560
2561 }
2562 if(true == finaldimnamelist.empty())
2563 break;
2564 }
2565
2566 // Finally set dimension newname
2567 for (auto &var:this->vars) {
2568 for (auto &dim:var->dims)
2569 dim->newname = dim->name;
2570 }
2571
2572}
2573
2574// Add dimension names for the case that has 1-D lat/lon or CoordAttr..
2575//
2576void GMFile::Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product() {
2577
2578 BESDEBUG("h5", "Coming to Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product()"<<endl);
2579 // Only need to add the fake dimension names
2580 for (auto &var:this->vars) {
2581 set<hsize_t> fakedimsize;
2582 pair<set<hsize_t>::iterator,bool> setsizeret;
2583 int num_dup_dim_size = 0;
2584 for (auto &dim:var->dims) {
2585 Add_One_FakeDim_Name(dim);
2586 setsizeret = fakedimsize.insert(dim->size);
2587 // Avoid the same size dimension sharing the same dimension name.
2588 // HYRAX-629, we not only want to have the unique dimension name for
2589 // the same dimension size of different dimensions in a var, we also
2590 // want to reduce the number of dimension names.
2591 // So foo[100][100], foo2[100][100][100] should be something like
2592 // foo[FakeDim0][FakeDim1] foo2[FakeDim0][FakeDim1][FakeDim2]
2593 if (false == setsizeret.second){
2594 num_dup_dim_size++;
2595 Adjust_Duplicate_FakeDim_Name2(dim,num_dup_dim_size);
2596 }
2597 // Comment out the original code for the time being.
2598#if 0
2599 if (false == setsizeret.second){
2600 num_dup_dim_size++;
2601 Adjust_Duplicate_FakeDim_Name(dim,num_dup_dim_size);
2602 }
2603#endif
2604 }
2605 }
2606
2607// leave it here for debugging purpose
2608#if 0
2609for (vector<Var *>::iterator irv = this->vars.begin();
2610 irv != this->vars.end(); ++irv) {
2611cerr<<"Var name is "<<(*irv)->newname<<endl;
2612 for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
2613 ird != (*irv)->dims.end(); ++ird)
2614cerr<<"Dimension name is "<<(*ird)->newname <<endl;
2615}
2616#endif
2617
2618}
2619
2620// For netCDF-4-like HDF5 products, we need to add the dimension scales.
2621void GMFile::Add_Dim_Name_Dimscale_General_Product() {
2622
2623 BESDEBUG("h5", "Coming to Add_Dim_Name_Dimscale_General_Product()"<<endl);
2624
2625 pair<set<string>::iterator,bool> setret;
2626 this->iscoard = true;
2627
2628 for (const auto &var:this->vars) {
2629
2630 // Obtain all the dimension names for this variable
2631 Handle_UseDimscale_Var_Dim_Names_General_Product(var);
2632
2633 // Need to update dimenamelist and dimname_to_dimsize and dimname_to_unlimited maps for future use.
2634 for (const auto &dim:var->dims) {
2635 setret = dimnamelist.insert(dim->name);
2636 if (true == setret.second)
2637 Insert_One_NameSizeMap_Element(dim->name,dim->size,dim->unlimited_dim);
2638 }
2639 } // end of for
2640
2641 if (true == dimnamelist.empty())
2642 throw1("This product should have the dimension names, but no dimension names are found");
2643
2644}
2645
2646// Obtain dimension names for this variable when netCDF-4 model(using dimension scales) is followed.
2647void GMFile::Handle_UseDimscale_Var_Dim_Names_General_Product(Var *var) {
2648
2649 BESDEBUG("h5", "Coming to Handle_UseDimscale_Var_Dim_Names_General_Product()"<<endl);
2650 const Attribute* dimlistattr = nullptr;
2651 bool has_dimlist = false;
2652 bool has_dimclass = false;
2653
2654 for(const auto &attr:var->attrs) {
2655 if ("DIMENSION_LIST" == attr->name) {
2656 dimlistattr = attr;
2657 has_dimlist = true;
2658 }
2659 if ("CLASS" == attr->name) {
2660
2661 Retrieve_H5_Attr_Value(attr,var->fullpath);
2662 string class_value;
2663 class_value.resize(attr->value.size());
2664 copy(attr->value.begin(),attr->value.end(),class_value.begin());
2665
2666 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2667 // "DIMENSION_SCALE", which is 15.
2668 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2669 has_dimclass = true;
2670 break;
2671 }
2672 }
2673
2674 } // end of for
2675
2676 // This is a general variable, we need to find the corresponding coordinate variables.
2677 if (true == has_dimlist)
2678 Add_UseDimscale_Var_Dim_Names_General_Product(var,dimlistattr);
2679
2680 // Dim name is the same as the variable name for dimscale variable
2681 else if(true == has_dimclass) {
2682 if (var->dims.size() !=1)
2683 throw2("Currently dimension scale dataset must be 1 dimension, this is not true for the dataset ",
2684 var->name);
2685
2686 // The var name is the object name, however, we would like the dimension name to be the full path.
2687 // so that the dim name can be served as the key for future handling.
2688 (var->dims)[0]->name = var->fullpath;
2689 (var->dims)[0]->newname = var->fullpath;
2690 pair<set<string>::iterator,bool> setret;
2691 setret = dimnamelist.insert((var->dims)[0]->name);
2692 if (true == setret.second)
2693 Insert_One_NameSizeMap_Element((var->dims)[0]->name,(var->dims)[0]->size,(var->dims)[0]->unlimited_dim);
2694 }
2695
2696 // No dimension, add fake dim names, this will rarely happen.
2697 else {
2698
2699 set<hsize_t> fakedimsize;
2700 pair<set<hsize_t>::iterator,bool> setsizeret;
2701 for (auto &dim:var->dims) {
2702 Add_One_FakeDim_Name(dim);
2703 setsizeret = fakedimsize.insert(dim->size);
2704 // Avoid the same size dimension sharing the same dimension name.
2705 if (false == setsizeret.second)
2706 Adjust_Duplicate_FakeDim_Name(dim);
2707 }
2708 }
2709
2710}
2711
2712// Add dimension names for the case when HDF5 dimension scale is followed(netCDF4-like)
2713void GMFile::Add_UseDimscale_Var_Dim_Names_General_Product(const Var *var,const Attribute*dimlistattr)
2714{
2715
2716 BESDEBUG("h5", "Coming to Add_UseDimscale_Var_Dim_Names_General_Product()"<<endl);
2717 ssize_t objnamelen = -1;
2718 hobj_ref_t rbuf;
2719
2720#if 0
2721 //hvl_t *vlbuf = nullptr;
2722#endif
2723 vector<hvl_t> vlbuf;
2724
2725 hid_t dset_id = -1;
2726 hid_t attr_id = -1;
2727 hid_t atype_id = -1;
2728 hid_t amemtype_id = -1;
2729 hid_t aspace_id = -1;
2730 hid_t ref_dset = -1;
2731
2732 if(nullptr == dimlistattr)
2733 throw2("Cannot obtain the dimension list attribute for variable ",var->name);
2734
2735 else if (0==var->rank)
2736 throw2("The number of dimension should NOT be 0 for the variable ",var->name);
2737
2738 else {
2739 try {
2740
2741 vlbuf.resize(var->rank);
2742
2743 dset_id = H5Dopen(this->fileid,(var->fullpath).c_str(),H5P_DEFAULT);
2744 if (dset_id < 0)
2745 throw2("Cannot open the dataset ",var->fullpath);
2746
2747 attr_id = H5Aopen(dset_id,(dimlistattr->name).c_str(),H5P_DEFAULT);
2748 if (attr_id <0 )
2749 throw4("Cannot open the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
2750
2751 atype_id = H5Aget_type(attr_id);
2752 if (atype_id <0)
2753 throw4("Cannot obtain the datatype of the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
2754
2755 amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
2756
2757 if (amemtype_id < 0)
2758 throw2("Cannot obtain the memory datatype for the attribute ",dimlistattr->name);
2759
2760
2761 if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0)
2762 throw2("Cannot obtain the referenced object for the variable ",var->name);
2763
2764
2765 vector<char> objname;
2766 int vlbuf_index = 0;
2767
2768 // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
2769 for (auto &dim:var->dims) {
2770
2771 if(vlbuf[vlbuf_index].p== nullptr)
2772 throw4("The dimension doesn't exist. Var name is ",var->name,"; the dimension index is ",vlbuf_index);
2773 rbuf =((hobj_ref_t*)vlbuf[vlbuf_index].p)[0];
2774 if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0)
2775 throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
2776
2777 if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0)
2778 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
2779 objname.resize(objnamelen+1);
2780 if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0)
2781 throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
2782
2783 auto objname_str = string(objname.begin(),objname.end());
2784
2785 // We need to remove the first character of the object name since the first character
2786 // of the object full path is always "/" and this will be changed to "_".
2787 // The convention of handling the dimension-scale general product is to remove the first "_".
2788 // Check the get_CF_string function of HDF5GMCF.cc.
2789 string trim_objname = objname_str.substr(0,objnamelen);
2790 dim->name = string(trim_objname.begin(),trim_objname.end());
2791
2792 pair<set<string>::iterator,bool> setret;
2793 setret = dimnamelist.insert(dim->name);
2794 if (true == setret.second)
2795 Insert_One_NameSizeMap_Element(dim->name,dim->size,dim->unlimited_dim);
2796 dim->newname = dim->name;
2797 H5Dclose(ref_dset);
2798#if 0
2799 ref_dset = -1;
2800#endif
2801 objname.clear();
2802 vlbuf_index++;
2803 }// end of for ird
2804
2805 if(vlbuf.empty() == false) {
2806
2807 if ((aspace_id = H5Aget_space(attr_id)) < 0)
2808 throw2("Cannot get hdf5 dataspace id for the attribute ",dimlistattr->name);
2809
2810 if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0)
2811 throw2("Cannot successfully clean up the variable length memory for the variable ",var->name);
2812
2813 H5Sclose(aspace_id);
2814
2815 }
2816
2817 H5Tclose(atype_id);
2818 H5Tclose(amemtype_id);
2819 H5Aclose(attr_id);
2820 H5Dclose(dset_id);
2821 }
2822
2823 catch(...) {
2824
2825 if(atype_id != -1)
2826 H5Tclose(atype_id);
2827
2828 if(amemtype_id != -1)
2829 H5Tclose(amemtype_id);
2830
2831 if(aspace_id != -1)
2832 H5Sclose(aspace_id);
2833
2834 if(attr_id != -1)
2835 H5Aclose(attr_id);
2836
2837 if(dset_id != -1)
2838 H5Dclose(dset_id);
2839
2840 throw;
2841 }
2842 }
2843
2844}
2845
2846// Handle coordinate variables
2848
2849 BESDEBUG("h5", "GMFile:: Coming to Handle_CVar()"<<endl);
2850 // No coordinate variables are generated for ACOS_L2S or OCO2_L1B
2851 // Currently we support the three patterns for the general products:
2852 // 1) Dimensions follow HDF5 dimension scale specification
2853 // 2) Dimensions don't follow HDF5 dimension scale specification but have 1D lat/lon
2854 // 3) Dimensions don't follow HDF5 dimension scale specification but have 2D lat/lon
2855 if (General_Product == this->product_type ||
2856 ACOS_L2S_OR_OCO2_L1B == this->product_type) {
2857 if (GENERAL_DIMSCALE == this->gproduct_pattern)
2858 Handle_CVar_Dimscale_General_Product();
2859 else if (GENERAL_LATLON1D == this->gproduct_pattern)
2860 Handle_CVar_LatLon1D_General_Product();
2861 else if (GENERAL_LATLON2D == this->gproduct_pattern)
2862 Handle_CVar_LatLon2D_General_Product();
2863 return;
2864 }
2865
2866 else if (Mea_SeaWiFS_L2 == this->product_type ||
2867 Mea_SeaWiFS_L3 == this->product_type)
2868 Handle_CVar_Mea_SeaWiFS();
2869 else if (Aqu_L3 == this->product_type)
2870 Handle_CVar_Aqu_L3();
2871 else if (OBPG_L3 == this->product_type)
2872 Handle_CVar_OBPG_L3();
2873 else if (OSMAPL2S == this->product_type)
2874 Handle_CVar_OSMAPL2S();
2875 else if (Mea_Ozone == this->product_type)
2876 Handle_CVar_Mea_Ozone();
2877 else if (GPMS_L3 == this->product_type || GPMM_L3 == this->product_type
2878 || GPM_L3_New == this->product_type )
2879 Handle_CVar_GPM_L3();
2880 else if (GPM_L1 == this->product_type)
2881 Handle_CVar_GPM_L1();
2882}
2883
2884// Handle GPM level 1 coordinate variables
2885void GMFile::Handle_CVar_GPM_L1() {
2886
2887 BESDEBUG("h5", "Coming to Handle_CVar_GPM_L1()"<<endl);
2888#if 0
2889 // Loop through the variable list to build the coordinates.
2890 for (vector<Var *>::iterator irv = this->vars.begin();
2891 irv != this->vars.end(); ++irv) {
2892 if((*irv)->name=="AlgorithmRuntimeInfo") {
2893 delete(*irv);
2894 this->vars.erase(irv);
2895 break;
2896 }
2897 }
2898#endif
2899
2900 // Loop through all variables to check 2-D "Latitude" and "Longitude".
2901 // Create coordinate variables based on 2-D "Latitude" and "Longitude".
2902 // Latitude[Xdim][YDim] Longitude[Xdim][YDim], Latitude <->Xdim, Longitude <->YDim.
2903 // Make sure to build cf dimension names cfdimname = latpath+ the lat dimension name.
2904 // We want to save dimension names of Latitude and Longitude since
2905 // the fake coordinate variables of these two dimensions should not be generated.
2906 // So we need to remember these dimension names.
2907 set<string> ll_dim_set;
2908 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
2909 if((*irv)->rank == 2 && (*irv)->name == "Latitude") {
2910 size_t lat_pos = (*irv)->fullpath.rfind("Latitude");
2911 string lat_path = (*irv)->fullpath.substr(0,lat_pos);
2912 auto GMcvar_unique = make_unique<GMCVar>(*irv);
2913 auto GMcvar = GMcvar_unique.release();
2914 GMcvar->cfdimname = lat_path + ((*irv)->dims)[0]->name;
2915 ll_dim_set.insert(((*irv)->dims)[0]->name);
2916 GMcvar->cvartype = CV_EXIST;
2917 GMcvar->product_type = product_type;
2918 this->cvars.push_back(GMcvar);
2919 delete(*irv);
2920 irv = this->vars.erase(irv);
2921 }
2922
2923 if((*irv)->rank == 2 && (*irv)->name == "Longitude") {
2924 size_t lon_pos = (*irv)->fullpath.rfind("Longitude");
2925 string lon_path = (*irv)->fullpath.substr(0,lon_pos);
2926 auto GMcvar_unique = make_unique<GMCVar>(*irv);
2927 auto GMcvar = GMcvar_unique.release();
2928 GMcvar->cfdimname = lon_path + ((*irv)->dims)[1]->name;
2929 ll_dim_set.insert(((*irv)->dims)[1]->name);
2930 GMcvar->cvartype = CV_EXIST;
2931 GMcvar->product_type = product_type;
2932 this->cvars.push_back(GMcvar);
2933 delete(*irv);
2934 irv = this->vars.erase(irv);
2935 }
2936 else {
2937 ++irv;
2938 }
2939 }// end of for irv
2940
2941
2942 // For each dimension, create a coordinate variable.
2943 // Here we just need to loop through the map dimname_to_dimsize,
2944 // use the name and the size to create coordinate variables.
2945 for (map<string,hsize_t>::const_iterator itd = dimname_to_dimsize.begin();
2946 itd!=dimname_to_dimsize.end();++itd) {
2947 // We will not create fake coordinate variables for the
2948 // dimensions of latitude and longitude.
2949 if((ll_dim_set.find(itd->first)) == ll_dim_set.end()) {
2950 auto GMcvar_unique = make_unique<GMCVar>();
2951 auto GMcvar = GMcvar_unique.release();
2952 Create_Missing_CV(GMcvar,itd->first);
2953 this->cvars.push_back(GMcvar);
2954 }
2955 }//end of for (map<string,hsize_t>.
2956
2957}
2958
2959// Handle coordinate variables for GPM level 3
2960void GMFile::Handle_CVar_GPM_L3() {
2961
2962 BESDEBUG("h5", "Coming to Handle_CVar_GPM_L3()"<<endl);
2963 iscoard = true;
2964
2965 // Here we just need to loop through the map dimname_to_dimsize,
2966 // use the name and the size to create coordinate variables.
2967 for (map<string,hsize_t>::const_iterator itd = dimname_to_dimsize.begin();
2968 itd!=dimname_to_dimsize.end();++itd) {
2969
2970 auto GMcvar_unique = make_unique<GMCVar>();
2971 auto GMcvar = GMcvar_unique.release();
2972 if("nlon" == itd->first || "nlat" == itd->first
2973 || "lnH" == itd->first || "ltH" == itd->first
2974 || "lnL" == itd->first || "ltL" == itd->first) {
2975 GMcvar->name = itd->first;
2976 GMcvar->newname = GMcvar->name;
2977 GMcvar->fullpath = GMcvar->name;
2978 GMcvar->rank = 1;
2979 GMcvar->dtype = H5FLOAT32;
2980 auto gmcvar_dim_unique = make_unique<Dimension>(itd->second);
2981 auto gmcvar_dim = gmcvar_dim_unique.release();
2982 gmcvar_dim->name = GMcvar->name;
2983 gmcvar_dim->newname = gmcvar_dim->name;
2984 GMcvar->dims.push_back(gmcvar_dim);
2985 GMcvar->cfdimname = gmcvar_dim->name;
2986 if ("nlat" ==GMcvar->name || "ltH" == GMcvar->name
2987 || "ltL" == GMcvar->name)
2988 GMcvar->cvartype = CV_LAT_MISS;
2989 else if ("nlon" == GMcvar->name || "lnH" == GMcvar->name
2990 || "lnL" == GMcvar->name)
2991 GMcvar->cvartype = CV_LON_MISS;
2992 GMcvar->product_type = product_type;
2993 }
2994 else if (("nlayer" == itd->first && (28 == itd->second || 19 == itd->second)) ||
2995 ("hgt" == itd->first && 5 == itd->second) ||
2996 ("nalt" == itd->first && 5 == itd->second)){
2997 GMcvar->name = itd->first;
2998 GMcvar->newname = GMcvar->name;
2999 GMcvar->fullpath = GMcvar->name;
3000 GMcvar->rank = 1;
3001 GMcvar->dtype = H5FLOAT32;
3002 auto gmcvar_dim_unique = make_unique<Dimension>(itd->second);
3003 auto gmcvar_dim = gmcvar_dim_unique.release();
3004 gmcvar_dim->name = GMcvar->name;
3005 gmcvar_dim->newname = gmcvar_dim->name;
3006 GMcvar->dims.push_back(gmcvar_dim);
3007 GMcvar->cfdimname = gmcvar_dim->name;
3008 GMcvar->cvartype = CV_SPECIAL;
3009 GMcvar->product_type = product_type;
3010 }
3011 else
3012 Create_Missing_CV(GMcvar,itd->first);
3013 this->cvars.push_back(GMcvar);
3014 }//end of for
3015}
3016
3017// Handle Coordinate variables for MeaSuRES SeaWiFS
3018void GMFile::Handle_CVar_Mea_SeaWiFS() {
3019
3020 BESDEBUG("h5", "Coming to Handle_CVar_Mea_SeaWiFS()"<<endl);
3021 pair<set<string>::iterator,bool> setret;
3022 set<string>tempdimnamelist = dimnamelist;
3023
3024 for (const auto &dimname:dimnamelist) {
3025 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3026 if (dimname== (*irv)->fullpath) {
3027
3028 if (!iscoard && (("/natrack" == dimname)
3029 || "/nxtrack" == dimname)) {
3030 ++irv;
3031 continue;
3032 }
3033
3034 if((*irv)->dims.size()!=1)
3035 throw3("Coard coordinate variable ",(*irv)->name, "is not 1D");
3036
3037 // Create Coordinate variables.
3038 tempdimnamelist.erase(dimname);
3039 auto GMcvar_unique = make_unique<GMCVar>(*irv);
3040 auto GMcvar = GMcvar_unique.release();
3041 GMcvar->cfdimname = dimname;
3042 GMcvar->cvartype = CV_EXIST;
3043 GMcvar->product_type = product_type;
3044 this->cvars.push_back(GMcvar);
3045 delete(*irv);
3046 irv = this->vars.erase(irv);
3047 } // end of if irs
3048 else if(false == iscoard) {
3049 // 2-D lat/lon, natrack maps to lat, nxtrack maps to lon.
3050 if (((dimname =="/natrack") && ((*irv)->fullpath == "/latitude"))
3051 ||((dimname =="/nxtrack") && ((*irv)->fullpath == "/longitude"))) {
3052 tempdimnamelist.erase(dimname);
3053 auto GMcvar_unique = make_unique<GMCVar>(*irv);
3054 auto GMcvar = GMcvar_unique.release();
3055 GMcvar->cfdimname = dimname;
3056 GMcvar->cvartype = CV_EXIST;
3057 GMcvar->product_type = product_type;
3058 this->cvars.push_back(GMcvar);
3059 delete(*irv);
3060 irv = this->vars.erase(irv);
3061 }
3062 else {
3063 ++irv;
3064 }
3065
3066 }// end of else if(false == iscoard)
3067 else {
3068 ++irv;
3069 }
3070 } // end of for irv ...
3071 } // end of for irs ...
3072
3073 // Creating the missing "third-dimension" according to the dimension names.
3074 // This may never happen for the current MeaSure SeaWiFS, but put it here for code coherence and completeness.
3075 // KY 12-30-2011
3076 for (auto &dimname:tempdimnamelist) {
3077 auto GMcvar_unique = make_unique<GMCVar>();
3078 auto GMcvar = GMcvar_unique.release();
3079 Create_Missing_CV(GMcvar,dimname);
3080 this->cvars.push_back(GMcvar);
3081 }
3082}
3083
3084// Handle Coordinate varibles for OSMAPL2S(Note: this function doesn't apply to SMAP)
3085void GMFile::Handle_CVar_OSMAPL2S() {
3086
3087 BESDEBUG("h5", "Coming to Handle_CVar_OSMAPL2S()"<<endl);
3088 pair<set<string>::iterator,bool> setret;
3089 set<string>tempdimnamelist = dimnamelist;
3090 string tempvarname;
3091 string key0 = "_lat";
3092 string key1 = "_lon";
3093 string osmapl2sdim0 ="YDim";
3094 string osmapl2sdim1 ="XDim";
3095
3096 bool foundkey0 = false;
3097 bool foundkey1 = false;
3098
3099 set<string> itset;
3100
3101 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3102
3103 tempvarname = (*irv)->name;
3104
3105 if ((tempvarname.size() > key0.size())&&
3106 (key0 == tempvarname.substr(tempvarname.size()-key0.size(),key0.size()))){
3107
3108 foundkey0 = true;
3109
3110 if (dimnamelist.find(osmapl2sdim0)== dimnamelist.end())
3111 throw5("variable ",tempvarname," must have dimension ",osmapl2sdim0," , but not found ");
3112
3113 tempdimnamelist.erase(osmapl2sdim0);
3114 auto GMcvar_unique = make_unique<GMCVar>(*irv);
3115 auto GMcvar = GMcvar_unique.release();
3116 GMcvar->newname = GMcvar->name; // Remove the path, just use the variable name
3117 GMcvar->cfdimname = osmapl2sdim0;
3118 GMcvar->cvartype = CV_EXIST;
3119 GMcvar->product_type = product_type;
3120 this->cvars.push_back(GMcvar);
3121 delete(*irv);
3122 irv = this->vars.erase(irv);
3123 }// end of if
3124
3125 else if ((tempvarname.size() > key1.size())&&
3126 (key1 == tempvarname.substr(tempvarname.size()-key1.size(),key1.size()))){
3127
3128 foundkey1 = true;
3129
3130 if (dimnamelist.find(osmapl2sdim1)== dimnamelist.end())
3131 throw5("variable ",tempvarname," must have dimension ",osmapl2sdim1," , but not found ");
3132
3133 tempdimnamelist.erase(osmapl2sdim1);
3134
3135 auto GMcvar_unique = make_unique<GMCVar>(*irv);
3136 auto GMcvar = GMcvar_unique.release();
3137 GMcvar->newname = GMcvar->name;
3138 GMcvar->cfdimname = osmapl2sdim1;
3139 GMcvar->cvartype = CV_EXIST;
3140 GMcvar->product_type = product_type;
3141 this->cvars.push_back(GMcvar);
3142 delete(*irv);
3143 irv = this->vars.erase(irv);
3144 }// end of else if ((tempvarname.size() > key1.size())&& ...
3145 else {
3146 ++irv;
3147 }
3148 if (true == foundkey0 && true == foundkey1)
3149 break;
3150 } // end of for
3151
3152 for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();++irs) {
3153
3154 auto GMcvar_unique = make_unique<GMCVar>();
3155 auto GMcvar = GMcvar_unique.release();
3156 Create_Missing_CV(GMcvar,*irs);
3157 this->cvars.push_back(GMcvar);
3158 }
3159
3160}
3161
3162// Handle coordinate variables for Aquarius level 3 products
3163void GMFile::Handle_CVar_Aqu_L3() {
3164
3165 BESDEBUG("h5", "Coming to Handle_CVar_Aqu_L3()"<<endl);
3166 iscoard = true;
3167 for (const auto &var:this->vars) {
3168
3169 if ( "l3m_data" == var->name) {
3170 for (const auto &dim:var->dims) {
3171
3172 auto GMcvar_unique = make_unique<GMCVar>();
3173 auto GMcvar = GMcvar_unique.release();
3174 GMcvar->name = dim->name;
3175 GMcvar->newname = GMcvar->name;
3176 GMcvar->fullpath = GMcvar->name;
3177 GMcvar->rank = 1;
3178 GMcvar->dtype = H5FLOAT32;
3179 auto gmcvar_dim_unique = make_unique<Dimension>(dim->size);
3180 auto gmcvar_dim = gmcvar_dim_unique.release();
3181 gmcvar_dim->name = GMcvar->name;
3182 gmcvar_dim->newname = gmcvar_dim->name;
3183 GMcvar->dims.push_back(gmcvar_dim);
3184 GMcvar->cfdimname = gmcvar_dim->name;
3185 if ("lat" ==GMcvar->name ) GMcvar->cvartype = CV_LAT_MISS;
3186 if ("lon" == GMcvar->name ) GMcvar->cvartype = CV_LON_MISS;
3187 GMcvar->product_type = product_type;
3188 this->cvars.push_back(GMcvar);
3189 } // end of for
3190 } // end of if
3191 }//end of for
3192
3193}
3194
3195//Handle coordinate variables for MeaSuRES Ozone products
3196void GMFile::Handle_CVar_Mea_Ozone() {
3197
3198 BESDEBUG("h5", "Coming to Handle_CVar_Mea_Ozone()"<<endl);
3199 pair<set<string>::iterator,bool> setret;
3200 set<string>tempdimnamelist = dimnamelist;
3201
3202 if(false == iscoard)
3203 throw1("Measure Ozone level 3 zonal average product must follow COARDS conventions");
3204
3205 for (const auto &dimname:dimnamelist) {
3206 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3207 if (dimname== (*irv)->fullpath) {
3208
3209 if((*irv)->dims.size()!=1)
3210 throw3("Coard coordinate variable",(*irv)->name, "is not 1D");
3211
3212 // Create Coordinate variables.
3213 tempdimnamelist.erase(dimname);
3214 auto GMcvar_unique = make_unique<GMCVar>(*irv);
3215 auto GMcvar = GMcvar_unique.release();
3216 GMcvar->cfdimname = dimname;
3217 GMcvar->cvartype = CV_EXIST;
3218 GMcvar->product_type = product_type;
3219 this->cvars.push_back(GMcvar);
3220 delete(*irv);
3221 irv = this->vars.erase(irv);
3222 } // end of if
3223 else {
3224 ++irv;
3225 }
3226 } // end of for irv
3227 } // end of for irs
3228
3229 for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();irs++) {
3230
3231 auto GMcvar_unique = make_unique<GMCVar>();
3232 auto GMcvar = GMcvar_unique.release();
3233 Create_Missing_CV(GMcvar,*irs);
3234 this->cvars.push_back(GMcvar);
3235 }
3236}
3237
3238// Handle coordinate variables for general products that use HDF5 dimension scales.
3239void GMFile::Handle_CVar_Dimscale_General_Product() {
3240
3241 BESDEBUG("h5", "Coming to Handle_CVar_Dimscale_General_Product"<<endl);
3242 pair<set<string>::iterator,bool> setret;
3243 set<string>tempdimnamelist = dimnamelist;
3244
3245 for (const auto &dimname:dimnamelist) {
3246 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3247
3248 // This is the dimension scale dataset; it should be changed to a coordinate variable.
3249 if (dimname== (*irv)->fullpath) {
3250 if((*irv)->dims.size()!=1)
3251 throw3("COARDS coordinate variable",(*irv)->name, "is not 1D");
3252
3253 // Create Coordinate variables.
3254 tempdimnamelist.erase(dimname);
3255
3256 auto GMcvar_unique = make_unique<GMCVar>(*irv);
3257 auto GMcvar = GMcvar_unique.release();
3258 GMcvar->cfdimname = dimname;
3259
3260 // Check if this is just a netCDF-4 dimension.
3261 bool is_netcdf_dimension = Is_netCDF_Dimension(*irv);
3262
3263 // If this is just the netcdf dimension, we
3264 // will fill in the index numbers.
3265 if (true == is_netcdf_dimension)
3266 GMcvar->cvartype = CV_FILLINDEX;
3267 else
3268 GMcvar->cvartype = CV_EXIST;
3269 GMcvar->product_type = product_type;
3270 this->cvars.push_back(GMcvar);
3271 delete(*irv);
3272 irv = this->vars.erase(irv);
3273 } // end of if irs
3274 else {
3275 ++irv;
3276 }
3277 } // end of for irv
3278 } // end of for irs
3279
3280 // Check if we have 2-D lat/lon CVs, and if yes, add those to the CV list.
3281 Update_M2DLatLon_Dimscale_CVs();
3282
3283 // Add other missing coordinate variables.
3284 for (auto &dimname:tempdimnamelist) {
3285 if ((this->have_nc4_non_coord == false) ||
3286 (nc4_sdimv_dv_path.find(dimname) == nc4_sdimv_dv_path.end())) {
3287
3288 auto GMcvar_unique = make_unique<GMCVar>();
3289 auto GMcvar = GMcvar_unique.release();
3290 Create_Missing_CV(GMcvar,dimname);
3291 this->cvars.push_back(GMcvar);
3292 }
3293 }
3294
3295
3296//Debugging
3297#if 0
3298for (set<string>::iterator irs = dimnamelist.begin();
3299 irs != dimnamelist.end();irs++) {
3300cerr<<"dimension name is "<<(*irs)<<endl;
3301}
3302#endif
3303
3304}
3305
3306
3307// Check if we have 2-D lat/lon CVs in a netCDF-4-like file, and if yes, add those to the CV list.
3308// This routine is a really complicate one. There are 9 steps to generate right 2-D lat/lon CVs.
3309void GMFile::Update_M2DLatLon_Dimscale_CVs() {
3310
3311 BESDEBUG("h5", "Coming to Update_M2DLatLon_Dimscale_CVs()"<<endl);
3312 // If this is not a file that only includes 1-D lat/lon CVs
3313 if(false == Check_1DGeolocation_Dimscale()) {
3314
3315 // Define temporary vectors to store 1-D lat/lon CVs
3316 vector<GMCVar*> tempcvar_1dlat;
3317 vector<GMCVar*> tempcvar_1dlon;
3318
3319 // 1. Obtain 1-D lat/lon CVs(only search the CF units and the reserved lat/lon names)
3320 Obtain_1DLatLon_CVs(tempcvar_1dlat,tempcvar_1dlon);
3321
3322 // Define temporary vectors to store 2-D lat/lon Vars
3323 vector<Var*> tempcvar_2dlat;
3324 vector<Var*> tempcvar_2dlon;
3325
3326 // This map remembers the positions of the latlon vars in the vector var.
3327 // Remembering the positions avoids the searching of these lat and lon again when
3328 // deleting them for the var vector and adding them(only the CVs) to the CV vector.
3329 // KY 2015-12-23
3330 map<string,int> latlon2d_path_to_index;
3331
3332 // 2. Obtain 2-D lat/lon variables(only search the CF units and the reserved names)
3333 Obtain_2DLatLon_Vars(tempcvar_2dlat,tempcvar_2dlon,latlon2d_path_to_index);
3334
3335#if 0
3336for(vector<GMCVar *>::iterator irv = tempcvar_1dlat.begin();irv != tempcvar_1dlat.end();++irv)
3337cerr<<"1-D lat variable full path is "<<(*irv)->fullpath <<endl;
3338for(vector<GMCVar *>::iterator irv = tempcvar_1dlon.begin();irv != tempcvar_1dlon.end();++irv)
3339cerr<<"1-D lon variable full path is "<<(*irv)->fullpath <<endl;
3340
3341for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3342cerr<<"2-D lat variable full path is "<<(*irv)->fullpath <<endl;
3343for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3344cerr<<"2-D lon variable full path is "<<(*irv)->fullpath <<endl;
3345#endif
3346
3347 // 3. Sequeeze the 2-D lat/lon vectors by removing the ones that share the same dims with 1-D lat/lon CVs.
3348 Obtain_2DLLVars_With_Dims_not_1DLLCVars(tempcvar_2dlat,tempcvar_2dlon,tempcvar_1dlat,tempcvar_1dlon,latlon2d_path_to_index);
3349
3350#if 0
3351for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3352cerr<<"2-D Left lat variable full path is "<<(*irv)->fullpath <<endl;
3353for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3354cerr<<"2-D Left lon variable full path is "<<(*irv)->fullpath <<endl;
3355#endif
3356
3357 // 4. Assemble the final 2-D lat/lon CV candidate vectors by checking if the corresponding 2-D lon of a 2-D lat shares
3358 // the same dimension and under the same group and if there is another pair of 2-D lat/lon under the same group.
3359 Obtain_2DLLCVar_Candidate(tempcvar_2dlat,tempcvar_2dlon,latlon2d_path_to_index);
3360
3361#if 0
3362for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3363cerr<<"Final candidate 2-D Left lat variable full path is "<<(*irv)->fullpath <<endl;
3364for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3365cerr<<"Final candidate 2-D Left lon variable full path is "<<(*irv)->fullpath <<endl;
3366#endif
3367
3368 // 5. Remove the 2-D lat/lon variables that are to be used as CVs from the vector that stores general variables
3369 // var2d_index, remembers the index of the 2-D lat/lon CVs in the original vector of vars.
3370 vector<int> var2d_index;
3371 for (map<string,int>::const_iterator it= latlon2d_path_to_index.begin();it!=latlon2d_path_to_index.end();++it)
3372 var2d_index.push_back(it->second);
3373
3374 Remove_2DLLCVar_Final_Candidate_from_Vars(var2d_index);
3375
3376 // 6. If we have 2-D CVs, COARDS should be turned off.
3377 if(tempcvar_2dlat.empty()==false)
3378 iscoard = false;
3379
3380 // 7. Add the CVs based on the final 2-D lat/lon CV candidates.
3381 // We need to remember the dim names that the 2-D lat/lon CVs are associated with.
3382 set<string>dim_names_2d_cvs;
3383
3384 for (auto &cvar_lat:tempcvar_2dlat){
3385
3386 auto lat_unique = make_unique<GMCVar>(cvar_lat);
3387 auto lat = lat_unique.release();
3388 // Latitude is always corresponding to the first dimension.
3389 lat->cfdimname = cvar_lat->getDimensions()[0]->name;
3390 dim_names_2d_cvs.insert(lat->cfdimname);
3391 lat->cvartype = CV_EXIST;
3392 lat->product_type = product_type;
3393 this->cvars.push_back(lat);
3394 }
3395 for (auto &cvar_lon:tempcvar_2dlon) {
3396
3397 auto lon_unique = make_unique<GMCVar>(cvar_lon);
3398 auto lon = lon_unique.release();
3399 // Longitude is always corresponding to the second dimension.
3400 lon->cfdimname = cvar_lon->getDimensions()[1]->name;
3401 dim_names_2d_cvs.insert(lon->cfdimname);
3402 lon->cvartype = CV_EXIST;
3403 lon->product_type = product_type;
3404 this->cvars.push_back(lon);
3405 }
3406
3407 // 8. Move the originally assigned 1-D CVs that are replaced by 2-D CVs back to the general variable list.
3408 // Also remove the CV created by the pure dimensions.
3409 // Dimension names are used to identify those 1-D CVs.
3410 for(auto ircv= this->cvars.begin();ircv !=this->cvars.end();) {
3411 if(1 == (*ircv)->rank) {
3412 if(dim_names_2d_cvs.find((*ircv)->cfdimname)!=dim_names_2d_cvs.end()) {
3413 if(CV_FILLINDEX == (*ircv)->cvartype) {// This is pure dimension
3414 delete(*ircv);
3415 ircv = this->cvars.erase(ircv);
3416 }
3417 else if(CV_EXIST == (*ircv)->cvartype) {// This var exists already
3418
3419 // Add this var. to the var list.
3420 auto var_unique = make_unique<Var>(*ircv);
3421 auto var = var_unique.release();
3422 this->vars.push_back(var);
3423
3424 // Remove this var. from the GMCVar list.
3425 delete(*ircv);
3426 ircv = this->cvars.erase(ircv);
3427
3428 }
3429 else {// the removed 1-D coordinate variable should be either the CV_FILLINDEX or CV_EXIST.
3430 if(CV_LAT_MISS == (*ircv)->cvartype)
3431 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_LAT_MISS");
3432 else if(CV_LON_MISS == (*ircv)->cvartype)
3433 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_LON_MISS");
3434 else if(CV_NONLATLON_MISS == (*ircv)->cvartype)
3435 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_NONLATLON_MISS");
3436 else if(CV_MODIFY == (*ircv)->cvartype)
3437 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_MODIFY");
3438 else if(CV_SPECIAL == (*ircv)->cvartype)
3439 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_SPECIAL");
3440 else
3441 throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_UNSUPPORTED");
3442 }
3443
3444 }
3445 else
3446 ++ircv;
3447
3448 }
3449 else
3450 ++ircv;
3451
3452 }
3453
3454
3455#if 0
3456//if(iscoard == true)
3457//cerr<<"COARD is true"<<endl;
3458for(set<string>::iterator irs = grp_cv_paths.begin();irs != grp_cv_paths.end();++irs) {
3459cerr<<"group path is "<< (*irs)<<endl;
3460
3461}
3462#endif
3463
3464#if 0
3465//Print CVs
3466cerr<<"File name is "<< this->path <<endl;
3467cerr<<"CV names are the following "<<endl;
3468for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ++i)
3469cerr<<(*i)->fullpath <<endl;
3470#endif
3471
3472
3473 // 9. release the resources allocated by the temporary vectors.
3474 release_standalone_GMCVar_vector(tempcvar_1dlat);
3475 release_standalone_GMCVar_vector(tempcvar_1dlon);
3476 release_standalone_var_vector(tempcvar_2dlat);
3477 release_standalone_var_vector(tempcvar_2dlon);
3478 }// if(false == Check_1DGeolocation_Dimscale())
3479#if 0
3480for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ++i)
3481cerr<<(*i)->fullpath <<endl;
3482#endif
3483
3484}
3485
3486// If Check_1DGeolocation_Dimscale() is true, no need to build 2-D lat/lon coordinate variables.
3487// This function is introduced to avoid the performance penalty caused by handling the general 2-D lat/lon case.
3488bool GMFile::Check_1DGeolocation_Dimscale() {
3489
3490 BESDEBUG("h5", "Coming to Check_1DGeolocation_Dimscale()"<<endl);
3491 bool has_only_1d_geolocation_cv = false;
3492 bool has_1d_lat_cv_flag = false;
3493 bool has_1d_lon_cv_flag = false;
3494
3495 string lat_dimname;
3496 hsize_t lat_size = 0;
3497
3498 string lon_dimname;
3499 hsize_t lon_size = 0;
3500
3501 // We need to consider both 1-D lat/lon and the 1-D zonal average case(1-D lat only).
3502 for (const auto &cvar:this->cvars) {
3503
3504 if(cvar->cvartype == CV_EXIST) {
3505
3506 string attr_name ="units";
3507 string lat_unit_value = "degrees_north";
3508 string lon_unit_value = "degrees_east";
3509
3510 for (const auto &attr:cvar->attrs) {
3511
3512 if(true == Is_Str_Attr(attr,cvar->fullpath,attr_name,lat_unit_value)) {
3513 lat_size = cvar->getDimensions()[0]->size;
3514 lat_dimname = cvar->getDimensions()[0]->name;
3515 has_1d_lat_cv_flag = true;
3516 break;
3517 }
3518 else if(true == Is_Str_Attr(attr,cvar->fullpath,attr_name,lon_unit_value)){
3519 lon_size = cvar->getDimensions()[0]->size;
3520 lon_dimname = cvar->getDimensions()[0]->name;
3521 has_1d_lon_cv_flag = true;
3522 break;
3523 }
3524 }
3525 }
3526 }
3527
3528 // If having 1-D lat/lon CVs, this is a good sign for only 1-D lat/lon CVs ,
3529 // just need to have a couple of checks.
3530 if(true == has_1d_lat_cv_flag ) {
3531
3532 if(true == has_1d_lon_cv_flag) {
3533
3534 // Come to the possible classic netCDF-4 case,
3535 if(true == this->groups.empty()) {
3536
3537 // Rarely happens when lat_size is the same as the lon_size.
3538 // However, still want to make sure there is a 2-D variable that uses both lat and lon dims.
3539 if(lat_size == lon_size) {
3540 bool var_has_latdim = false;
3541 bool var_has_londim = false;
3542 for (const auto &var:this->vars) {
3543 if(var->rank >= 2) {
3544 for (const auto &dim:var->dims) {
3545 if(dim->name == lat_dimname)
3546 var_has_latdim = true;
3547 else if(dim->name == lon_dimname)
3548 var_has_londim = true;
3549 }
3550 if(true == var_has_latdim && true == var_has_londim) {
3551 has_only_1d_geolocation_cv = true;
3552 break;
3553 }
3554 else {
3555 var_has_latdim = false;
3556 var_has_londim = false;
3557 }
3558 }
3559 }
3560 }
3561 else
3562 has_only_1d_geolocation_cv = true;
3563 }// if(0 == this->groups.size())
3564 else {
3565 // Multiple groups, need to check if having 2-D lat/lon pairs
3566 bool has_2d_latname_flag = false;
3567 bool has_2d_lonname_flag = false;
3568 for (const auto &var:this->vars) {
3569 if(var->rank == 2) {
3570
3571 //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
3572 if(true == Is_geolatlon(var->name,true))
3573 has_2d_latname_flag = true;
3574
3575 //Note: When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
3576 else if(true == Is_geolatlon(var->name,false))
3577 has_2d_lonname_flag = true;
3578
3579 if((true == has_2d_latname_flag) && (true == has_2d_lonname_flag))
3580 break;
3581 }
3582 }
3583
3584 if(has_2d_latname_flag != true || has_2d_lonname_flag != true) {
3585
3586 //Check if having the 2-D lat/lon by checking if having lat/lon CF units(lon's units: degrees_east lat's units: degrees_north)
3587 has_2d_latname_flag = false;
3588 has_2d_lonname_flag = false;
3589
3590 for (const auto &var:this->vars) {
3591 if(var->rank == 2) {
3592 for (const auto &attr:var->attrs) {
3593
3594 if (false == has_2d_latname_flag) {
3595
3596 // When the third parameter of the function has_latlon_cf_units is set to true, it checks latitude
3597 has_2d_latname_flag = has_latlon_cf_units(attr,var->fullpath,true);
3598 if(true == has_2d_latname_flag)
3599 break;
3600 else if(false == has_2d_lonname_flag) {
3601
3602 // When the third parameter of the function has_latlon_cf_units is set to false, it checks longitude
3603 has_2d_lonname_flag = has_latlon_cf_units(attr,var->fullpath,false);
3604 if(true == has_2d_lonname_flag)
3605 break;
3606 }
3607 }
3608 else if(false == has_2d_lonname_flag) {
3609
3610 // Now has_2d_latname_flag is true, just need to check the has_2d_lonname_flag
3611 // When the third parameter of has_latlon_cf_units is set to false, it checks longitude
3612 has_2d_lonname_flag = has_latlon_cf_units(attr,var->fullpath,false);
3613 if(true == has_2d_lonname_flag)
3614 break;
3615 }
3616 }
3617 if(true == has_2d_latname_flag && true == has_2d_lonname_flag)
3618 break;
3619 }
3620 }
3621 }// if(has_2d_latname_flag != true || has_2d_lonname_flag != true)
3622
3623 // If we cannot find either of 2-D any lat/lon variables, this file is treated as having only 1-D lat/lon.
3624 if(has_2d_latname_flag != true || has_2d_lonname_flag != true)
3625 has_only_1d_geolocation_cv = true;
3626 }
3627
3628 }//
3629 else {//Zonal average case, we do not need to find 2-D lat/lon CVs.
3630 has_only_1d_geolocation_cv = true;
3631 }
3632
3633 }
3634
3635#if 0
3636if(has_only_1d_geolocation_cv == true)
3637cerr <<"has only 1D lat/lon CVs. "<<endl;
3638else
3639cerr<<"Possibly has 2D lat/lon CVs. "<<endl;
3640#endif
3641
3642 return has_only_1d_geolocation_cv;
3643
3644}
3645
3646// Obtain the originally assigned 1-D lat/lon coordinate variables.
3647// This function should be used before generating any 2-D lat/lon CVs.
3648void GMFile::Obtain_1DLatLon_CVs(vector<GMCVar*> &cvar_1dlat,vector<GMCVar*> &cvar_1dlon) {
3649
3650 BESDEBUG("h5", "Coming to Obtain_1DLatLon_CVs()"<<endl);
3651 for (const auto &cvar:this->cvars) {
3652
3653 if (cvar->cvartype == CV_EXIST) {
3654
3655 string attr_name ="units";
3656 string lat_unit_value = "degrees_north";
3657 string lon_unit_value = "degrees_east";
3658
3659 for (const auto &attr:cvar->attrs) {
3660
3661 // 1-D latitude
3662 if(true == Is_Str_Attr(attr,cvar->fullpath,attr_name,lat_unit_value)) {
3663 auto lat_unique = make_unique<GMCVar>(cvar);
3664 auto lat = lat_unique.release();
3665 lat->cfdimname = cvar->getDimensions()[0]->name;
3666 lat->cvartype = cvar->cvartype;
3667 lat->product_type = cvar->product_type;
3668 cvar_1dlat.push_back(lat);
3669 }
3670 // 1-D longitude
3671 else if(true == Is_Str_Attr(attr,cvar->fullpath,attr_name,lon_unit_value)){
3672 auto lon_unique = make_unique<GMCVar>(cvar);
3673 auto lon = lon_unique.release();
3674 lon->cfdimname = cvar->getDimensions()[0]->name;
3675 lon->cvartype = cvar->cvartype;
3676 lon->product_type = cvar->product_type;
3677 cvar_1dlon.push_back(lon);
3678 }
3679 }
3680 }// end of if(cvar->cvartype == CV_EXIST)
3681 }// end of for (cvar = this->cvars
3682
3683}
3684
3685// Obtain all 2-D lat/lon variables. We first check the lat/latitude/Latitude names, if not found, we check if CF lat/lon units are present.
3686// Latitude variables are saved in the vector var_2dlat. Longitude variables are saved in the vector var_2dlon.
3687// We also remember the index of these lat/lon in the original var vector.
3688void GMFile::Obtain_2DLatLon_Vars(vector<Var*> &var_2dlat,vector<Var*> &var_2dlon,map<string,int> & latlon2d_path_to_index) {
3689
3690 BESDEBUG("h5", "Coming to Obtain_2DLatLon_Vars()"<<endl);
3691 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3692 if((*irv)->rank == 2) {
3693
3694 //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
3695 if(true == Is_geolatlon((*irv)->name,true)) {
3696 auto lat_unique = make_unique<Var>(*irv);
3697 auto lat = lat_unique.release();
3698 var_2dlat.push_back(lat);
3699 latlon2d_path_to_index[(*irv)->fullpath]= distance(this->vars.begin(),irv);
3700 continue;
3701 }
3702 else {
3703
3704 bool has_2dlat = false;
3705 for (const auto &attr:(*irv)->attrs) {
3706
3707 // When the third parameter of has_latlon_cf_units is set to true, it checks latitude
3708 if(true == has_latlon_cf_units(attr,(*irv)->fullpath,true)) {
3709 auto lat_unique = make_unique<Var>(*irv);
3710 auto lat = lat_unique.release();
3711 var_2dlat.push_back(lat);
3712 latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3713 has_2dlat = true;
3714 break;
3715 }
3716 }
3717
3718 if(true == has_2dlat)
3719 continue;
3720 }
3721
3722 //Note: When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
3723 if (true == Is_geolatlon((*irv)->name,false)) {
3724 latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3725 auto lon_unique = make_unique<Var>(*irv);
3726 auto lon = lon_unique.release();
3727 var_2dlon.push_back(lon);
3728 }
3729 else {
3730 for (const auto &attr:(*irv)->attrs) {
3731
3732 // When the third parameter of has_latlon_cf_units is set to false, it checks longitude
3733 if(true == has_latlon_cf_units(attr,(*irv)->fullpath,false)) {
3734 latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3735 auto lon_unique = make_unique<Var>(*irv);
3736 auto lon = lon_unique.release();
3737 var_2dlon.push_back(lon);
3738 break;
3739 }
3740 }
3741 }
3742 } // if((*irv)->rank == 2)
3743 } // for (auto irv
3744}
3745
3746// Squeeze the 2-D lat/lon vectors by removing the ones that share the same dims with 1-D lat/lon CVs.
3747// The latlon2d_path_to_index map also needs to be updated.
3748void GMFile::Obtain_2DLLVars_With_Dims_not_1DLLCVars(vector<Var*> &var_2dlat,
3749 vector<Var*> &var_2dlon,
3750 const vector<GMCVar*> &cvar_1dlat,
3751 const vector<GMCVar*> &cvar_1dlon,
3752 map<string,int> &latlon2d_path_to_index) {
3753
3754 BESDEBUG("h5", "Coming to Obtain_2DLLVars_With_Dims_not_1DLLCVars()"<<endl);
3755 // First latitude at var_2dlat
3756 for(auto irv = var_2dlat.begin();irv != var_2dlat.end();) {
3757 bool remove_2dlat = false;
3758 for(const auto &lat:cvar_1dlat) {
3759 for (auto &dim:(*irv)->dims) {
3760 if(dim->name == lat->getDimensions()[0]->name &&
3761 dim->size == lat->getDimensions()[0]->size) {
3762 latlon2d_path_to_index.erase((*irv)->fullpath);
3763 delete(*irv);
3764 irv = var_2dlat.erase(irv);
3765 remove_2dlat = true;
3766 break;
3767 }
3768 }
3769 if(true == remove_2dlat)
3770 break;
3771 }
3772
3773 if(false == remove_2dlat)
3774 ++irv;
3775 }// end of for(auto irv = var_2dlat.begin()
3776
3777 // Second longitude
3778 for(auto irv = var_2dlon.begin();irv != var_2dlon.end();) {
3779 bool remove_2dlon = false;
3780 for(const auto &lon:cvar_1dlon) {
3781 for (auto &dim:(*irv)->dims) {
3782 if(dim->name == lon->getDimensions()[0]->name &&
3783 dim->size == lon->getDimensions()[0]->size) {
3784 latlon2d_path_to_index.erase((*irv)->fullpath);
3785 delete(*irv);
3786 irv = var_2dlon.erase(irv);
3787 remove_2dlon = true;
3788 break;
3789 }
3790 }
3791 if(true == remove_2dlon)
3792 break;
3793 }
3794
3795 if(false == remove_2dlon)
3796 ++irv;
3797 } // end of for(auto irv = var_2dlon.begin()
3798
3799}
3800
3801//Out of the collected 2-D lat/lon variables, we will select the final qualified 2-D lat/lon as CVs.
3802void GMFile::Obtain_2DLLCVar_Candidate(vector<Var*> &var_2dlat,
3803 vector<Var*> &var_2dlon,
3804 map<string,int>& latlon2d_path_to_index) {
3805 BESDEBUG("h5", "Coming to Obtain_2DLLCVar_Candidate()"<<endl);
3806 // First check 2-D lat, see if we have the corresponding 2-D lon(same dims, under the same group).
3807 // If no, remove that lat from the vector.
3808 vector<string> lon2d_group_paths;
3809
3810 for (auto irv_2dlat = var_2dlat.begin();irv_2dlat !=var_2dlat.end();) {
3811 for (const auto &lon:var_2dlon) {
3812 if(((*irv_2dlat)->getDimensions()[0]->name == lon->getDimensions()[0]->name) &&
3813 ((*irv_2dlat)->getDimensions()[0]->size == lon->getDimensions()[0]->size) &&
3814 ((*irv_2dlat)->getDimensions()[1]->name == lon->getDimensions()[1]->name) &&
3815 ((*irv_2dlat)->getDimensions()[1]->size == lon->getDimensions()[1]->size))
3816 lon2d_group_paths.push_back(HDF5CFUtil::obtain_string_before_lastslash(lon->fullpath));
3817 }
3818 // Doesn't find any lons that shares the same dims,remove this lat from the 2dlat vector,
3819 // also update the latlon2d_path_to_index map
3820 if (true == lon2d_group_paths.empty()) {
3821 latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3822 delete(*irv_2dlat);
3823 irv_2dlat = var_2dlat.erase(irv_2dlat);
3824 }
3825 else {// Find lons,check if they are under the same group
3826#if 0
3827 //string lat2d_group_path = (*irv_2dlat)->fullpath.substr(0,(*irv_2dlat)->fullpath.find_last_of("/"));
3828#endif
3829 string lat2d_group_path = HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlat)->fullpath);
3830
3831 // Check how many lon2d shares the same group with the lat2d
3832 short lon2d_has_lat2d_group_path_flag = 0;
3833 for(const auto &lon2d_group_path:lon2d_group_paths) {
3834 if(lon2d_group_path==lat2d_group_path)
3835 lon2d_has_lat2d_group_path_flag++;
3836 }
3837
3838 // No lon2d shares the same group with the lat2d, remove this lat2d
3839 if(0 == lon2d_has_lat2d_group_path_flag) {
3840 latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3841 delete(*irv_2dlat);
3842 irv_2dlat = var_2dlat.erase(irv_2dlat);
3843 }
3844 // Only one lon2d, yes, keep it.
3845 else if (1== lon2d_has_lat2d_group_path_flag) {
3846 ++irv_2dlat;
3847 }
3848 // More than 1 lon2d, we will remove the lat2d, but save the group path so that we may
3849 // flatten the variable path stored in the coordinates attribute under this group.
3850 else {
3851 // Save the group path for the future use.
3852 grp_cv_paths.insert(lat2d_group_path);
3853 latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3854 delete(*irv_2dlat);
3855 irv_2dlat = var_2dlat.erase(irv_2dlat);
3856 }
3857 }
3858
3859 //Clear the vector that stores the same dim. since it is only applied to this lat,
3860 lon2d_group_paths.clear();
3861 }
3862
3863#if 0
3864for(auto irv_2dlat = var_2dlat.begin();irv_2dlat !=var_2dlat.end();++irv_2dlat)
3865cerr<<"2 left 2-D lat variable full path is: "<<(*irv_2dlat)->fullpath <<endl;
3866#endif
3867
3868
3869 // Second check 2-D lon, see if we have the corresponding 2-D lat(same dims, under the same group).
3870 // If no, remove that lon from the vector.
3871 vector<string> lat2d_group_paths;
3872
3873 // Check the longitude
3874 for(auto irv_2dlon = var_2dlon.begin();irv_2dlon !=var_2dlon.end();) {
3875 for(const auto &lat:var_2dlat) {
3876 if((lat->getDimensions()[0]->name == (*irv_2dlon)->getDimensions()[0]->name) &&
3877 (lat->getDimensions()[0]->size == (*irv_2dlon)->getDimensions()[0]->size) &&
3878 (lat->getDimensions()[1]->name == (*irv_2dlon)->getDimensions()[1]->name) &&
3879 (lat->getDimensions()[1]->size == (*irv_2dlon)->getDimensions()[1]->size))
3880 lat2d_group_paths.push_back(HDF5CFUtil::obtain_string_before_lastslash(lat->fullpath));
3881#if 0
3882 //lat2d_group_paths.push_back(lat->fullpath.substr(0,lat->fullpath.find_last_of("/")));
3883#endif
3884 }
3885 // Doesn't find any lats that shares the same dims,remove this lon from this vector
3886 if(lat2d_group_paths.empty()) {
3887 latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3888 delete(*irv_2dlon);
3889 irv_2dlon = var_2dlon.erase(irv_2dlon);
3890 }
3891 else {
3892 string lon2d_group_path = HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlon)->fullpath);
3893
3894 // Check how many lat2d shares the same group with the lon2d
3895 short lat2d_has_lon2d_group_path_flag = 0;
3896 for(const auto &lat2d_group_path:lat2d_group_paths) {
3897 if(lat2d_group_path == lon2d_group_path)
3898 lat2d_has_lon2d_group_path_flag++;
3899 }
3900
3901 // No lat2d shares the same group with the lon2d, remove this lon2d
3902 if(0 == lat2d_has_lon2d_group_path_flag) {
3903 latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3904 delete(*irv_2dlon);
3905 irv_2dlon = var_2dlon.erase(irv_2dlon);
3906 }
3907 // Only one lat2d shares the same group with the lon2d, yes, keep it.
3908 else if (1== lat2d_has_lon2d_group_path_flag) {
3909 ++irv_2dlon;
3910 }
3911 // more than 1 lat2d, we will remove the lon2d, but save the group path so that we can
3912 // change the coordinates attribute for variables under this group later.
3913 else {
3914 // Save the group path for future "coordinates" modification.
3915 grp_cv_paths.insert(lon2d_group_path);
3916 latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3917 delete(*irv_2dlon);
3918 irv_2dlon = var_2dlon.erase(irv_2dlon);
3919 }
3920 }
3921 //Clear the vector that stores the same dim. since it is only applied to this lon,
3922 lat2d_group_paths.clear();
3923 }
3924#if 0
3925for(vector<Var*>::iterator itv = var_2dlat.begin(); itv!= var_2dlat.end();++itv) {
3926cerr<<"Before unique, 2-D CV latitude name is "<<(*itv)->fullpath <<endl;
3927}
3928for(vector<Var*>::iterator itv = var_2dlon.begin(); itv!= var_2dlon.end();++itv) {
3929cerr<<"Before unique, 2-D CV longitude name is "<<(*itv)->fullpath <<endl;
3930}
3931#endif
3932
3933 // Final check var_2dlat and var_2dlon to remove non-qualified CVs.
3934 Obtain_unique_2dCV(var_2dlat,latlon2d_path_to_index);
3935 Obtain_unique_2dCV(var_2dlon,latlon2d_path_to_index);
3936#if 0
3937for(vector<Var*>::iterator itv = var_2dlat.begin(); itv!= var_2dlat.end();++itv) {
3938cerr<<"2-D CV latitude name is "<<(*itv)->fullpath <<endl;
3939}
3940for(vector<Var*>::iterator itv = var_2dlon.begin(); itv!= var_2dlon.end();++itv) {
3941cerr<<"2-D CV longitude name is "<<(*itv)->fullpath <<endl;
3942}
3943#endif
3944
3945 // This is to serve as a sanity check. This can help us find bugs in the first place.
3946 if(var_2dlat.size() != var_2dlon.size()) {
3947 throw1("Error in generating 2-D lat/lon CVs. The size of 2d-lat should be the same as that of 2d-lon.");
3948 }
3949}
3950
3951// If two vars in the 2-D lat or 2-D lon CV candidate vector share the same dim, these two vars cannot be CVs.
3952// The group they belong to is the group candidate that the coordinates attribute of the variable under that group
3953// may be modified.
3954void GMFile::Obtain_unique_2dCV(vector<Var*> &var_ll,map<string,int>&latlon2d_path_to_index){
3955
3956 BESDEBUG("h5", "Coming to Obtain_unique_2dCV()"<<endl);
3957 vector<bool> var_share_dims(var_ll.size(),false);
3958
3959 for(unsigned int i = 0; i <var_ll.size();i++) {
3960
3961 // obtain the path of var_ll
3962 string var_ll_i_path = HDF5CFUtil::obtain_string_before_lastslash(var_ll[i]->fullpath);
3963
3964 // Check if two vars share the same dims.
3965 for(unsigned int j = i+1; j<var_ll.size();j++) {
3966 if((var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[0]->name)
3967 ||(var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[1]->name)
3968 ||(var_ll[i]->getDimensions()[1]->name == var_ll[j]->getDimensions()[0]->name)
3969 ||(var_ll[i]->getDimensions()[1]->name == var_ll[j]->getDimensions()[1]->name)){
3970 string var_ll_j_path = HDF5CFUtil::obtain_string_before_lastslash(var_ll[j]->fullpath);
3971
3972 // Compare var_ll_i_path and var_ll_j_path,only set the child group path be true and remember the path.
3973 // The variable at the parent group can be the coordinate variable.
3974 // Obtain the string size,
3975 // compare the string size, long.compare(0,short-length,short)==0,
3976 // yes, save the long path(child group path), set the long path one true. Else save two paths, set both true
3977 if(var_ll_i_path.size() > var_ll_j_path.size()) {
3978
3979 // If var_ll_j_path is the parent group of var_ll_i_path,
3980 // set the shared dim. be true for the child group only,remember the path.
3981 if(var_ll_i_path.compare(0,var_ll_j_path.size(),var_ll_j_path)==0) {
3982 var_share_dims[i] = true;
3983 grp_cv_paths.insert(var_ll_i_path);
3984 }
3985 else {// Save both as shared, they cannot be CVs.
3986 var_share_dims[i] = true;
3987 var_share_dims[j] = true;
3988
3989 grp_cv_paths.insert(var_ll_i_path);
3990 grp_cv_paths.insert(var_ll_j_path);
3991 }
3992 }
3993 else if (var_ll_i_path.size() == var_ll_j_path.size()) {// Share the same group, remember both group paths.
3994 var_share_dims[i] = true;
3995 var_share_dims[j] = true;
3996 if(var_ll_i_path == var_ll_j_path)
3997 grp_cv_paths.insert(var_ll_i_path);
3998 else {
3999 grp_cv_paths.insert(var_ll_i_path);
4000 grp_cv_paths.insert(var_ll_j_path);
4001 }
4002 }
4003 else {
4004 // var_ll_i_path is the parent group of var_ll_j_path,
4005 // set the shared dim. be true for the child group,remember the path.
4006 if(var_ll_j_path.compare(0,var_ll_i_path.size(),var_ll_i_path)==0) {
4007 var_share_dims[j] = true;
4008 grp_cv_paths.insert(var_ll_j_path);
4009 }
4010 else {// Save both as shared, they cannot be CVs.
4011 var_share_dims[i] = true;
4012 var_share_dims[j] = true;
4013
4014 grp_cv_paths.insert(var_ll_i_path);
4015 grp_cv_paths.insert(var_ll_j_path);
4016
4017 }
4018 }
4019 }// end of if((var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[0]->name)
4020 }// end of for(int j = i+1; j<var_ll.size();j++)
4021 }// end of for( int i = 0; i <var_ll.size();i++)
4022
4023 // Remove the shared 2-D lat/lon CVs from the 2-D lat/lon CV candidates.
4024 int var_index = 0;
4025 for(auto itv = var_ll.begin(); itv!= var_ll.end();) {
4026 if(true == var_share_dims[var_index]) {
4027 latlon2d_path_to_index.erase((*itv)->fullpath);
4028 delete(*itv);
4029 itv = var_ll.erase(itv);
4030 }
4031 else {
4032 ++itv;
4033 }
4034 ++var_index;
4035 }
4036
4037}
4038
4039// When promoting a 2-D lat or lon to a coordinate variable, we need to remove them from the general variable vector.
4040void GMFile::Remove_2DLLCVar_Final_Candidate_from_Vars(vector<int> &var2d_index) {
4041
4042 BESDEBUG("h5", "Coming to Remove_2DLLCVar_Final_Candidate_from_Vars()"<<endl);
4043 //Sort the 2-D lat/lon var index according to the ascending order before removing the 2-D lat/lon vars
4044 sort(var2d_index.begin(),var2d_index.end());
4045 auto it = this->vars.begin();
4046
4047 // This is a performance optimization operation.
4048 // We find it is typical for swath files that have many general variables but only have very few lat/lon CVs.
4049 // To reduce the looping through all variables and comparing the fullpath(string), we use index and remember
4050 // the position of 2-D CVs in the iterator. In this way, only a few operations are needed.
4051 for (unsigned int i = 0; i <var2d_index.size();i++) {
4052 if ( i == 0)
4053 advance(it,var2d_index[i]);
4054 else
4055 advance(it,var2d_index[i]-var2d_index[i-1]-1);
4056
4057 if(it == this->vars.end())
4058 throw1("Out of range to obtain 2D lat/lon variables");
4059 else {
4060 delete(*it);
4061 it = this->vars.erase(it);
4062 }
4063 }
4064}
4065
4066//This function is for generating the coordinates attribute for the 2-D lat/lon.
4067//It will check if this var can keep its "coordinates" attribute rather than rebuilding it.
4068//This function is used by Handle_Coor_Attr().
4069bool GMFile::Check_Var_2D_CVars(Var *var) const {
4070
4071 BESDEBUG("h5", "Coming to Check_Var_2D_CVars()"<<endl);
4072 bool ret_value = true;
4073 for (const auto &cvar:this->cvars) {
4074 if(cvar->rank==2) {
4075 short first_dim_index = 0;
4076 short first_dim_times = 0;
4077 short second_dim_index = 0;
4078 short second_dim_times = 0;
4079 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
4080 if((*ird)->name == (cvar->getDimensions()[0])->name) {
4081 first_dim_index = distance(var->dims.begin(),ird);
4082 first_dim_times++;
4083 }
4084 else if((*ird)->name == (cvar->getDimensions()[1])->name) {
4085 second_dim_index = distance(var->dims.begin(),ird);
4086 second_dim_times++;
4087 }
4088 }
4089 // The 2-D CV dimensions must only appear once as the dimension of the variable
4090 // It also must follow the dimension order of the 2-D lat/lon dimensions.
4091 if(first_dim_times == 1 && second_dim_times == 1) {
4092 if(first_dim_index < second_dim_index) {
4093 ret_value = false;
4094 break;
4095 }
4096 }
4097 }
4098 }
4099 return ret_value;
4100
4101}
4102
4103// This function flattens the variable path in the "coordinates" attribute.
4104// It is also used by Handle_Coor_Attr().
4105bool GMFile::Flatten_VarPath_In_Coordinates_Attr(Var *var) {
4106
4107 BESDEBUG("h5", "Coming to Flatten_VarPath_In_Coordinates_Attr()"<<endl);
4108 string co_attrname = "coordinates";
4109 bool need_flatten_coor_attr = false;
4110 string orig_coor_value;
4111 string flatten_coor_value;
4112 // Assume the separator is always a space.
4113 char sc = ' ';
4114 char backslash = '/';
4115
4116 for (auto ira =var->attrs.begin(); ira !=var->attrs.end();) {
4117
4118 // We only check the original attribute name
4119 // Remove the original "coordinates" attribute.
4120 // There is a case that the "coordinates" attribute doesn't need to be flattened.
4121 // We will skip this case. This is necessary since the "coordinates" may be revised
4122 // to add a path to follow the CF. Without skipping the case, the "coordinates" is
4123 // actually not handled but "makes the impression" it is handled. That makes
4124 // this case ignored when the EnableCoorattrAddPath BES key is supposed to work on.
4125 // https://bugs.earthdata.nasa.gov/browse/HDFFEATURE-45
4126 if((*ira)->name == co_attrname) {
4127 Retrieve_H5_Attr_Value((*ira),var->fullpath);
4128 string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
4129 if(orig_attr_value.find_first_of(backslash)!=string::npos){
4130 orig_coor_value = orig_attr_value;
4131 need_flatten_coor_attr = true;
4132 delete(*ira);
4133 ira = var->attrs.erase(ira);
4134 }
4135 break;
4136 }
4137 else
4138 ++ira;
4139 }
4140
4141 if(true == need_flatten_coor_attr) {
4142
4143 // We need to loop through each element in the "coordinates".
4144 size_t ele_start_pos = 0;
4145 size_t cur_pos = orig_coor_value.find_first_of(sc);
4146 while(cur_pos !=string::npos) {
4147 string tempstr = orig_coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
4148 tempstr = get_CF_string(tempstr);
4149 flatten_coor_value += tempstr + sc;
4150 ele_start_pos = cur_pos+1;
4151 cur_pos = orig_coor_value.find_first_of(sc,cur_pos+1);
4152 }
4153 // Only one element
4154 if(ele_start_pos == 0)
4155 flatten_coor_value = get_CF_string(orig_coor_value);
4156 else // Add the last element
4157 flatten_coor_value += get_CF_string(orig_coor_value.substr(ele_start_pos));
4158
4159 // Generate the new "coordinates" attribute.
4160 auto attr_unique = make_unique<Attribute>();
4161 auto attr = attr_unique.release();
4162 Add_Str_Attr(attr,co_attrname,flatten_coor_value);
4163 var->attrs.push_back(attr);
4164 var->coord_attr_add_path = false;
4165
4166 }
4167
4168 return true;
4169
4170}
4171// This function flattens the variable path in the "coordinates" attribute for hybrid EOS5.
4172// Will implement it later.
4173// It is also used by Handle_Coor_Attr().
4174#if 0
4175bool GMFile::Flatten_VarPath_In_Coordinates_Attr_EOS5(Var *var) {
4176
4177 BESDEBUG("h5", "Coming to Flatten_VarPath_In_Coordinates_Attr_EOS5()"<<endl);
4178 string co_attrname = "coordinates";
4179 bool has_coor_attr = false;
4180 string orig_coor_value;
4181 string flatten_coor_value;
4182 // Assume the separator is always a space.
4183 char sc = ' ';
4184
4185 for (auto ira =var->attrs.begin(); ira !=var->attrs.end();) {
4186
4187 // We only check the original attribute name
4188 // Remove the original "coordinates" attribute.
4189 if((*ira)->name == co_attrname) {
4190 Retrieve_H5_Attr_Value((*ira),var->fullpath);
4191
4192 string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
4193 orig_coor_value = orig_attr_value;
4194 has_coor_attr = true;
4195 delete(*ira);
4196 ira = var->attrs.erase(ira);
4197 break;
4198 }
4199 else
4200 ++ira;
4201 }
4202
4203 if(true == has_coor_attr) {
4204
4205 // We need to loop through each element in the "coordinates".
4206 // For EOS5: we need to find the swath name.
4207 size_t ele_start_pos = 0;
4208// cerr<<"orig_coor_value is "<<orig_coor_value <<endl;
4209 size_t cur_pos = orig_coor_value.find_first_of(sc);
4210 while(cur_pos !=string::npos) {
4211 string tempstr = orig_coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
4212 // Find the swath name
4213 // tempstr = "swath name" +"_"+tempstr;
4214 tempstr = get_CF_string(tempstr);
4215 flatten_coor_value += tempstr + sc;
4216 ele_start_pos = cur_pos+1;
4217 cur_pos = orig_coor_value.find_first_of(sc,cur_pos+1);
4218 }
4219 // Only one element
4220 if(ele_start_pos == 0) {
4221 // Find the swath name
4222 // tempstr = "swath name" +"_"+tempstr;
4223 flatten_coor_value = get_CF_string(tempstr);
4224 }
4225 else // Add the last element
4226 flatten_coor_value += get_CF_string(orig_coor_value.substr(ele_start_pos));
4227
4228 // Generate the new "coordinates" attribute.
4229 Attribute *attr = new Attribute();
4230 Add_Str_Attr(attr,co_attrname,flatten_coor_value);
4231 var->attrs.push_back(attr);
4232 }
4233
4234 return true;
4235
4236}
4237#endif
4238
4239
4240// The following two routines only handle one 2-D lat/lon CVs. It is replaced by the more general
4241// multi 2-D lat/lon CV routines. Leave it here just for references.
4242#if 0
4243bool GMFile::Check_2DLatLon_Dimscale(string & latname, string &lonname) {
4244
4245 // New code to support 2-D lat/lon, still in development.
4246 // Need to handle 2-D latitude and longitude cases.
4247 // 1. Searching only the coordinate variables and if getting either of the following, keep the current way,
4248 // (A) GMcvar no CV_FILLINDEX:(The 2-D latlon case should have fake CVs)
4249 // (B) CV_EXIST: Attributes contain units and units value is degrees_east or degrees_north(have lat/lon)
4250 // (B) CV_EXIST: variables have name pair{lat/latitude/Latitude,lon/longitude/Longitude}(have lat/lon)
4251 //
4252 // 2. if not 1), searching all the variables and see if finding variables {lat/latitude/Latitude,lon/longitude/Longitude};
4253 // If finding {lat/lon},{latitude,longitude},{latitude,Longitude} pair,
4254 // if the number of dimension of either variable is not 2, keep the current way.
4255 // else check if the dimension name of latitude and longitude are the same, not, keep the current way
4256 // check the units of this CV pair, if units of the latitude is not degrees_north,
4257 // change it to degrees_north.
4258 // if units of the longitude is not degrees_east, change it to degrees_east.
4259 // make iscoard false.
4260
4261 bool latlon_2d_cv_check1 = false;
4262
4263 // Some products(TOM MEaSURE) provide the true dimension scales for 2-D lat,lon. So relax this check.
4264 latlon_2d_cv_check1 = true;
4265#if 0
4266 // If having 2-D lat/lon, the corresponding dimension must be pure and the CV type must be FILLINDEX.
4267 for (auto ircv = this->cvars.begin();
4268 ircv != this->cvars.end(); ++ircv) {
4269 if((*ircv)->cvartype == CV_FILLINDEX){
4270 latlon_2d_cv_check1 = true;
4271 break;
4272 }
4273 }
4274#endif
4275
4276 bool latlon_2d_cv_check2 = true;
4277
4278 // There may still not be 2-D lat/lon. Check the units attributes and lat/lon pairs.
4279 if(true == latlon_2d_cv_check1) {
4280 BESDEBUG("h5","Coming to check if having 2d latlon coordinates for a netCDF-4 like product. "<<endl);
4281
4282 // check if units attribute values have CF lat/lon units "degrees_north" or "degrees_east".
4283 for (auto ircv = this->cvars.begin();
4284 ircv != this->cvars.end(); ++ircv) {
4285 if((*ircv)->cvartype == CV_EXIST) {
4286 for(auto ira = (*ircv)->attrs.begin();
4287 ira != (*ircv)->attrs.end();ira++) {
4288 string attr_name ="units";
4289 string lat_unit_value = "degrees_north";
4290 string lon_unit_value = "degrees_east";
4291
4292 // Considering the cross-section case, either is fine.
4293 if((true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lat_unit_value)) ||
4294 (true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lon_unit_value))) {
4295 latlon_2d_cv_check2= false;
4296 break;
4297 }
4298 }
4299 }
4300
4301 if(false == latlon_2d_cv_check2)
4302 break;
4303 }
4304 }
4305
4306 bool latlon_2d_cv_check3 = true;
4307
4308 // Even we cannot find the CF lat/lon attributes, we may still find lat/lon etc pairs.
4309 if(true == latlon_2d_cv_check1 && true == latlon_2d_cv_check2) {
4310
4311 short latlon_flag = 0;
4312 short LatLon_flag = 0;
4313 short latilong_flag = 0;
4314
4315 for (auto ircv = this->cvars.begin();
4316 ircv != this->cvars.end(); ++ircv) {
4317 if((*ircv)->cvartype == CV_EXIST) {
4318 if((*ircv)->name == "lat")
4319 latlon_flag++;
4320 else if((*ircv)->name == "lon")
4321 latlon_flag++;
4322 else if((*ircv)->name == "latitude")
4323 latilong_flag++;
4324 else if((*ircv)->name == "longitude")
4325 latilong_flag++;
4326 else if((*ircv)->name == "Latitude")
4327 LatLon_flag++;
4328 else if((*ircv)->name == "Longitude")
4329 LatLon_flag++;
4330 }
4331
4332 }
4333 if((2== latlon_flag) || (2 == latilong_flag) || (2 == LatLon_flag ))
4334 latlon_2d_cv_check3 = false;
4335 }
4336
4337 bool latlon_2d = false;
4338 short latlon_flag = 0;
4339 string latdim1,latdim2,londim1,londim2;
4340
4341 short LatLon_flag = 0;
4342 string Latdim1,Latdim2,Londim1,Londim2;
4343
4344 short latilong_flag = 0;
4345 string latidim1,latidim2,longdim1,longdim2;
4346
4347
4348 // Final check, we need to check if we have 2-D {lat/latitude/Latitude, lon/longitude/Longitude}
4349 // in the general variable list.
4350 // Here, depending on the future support, lat/lon pairs with other names(cell_lat,cell_lon etc) may be supported.
4351 // KY 2015-12-03
4352 if(true == latlon_2d_cv_check1 && true == latlon_2d_cv_check2 && true == latlon_2d_cv_check3) {
4353
4354 for (auto irv = this->vars.begin();
4355 irv != this->vars.end(); ++irv) {
4356
4357 //
4358 if((*irv)->rank == 2) {
4359 if((*irv)->name == "lat") {
4360 latlon_flag++;
4361 latdim1 = (*irv)->getDimensions()[0]->name;
4362 latdim2 = (*irv)->getDimensions()[1]->name;
4363
4364 }
4365 else if((*irv)->name == "lon") {
4366 latlon_flag++;
4367 londim1 = (*irv)->getDimensions()[0]->name;
4368 londim2 = (*irv)->getDimensions()[1]->name;
4369 }
4370 else if((*irv)->name == "latitude"){
4371 latilong_flag++;
4372 latidim1 = (*irv)->getDimensions()[0]->name;
4373 latidim2 = (*irv)->getDimensions()[1]->name;
4374 }
4375 else if((*irv)->name == "longitude"){
4376 latilong_flag++;
4377 longdim1 = (*irv)->getDimensions()[0]->name;
4378 longdim2 = (*irv)->getDimensions()[1]->name;
4379
4380 }
4381 else if((*irv)->name == "Latitude"){
4382 LatLon_flag++;
4383 Latdim1 = (*irv)->getDimensions()[0]->name;
4384 Latdim2 = (*irv)->getDimensions()[1]->name;
4385
4386 }
4387 else if((*irv)->name == "Longitude"){
4388 LatLon_flag++;
4389 Londim1 = (*irv)->getDimensions()[0]->name;
4390 Londim2 = (*irv)->getDimensions()[1]->name;
4391 }
4392
4393 }
4394 }
4395
4396 // Here we ensure that only one lat/lon(lati/long,Lati/Long) is in the file.
4397 // If we find >=2 pairs lat/lon and latitude/longitude, or Latitude/Longitude,
4398 // we will not treat this as a 2-D latlon Dimscale case. The data producer
4399 // should correct their mistakes.
4400 if(2 == latlon_flag) {
4401 if((2 == latilong_flag) || ( 2 == LatLon_flag))
4402 latlon_2d = false;
4403 else if((latdim1 == londim1) && (latdim2 == londim2)) {
4404 latname = "lat";
4405 lonname = "lon";
4406 latlon_2d = true;
4407 }
4408 }
4409 else if ( 2 == latilong_flag) {
4410 if( 2 == LatLon_flag)
4411 latlon_2d = false;
4412 else if ((latidim1 == longdim1) ||(latidim2 == longdim2)) {
4413 latname = "latitude";
4414 lonname = "longitude";
4415 latlon_2d = true;
4416 }
4417 }
4418 else if (2 == LatLon_flag){
4419 if ((Latdim1 == Londim1) ||(Latdim2 == Londim2)) {
4420 latname = "Latitude";
4421 lonname = "Longitude";
4422 latlon_2d = true;
4423 }
4424 }
4425 }
4426
4427 return latlon_2d;
4428}
4429
4430
4431// Update the coordinate variables for files that use HDF5 dimension scales and have 2-D lat/lon.
4432void GMFile::Update_2DLatLon_Dimscale_CV(const string &latname,const string &lonname) {
4433
4434 iscoard = false;
4435
4436 // Update latitude.
4437 for (auto irv = this->vars.begin();
4438 irv != this->vars.end(); ++irv) {
4439
4440 if((*irv)->rank == 2) {
4441
4442 // Find 2-D latitude
4443 if((*irv)->name == latname) {
4444
4445 // Obtain the first dimension of this variable
4446 string latdim0 = (*irv)->getDimensions()[0]->name;
4447//cerr<<"latdim0 is "<<latdim0 <<endl;
4448
4449 // Remove the CV corresponding to latdim0
4450 for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ) {
4451 if((*i)->cfdimname == latdim0) {
4452 if(CV_FILLINDEX == (*i)->cvartype) {
4453 delete(*i);
4454 i = this->cvars.erase(i);
4455 }
4456 else if(CV_EXIST == (*i)->cvartype) {
4457 // Add this var. to the var list.
4458 Var *var = new Var(*i);
4459 this->vars.push_back(var);
4460 // Remove this var. from the GMCVar list.
4461 delete(*i);
4462 i = this->cvars.erase(i);
4463
4464 }
4465 else {// the latdimname should be either the CV_FILLINDEX or CV_EXIST.
4466 if(CV_LAT_MISS == (*i)->cvartype)
4467 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_LAT_MISS");
4468 else if(CV_LON_MISS == (*i)->cvartype)
4469 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_LON_MISS");
4470 else if(CV_NONLATLON_MISS == (*i)->cvartype)
4471 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_NONLATLON_MISS");
4472 else if(CV_MODIFY == (*i)->cvartype)
4473 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_MODIFY");
4474 else if(CV_SPECIAL == (*i)->cvartype)
4475 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_SPECIAL");
4476 else
4477 throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_UNSUPPORTED");
4478
4479 }
4480 }
4481 else
4482 ++i;
4483 }
4484 // Add the 2-D latitude(latname) to the CV list.
4485 GMCVar* GMcvar = new GMCVar(*irv);
4486 GMcvar->cfdimname = latdim0;
4487 GMcvar->cvartype = CV_EXIST;
4488 GMcvar->product_type = product_type;
4489 this->cvars.push_back(GMcvar);
4490 delete(*irv);
4491 this->vars.erase(irv);
4492 break;
4493 }
4494 }
4495 }
4496
4497 // Update longitude.
4498 for (auto irv = this->vars.begin();
4499 irv != this->vars.end(); ++irv) {
4500
4501 if((*irv)->rank == 2) {
4502
4503 // Find 2-D longitude
4504 if((*irv)->name == lonname) {
4505
4506 // Obtain the second dimension of this variable
4507 string londim0 = (*irv)->getDimensions()[1]->name;
4508
4509 // Remove the CV corresponding to londim0
4510 for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ) {
4511 // NEED more work!!! should also remove ntime from the GMCVar list but add it to the cvar list.Same for Lon.
4512 if((*i)->cfdimname == londim0) {
4513 if(CV_FILLINDEX == (*i)->cvartype) {
4514 delete(*i);
4515 i= this->cvars.erase(i);
4516 }
4517 else if(CV_EXIST == (*i)->cvartype) {
4518 // Add this var. to the var list.
4519 Var *var = new Var(*i);
4520 this->vars.push_back(var);
4521 // Remove this var. from the GMCVar list.
4522 delete(*i);
4523 i = this->cvars.erase(i);
4524 }
4525 else {// the latdimname should be either the CV_FILLINDEX or CV_EXIST.
4526 if(CV_LAT_MISS == (*i)->cvartype)
4527 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_LAT_MISS");
4528 else if(CV_LON_MISS == (*i)->cvartype)
4529 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_LON_MISS");
4530 else if(CV_NONLATLON_MISS == (*i)->cvartype)
4531 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_NONLATLON_MISS");
4532 else if(CV_MODIFY == (*i)->cvartype)
4533 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_MODIFY");
4534 else if(CV_SPECIAL == (*i)->cvartype)
4535 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_SPECIAL");
4536 else
4537 throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_UNSUPPORTED");
4538 }
4539 }
4540 else
4541 ++i;
4542 }
4543
4544 // Add the 2-D longitude(lonname) to the CV list.
4545 GMCVar* GMcvar = new GMCVar(*irv);
4546 GMcvar->cfdimname = londim0;
4547 GMcvar->cvartype = CV_EXIST;
4548 GMcvar->product_type = product_type;
4549 this->cvars.push_back(GMcvar);
4550 delete(*irv);
4551 this->vars.erase(irv);
4552 break;
4553 }
4554 }
4555 }
4556}
4557#endif
4558
4559// Handle coordinate variables for general HDF5 products that have 1-D lat/lon
4560void GMFile::Handle_CVar_LatLon1D_General_Product() {
4561
4562 BESDEBUG("h5", "Coming to Handle_CVar_LatLon1D_General_Product()"<<endl);
4563 this->iscoard = true;
4564 Handle_CVar_LatLon_General_Product();
4565
4566}
4567
4568// Handle coordinate variables for general HDF5 products that have 2-D lat/lon
4569void GMFile::Handle_CVar_LatLon2D_General_Product() {
4570
4571 BESDEBUG("h5", "Coming to Handle_CVar_LatLon2D_General_Product()"<<endl);
4572 Handle_CVar_LatLon_General_Product();
4573
4574}
4575
4576// Routine to handle coordinate variables for general HDF5 product
4577// that have either 1-D or 2-D lat/lon
4578void GMFile::Handle_CVar_LatLon_General_Product() {
4579
4580 BESDEBUG("h5", "Coming to Handle_CVar_LatLon_General_Product()"<<endl);
4581 if((GENERAL_LATLON2D != this->gproduct_pattern)
4582 && GENERAL_LATLON1D != this->gproduct_pattern)
4583 throw1("This function only supports latlon 1D or latlon 2D general products");
4584
4585 pair<set<string>::iterator,bool> setret;
4586 set<string>tempdimnamelist = dimnamelist;
4587
4588 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4589
4590 // This is the dimension scale dataset; it should be changed to a coordinate variable.
4591 if (gp_latname== (*irv)->name) {
4592
4593 // For latitude, regardless 1D or 2D, the first dimension needs to be updated.
4594 // Create Coordinate variables.
4595 tempdimnamelist.erase(((*irv)->dims[0])->name);
4596 auto GMcvar_unique = make_unique<GMCVar>(*irv);
4597 auto GMcvar = GMcvar_unique.release();
4598 GMcvar->cfdimname = ((*irv)->dims[0])->name;
4599 GMcvar->cvartype = CV_EXIST;
4600 GMcvar->product_type = product_type;
4601 this->cvars.push_back(GMcvar);
4602 delete(*irv);
4603 this->vars.erase(irv);
4604 break;
4605 } // end of if
4606 } // end of for
4607
4608 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4609
4610 // This is the dimension scale dataset; it should be changed to a coordinate variable.
4611 if (gp_lonname== (*irv)->name) {
4612
4613 // For 2-D lat/lon, the londimname should be the second dimension of the longitude
4614 // For 1-D lat/lon, the londimname should be the first dimension of the longitude
4615 // Create Coordinate variables.
4616 string londimname;
4617 if(GENERAL_LATLON2D == this->gproduct_pattern)
4618 londimname = ((*irv)->dims[1])->name;
4619 else
4620 londimname = ((*irv)->dims[0])->name;
4621
4622 tempdimnamelist.erase(londimname);
4623 auto GMcvar_unique = make_unique<GMCVar>(*irv);
4624 auto GMcvar = GMcvar_unique.release();
4625 GMcvar->cfdimname = londimname;
4626 GMcvar->cvartype = CV_EXIST;
4627 GMcvar->product_type = product_type;
4628 this->cvars.push_back(GMcvar);
4629 delete(*irv);
4630 this->vars.erase(irv);
4631 break;
4632 } // end of if
4633 } // end of for
4634
4635 //
4636 // Add other missing coordinate variables.
4637 for (const auto &dimname:tempdimnamelist) {
4638 auto GMcvar_unique = make_unique<GMCVar>();
4639 auto GMcvar = GMcvar_unique.release();
4640 Create_Missing_CV(GMcvar,dimname);
4641 this->cvars.push_back(GMcvar);
4642 }
4643
4644}
4645
4646// Handle coordinate variables for OBPG level 3
4647void GMFile::Handle_CVar_OBPG_L3() {
4648
4649 BESDEBUG("h5", "Coming to Handle_CVar_OBPG_L3()"<<endl);
4650 if (GENERAL_DIMSCALE == this->gproduct_pattern)
4651 Handle_CVar_Dimscale_General_Product();
4652
4653 // Change the CV Type of the corresponding CVs of lat and lon from CV_FILLINDEX to CV_LATMISS or CV_LONMISS
4654 for (const auto &var:this->vars) {
4655
4656 // Here I try to avoid using the dimension name row and column to find the lat/lon dimension size.
4657 // So I am looking for a 2-D floating-point array or a 2-D array under the group geophysical_data.
4658 // This may be subject to change if OBPG level 3 change its arrangement of variables.
4659 // KY 2014-09-29
4660
4661 if(var->rank == 2) {
4662
4663 if((var->fullpath.find("/geophysical_data") == 0) || (var->dtype == H5FLOAT32)) {
4664
4665 size_t lat_size = var->getDimensions()[0]->size;
4666 string lat_name = var->getDimensions()[0]->name;
4667 size_t lon_size = var->getDimensions()[1]->size;
4668 string lon_name = var->getDimensions()[1]->name;
4669 size_t temp_size = 0;
4670 string temp_name;
4671 H5DataType ll_dtype = var->dtype;
4672
4673 // We always assume that longitude size is greater than latitude size.
4674 if(lat_size >lon_size) {
4675 temp_size = lon_size;
4676 temp_name = lon_name;
4677 lon_size = lat_size;
4678 lon_name = lat_name;
4679 lat_size = temp_size;
4680 lat_name = temp_name;
4681 }
4682 for (auto &cvar:this->cvars) {
4683 if(cvar->cvartype == CV_FILLINDEX) {
4684 if(cvar->getDimensions()[0]->size == lat_size &&
4685 cvar->getDimensions()[0]->name == lat_name) {
4686 cvar->cvartype = CV_LAT_MISS;
4687 cvar->dtype = ll_dtype;
4688 for (auto ira = cvar->attrs.begin(); ira != cvar->attrs.end(); ++ira) {
4689 if ((*ira)->name == "NAME") {
4690 delete (*ira);
4691 cvar->attrs.erase(ira);
4692 break;
4693 }
4694 }
4695 }
4696 else if(cvar->getDimensions()[0]->size == lon_size &&
4697 cvar->getDimensions()[0]->name == lon_name) {
4698 cvar->cvartype = CV_LON_MISS;
4699 cvar->dtype = ll_dtype;
4700 for (auto ira = cvar->attrs.begin(); ira != cvar->attrs.end(); ++ira) {
4701 if ((*ira)->name == "NAME") {
4702 delete (*ira);
4703 cvar->attrs.erase(ira);
4704 break;
4705 }
4706 }
4707 }
4708 }
4709 }
4710 break;
4711
4712 } // end of if((var->fullpath.find("/geophysical_data") == 0) || (var->dtype == H5FLOAT32))
4713 } // end of if(var->rank == 2)
4714 } // end of for (auto irv = this->vars.begin();
4715
4716}
4717
4718// Handle some special variables. Currently only GPM and ACOS have these variables.
4720
4721 BESDEBUG("h5", "Coming to Handle_SpVar()"<<endl);
4722 if (ACOS_L2S_OR_OCO2_L1B == product_type)
4723 Handle_SpVar_ACOS_OCO2();
4724 else if(GPM_L1 == product_type) {
4725 // Loop through the variable list to build the coordinates.
4726 // These variables need to be removed.
4727 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4728 if((*irv)->name=="AlgorithmRuntimeInfo") {
4729 delete(*irv);
4730 this->vars.erase(irv);
4731 break;
4732 }
4733 }
4734 }
4735
4736 // GPM level-3 These variables need to be removed.
4737 else if(GPMM_L3 == product_type || GPMS_L3 == product_type || GPM_L3_New==product_type) {
4738
4739 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
4740 if((*irv)->name=="InputFileNames") {
4741 delete(*irv);
4742 irv = this->vars.erase(irv);
4743 }
4744 else if((*irv)->name=="InputAlgorithmVersions") {
4745 delete(*irv);
4746 irv = this->vars.erase(irv);
4747 }
4748 else if((*irv)->name=="InputGenerationDateTimes") {
4749 delete(*irv);
4750 irv = this->vars.erase(irv);
4751 }
4752 else {
4753 ++irv;
4754 }
4755 }
4756 }
4757
4758}
4759
4760// Handle special variables for ACOS.
4761void GMFile::Handle_SpVar_ACOS_OCO2() {
4762
4763 BESDEBUG("h5", "Coming to Handle_SpVar_ACOS_OCO2()"<<endl);
4764 //The ACOS or OCO2 have 64-bit variables. DAP2 doesn't support 64-bit variables.
4765 // So we will not handle attributes yet.
4766 for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
4767
4768 if (H5INT64 == (*irv)->getType()) {
4769
4770 // First: Time Part of soundingid
4771 auto spvar_unique = make_unique<GMSPVar>(*irv);
4772 auto spvar = spvar_unique.release();
4773 spvar->name = (*irv)->name +"_Time";
4774 spvar->newname = (*irv)->newname+"_Time";
4775 spvar->dtype = H5INT32;
4776 spvar->otype = (*irv)->getType();
4777 spvar->sdbit = 1;
4778
4779 // 2 digit hour, 2 digit min, 2 digit seconds
4780 spvar->numofdbits = 6;
4781 this->spvars.push_back(spvar);
4782
4783 // Second: Date Part of soundingid
4784 auto spvar2_unique = make_unique<GMSPVar>(*irv);
4785 auto spvar2 = spvar2_unique.release();
4786 spvar2->name = (*irv)->name +"_Date";
4787 spvar2->newname = (*irv)->newname+"_Date";
4788 spvar2->dtype = H5INT32;
4789 spvar2->otype = (*irv)->getType();
4790 spvar2->sdbit = 7;
4791
4792 // 4 digit year, 2 digit month, 2 digit day
4793 spvar2->numofdbits = 8;
4794 this->spvars.push_back(spvar2);
4795
4796 delete(*irv);
4797 irv = this->vars.erase(irv);
4798 } // end of if (H5INT64 == (*irv)->getType())
4799 else {
4800 ++irv;
4801 }
4802 } // end of for (auto irv = this->vars.begin(); ...
4803}
4804
4805// Adjust Object names, For some products, NASA data centers don't need
4806// the fullpath of objects.
4808
4809 BESDEBUG("h5", "Coming to Adjust_Obj_Name()"<<endl);
4810 if(Mea_Ozone == product_type)
4811 Adjust_Mea_Ozone_Obj_Name();
4812
4813 if(GPMS_L3 == product_type || GPMM_L3 == product_type)
4814 Adjust_GPM_L3_Obj_Name();
4815
4816// Just for debugging
4817#if 0
4818for (auto irv2 = this->vars.begin();
4819 irv2 != this->vars.end(); irv2++) {
4820 for (auto ird = (*irv2)->dims.begin();
4821 ird !=(*irv2)->dims.end(); ird++) {
4822 cerr<<"Dimension name afet Adjust_Obj_Name "<<(*ird)->newname <<endl;
4823 }
4824}
4825#endif
4826
4827}
4828
4829// Adjust object names for GPM level 3 products
4830void GMFile:: Adjust_GPM_L3_Obj_Name() const {
4831
4832 BESDEBUG("h5", "Coming to Adjust_GPM_L3_Obj_Name()"<<endl);
4833 string objnewname;
4834 // In this definition, root group is not considered as a group.
4835 if(this->groups.size() <= 1) {
4836 for (auto &var:this->vars) {
4837 objnewname = HDF5CFUtil::obtain_string_after_lastslash(var->newname);
4838 if (objnewname !="")
4839 var->newname = objnewname;
4840 }
4841 }
4842 else {
4843 for (auto &var:this->vars) {
4844 size_t grid_group_path_pos = (var->newname.substr(1)).find_first_of("/");
4845 objnewname = (var->newname).substr(grid_group_path_pos+2);
4846 var->newname = objnewname;
4847 }
4848 }
4849}
4850
4851// Adjust object names for MeaSUREs OZone
4852void GMFile:: Adjust_Mea_Ozone_Obj_Name() const {
4853
4854 BESDEBUG("h5", "Coming to Adjust_Mea_Ozone_Obj_Name()"<<endl);
4855 string objnewname;
4856 for (auto &var:this->vars) {
4857 objnewname = HDF5CFUtil::obtain_string_after_lastslash(var->newname);
4858 if (objnewname !="")
4859 var->newname = objnewname;
4860
4861#if 0
4862//Just for debugging
4863for (auto ird = (*irv)->dims.begin();
4864 ird !=(*irv)->dims.end();++ird) {
4865 cerr<<"Ozone dim. name "<<(*ird)->name <<endl;
4866 cerr<<"Ozone dim. new name "<<(*ird)->newname <<endl;
4867}
4868#endif
4869
4870 }
4871
4872 for (auto &cvar:this->cvars) {
4873 objnewname = HDF5CFUtil::obtain_string_after_lastslash(cvar->newname);
4874 if (objnewname !="")
4875 cvar->newname = objnewname;
4876#if 0
4877 //Just for debugging
4878for (auto ird = (*irv)->dims.begin();
4879 ird !=(*irv)->dims.end();++ird) {
4880 cerr<<"Ozone CV dim. name "<<(*ird)->name <<endl;
4881 cerr<<"Ozone CV dim. new name "<<(*ird)->newname <<endl;
4882}
4883#endif
4884 }
4885}
4886
4887// Flatten object names.
4888void GMFile::Flatten_Obj_Name(bool include_attr) {
4889
4890 BESDEBUG("h5", "GMFile::Coming to Flatten_Obj_Name()"<<endl);
4891 // General variables
4892 File::Flatten_Obj_Name(include_attr);
4893
4894 // Coordinate variables
4895 for (auto &cvar:this->cvars) {
4896 cvar->newname = get_CF_string(cvar->newname);
4897
4898 for (auto &dim:cvar->dims)
4899 dim->newname = get_CF_string(dim->newname);
4900
4901 if (true == include_attr) {
4902 for (auto &attr: cvar->attrs)
4903 attr->newname = File::get_CF_string(attr->newname);
4904 }
4905
4906 }
4907
4908 // Special variables
4909 for (auto &spvar:this->spvars) {
4910
4911 spvar->newname = get_CF_string(spvar->newname);
4912 for (auto &dim:spvar->dims)
4913 dim->newname = get_CF_string(dim->newname);
4914
4915 if (true == include_attr) {
4916 for (auto &attr:spvar->attrs)
4917 attr->newname = File::get_CF_string(attr->newname);
4918 }
4919 }
4920
4921// Just for debugging
4922#if 0
4923for (auto irv2 = this->vars.begin();
4924 irv2 != this->vars.end(); irv2++) {
4925 for (auto ird = (*irv2)->dims.begin();
4926 ird !=(*irv2)->dims.end(); ird++) {
4927 cerr<<"Dimension name afet Flatten_Obj_Name "<<(*ird)->newname <<endl;
4928 }
4929}
4930#endif
4931
4932
4933}
4934
4935// Rarely object name clashing may occur. This routine makes sure
4936// all object names are unique.
4937void GMFile::Handle_Obj_NameClashing(bool include_attr) {
4938
4939 BESDEBUG("h5", "GMFile::Coming to Handle_Obj_NameClashing()"<<endl);
4940 // objnameset will be filled with all object names that we are going to check the name clashing.
4941 // For example, we want to see if there are any name clashing for all variable names in this file.
4942 // objnameset will include all variable names. If a name clashing occurs, we can figure out from the set operation immediately.
4943 set<string>objnameset;
4944 Handle_GMCVar_NameClashing(objnameset);
4945 Handle_GMSPVar_NameClashing(objnameset);
4946 File::Handle_GeneralObj_NameClashing(include_attr,objnameset);
4947 if (true == include_attr) {
4948 Handle_GMCVar_AttrNameClashing();
4949 Handle_GMSPVar_AttrNameClashing();
4950 }
4951 // Moving to h5gmcfdap.cc, right after Adjust_Dim_Name
4952 //Handle_DimNameClashing();
4953}
4954
4955// Name clashing for coordinate variables
4956void GMFile::Handle_GMCVar_NameClashing(set<string> &objnameset ) {
4957
4958 GMHandle_General_NameClashing(objnameset,this->cvars);
4959}
4960
4961// Name clashing for special variables(like 64-bit integer variables)
4962void GMFile::Handle_GMSPVar_NameClashing(set<string> &objnameset ) {
4963
4964 GMHandle_General_NameClashing(objnameset,this->spvars);
4965}
4966
4967// This routine handles attribute name clashing for coordinate variables.
4968void GMFile::Handle_GMCVar_AttrNameClashing() {
4969
4970 BESDEBUG("h5", "Coming to Handle_GMCVar_AttrNameClashing()"<<endl);
4971 set<string> objnameset;
4972
4973 for (auto &cvar:this->cvars) {
4974 Handle_General_NameClashing(objnameset,cvar->attrs);
4975 objnameset.clear();
4976 }
4977}
4978
4979// Attribute name clashing for special variables
4980void GMFile::Handle_GMSPVar_AttrNameClashing() {
4981
4982 BESDEBUG("h5", "Coming to Handle_GMSPVar_AttrNameClashing()"<<endl);
4983 set<string> objnameset;
4984
4985 for (auto &spvar:this->spvars) {
4986 Handle_General_NameClashing(objnameset,spvar->attrs);
4987 objnameset.clear();
4988 }
4989}
4990
4991//class T must have member string newname
4992// The subroutine to handle name clashing,
4993// it builds up a map from original object names to clashing-free object names.
4994template<class T> void
4995GMFile::GMHandle_General_NameClashing(set <string>&objnameset, vector<T*>& objvec) {
4996
4997 BESDEBUG("h5", "Coming to GMHandle_General_NameClashing()"<<endl);
4998 pair<set<string>::iterator,bool> setret;
4999 set<string>::iterator iss;
5000
5001 vector<string> clashnamelist;
5002
5003 map<int,int> cl_to_ol;
5004 int ol_index = 0;
5005 int cl_index = 0;
5006
5007 typename vector<T*>::iterator irv;
5008
5009 for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
5010
5011 setret = objnameset.insert((*irv)->newname);
5012 if (false == setret.second ) {
5013 clashnamelist.insert(clashnamelist.end(),(*irv)->newname);
5014 cl_to_ol[cl_index] = ol_index;
5015 cl_index++;
5016 }
5017 ol_index++;
5018 }
5019
5020
5021 // Now change the clashed elements to unique elements;
5022 // Generate the set which has the same size as the original vector.
5023 for (auto &clashname:clashnamelist) {
5024 int clash_index = 1;
5025 string temp_clashname = clashname +'_';
5026 HDF5CFUtil::gen_unique_name(temp_clashname,objnameset,clash_index);
5027 clashname = temp_clashname;
5028 }
5029
5030
5031 // Now go back to the original vector, make it unique.
5032 for (unsigned int i =0; i <clashnamelist.size(); i++)
5033 objvec[cl_to_ol[i]]->newname = clashnamelist[i];
5034
5035}
5036
5037// Handle dimension name clashing
5039
5040
5041 BESDEBUG("h5", "GMFile: Coming to Handle_DimNameClashing()"<<endl);
5042 // ACOS L2S or OCO2 L1B products doesn't need the dimension name clashing check based on our current understanding. KY 2012-5-16
5043 if (ACOS_L2S_OR_OCO2_L1B == product_type)
5044 return;
5045
5046 map<string,string>dimname_to_dimnewname;
5047 pair<map<string,string>::iterator,bool>mapret;
5048 set<string> dimnameset;
5049 vector<Dimension*>vdims;
5050 set<string> dimnewnameset;
5051 pair<set<string>::iterator,bool> setret;
5052
5053 // First: Generate the dimset/dimvar based on coordinate variables.
5054 for (const auto &cvar:this->cvars) {
5055 for (const auto &dim:cvar->dims) {
5056 setret = dimnameset.insert(dim->name);
5057 if (true == setret.second)
5058 vdims.push_back(dim);
5059 }
5060 }
5061
5062 // For some cases, dimension names are provided but there are no corresponding coordinate
5063 // variables. For now, we will assume no such a case.
5064 // Actually, we find such a case in our fake testsuite. So we need to fix it.
5065 for (const auto &var:this->vars) {
5066 for (const auto &dim:var->dims) {
5067#if 0
5068 //setret = dimnameset.insert((*ird)->newname);
5069#endif
5070 setret = dimnameset.insert(dim->name);
5071 if (setret.second) vdims.push_back(dim);
5072 }
5073 }
5074
5075 GMHandle_General_NameClashing(dimnewnameset,vdims);
5076
5077 // Third: Make dimname_to_dimnewname map
5078 for (const auto& dim:vdims) {
5079 mapret = dimname_to_dimnewname.insert(pair<string,string>(dim->name,dim->newname));
5080 if (false == mapret.second)
5081 throw4("The dimension name ",dim->name," should map to ",
5082 dim->newname);
5083 }
5084
5085 // Fourth: Change the original dimension new names to the unique dimension new names
5086 for (auto &cvar:this->cvars)
5087 for (auto &dim:cvar->dims)
5088 dim->newname = dimname_to_dimnewname[dim->name];
5089
5090 for (auto &var:this->vars)
5091 for (auto &dim:var->dims)
5092 dim->newname = dimname_to_dimnewname[dim->name];
5093
5094}
5095
5096// For COARDS, dim. names need to be the same as obj. names.
5098
5099 BESDEBUG("h5", "GMFile:Coming to Adjust_Dim_Name()"<<endl);
5100#if 0
5101 // Just for debugging
5102for (auto irv2 = this->vars.begin();
5103 irv2 != this->vars.end(); irv2++) {
5104 for (auto ird = (*irv2)->dims.begin();
5105 ird !=(*irv2)->dims.end(); ird++) {
5106 cerr<<"Dimension new name "<<(*ird)->newname <<endl;
5107 }
5108}
5109#endif
5110
5111 // Only need for COARD conventions.
5112 if( true == iscoard) {
5113 for (const auto &cvar:this->cvars) {
5114#if 0
5115cerr<<"1D Cvariable name is "<<cvar->name <<endl;
5116cerr<<"1D Cvariable new name is "<<cvar->newname <<endl;
5117cerr<<"1D Cvariable dim name is "<<(cvar->dims)[0]->name <<endl;
5118cerr<<"1D Cvariable dim new name is "<<(cvar->dims)[0]->newname <<endl;
5119#endif
5120 if (cvar->dims.size()!=1)
5121 throw3("Coard coordinate variable ",cvar->name, "is not 1D");
5122 if (cvar->newname != ((cvar->dims)[0]->newname)) {
5123 (cvar->dims)[0]->newname = cvar->newname;
5124
5125 // For all variables that have this dimension,the dimension newname should also change.
5126 for (auto &var:this->vars) {
5127 for (auto &dim:var->dims) {
5128 // This is the key, the dimension name of this dimension
5129 // should be equal to the dimension name of the coordinate variable.
5130 // Then the dimension name matches and the dimension name should be changed to
5131 // the new dimension name.
5132 if (dim->name == (cvar->dims)[0]->name)
5133 dim->newname = (cvar->dims)[0]->newname;
5134 }
5135 }
5136 } // end of if (cvar->newname != ((cvar->dims)[0]->newname))
5137 }// end of for (auto irv = this->cvars.begin(); ...
5138 } // end of if( true == iscoard)
5139
5140// Just for debugging
5141#if 0
5142for (auto irv2 = this->vars.begin();
5143 irv2 != this->vars.end(); irv2++) {
5144 for (auto ird = (*irv2)->dims.begin();
5145 ird !=(*irv2)->dims.end(); ird++) {
5146 cerr<<"Dimension name afet Adjust_Dim_Name "<<(*ird)->newname <<endl;
5147 }
5148}
5149#endif
5150
5151
5152}
5153
5154// Add supplemental CF attributes for some products.
5155void
5157
5158 BESDEBUG("h5", "GMFile::Coming to Add_Supplement_Attrs()"<<endl);
5159 if (General_Product == product_type || true == add_path) {
5160 File::Add_Supplement_Attrs(add_path);
5161
5162 if (true == add_path) {
5163 // Adding variable original name(origname) and full path(fullpath)
5164 for (auto &cvar:this->cvars) {
5165 if ((cvar->cvartype == CV_EXIST) || (cvar->cvartype == CV_MODIFY)) {
5166 const string varname = cvar->name;
5167 const string attrname = "origname";
5168 auto attr_unique = make_unique<Attribute>();
5169 auto attr = attr_unique.release();
5170 Add_Str_Attr(attr,attrname,varname);
5171 cvar->attrs.push_back(attr);
5172 }
5173 }
5174
5175 for (auto &cvar:this->cvars) {
5176 // Turn off the fullnamepath attribute when zero_storage_size is 0.
5177 // Use the BES key since quite a few testing cases will be affected.
5178 // KY 2020-03-23
5179 if(cvar->zero_storage_size == false
5180 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
5181 if ((cvar->cvartype == CV_EXIST) || (cvar->cvartype == CV_MODIFY)) {
5182 const string varname = cvar->fullpath;
5183 const string attrname = "fullnamepath";
5184 auto attr_unique = make_unique<Attribute>();
5185 auto attr = attr_unique.release();
5186 Add_Str_Attr(attr,attrname,varname);
5187 cvar->attrs.push_back(attr);
5188 }
5189 }
5190 }
5191
5192 for (auto &spvar:this->spvars) {
5193 const string varname = spvar->name;
5194 const string attrname = "origname";
5195 auto attr_unique = make_unique<Attribute>();
5196 auto attr = attr_unique.release();
5197 Add_Str_Attr(attr,attrname,varname);
5198 spvar->attrs.push_back(attr);
5199 }
5200
5201 for (auto &spvar:this->spvars) {
5202 // Turn off the fullnamepath attribute when zero_storage_size is 0.
5203 // Use the BES key since quite a few testing cases will be affected.
5204 // KY 2020-03-23
5205 if(spvar->zero_storage_size == false
5206 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
5207 const string varname = spvar->fullpath;
5208 const string attrname = "fullnamepath";
5209 auto attr_unique = make_unique<Attribute>();
5210 auto attr = attr_unique.release();
5211 Add_Str_Attr(attr,attrname,varname);
5212 spvar->attrs.push_back(attr);
5213 }
5214 }
5215 }
5216 } // if (General_Product == product_type || true == add_path)
5217
5218 if(GPM_L1 == product_type || GPMS_L3 == product_type || GPMM_L3 == product_type)
5219 Add_GPM_Attrs();
5220 else if (Aqu_L3 == product_type)
5221 Add_Aqu_Attrs();
5222 else if (Mea_SeaWiFS_L2 == product_type || Mea_SeaWiFS_L3 == product_type)
5223 Add_SeaWiFS_Attrs();
5224
5225}
5226
5227// Add CF attributes for GPM products
5228void
5229GMFile:: Add_GPM_Attrs() {
5230
5231 BESDEBUG("h5", "Coming to Add_GPM_Attrs()"<<endl);
5232 const string attr_name_be_replaced = "CodeMissingValue";
5233 const string attr_new_name = "_FillValue";
5234 const string attr2_name_be_replaced = "Units";
5235 const string attr2_new_name ="units";
5236
5237 // Need to convert String type CodeMissingValue to the corresponding _FilLValue
5238 // Create a function at HDF5CF.cc. use strtod,strtof,strtol etc. function to convert
5239 // string to the corresponding type.
5240 for (const auto &var:this->vars) {
5241 bool has_fvalue_attr = false;
5242 for (const auto &attr:var->attrs) {
5243 if(attr_new_name == attr->name) {
5244 has_fvalue_attr = true;
5245 break;
5246 }
5247 }
5248
5249 if(false == has_fvalue_attr) {
5250 for(auto &attr:var->attrs) {
5251 if(attr_name_be_replaced == attr->name) {
5252 if(attr->dtype == H5FSTRING)
5253 Change_Attr_One_Str_to_Others(attr,var);
5254 attr->name = attr_new_name;
5255 attr->newname = attr_new_name;
5256 }
5257 }
5258 }
5259
5260 }
5261
5262
5263 for (auto &cvar: this->cvars) {
5264
5265 bool has_fvalue_attr = false;
5266
5267 for(const auto &attr:cvar->attrs) {
5268
5269 if(attr_new_name == attr->name) {
5270 has_fvalue_attr = true;
5271 break;
5272 }
5273 }
5274
5275 if(false == has_fvalue_attr) {
5276
5277 for(auto &attr:cvar->attrs) {
5278
5279 if(attr_name_be_replaced == attr->name) {
5280 if(attr->dtype == H5FSTRING)
5281 Change_Attr_One_Str_to_Others(attr,cvar);
5282 attr->name = attr_new_name;
5283 attr->newname = attr_new_name;
5284 break;
5285 }
5286 }
5287 }
5288
5289
5290 if(product_type == GPM_L1) {
5291
5292 if (cvar->cvartype == CV_EXIST) {
5293 if(cvar->name.find("Latitude") !=string::npos) {
5294 string unit_value = "degrees_north";
5295 Correct_GPM_L1_LatLon_units(cvar,unit_value);
5296
5297 }
5298 else if(cvar->name.find("Longitude") !=string::npos) {
5299 string unit_value = "degrees_east";
5300 Correct_GPM_L1_LatLon_units(cvar,unit_value);
5301 }
5302 }
5303
5304
5305 else if (cvar->cvartype == CV_NONLATLON_MISS) {
5306
5307 string comment;
5308 const string attrname = "comment";
5309
5310 {
5311 if(cvar->name == "nchannel1")
5312 comment = "Number of Swath S1 channels (10V 10H 19V 19H 23V 37V 37H 89V 89H).";
5313 else if(cvar->name == "nchannel2")
5314 comment = "Number of Swath S2 channels (166V 166H 183+/-3V 183+/-8V).";
5315 else if(cvar->name == "nchan1")
5316 comment = "Number of channels in Swath 1.";
5317 else if(cvar->name == "nchan2")
5318 comment = "Number of channels in Swath 2.";
5319 else if(cvar->name == "VH")
5320 comment = "Number of polarizations.";
5321 else if(cvar->name == "GMIxyz")
5322 comment = "x, y, z components in GMI instrument coordinate system.";
5323 else if(cvar->name == "LNL")
5324 comment = "Linear and non-linear.";
5325 else if(cvar->name == "nscan")
5326 comment = "Number of scans in the granule.";
5327 else if(cvar->name == "nscan1")
5328 comment = "Typical number of Swath S1 scans in the granule.";
5329 else if(cvar->name == "nscan2")
5330 comment = "Typical number of Swath S2 scans in the granule.";
5331 else if(cvar->name == "npixelev")
5332 comment = "Number of earth view pixels in one scan.";
5333 else if(cvar->name == "npixelht")
5334 comment = "Number of hot load pixels in one scan.";
5335 else if(cvar->name == "npixelcs")
5336 comment = "Number of cold sky pixels in one scan.";
5337 else if(cvar->name == "npixelfr")
5338 comment = "Number of full rotation earth view pixels in one scan.";
5339 else if(cvar->name == "nfreq1")
5340 comment = "Number of frequencies in Swath 1.";
5341 else if(cvar->name == "nfreq2")
5342 comment = "Number of frequencies in Swath 2.";
5343 else if(cvar->name == "npix1")
5344 comment = "Number of pixels in Swath 1.";
5345 else if(cvar->name == "npix2")
5346 comment = "Number of pixels in Swath 2.";
5347 else if(cvar->name == "npix3")
5348 comment = "Number of pixels in Swath 3.";
5349 else if(cvar->name == "npix4")
5350 comment = "Number of pixels in Swath 4.";
5351 else if(cvar->name == "ncolds1")
5352 comment = "Maximum number of cold samples in Swath 1.";
5353 else if(cvar->name == "ncolds2")
5354 comment = "Maximum number of cold samples in Swath 2.";
5355 else if(cvar->name == "nhots1")
5356 comment = "Maximum number of hot samples in Swath 1.";
5357 else if(cvar->name == "nhots2")
5358 comment = "Maximum number of hot samples in Swath 2.";
5359 else if(cvar->name == "ntherm")
5360 comment = "Number of hot load thermisters.";
5361 else if(cvar->name == "ntach")
5362 comment = "Number of tachometer readings.";
5363 else if(cvar->name == "nsamt"){
5364 comment = "Number of sample types. ";
5365 comment = +"The types are: total science GSDR, earthview,hot load, cold sky.";
5366 }
5367 else if(cvar->name == "nndiode")
5368 comment = "Number of noise diodes.";
5369 else if(cvar->name == "n7")
5370 comment = "Number seven.";
5371 else if(cvar->name == "nray")
5372 comment = "Number of angle bins in each NS scan.";
5373 else if(cvar->name == "nrayMS")
5374 comment = "Number of angle bins in each MS scan.";
5375 else if(cvar->name == "nrayHS")
5376 comment = "Number of angle bins in each HS scan.";
5377 else if(cvar->name == "nbin")
5378 comment = "Number of range bins in each NS and MS ray. Bin interval is 125m.";
5379 else if(cvar->name == "nbinHS")
5380 comment = "Number of range bins in each HS ray. Bin interval is 250m.";
5381 else if(cvar->name == "nbinSZP")
5382 comment = "Number of range bins for sigmaZeroProfile.";
5383 else if(cvar->name == "nbinSZPHS")
5384 comment = "Number of range bins for sigmaZeroProfile in each HS scan.";
5385 else if(cvar->name == "nNP")
5386 comment = "Number of NP kinds.";
5387 else if(cvar->name == "nearFar")
5388 comment = "Near reference, Far reference.";
5389 else if(cvar->name == "foreBack")
5390 comment = "Forward, Backward.";
5391 else if(cvar->name == "method")
5392 comment = "Number of SRT methods.";
5393 else if(cvar->name == "nNode")
5394 comment = "Number of binNode.";
5395 else if(cvar->name == "nDSD")
5396 comment = "Number of DSD parameters. Parameters are N0 and D0";
5397 else if(cvar->name == "LS")
5398 comment = "Liquid, solid.";
5399 }
5400
5401 if (!comment.empty()) {
5402 auto attr_unique = make_unique<Attribute>();
5403 auto attr = attr_unique.release();
5404 Add_Str_Attr(attr,attrname,comment);
5405 cvar->attrs.push_back(attr);
5406 }
5407 }
5408 }
5409
5410 if(product_type == GPMS_L3 || product_type == GPMM_L3) {
5411 if (cvar->cvartype == CV_NONLATLON_MISS) {
5412
5413 string comment;
5414 const string attrname = "comment";
5415 {
5416 if(cvar->name == "chn")
5417 comment = "Number of channels:Ku,Ka,KaHS,DPR.";
5418 else if(cvar->name == "inst")
5419 comment = "Number of instruments:Ku,Ka,KaHS.";
5420 else if(cvar->name == "tim")
5421 comment = "Number of hours(local time).";
5422 else if(cvar->name == "ang"){
5423 comment = "Number of angles.The meaning of ang is different for each channel.";
5424 comment +=
5425 "For Ku channel all indices are used with the meaning 0,1,2,..6 =angle bins 24,";
5426 comment +=
5427 "(20,28),(16,32),(12,36),(8,40),(3,44),and (0,48).";
5428 comment +=
5429 "For Ka channel 4 indices are used with the meaning 0,1,2,3 = angle bins 12,(8,16),";
5430 comment +=
5431 "(4,20),and (0,24). For KaHS channel 4 indices are used with the meaning 0,1,2,3 =";
5432 comment += "angle bins(11,2),(7,16),(3,20),and (0.23).";
5433
5434 }
5435 else if(cvar->name == "rt")
5436 comment = "Number of rain types: stratiform, convective,all.";
5437 else if(cvar->name == "st")
5438 comment = "Number of surface types:ocean,land,all.";
5439 else if(cvar->name == "bin"){
5440 comment = "Number of bins in histogram. The thresholds are different for different";
5441 comment +=" variables. see the file specification for this algorithm.";
5442 }
5443 else if(cvar->name == "nvar") {
5444 comment = "Number of phase bins. Bins are counts of phase less than 100, ";
5445 comment +="counts of phase greater than or equal to 100 and less than 200, ";
5446 comment +="counts of phase greater than or equal to 200.";
5447 }
5448 else if(cvar->name == "AD")
5449 comment = "Ascending or descending half of the orbit.";
5450 }
5451
5452 if (comment.empty() == false) {
5453 auto attr_unique = make_unique<Attribute>();
5454 auto attr = attr_unique.release();
5455 Add_Str_Attr(attr,attrname,comment);
5456 cvar->attrs.push_back(attr);
5457 }
5458 }
5459 }
5460
5461 if (cvar->cvartype == CV_SPECIAL) {
5462 if(cvar->name == "nlayer" || cvar->name == "hgt"
5463 || cvar->name == "nalt") {
5464 string unit_value = "km";
5465 auto attr_unique = make_unique<Attribute>();
5466 auto attr = attr_unique.release();
5467 Add_Str_Attr(attr,attr2_new_name,unit_value);
5468 cvar->attrs.push_back(attr);
5469
5470 string attr1_axis="axis";
5471 string attr1_value = "Z";
5472 auto attr1_unique = make_unique<Attribute>();
5473 auto attr1 = attr1_unique.release();
5474 Add_Str_Attr(attr1,attr1_axis,attr1_value);
5475 cvar->attrs.push_back(attr1);
5476
5477 string attr2_positive="positive";
5478 string attr2_value = "up";
5479 auto attr2_unique = make_unique<Attribute>();
5480 auto attr2 = attr2_unique.release();
5481 Add_Str_Attr(attr2,attr2_positive,attr2_value);
5482 cvar->attrs.push_back(attr2);
5483
5484 }
5485 if(cvar->name == "hgt" || cvar->name == "nalt"){
5486 string comment ="Number of heights above the earth ellipsoid";
5487 auto attr1_unique = make_unique<Attribute>();
5488 auto attr1 = attr1_unique.release();
5489 Add_Str_Attr(attr1,"comment",comment);
5490 cvar->attrs.push_back(attr1);
5491 }
5492
5493 }
5494
5495 }
5496
5497
5498// Old code, leave it for the time being
5499#if 0
5500 const string fill_value_attr_name = "_FillValue";
5501 vector<HDF5CF::Var *>::const_iterator it_v;
5502 vector<HDF5CF::Attribute *>::const_iterator ira;
5503
5504 for (it_v = vars.begin();
5505 it_v != vars.end(); ++it_v) {
5506
5507 bool has_fillvalue = false;
5508 for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5509 if (fill_value_attr_name == (*ira)->name){
5510 has_fillvalue = true;
5511 break;
5512 }
5513
5514 }
5515
5516 // Add the fill value
5517 if (has_fillvalue != true ) {
5518
5519 if(H5FLOAT32 == (*it_v)->dtype) {
5520 Attribute* attr = new Attribute();
5521 float _FV = -9999.9;
5522 Add_One_Float_Attr(attr,fill_value_attr_name,_FV);
5523 (*it_v)->attrs.push_back(attr);
5524 }
5525 }
5526 }// for (it_v = vars.begin(); ...
5527#endif
5528
5529}
5530
5531// For GPM level 1 data, var must have names that contains either "Latitude" nor "Longitude".
5532void
5533GMFile:: Correct_GPM_L1_LatLon_units(Var *var, const string & unit_value) {
5534
5535 BESDEBUG("h5", "Coming to Correct_GPM_L1_LatLon_units()"<<endl);
5536 const string Unit_name = "Units";
5537 const string unit_name = "units";
5538
5539 // Delete "units" and "Units"
5540 for(auto ira = var->attrs.begin(); ira!= var->attrs.end();) {
5541 if(unit_name == (*ira)->name) {
5542 delete(*ira);
5543 ira = var->attrs.erase(ira);
5544 }
5545 else if(Unit_name == (*ira)->name) {
5546 delete(*ira);
5547 ira = var->attrs.erase(ira);
5548 }
5549 else
5550 ++ira;
5551 }
5552 // Add the correct units for Latitude and Longitude
5553 // Note: the reason we do this way, for some versions of GPM, units is degrees,
5554 // rather than degrees_north. So units also needs to be corrected to follow CF.
5555 auto attr_unique = make_unique<Attribute>();
5556 auto attr = attr_unique.release();
5557 Add_Str_Attr(attr,unit_name,unit_value);
5558 var->attrs.push_back(attr);
5559}
5560
5561
5562
5563// Add attributes for Aquarius products
5564void
5565GMFile:: Add_Aqu_Attrs() {
5566
5567 BESDEBUG("h5", "Coming to Add_Aqu_Attrs()"<<endl);
5568 vector<HDF5CF::Var *>::const_iterator it_v;
5569 vector<HDF5CF::Attribute *>::const_iterator ira;
5570
5571 const string orig_longname_attr_name = "Parameter";
5572 const string longname_attr_name ="long_name";
5573 bool has_orig_longname = false;
5574 string longname_value;
5575
5576
5577 const string orig_units_attr_name = "Units";
5578 const string units_attr_name = "units";
5579 bool has_orig_units = false;
5580 string units_value;
5581
5582 const string orig_valid_min_attr_name = "Data Minimum";
5583 const string orig_valid_min_attr_name2 = "data_minimum";
5584 const string valid_min_attr_name = "valid_min";
5585 bool has_orig_valid_min = false;
5586 float valid_min_value = 0;
5587
5588 const string orig_valid_max_attr_name = "Data Maximum";
5589 const string orig_valid_max_attr_name2 = "data_maximum";
5590 const string valid_max_attr_name = "valid_max";
5591 bool has_orig_valid_max = false;
5592 float valid_max_value = 0;
5593
5594 // The fill value is -32767.0. However, No _FillValue attribute is added.
5595 // So add it here. KY 2012-2-16
5596
5597 const string fill_value_attr_name = "_FillValue";
5598 float _FV = -32767.0;
5599
5600 for (ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
5601 if (orig_longname_attr_name == (*ira)->name) {
5602 Retrieve_H5_Attr_Value(*ira,"/");
5603 longname_value.resize((*ira)->value.size());
5604 copy((*ira)->value.begin(),(*ira)->value.end(),longname_value.begin());
5605 has_orig_longname = true;
5606 }
5607 else if (orig_units_attr_name == (*ira)->name) {
5608 Retrieve_H5_Attr_Value(*ira,"/");
5609 units_value.resize((*ira)->value.size());
5610 copy((*ira)->value.begin(),(*ira)->value.end(),units_value.begin());
5611 has_orig_units = true;
5612 }
5613 else if (orig_valid_min_attr_name == (*ira)->name || orig_valid_min_attr_name2 == (*ira)->name) {
5614 Retrieve_H5_Attr_Value(*ira,"/");
5615 memcpy(&valid_min_value,(void*)(&((*ira)->value[0])),(*ira)->value.size());
5616 has_orig_valid_min = true;
5617 }
5618
5619 else if (orig_valid_max_attr_name == (*ira)->name || orig_valid_max_attr_name2 == (*ira)->name) {
5620 Retrieve_H5_Attr_Value(*ira,"/");
5621 memcpy(&valid_max_value,(void*)(&((*ira)->value[0])),(*ira)->value.size());
5622 has_orig_valid_max = true;
5623 }
5624
5625 }// end of for (ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
5626
5627 // New version Aqu(Q20112132011243.L3m_MO_SCI_V3.0_SSS_1deg.bz2) files seem to have CF attributes added.
5628 // In this case, we should not add extra CF attributes, or duplicate values may appear. KY 2015-06-20
5629 bool has_long_name = false;
5630 bool has_units = false;
5631 bool has_valid_min = false;
5632 bool has_valid_max = false;
5633 bool has_fillvalue = false;
5634
5635 for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5636 if ("l3m_data" == (*it_v)->name) {
5637 for (ira = (*it_v)->attrs.begin(); ira != (*it_v)->attrs.end(); ++ira) {
5638 if (longname_attr_name == (*ira)->name)
5639 has_long_name = true;
5640 else if(units_attr_name == (*ira)->name)
5641 has_units = true;
5642 else if(valid_min_attr_name == (*ira)->name)
5643 has_valid_min = true;
5644 else if(valid_max_attr_name == (*ira)->name)
5645 has_valid_max = true;
5646 else if(fill_value_attr_name == (*ira)->name)
5647 has_fillvalue = true;
5648 }
5649 break;
5650 }
5651 } // end of for (it_v = vars.begin(); ...
5652
5653
5654 // Level 3 variable name is l3m_data
5655 for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5656 if ("l3m_data" == (*it_v)->name) {
5657
5658 // 1. Add the long_name attribute if no
5659 if(false == has_long_name && true == has_orig_longname) {
5660 auto attr_unique = make_unique<Attribute>();
5661 auto attr = attr_unique.release();
5662 Add_Str_Attr(attr,longname_attr_name,longname_value);
5663 (*it_v)->attrs.push_back(attr);
5664 }
5665
5666 // 2. Add the units attribute
5667 if(false == has_units && true == has_orig_units) {
5668 auto attr_unique = make_unique<Attribute>();
5669 auto attr = attr_unique.release();
5670 Add_Str_Attr(attr,units_attr_name,units_value);
5671 (*it_v)->attrs.push_back(attr);
5672 }
5673
5674 // 3. Add the valid_min attribute
5675 if (false == has_valid_min && has_orig_valid_min == true) {
5676 auto attr_unique = make_unique<Attribute>();
5677 auto attr = attr_unique.release();
5678 Add_One_Float_Attr(attr,valid_min_attr_name,valid_min_value);
5679 (*it_v)->attrs.push_back(attr);
5680 }
5681
5682 // 4. Add the valid_max attribute
5683 if(false == has_valid_max && has_orig_valid_max == true) {
5684 auto attr_unique = make_unique<Attribute>();
5685 auto attr = attr_unique.release();
5686 Add_One_Float_Attr(attr,valid_max_attr_name,valid_max_value);
5687 (*it_v)->attrs.push_back(attr);
5688 }
5689
5690 // 5. Add the _FillValue attribute
5691 if(false == has_fillvalue) {
5692 auto attr_unique = make_unique<Attribute>();
5693 auto attr = attr_unique.release();
5694 Add_One_Float_Attr(attr,fill_value_attr_name,_FV);
5695 (*it_v)->attrs.push_back(attr);
5696 }
5697
5698 break;
5699 }
5700 } // for (it_v = vars.begin(); ...
5701}
5702
5703// Add SeaWiFS attributes
5704void
5705GMFile:: Add_SeaWiFS_Attrs() const {
5706
5707 BESDEBUG("h5", "Coming to Add_SeaWiFS_Attrs()"<<endl);
5708 // The fill value is -999.0. However, No _FillValue attribute is added.
5709 // So add it here. KY 2012-2-16
5710 const string fill_value_attr_name = "_FillValue";
5711 float _FV = -999.0;
5712 const string valid_range_attr_name = "valid_range";
5713
5714 for (auto &var:this->vars) {
5715 if (H5FLOAT32 == var->dtype) {
5716 bool has_fillvalue = false;
5717 bool has_validrange = false;
5718 for(const auto &attr:var->attrs) {
5719 if (fill_value_attr_name == attr->name){
5720 has_fillvalue = true;
5721 break;
5722 }
5723 else if(valid_range_attr_name == attr->name) {
5724 has_validrange = true;
5725 break;
5726 }
5727 }
5728 // Add the fill value
5729 if (has_fillvalue != true && has_validrange != true ) {
5730 auto attr_unique = make_unique<Attribute>();
5731 auto attr = attr_unique.release();
5732 Add_One_Float_Attr(attr,fill_value_attr_name,_FV);
5733 var->attrs.push_back(attr);
5734 }
5735 }
5736 }
5737}
5738
5739// Leave the following code for the time being
5740#if 0
5741// Handle the "coordinates" and "units" attributes of coordinate variables.
5743
5744 string co_attrname = "coordinates";
5745 string co_attrvalue="";
5746 string unit_attrname = "units";
5747 string nonll_unit_attrvalue ="level";
5748 string lat_unit_attrvalue ="degrees_north";
5749 string lon_unit_attrvalue ="degrees_east";
5750
5751 for (auto ircv = this->cvars.begin();
5752 ircv != this->cvars.end(); ++ircv) {
5753//cerr<<"CV name is "<<(*ircv)->name << " cv type is "<<(*ircv)->cvartype <<endl;
5754
5755 if ((*ircv)->cvartype == CV_NONLATLON_MISS) {
5756 Attribute * attr = new Attribute();
5757 Add_Str_Attr(attr,unit_attrname,nonll_unit_attrvalue);
5758 (*ircv)->attrs.push_back(attr);
5759 }
5760
5761 else if ((*ircv)->cvartype == CV_LAT_MISS) {
5762//cerr<<"Should add new attribute "<<endl;
5763 Attribute * attr = new Attribute();
5764// float temp = -999.9;
5765// Add_One_Float_Attr(attr,unit_attrname,temp);
5766 Add_Str_Attr(attr,unit_attrname,lat_unit_attrvalue);
5767 (*ircv)->attrs.push_back(attr);
5768//cerr<<"After adding new attribute "<<endl;
5769 }
5770
5771 else if ((*ircv)->cvartype == CV_LON_MISS) {
5772 Attribute * attr = new Attribute();
5773 Add_Str_Attr(attr,unit_attrname,lon_unit_attrvalue);
5774 (*ircv)->attrs.push_back(attr);
5775 }
5776 } // for (auto ircv = this->cvars.begin(); ...
5777
5778 // No need to handle MeaSUREs SeaWiFS level 2 products
5779 if(product_type == Mea_SeaWiFS_L2)
5780 return;
5781
5782 // GPM level 1 needs to be handled separately
5783 else if(product_type == GPM_L1) {
5784 Handle_GPM_l1_Coor_Attr();
5785 return;
5786 }
5787 // No need to handle products that follow COARDS.
5788 else if (true == iscoard) {
5789 // May need to check coordinates for 2-D lat/lon but cannot treat those lat/lon as CV case. KY 2015-12-10-TEMPPP
5790 return;
5791 }
5792
5793
5794 // Now handle the 2-D lat/lon case(note: this only applies to the one that dim. scale doesn't apply)
5795 for (auto ircv = this->cvars.begin();
5796 ircv != this->cvars.end(); ++ircv) {
5797 if((*ircv)->rank == 2) {
5798
5799 // The following code makes sure that the replacement only happens with the general 2-D lat/lon case.
5800 if(gp_latname == (*ircv)->name)
5801 Replace_Var_Str_Attr((*ircv),unit_attrname,lat_unit_attrvalue);
5802 else if(gp_lonname ==(*ircv)->name)
5803 Replace_Var_Str_Attr((*ircv),unit_attrname,lon_unit_attrvalue);
5804 }
5805 }
5806
5807 // Check the dimension names of 2-D lat/lon CVs
5808 string ll2d_dimname0,ll2d_dimname1;
5809 bool has_ll2d_coords = false;
5810 for (auto ircv = this->cvars.begin();
5811 ircv != this->cvars.end(); ++ircv) {
5812 if((*ircv)->rank == 2) {
5813 // Note: we should still use the original dim. name to match the general variables.
5814 ll2d_dimname0 = (*ircv)->getDimensions()[0]->name;
5815 ll2d_dimname1 = (*ircv)->getDimensions()[1]->name;
5816 if(ll2d_dimname0 !="" && ll2d_dimname1 !="")
5817 has_ll2d_coords = true;
5818 break;
5819 }
5820 }
5821
5822 if(true == has_ll2d_coords) {
5823
5824 for (auto irv = this->vars.begin();
5825 irv != this->vars.end(); ++irv) {
5826
5827 bool coor_attr_keep_exist = false;
5828
5829 // May need to delete only the "coordinates" with both 2-D lat/lon dim. KY 2015-12-07
5830 if(((*irv)->rank >=2)) {
5831
5832 short ll2dim_flag = 0;
5833 for (auto ird = (*irv)->dims.begin();
5834 ird != (*irv)->dims.end(); ++ ird) {
5835 if((*ird)->name == ll2d_dimname0)
5836 ll2dim_flag++;
5837 else if((*ird)->name == ll2d_dimname1)
5838 ll2dim_flag++;
5839 }
5840
5841 if(ll2dim_flag != 2)
5842 coor_attr_keep_exist = true;
5843
5844 // The following line doesn't apply to SMAP,it applies to Old SMAP Level 2 Simulation files.
5845 if(product_type == OSMAPL2S)
5846 coor_attr_keep_exist = true;
5847
5848 if (false == coor_attr_keep_exist) {
5849 for (auto ira =(*irv)->attrs.begin();
5850 ira !=(*irv)->attrs.end();) {
5851 if ((*ira)->newname == co_attrname) {
5852 delete (*ira);
5853 ira = (*irv)->attrs.erase(ira);
5854 }
5855 else {
5856 ++ira;
5857 }
5858 }// for (auto ira =(*irv)->attrs.begin(); ...
5859
5860 // Generate the "coordinates" attribute only for variables that have both 2-D lat/lon dim. names.
5861 for (auto ird = (*irv)->dims.begin();
5862 ird != (*irv)->dims.end(); ++ ird) {
5863 for (auto ircv = this->cvars.begin();
5864 ircv != this->cvars.end(); ++ircv) {
5865 if ((*ird)->name == (*ircv)->cfdimname)
5866 co_attrvalue = (co_attrvalue.empty())
5867 ?(*ircv)->newname:co_attrvalue + " "+(*ircv)->newname;
5868 }
5869 }
5870
5871 if (false == co_attrvalue.empty()) {
5872 Attribute * attr = new Attribute();
5873 Add_Str_Attr(attr,co_attrname,co_attrvalue);
5874 (*irv)->attrs.push_back(attr);
5875 }
5876
5877 co_attrvalue.clear();
5878 } // for (auto irv = this->vars.begin(); ...
5879 }
5880 }
5881 }
5882}
5883#endif
5884
5885
5886// Handle the "coordinates" and "units" attributes of coordinate variables.
5888
5889 BESDEBUG("h5", "GMFile::Coming to Handle_Coor_Attr()"<<endl);
5890 string co_attrname = "coordinates";
5891 string co_attrvalue;
5892 string unit_attrname = "units";
5893 string nonll_unit_attrvalue ="level";
5894 string lat_unit_attrvalue ="degrees_north";
5895 string lon_unit_attrvalue ="degrees_east";
5896
5897 // Attribute units should be added for coordinate variables that
5898 // have the type CV_NONLATLON_MISS,CV_LAT_MISS and CV_LON_MISS.
5899 for (auto &cvar:this->cvars) {
5900
5901 if (cvar->cvartype == CV_NONLATLON_MISS) {
5902 auto attr_unique = make_unique<Attribute>();
5903 auto attr = attr_unique.release();
5904 Add_Str_Attr(attr,unit_attrname,nonll_unit_attrvalue);
5905 cvar->attrs.push_back(attr);
5906 }
5907 else if (cvar->cvartype == CV_LAT_MISS) {
5908 auto attr_unique = make_unique<Attribute>();
5909 auto attr = attr_unique.release();
5910 Add_Str_Attr(attr,unit_attrname,lat_unit_attrvalue);
5911 cvar->attrs.push_back(attr);
5912 }
5913 else if (cvar->cvartype == CV_LON_MISS) {
5914 auto attr_unique = make_unique<Attribute>();
5915 auto attr = attr_unique.release();
5916 Add_Str_Attr(attr,unit_attrname,lon_unit_attrvalue);
5917 cvar->attrs.push_back(attr);
5918 }
5919 } // end of for
5920
5921 // No need to handle MeaSUREs SeaWiFS level 2 products
5922 if(product_type == Mea_SeaWiFS_L2)
5923 return;
5924
5925 // GPM level 1 needs to be handled separately
5926 else if(product_type == GPM_L1) {
5927 Handle_GPM_l1_Coor_Attr();
5928 return;
5929 }
5930
5931 // Handle Lat/Lon with "coordinates" attribute.
5932 else if(product_type == General_Product && gproduct_pattern == GENERAL_LATLON_COOR_ATTR){
5933 Handle_LatLon_With_CoordinateAttr_Coor_Attr();
5934 return;
5935 }
5936 // No need to handle products that follow COARDS.
5937 else if (true == iscoard) {
5938
5939 // If we find that there are groups that should check the coordinates attribute of the variable.
5940 // We should flatten the path inside the coordinates.(this is the case mainly for netcdf-4 2D lat/lon case)
5941 if(grp_cv_paths.empty() == false) {
5942 for (auto &var:this->vars) {
5943 if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash(var->fullpath)) != grp_cv_paths.end()){
5944 // Check the "coordinates" attribute and flatten the values.
5945 Flatten_VarPath_In_Coordinates_Attr(var);
5946 }
5947 }
5948 }
5949 return;
5950 }
5951
5952 // Now handle the 2-D lat/lon case
5953 for (auto &cvar:this->cvars) {
5954
5955 if(cvar->rank == 2 && cvar->cvartype == CV_EXIST) {
5956 //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
5957 // When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
5958 // The following code makes sure that the replacement only happens with the general 2-D lat/lon case.
5959 // The following code is commented out since we find an OMPS-NPP case that has the no-CF unit for
5960 // "Latitude". So just to check the latitude and longitude and if the units are not CF-compliant,
5961 // change them. KY 2020-02-27
5962#if 0
5963 if(gp_latname == cvar->name) {
5964 // Only if gp_latname is not lat/latitude/Latitude, change the units
5965 if(false == Is_geolatlon(gp_latname,true))
5966 Replace_Var_Str_Attr(cvar,unit_attrname,lat_unit_attrvalue);
5967 }
5968 else if(gp_lonname ==cvar->name) {
5969 // Only if gp_lonname is not lon/longitude/Longitude, change the units
5970 if(false == Is_geolatlon(gp_lonname,false))
5971 Replace_Var_Str_Attr(cvar,unit_attrname,lon_unit_attrvalue);
5972 }
5973#endif
5974
5975 // We meet several products that miss the 2-D latitude and longitude CF units, although they
5976 // have the CV names like latitude/longitude, we should double-check this case,
5977 // and add the correct CF units if possible. We will watch if this is the right way.
5978 //else if(true == Is_geolatlon(cvar->name,true))
5979 if(true == Is_geolatlon(cvar->name,true))
5980 Replace_Var_Str_Attr(cvar,unit_attrname,lat_unit_attrvalue);
5981 else if(true == Is_geolatlon(cvar->name,false))
5982 Replace_Var_Str_Attr(cvar,unit_attrname,lon_unit_attrvalue);
5983 }
5984 } // for (auto ircv = this->cvars.begin()
5985
5986 // If we find that there are groups that we should check the coordinates attribute of the variable under,
5987 // we should flatten the path inside the coordinates. Note this is for 2D-latlon CV netCDF-4-like case.
5988 if (grp_cv_paths.empty() == false) {
5989 for (auto &var:this->vars) {
5990 if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash(var->fullpath)) != grp_cv_paths.end()){
5991 // Check the "coordinates" attribute and flatten the values.
5992 Flatten_VarPath_In_Coordinates_Attr(var);
5993 }
5994 }
5995 }
5996
5997 // Check if having 2-D lat/lon CVs
5998 bool has_ll2d_coords = false;
5999
6000 // Since iscoard is false up to this point, So the netCDF-4 like 2-D lat/lon case must fulfill if the program comes here.
6001 if(General_Product == this->product_type && GENERAL_DIMSCALE == this->gproduct_pattern)
6002 has_ll2d_coords = true;
6003 else {// For other cases. Need to see if there is a case. KY 2016-07-07
6004 string ll2d_dimname0;
6005 string ll2d_dimname1;
6006 for (const auto &cvar:this->cvars) {
6007 if(cvar->rank == 2) {
6008 // Note: we should still use the original dim. name to match the general variables.
6009 ll2d_dimname0 = cvar->getDimensions()[0]->name;
6010 ll2d_dimname1 = cvar->getDimensions()[1]->name;
6011 if(ll2d_dimname0 !="" && ll2d_dimname1 !="")
6012 has_ll2d_coords = true;
6013 break;
6014 }
6015 }
6016 }
6017
6018 // We now walk through all the >=2 vars and flatten the "coordinates"
6019 if(true == has_ll2d_coords) {
6020
6021 // For some netCDF-4-like 2-D lat/lon cases, we may need to forcely flatten the coordinates.
6022 // This case usually happens when the data producers follow the CF and the NASA DIWG guideline to
6023 // provide the absolute path of the coordinates as the value of the "coordinates" attribute.
6024 // The handler doesn't need to figure out the contents of the coordinates attribute but to
6025 // flatten the path inside.
6026 // However, the BES Key FORCENDCoorAttr must be set.
6027 bool force_flatten_coor_attr = HDF5RequestHandler::get_force_flatten_coor_attr();
6028
6029 // We also need to find if we have coordinates attribute for >=2D variables.
6030 // If not, the handler has to figure out the coordinates.
6031 bool has_coor_attr_ge_2d_vars = false;
6032 for (const auto &var:this->vars) {
6033 if(var->rank >=2){
6034 for (const auto &attr:var->attrs) {
6035 // We will check if we have the coordinate attribute
6036 if(attr->name == co_attrname) {
6037 has_coor_attr_ge_2d_vars = true;
6038 break;
6039 }
6040 }
6041 if(has_coor_attr_ge_2d_vars == true)
6042 break;
6043 }
6044 }
6045#if 0
6046 // Here we may need to consider the special case for HDF-EOS5. The "Data Fields" etc should not be
6047 // in the group path. May need to let DIWG provide a guideline for this issue.
6048 bool is_hybrid_eos5= false;
6049 if(force_flatten_coor_attr == true && has_coor_attr_ge_2d_vars == true)
6050 is_hybrid_eos5 = Is_Hybrid_EOS5();
6051#endif
6052 for (auto &var:this->vars) {
6053
6054 bool has_coor = false;
6055 for (const auto &attr:var->attrs) {
6056 // We will check if we have the coordinate attribute
6057 if(attr->name == co_attrname) {
6058 has_coor = true;
6059 break;
6060 }
6061 }
6062
6063 // The coordinates attribute is flattened by force.
6064 if(true == force_flatten_coor_attr && true == has_coor) {
6065#if 0
6066 if(is_hybrid_eos5 == true) {
6067 Flatten_VarPath_In_Coordinates_Attr_EOS5(var);
6068 }
6069 else
6070#endif
6071 Flatten_VarPath_In_Coordinates_Attr(var);
6072 }
6073
6074 else if((var->rank >=2) && (has_coor_attr_ge_2d_vars == false || false == force_flatten_coor_attr)) {
6075
6076 bool coor_attr_keep_exist = false;
6077
6078 // Check if this var is under group_cv_paths, no, then check if this var's dims are the same as the dims of 2-D CVars
6079 if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash(var->fullpath)) == grp_cv_paths.end())
6080
6081 // If finding this var is associated with 2-D lat/lon CVs, not keep the original "coordinates" attribute.
6082 coor_attr_keep_exist = Check_Var_2D_CVars(var);
6083 else {
6084 coor_attr_keep_exist = true;
6085 }
6086
6087 // The following two lines are just for old smap level 2 case.
6088 if(product_type == OSMAPL2S)
6089 coor_attr_keep_exist = true;
6090
6091 // Need to delete the original "coordinates" and rebuild the "coordinates" if this var is associated with the 2-D lat/lon CVs.
6092 if (false == coor_attr_keep_exist) {
6093 for (auto ira =var->attrs.begin(); ira !=var->attrs.end();) {
6094 if ((*ira)->newname == co_attrname) {
6095 delete (*ira);
6096 ira = var->attrs.erase(ira);
6097 }
6098 else {
6099 ++ira;
6100 }
6101 }// for (auto ira =var->attrs.begin(); ...
6102
6103 // Generate the new "coordinates" attribute.
6104 for (const auto &dim:var->dims) {
6105 for (const auto &cvar:this->cvars) {
6106 if (dim->name == cvar->cfdimname)
6107 co_attrvalue = (co_attrvalue.empty())
6108 ?cvar->newname:co_attrvalue + " "+cvar->newname;
6109 }
6110 }
6111
6112 if (false == co_attrvalue.empty()) {
6113 auto attr_unique = make_unique<Attribute>();
6114 auto attr = attr_unique.release();
6115 Add_Str_Attr(attr,co_attrname,co_attrvalue);
6116 var->attrs.push_back(attr);
6117 }
6118
6119 co_attrvalue.clear();
6120 var->coord_attr_add_path = false;
6121 } // for (auto irv = this->vars.begin(); ...
6122 }
6123 }
6124 }
6125}
6126
6127// Handle GPM level 1 coordinates attributes.
6128void GMFile:: Handle_GPM_l1_Coor_Attr() const {
6129
6130 BESDEBUG("h5", "Coming to Handle_GPM_l1_Coor_Attr()"<<endl);
6131 // Build a map from CFdimname to 2-D lat/lon variable name, should be something like: aa_list[cfdimname]=s1_latitude .
6132 // Loop all variables
6133 // Inner loop: for all dims of a var
6134 // if (dimname matches the dim(not cfdim) name) of one of 2-D lat/lon,
6135 // check if the variable's full path contains the path of one of 2-D lat/lon,
6136 // yes, build its cfdimname = path+ dimname, check this cfdimname with the cfdimname of the corresponding 2-D lat/lon
6137 // If matched, save this latitude variable name as one of the coordinate variable.
6138 // else this is a 3rd-dimension cv, just use the dimension name(or the corresponding cv name maybe through a map).
6139
6140 // Prepare 1) 2-D CVar(lat,lon) corresponding dimension name set.
6141 // 2) cfdim name to cvar name map(don't need to use a map, just a holder). It should be fine.
6142
6143 // "coordinates" attribute name and value. We only need to provide this attribute for variables that have 2-D lat/lon
6144 string co_attrname = "coordinates";
6145 string co_attrvalue;
6146
6147 // 2-D cv dimname set.
6148 set<string> cvar_2d_dimset;
6149
6150 pair<map<string,string>::iterator,bool>mapret;
6151
6152 // Hold the mapping from cfdimname to 2-D cvar name. Something like nscan->lat, npixel->lon
6153 map<string,string>cfdimname_to_cvar2dname;
6154
6155 // Loop through cv variables to build 2-D cv dimname set and the mapping from cfdimname to 2-D cvar name.
6156 for (const auto &cvar:this->cvars) {
6157
6158 //This CVar must be 2-D array.
6159 if(cvar->rank == 2) {
6160
6161#if 0
6162//cerr<<"2-D cv name is "<<cvar->name <<endl;
6163//cerr<<"2-D cv new name is "<<cvar->newname <<endl;
6164//cerr<<"cvar->cfdimname is "<<cvar->cfdimname <<endl;
6165#endif
6166
6167 for (const auto &dim:cvar->dims)
6168 cvar_2d_dimset.insert(dim->name);
6169
6170 // Generate cfdimname to cvar2d map
6171 mapret = cfdimname_to_cvar2dname.insert(pair<string,string>(cvar->cfdimname,cvar->newname));
6172 if (false == mapret.second)
6173 throw4("The cf dimension name ",cvar->cfdimname," should map to 2-D coordinate variable",
6174 cvar->newname);
6175 }
6176 }
6177
6178 // Loop through the variable list to build the coordinates.
6179 for (auto &var:this->vars) {
6180
6181 // Only apply to >2D variables.
6182 if(var->rank >=2) {
6183
6184 // The variable dimension names must be found in the 2D cvar dim. nameset.
6185 // The flag must be at least 2.
6186 short have_2d_dimnames_flag = 0;
6187 for (const auto &dim:var->dims) {
6188 if (cvar_2d_dimset.find(dim->name)!=cvar_2d_dimset.end())
6189 have_2d_dimnames_flag++;
6190 }
6191
6192 // Final candidates to have 2-D CVar coordinates.
6193 if(have_2d_dimnames_flag >=2) {
6194
6195 // Obtain the variable path
6196 string var_path;
6197 if(var->fullpath.size() > var->name.size())
6198 var_path=var->fullpath.substr(0,var->fullpath.size()-var->name.size());
6199 else
6200 throw4("The variable full path ",var->fullpath," doesn't contain the variable name ",
6201 var->name);
6202
6203 // A flag to identify if this variable really needs the 2-D coordinate variables.
6204 short cv_2d_flag = 0;
6205
6206 // 2-D coordinate variable names for the potential variable candidate
6207 vector<string> cv_2d_names;
6208
6209 // Dimension names of the 2-D coordinate variables.
6210 set<string> cv_2d_dimnames;
6211
6212 // Loop through the map from dim. name to coordinate name.
6213 for(map<string,string>::const_iterator itm = cfdimname_to_cvar2dname.begin();
6214 itm != cfdimname_to_cvar2dname.end();++itm) {
6215
6216 // Obtain the dimension name from the cfdimname.
6217 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(itm->first);
6218 string cfdim_path;
6219 if(itm->first.size() <= reduced_dimname.size())
6220 throw2("The cf dim. name of this dimension is not right.",itm->first);
6221 else
6222 cfdim_path= itm->first.substr(0,itm->first.size() - reduced_dimname.size());
6223 // cfdim_path will not be nullptr only when the cfdim name is for the 2-D cv var.
6224
6225 // Find the correct path,
6226 // Note:
6227 // var_path doesn't have to be the same as cfdim_path
6228 // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6229 // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6230 // But we want to check if var_path is the same as cfdim_path first. So we check cfdimname_to_cvarname again.
6231 if(var_path == cfdim_path) {
6232 for (const auto &dim:var->dims) {
6233 if(reduced_dimname == dim->name) {
6234 cv_2d_flag++;
6235 cv_2d_names.push_back(itm->second);
6236 cv_2d_dimnames.insert(dim->name);
6237 }
6238 }
6239 }
6240 }
6241
6242 // Note:
6243 // var_path doesn't have to be the same as cfdim_path
6244 // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6245 // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6246 // The variable has 2 coordinates(dimensions) if the flag is 2
6247 // If the flag is not 2, we will see if the above case stands.
6248 if(cv_2d_flag != 2) {
6249 cv_2d_flag = 0;
6250 // Loop through the map from dim. name to coordinate name.
6251 for(map<string,string>::const_iterator itm = cfdimname_to_cvar2dname.begin();
6252 itm != cfdimname_to_cvar2dname.end();++itm) {
6253 // Obtain the dimension name from the cfdimname.
6254 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(itm->first);
6255 string cfdim_path;
6256 if(itm->first.size() <= reduced_dimname.size())
6257 throw2("The cf dim. name of this dimension is not right.",itm->first);
6258 else
6259 cfdim_path= itm->first.substr(0,itm->first.size() - reduced_dimname.size());
6260 // cfdim_path will not be nullptr only when the cfdim name is for the 2-D cv var.
6261
6262 // Find the correct path,
6263 // Note:
6264 // var_path doesn't have to be the same as cfdim_path
6265 // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6266 // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6267 //
6268 if(var_path.find(cfdim_path)!=string::npos) {
6269 for (const auto &dim:var->dims) {
6270 if(reduced_dimname == dim->name) {
6271 cv_2d_flag++;
6272 cv_2d_names.push_back(itm->second);
6273 cv_2d_dimnames.insert(dim->name);
6274 }
6275 }
6276 }
6277
6278 }
6279 }
6280
6281 // Now we got all cases.
6282 if(2 == cv_2d_flag) {
6283
6284 // Add latitude and longitude to the 'coordinates' attribute.
6285 co_attrvalue = cv_2d_names[0] + " " + cv_2d_names[1];
6286 if(var->rank >2) {
6287 for (const auto &dim:var->dims) {
6288
6289 // Add 3rd-dimension to the 'coordinates' attribute.
6290 if(cv_2d_dimnames.find(dim->name) == cv_2d_dimnames.end())
6291 co_attrvalue = co_attrvalue + " " +dim->newname;
6292 }
6293 }
6294 auto attr_unique = make_unique<Attribute>();
6295 auto attr = attr_unique.release();
6296 Add_Str_Attr(attr,co_attrname,co_attrvalue);
6297 var->attrs.push_back(attr);
6298 var->coord_attr_add_path = false;
6299 }
6300 }
6301 }
6302 }
6303}
6304
6305// This routine is for handling "coordinates" for the GENERAL_LATLON_COOR_ATTR pattern of General_Product.
6306void GMFile::Handle_LatLon_With_CoordinateAttr_Coor_Attr() {
6307
6308 BESDEBUG("h5", "Coming to Handle_LatLon_With_CoordinateAttr_Coor_Attr()"<<endl);
6309 string co_attrname = "coordinates";
6310
6311 // Loop through all rank >1 variables
6312 for (auto &var:this->vars) {
6313 if(var->rank >= 2) {
6314 for (auto &attr:var->attrs) {
6315 if(attr->name == co_attrname) {
6316 // If having the coordinates attribute, check if the "coordinates" variables match 2-D lat/lon CV condition,
6317 // if yes, flatten the coordinates attribute.
6318 string coor_value = Retrieve_Str_Attr_Value(attr,var->fullpath);
6319 if(Coord_Match_LatLon_NameSize(coor_value) == true)
6320 Flatten_VarPath_In_Coordinates_Attr(var);
6321 // If the "coordinates" variables don't match the first condition, we can still check
6322 // if we can find the corresponding "coordinates" variables that match the names under the same group,
6323 // if yes, we add the path to the attribute "coordinates".
6324 else if(Coord_Match_LatLon_NameSize_Same_Group(coor_value,HDF5CFUtil::obtain_string_before_lastslash(var->fullpath)) == true)
6325 Add_VarPath_In_Coordinates_Attr(var,coor_value);
6326 // For other cases, we don't do anything with the "coordinates".
6327 break;
6328 }
6329 }
6330 }
6331 }
6332
6333}
6334
6335// We will check the "coordinates variables" stored in the coordinate attribute match the
6336// checked latlon_name_pairs for the GENERAL_LATLON_COOR_ATTR case.
6337bool GMFile::Coord_Match_LatLon_NameSize(const string & coord_values) {
6338
6339 BESDEBUG("h5", "Coming to Coord_Match_LatLon_NameSize()"<<endl);
6340 bool ret_value =false;
6341 vector<string> coord_values_vec;
6342 char sep=' ';
6343 int match_lat_name_pair_index = -1;
6344 int match_lon_name_pair_index = -2;
6345 int num_match_lat = 0;
6346 int num_match_lon = 0;
6347
6348
6349 // Decompose the coordinates attribute into a string vector.
6350 HDF5CFUtil::Split_helper(coord_values_vec,coord_values,sep);
6351
6352 // Some products ignore the first "/" of the coordinate path in the coordinate attribute, we will correct this
6353 if((coord_values_vec[0])[0] !='/') {
6354 for(auto &coor_value:coord_values_vec){
6355 if((coor_value.find_first_of('/'))!=string::npos) {
6356 coor_value = '/' + coor_value;
6357 }
6358 }
6359 }
6360
6361 //Loop through all coordinate path stored in the coordinate patch vector,
6362 for (const auto &coor_value:coord_values_vec){
6363
6364 // Loop through all the lat/lon pairs generated in the Check_LatLon_With_Coordinate_Attr routine
6365 // Remember the index and number appeared for both lat and lon.
6366 for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
6367 if(coor_value == (*ivs).name1){
6368 match_lat_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6369 num_match_lat++;
6370 }
6371 else if (coor_value == (*ivs).name2) {
6372 match_lon_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6373 num_match_lon++;
6374 }
6375 }
6376 }
6377 //Only when both index and the number of appearance match, we can set this be true.
6378 if((match_lat_name_pair_index == match_lon_name_pair_index) && (num_match_lat ==1) && (num_match_lon ==1))
6379 ret_value = true;
6380
6381 return ret_value;
6382
6383}
6384
6385//Some products only store the coordinate name(not full path) in the attribute coordinates, as
6386//long as it is valid, we should add the path to these coordinates.
6387bool GMFile::Coord_Match_LatLon_NameSize_Same_Group(const string &coord_values,const string &var_path) {
6388
6389 BESDEBUG("h5", "Coming to Coord_Match_LatLon_NameSize_Same_Group()"<<endl);
6390 bool ret_value =false;
6391 vector<string> coord_values_vec;
6392 char sep=' ';
6393 int match_lat_name_pair_index = -1;
6394 int match_lon_name_pair_index = -2;
6395 int num_match_lat = 0;
6396 int num_match_lon = 0;
6397
6398 HDF5CFUtil::Split_helper(coord_values_vec,coord_values,sep);
6399
6400 // Assume the 3rd-variable is also located under the same group if rank >=2
6401 for (const auto &coord_value:coord_values_vec){
6402#if 0
6403//cerr<<"coordinate values are "<<*irs <<endl;
6404#endif
6405 for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
6406 string lat_name = HDF5CFUtil::obtain_string_after_lastslash((*ivs).name1);
6407 string lat_path = HDF5CFUtil::obtain_string_before_lastslash((*ivs).name1);
6408 string lon_name = HDF5CFUtil::obtain_string_after_lastslash((*ivs).name2);
6409 string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*ivs).name2);
6410 if(coord_value == lat_name && lat_path == var_path){
6411 match_lat_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6412 num_match_lat++;
6413 }
6414 else if (coord_value == lon_name && lon_path == var_path) {
6415 match_lon_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6416 num_match_lon++;
6417 }
6418 }
6419 }
6420
6421 if((match_lat_name_pair_index == match_lon_name_pair_index) && (num_match_lat ==1) && (num_match_lon ==1))
6422 ret_value = true;
6423
6424 return ret_value;
6425}
6426
6427// This is for the GENERAL_LATLON_COOR_ATTR pattern of General_Product.
6428void GMFile::Add_VarPath_In_Coordinates_Attr(Var *var, const string &coor_value) {
6429
6430 BESDEBUG("h5", "Coming to Add_VarPath_In_Coordinates_Attr()"<<endl);
6431 string new_coor_value;
6432 char sep =' ';
6433 string var_path = HDF5CFUtil::obtain_string_before_lastslash(var->fullpath) ;
6434 string var_flatten_path = get_CF_string(var_path);
6435
6436 // We need to loop through each element in the "coor_value".
6437 size_t ele_start_pos = 0;
6438 size_t cur_pos = coor_value.find_first_of(sep);
6439 while(cur_pos !=string::npos) {
6440 string tempstr = coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
6441 tempstr = var_flatten_path + tempstr;
6442 new_coor_value += tempstr + sep;
6443 ele_start_pos = cur_pos+1;
6444 cur_pos = coor_value.find_first_of(sep,cur_pos+1);
6445 }
6446
6447 if(ele_start_pos == 0)
6448 new_coor_value = var_flatten_path + coor_value;
6449 else
6450 new_coor_value += var_flatten_path + coor_value.substr(ele_start_pos);
6451
6452 string coor_attr_name = "coordinates";
6453 Replace_Var_Str_Attr(var,coor_attr_name,new_coor_value);
6454 var->coord_attr_add_path = false;
6455
6456}
6457
6458// Create Missing coordinate variables. Index numbers are used for these variables.
6459void GMFile:: Create_Missing_CV(GMCVar *GMcvar, const string& dimname) {
6460
6461 BESDEBUG("h5", "GMFile::Coming to Create_Missing_CV()"<<endl);
6462
6463 GMcvar->name = dimname;
6464 GMcvar->newname = GMcvar->name;
6465 GMcvar->fullpath = GMcvar->name;
6466 GMcvar->rank = 1;
6467 GMcvar->dtype = H5INT32;
6468 hsize_t gmcvar_dimsize = dimname_to_dimsize[dimname];
6469 bool unlimited_flag = dimname_to_unlimited[dimname];
6470
6471 auto gmcvar_dim_unique = make_unique<Dimension>(gmcvar_dimsize);
6472 auto gmcvar_dim = gmcvar_dim_unique.release();
6473
6474 gmcvar_dim->unlimited_dim = unlimited_flag;
6475 gmcvar_dim->name = dimname;
6476 gmcvar_dim->newname = dimname;
6477 GMcvar->dims.push_back(gmcvar_dim);
6478 GMcvar->cfdimname = dimname;
6479 GMcvar->cvartype = CV_NONLATLON_MISS;
6480 GMcvar->product_type = product_type;
6481}
6482
6483 // Check if this is just a netCDF-4 dimension. We need to check the dimension scale dataset attribute "NAME",
6484 // the value should start with "This is a netCDF dimension but not a netCDF variable".
6485bool GMFile::Is_netCDF_Dimension(const Var *var) const {
6486
6487 string netcdf_dim_mark = "This is a netCDF dimension but not a netCDF variable";
6488
6489 bool is_only_dimension = false;
6490
6491 for(const auto &attr:var->attrs) {
6492
6493 if ("NAME" == attr->name) {
6494
6495 Retrieve_H5_Attr_Value(attr,var->fullpath);
6496 string name_value;
6497 name_value.resize(attr->value.size());
6498 copy(attr->value.begin(),attr->value.end(),name_value.begin());
6499
6500 // Compare the attribute "NAME" value with the string netcdf_dim_mark. We only compare the string with the size of netcdf_dim_mark
6501 if (0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark))
6502 is_only_dimension = true;
6503
6504 break;
6505 }
6506 } // for(auto ira = var->attrs.begin(); ...
6507
6508 return is_only_dimension;
6509}
6510
6511// Handle attributes for special variables.
6512void
6516
6517bool
6518GMFile::Is_Hybrid_EOS5() const {
6519
6520 bool has_group_hdfeos = false;
6521 bool has_group_hdfeos_info = false;
6522
6523 // Too costly to check the dataset.
6524 // We will just check the attribute under /HDFEOS INFORMATION.
6525
6526 // First check if the HDFEOS groups are included
6527 for (const auto &grp:this->groups) {
6528 if ("/HDFEOS" == grp->path)
6529 has_group_hdfeos = true;
6530 else if("/HDFEOS INFORMATION" == grp->path) {
6531 for(const auto &attr:grp->attrs) {
6532 if("HDFEOSVersion" == attr->name)
6533 has_group_hdfeos_info = true;
6534 }
6535 }
6536 if(true == has_group_hdfeos && true == has_group_hdfeos_info)
6537 break;
6538 }
6539
6540
6541 if(true == has_group_hdfeos && true == has_group_hdfeos_info)
6542 return true;
6543 else
6544 return false;
6545}
6546
6547void GMFile::Handle_Hybrid_EOS5() {
6548
6549 string eos_str="HDFEOS_";
6550 string eos_info_str="HDFEOS_INFORMATION_";
6551 string grid_str="GRIDS_";
6552 string swath_str="SWATHS_";
6553 string zas_str="ZAS_";
6554 string df_str="Data_Fields_";
6555 string gf_str="Geolocation_Fields_";
6556 for (auto &var:this->vars) {
6557 string temp_var_name = var->newname;
6558
6559 bool remove_eos = Remove_EOS5_Strings(temp_var_name);
6560
6561 if(true == remove_eos)
6562 var->newname = get_CF_string(temp_var_name);
6563 else {//HDFEOS info and extra fields
6564 string::size_type eos_info_pos = temp_var_name.find(eos_info_str);
6565 if(eos_info_pos !=string::npos)
6566 var->newname = temp_var_name.erase(eos_info_pos,eos_info_str.size());
6567 else {// Check the extra fields
6568 if(Remove_EOS5_Strings_NonEOS_Fields(temp_var_name)==true)
6569 var->newname = get_CF_string(temp_var_name);
6570 }
6571 }
6572 }
6573
6574 // Now we need to handle the dimension names.
6575 for (auto &var:this->vars) {
6576 for (auto &dim:var->dims) {
6577 string temp_dim_name = dim->newname;
6578 bool remove_eos = Remove_EOS5_Strings(temp_dim_name);
6579
6580 if(true == remove_eos)
6581 dim->newname = get_CF_string(temp_dim_name);
6582 else {//HDFEOS info and extra fields
6583 string::size_type eos_info_pos = temp_dim_name.find(eos_info_str);
6584 if(eos_info_pos !=string::npos)
6585 dim->newname = temp_dim_name.erase(eos_info_pos,eos_info_str.size());
6586 else {// Check the extra fields
6587 if(Remove_EOS5_Strings_NonEOS_Fields(temp_dim_name)==true)
6588 dim->newname = get_CF_string(temp_dim_name);
6589 }
6590 }
6591 }
6592 }
6593
6594 // We have to loop through all CVs.
6595 for (auto &cvar:this->cvars) {
6596 string temp_var_name = cvar->newname;
6597
6598 bool remove_eos = Remove_EOS5_Strings(temp_var_name);
6599
6600 if(true == remove_eos)
6601 cvar->newname = get_CF_string(temp_var_name);
6602 else {//HDFEOS info and extra "fields"
6603 string::size_type eos_info_pos = temp_var_name.find(eos_info_str);
6604 if(eos_info_pos !=string::npos)
6605 cvar->newname = temp_var_name.erase(eos_info_pos,eos_info_str.size());
6606 else {// Check the extra fields
6607 if(Remove_EOS5_Strings_NonEOS_Fields(temp_var_name)==true)
6608 cvar->newname = get_CF_string(temp_var_name);
6609 }
6610 }
6611 }
6612 // Now we need to handle the dimension names.
6613 for (const auto &cvar:this->cvars) {
6614 for (const auto &dim:cvar->dims) {
6615 string temp_dim_name = dim->newname;
6616 bool remove_eos = Remove_EOS5_Strings(temp_dim_name);
6617
6618 if(true == remove_eos)
6619 dim->newname = get_CF_string(temp_dim_name);
6620 else {// HDFEOS info and extra "fields"
6621 string::size_type eos_info_pos = temp_dim_name.find(eos_info_str);
6622 if(eos_info_pos !=string::npos)
6623 dim->newname = temp_dim_name.erase(eos_info_pos,eos_info_str.size());
6624 else {// Check the extra "fields"
6625 if(Remove_EOS5_Strings_NonEOS_Fields(temp_dim_name)==true)
6626 dim->newname = get_CF_string(temp_dim_name);
6627 }
6628 }
6629 }
6630 }
6631
6632 // Update the coordinate attribute values
6633 // We need to remove the HDFEOS special information from the coordinates attributes
6634 // since the variable names in the DAP output are already updated.
6635 for (const auto &var:this->vars) {
6636 for (const auto &attr:var->attrs) {
6637 // We cannot use Retrieve_Str_Attr_value for "coordinates" since "coordinates" may be added by the handler.
6638 // KY 2017-11-3
6639 if(attr->name == "coordinates") {
6640 string cor_values(attr->value.begin(),attr->value.end()) ;
6641 bool change_cor_values = false;
6642 // Find the HDFEOS string
6643 if(cor_values.find(eos_str)==0) {
6644 if(cor_values.find(grid_str)!=string::npos) {// Grid
6645 cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6646 cor_values = HDF5CFUtil::remove_substrings(cor_values,grid_str);
6647 string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6648 if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6649 change_cor_values = true;
6650 cor_values = new_cor_values;
6651 }
6652 }
6653 else if(cor_values.find(zas_str)!=string::npos) {//ZA
6654 cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6655 cor_values = HDF5CFUtil::remove_substrings(cor_values,zas_str);
6656 string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6657 if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6658 change_cor_values = true;
6659 cor_values = new_cor_values;
6660 }
6661 }
6662 else if(cor_values.find(swath_str)!=string::npos) {//Swath
6663 cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6664 cor_values = HDF5CFUtil::remove_substrings(cor_values,swath_str);
6665 string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6666 if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6667 change_cor_values = true;
6668 cor_values = new_cor_values;
6669 }
6670 else {
6671 new_cor_values = HDF5CFUtil::remove_substrings(cor_values,gf_str);
6672 if(new_cor_values.size() < cor_values.size()){//gf_str is also removed.
6673 change_cor_values = true;
6674 cor_values = new_cor_values;
6675 }
6676 }
6677 }
6678 }
6679 if(true == change_cor_values) {//Update the coordinate values
6680 attr->value.resize(cor_values.size());
6681 attr->fstrsize=cor_values.size();
6682 attr->strsize[0] = cor_values.size();
6683 copy(cor_values.begin(), cor_values.end(), attr->value.begin());
6684 var->coord_attr_add_path = false;
6685 }
6686
6687 break;
6688 }
6689 }
6690 }
6691}
6692
6693// This routine is for handling the hybrid-HDFEOS5 products that have to be treated as "general products"
6694bool GMFile:: Remove_EOS5_Strings(string &var_name) const {
6695
6696 string eos_str="HDFEOS_";
6697 string grid_str="GRIDS_";
6698 string swath_str="SWATHS_";
6699 string zas_str="ZAS_";
6700 string df_str="Data_Fields_";
6701 string gf_str="Geolocation_Fields_";
6702 string temp_var_name = var_name;
6703
6704 bool remove_eos = false;
6705
6706 string::size_type eos_pos = temp_var_name.find(eos_str);
6707 if(eos_pos!=string::npos) {
6708 temp_var_name.erase(eos_pos,eos_str.size());
6709 // Check grid,swath and zonal
6710 string::size_type grid_pos=temp_var_name.find(grid_str);
6711 string::size_type grid_df_pos=string::npos;
6712 if(grid_pos!=string::npos)
6713 grid_df_pos = temp_var_name.find(df_str,grid_pos);
6714 string::size_type zas_pos = string::npos;
6715 string::size_type zas_df_pos=string::npos;
6716 if(grid_pos==string::npos || grid_df_pos ==string::npos)
6717 zas_pos=temp_var_name.find(zas_str);
6718 if(zas_pos!=string::npos)
6719 zas_df_pos=temp_var_name.find(df_str,zas_pos);
6720
6721 if(grid_pos !=string::npos && grid_df_pos!=string::npos) {
6722 temp_var_name.erase(grid_pos,grid_str.size());
6723 grid_df_pos = temp_var_name.find(df_str);
6724 temp_var_name.erase(grid_df_pos,df_str.size());
6725 remove_eos = true;
6726 }
6727 else if(zas_pos!=string::npos && zas_df_pos!=string::npos){
6728 temp_var_name.erase(zas_pos,zas_str.size());
6729 zas_df_pos = temp_var_name.find(df_str);
6730 temp_var_name.erase(zas_df_pos,df_str.size());
6731 remove_eos = true;
6732 }
6733 else {//Check both Geolocation and Data for Swath
6734
6735 string::size_type swath_pos=temp_var_name.find(swath_str);
6736 string::size_type swath_df_pos=string::npos;
6737 if(swath_pos!=string::npos)
6738 swath_df_pos=temp_var_name.find(df_str,swath_pos);
6739
6740 string::size_type swath_gf_pos=string::npos;
6741 if(swath_pos!=string::npos && swath_df_pos == string::npos)
6742 swath_gf_pos=temp_var_name.find(gf_str,swath_pos);
6743
6744 if(swath_pos !=string::npos) {
6745
6746 if(swath_df_pos!=string::npos) {
6747 temp_var_name.erase(swath_pos,swath_str.size());
6748 swath_df_pos = temp_var_name.find(df_str);
6749 temp_var_name.erase(swath_df_pos,df_str.size());
6750 remove_eos = true;
6751 }
6752 else if(swath_gf_pos!=string::npos) {
6753 temp_var_name.erase(swath_pos,swath_str.size());
6754 swath_gf_pos = temp_var_name.find(gf_str);
6755 temp_var_name.erase(swath_gf_pos,gf_str.size());
6756 remove_eos = true;
6757 }
6758 }
6759 }
6760 }
6761 if(true == remove_eos)
6762 var_name = temp_var_name;
6763
6764 return remove_eos;
6765}
6766
6767bool GMFile:: Remove_EOS5_Strings_NonEOS_Fields(string &var_name) const{
6768
6769 string eos_str="HDFEOS_";
6770 string grid_str="GRIDS_";
6771 string swath_str="SWATHS_";
6772 string zas_str="ZAS_";
6773 string temp_var_name = var_name;
6774
6775 bool remove_eos = false;
6776
6777 string::size_type eos_pos = temp_var_name.find(eos_str);
6778 if(eos_pos!=string::npos) {
6779 temp_var_name.erase(eos_pos,eos_str.size());
6780 remove_eos = true;
6781
6782 // See if we need to further remove some fields
6783 if(temp_var_name.find(grid_str)==0)
6784 temp_var_name.erase(0,grid_str.size());
6785 else if(temp_var_name.find(swath_str)==0)
6786 temp_var_name.erase(0,swath_str.size());
6787 else if(temp_var_name.find(zas_str)==0)
6788 temp_var_name.erase(0,zas_str.size());
6789 }
6790 if(true == remove_eos)
6791 var_name = temp_var_name;
6792
6793
6794 return remove_eos;
6795}
6796
6797// We do have an AirMSPI HDF-EOS5 hybrid UTM product that has grid_mapping attribute.
6801
6805
6807
6808 // We need to remove the FakeDim added for the unsupported variables.
6809 // We found such a case in the AirMSPR product. A compound dataype array
6810 // is assigned to a FakeDim. We need to remove them.
6811 // KY 2017-11-2: no need to even check the unsupported_var_dspace now.
6812 if(this->unsupported_var_dtype == true) {
6813
6814 // Need to check if we have coordinate variables such as FakeDim?
6815 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();) {
6816 if((*ircv)->newname.find("FakeDim")==0) {
6817 bool var_has_fakedim = false;
6818 for (const auto &var:this->vars) {
6819 for (const auto &dim:var->dims) {
6820 if(dim->newname == (*ircv)->newname){
6821 var_has_fakedim = true;
6822 break;
6823 }
6824 }
6825 if(var_has_fakedim == true)
6826 break;
6827 }
6828 if(var_has_fakedim == false) {
6829 // Remove this cv, the variable is unsupported.
6830 delete(*ircv);
6831 ircv = this->cvars.erase(ircv);
6832 }
6833 else
6834 ++ircv;
6835 }
6836 else
6837 ++ircv;
6838 }
6839#if 0
6840 // We need to handle unlimited dimensions
6841 //if(removed_fakedim_vars.size()!=0) {
6842 //}
6843#endif
6844 }
6845
6846}
6847
6848//Rename NC4 NonCoordVars back to the original name. This is detected by CAR_ARCTAS files.
6849//By handling this way, the output will be the same as the netCDF handler output.
6850//Check HFVHANDLER-254 for more information.
6852
6853 if(true == this->have_nc4_non_coord) {
6854 string nc4_non_coord="_nc4_non_coord_";
6855 size_t nc4_non_coord_size= nc4_non_coord.size();
6856 for (const auto &var:this->vars) {
6857 if(var->name.find(nc4_non_coord)==0) {
6858 size_t rm_str_pos = var->newname.find(nc4_non_coord);
6859 if (rm_str_pos == string::npos)
6860 throw4(var->name," variable's new name ",var->newname, "doesn't contain nc4_non_coord");
6861 else {
6862 string var_new_first_part;
6863 if (rm_str_pos !=0)
6864 var_new_first_part = var->newname.substr(0,rm_str_pos);
6865 string var_new_second_part = var->newname.substr(rm_str_pos+nc4_non_coord_size);
6866 var->newname = var_new_first_part + var_new_second_part;
6867 }
6868 }
6869 }
6870
6871 // Coordinate variables should never contain "_nc4_non_coord_"
6872#if 0
6873 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();++ircv) {
6874 if((*ircv)->name.find(nc4_non_coord)==0)
6875 (*ircv)->newname = (*ircv)->newname.substr(nc4_non_coord_size,(*ircv)->newname.size()-nc4_non_coord_size);
6876 }
6877#endif
6878 }
6879
6880}
6881
6883
6884 BESDEBUG("h5", "GMFile::Coming to Add_Path_Coor_Attr()"<<endl);
6885 string co_attrname = "coordinates";
6886 for (const auto &var:this->vars) {
6887 if(var->coord_attr_add_path == true) {
6888 for (const auto &attr:var->attrs) {
6889 // We will check if we have the coordinate attribute
6890 if(attr->name == co_attrname) {
6891 string coor_value = Retrieve_Str_Attr_Value(attr,var->fullpath);
6892 char sep=' ';
6893 vector<string>cvalue_vec;
6894 HDF5CFUtil::Split_helper(cvalue_vec,coor_value,sep);
6895 string new_coor_value;
6896 for (int i = 0; i<cvalue_vec.size();i++) {
6897 HDF5CFUtil::cha_co(cvalue_vec[i],var->fullpath);
6898 cvalue_vec[i] = get_CF_string(cvalue_vec[i]);
6899 if(i == 0)
6900 new_coor_value = cvalue_vec[i];
6901 else
6902 new_coor_value += sep+cvalue_vec[i];
6903
6904#if 0
6905//cout<<"co1["<<i<<"]= "<<cvalue_vec[i]<<endl;
6906#endif
6907 }
6908#if 0
6909//cout<<"new_coor_value is "<<new_coor_value<<endl;
6910#endif
6911 Replace_Var_Str_Attr(var,co_attrname,new_coor_value);
6912 break;
6913 }
6914 }
6915 }
6916 }
6917}
6918
6920
6921 BESDEBUG("h5", "GMFile::Coming to Add_Path_Coor_Attr()"<<endl);
6922 if (product_type == GPMS_L3 || product_type == GPMM_L3 || product_type == GPM_L3_New || special_gpm_l3)
6923 return;
6924 for (auto &var:this->vars) {
6925 for (auto &attr:var->attrs) {
6926 if (attr->name == "bounds") {
6927 string bnd_values = Retrieve_Str_Attr_Value(attr,var->fullpath);
6928 HDF5CFUtil::cha_co(bnd_values,var->fullpath);
6929 bnd_values = get_CF_string(bnd_values);
6930 Replace_Var_Str_Attr(var,"bounds",bnd_values);
6931 break;
6932 }
6933 }
6934 }
6935
6936 for (auto &var:this->cvars) {
6937 for (auto &attr:var->attrs) {
6938 if (attr->name == "bounds") {
6939 string bnd_values = Retrieve_Str_Attr_Value(attr,var->fullpath);
6940 HDF5CFUtil::cha_co(bnd_values,var->fullpath);
6941 bnd_values = get_CF_string(bnd_values);
6942 Replace_Var_Str_Attr(var,"bounds",bnd_values);
6943 break;
6944 }
6945 }
6946 }
6947
6948}
6949
6951
6952 for (auto &cvar:this->cvars) {
6953 if (cvar->getRank() == 1) {
6954 Dimension *dim = cvar->dims[0];
6955 if (dim->HaveUnlimitedDim()==true && dim->getSize()==0) {
6956 bool correct_dim_size = false;
6957 for (const auto &var:this->vars) {
6958 for (auto & vdim:var->dims) {
6959 if (vdim->unlimited_dim && vdim->newname == dim->newname) {
6960 dim->size = vdim->size;
6961 correct_dim_size = true;
6962 break;
6963 }
6964 }
6965 if (correct_dim_size)
6966 break;
6967 }
6968 }
6969 }
6970 }
6971
6972}
6973
6974// We will create some temporary coordinate variables. The resource allocated
6975// for these variables need to be released.
6976void
6977GMFile::release_standalone_GMCVar_vector(vector<GMCVar*>&tempgc_vars){
6978
6979 for (auto i = tempgc_vars.begin(); i != tempgc_vars.end(); ) {
6980 delete(*i);
6981 i = tempgc_vars.erase(i);
6982 }
6983
6984}
6985
6986#if 0
6987void
6988GMFile::add_ignored_info_attrs(bool is_grp,bool is_first){
6989
6990}
6991void
6992GMFile::add_ignored_info_objs(bool is_dim_related, bool is_first) {
6993
6994}
6995#endif
6996
6997#if 0
6998bool
6999GMFile::ignored_dimscale_ref_list(Var *var) {
7000
7001 bool ignored_dimscale = true;
7002 if(General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern) {
7003
7004 bool has_dimscale = false;
7005 bool has_reference_list = false;
7006
7007 for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
7008
7009 if((*ira)->name == "REFERENCE_LIST" &&
7010 false == HDF5CFUtil::cf_strict_support_type((*ira)->getType()))
7011 has_reference_list = true;
7012 if((*ira)->name == "CLASS") {
7013 Retrieve_H5_Attr_Value(*ira,var->fullpath);
7014 string class_value;
7015 class_value.resize((*ira)->value.size());
7016 copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
7017
7018 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
7019 // "DIMENSION_SCALE", which is 15.
7020 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
7021 has_dimscale = true;
7022 }
7023 }
7024
7025 if(true == has_dimscale && true == has_reference_list) {
7026 ignored_dimscale= false;
7027 break;
7028 }
7029
7030 }
7031 }
7032 return ignored_dimscale;
7033}
7034
7035#endif
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
include the entry functions to execute the handlers
This class represents one attribute.
Definition HDF5CF.h:185
This class repersents one dimension of an HDF5 dataset(variable).
Definition HDF5CF.h:142
bool HaveUnlimitedDim() const
Has unlimited dimensions.
Definition HDF5CF.h:158
This class retrieves all information from an HDF5 file.
Definition HDF5CF.h:559
std::vector< Group * > groups
Non-root group vectors.
Definition HDF5CF.h:791
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition HDF5CF.cc:1279
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition HDF5CF.cc:2208
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition HDF5CF.cc:1326
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition HDF5CF.cc:735
std::vector< Var * > vars
Var vectors.
Definition HDF5CF.h:785
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition HDF5CF.cc:2027
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition HDF5CF.cc:170
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition HDF5CF.h:788
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition HDF5CF.cc:922
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition HDF5CF.cc:1376
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition HDF5CF.cc:2191
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition HDF5CF.h:409
void Add_Supplement_Attrs(bool) override
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition HDF5GMCF.cc:5156
void Add_Path_Coord_Attr()
Update the coordinate attribute to include path and also flatten.
Definition HDF5GMCF.cc:6882
void Rename_NC4_NonCoordVars() const
Remove the _nc4_non_coord from the variable new names.
Definition HDF5GMCF.cc:6851
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition HDF5GMCF.cc:4937
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition HDF5GMCF.cc:6806
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition HDF5GMCF.cc:199
bool Have_Grid_Mapping_Attrs() override
Check if having Grid Mapping Attrs.
Definition HDF5GMCF.cc:6798
void Handle_SpVar_Attr() override
Handle special variable attributes for general NASA HDF5 products.
Definition HDF5GMCF.cc:6513
void Handle_DimNameClashing() override
Definition HDF5GMCF.cc:5038
void Handle_Grid_Mapping_Vars() override
Handle Grid Mapping Vars.
Definition HDF5GMCF.cc:6802
void Retrieve_H5_CVar_Supported_Attr_Values() override
Retrieve coordinate variable attributes.
Definition HDF5GMCF.cc:298
void Update_Bounds_Attr()
Update the Bounds attribute to follow the CF conventions.
Definition HDF5GMCF.cc:6919
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr) override
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition HDF5GMCF.cc:180
void Handle_SpVar() override
Handle special variables for general NASA HDF5 products.
Definition HDF5GMCF.cc:4719
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition HDF5GMCF.cc:223
void Update_NC4_PureDimSize()
Update the netCDF-4 pure dimension size when the pure dimension is an unlimited dimension.
Definition HDF5GMCF.cc:6950
void Handle_CVar() override
Handle coordinate variables for general NASA HDF5 products.
Definition HDF5GMCF.cc:2847
void Add_Dim_Name()
Add dimension name.
Definition HDF5GMCF.cc:766
void Retrieve_H5_Supported_Attr_Values() override
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition HDF5GMCF.cc:311
void Handle_Coor_Attr() override
Handle "coordinates" attributes for general HDF5 products.
Definition HDF5GMCF.cc:5887
void Handle_Unsupported_Dspace(bool) override
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition HDF5GMCF.cc:538
void Handle_Unsupported_Others(bool) override
Handle other unmapped objects/attributes for general HDF5 products.
Definition HDF5GMCF.cc:629
void Handle_Unsupported_Dtype(bool) override
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition HDF5GMCF.cc:351
void Flatten_Obj_Name(bool include_attr) override
Flatten the object name for general NASA HDF5 products.
Definition HDF5GMCF.cc:4888
void Adjust_H5_Attr_Value(Attribute *attr) const
Adjust attribute values for general HDF5 products.
Definition HDF5GMCF.cc:333
void Adjust_Dim_Name() override
Adjust dimension name for general NASA HDF5 products.
Definition HDF5GMCF.cc:5097
void Adjust_Obj_Name() override
Adjust object names based on different general NASA HDF5 products.
Definition HDF5GMCF.cc:4807
This class represents one HDF5 dataset(CF variable)
Definition HDF5CF.h:252
int getRank() const
Get the dimension rank of this variable.
Definition HDF5CF.h:294
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition HDF5CF.h:311
STL iterator class.
static void Split(const char *s, int len, char sep, std::vector< std::string > &names)