bes Updated for version 3.21.1
The Backend Server (BES) is the lower two tiers of the Hyrax data server
HDFEOS5CF.cc
Go to the documentation of this file.
1// This file is part of the hdf5_handler implementing for the CF-compliant
2// Copyright (c) 2011-2023 The HDF Group, Inc. and OPeNDAP, Inc.
3//
4// This is free software; you can redistribute it and/or modify it under the
5// terms of the GNU Lesser General Public License as published by the Free
6// Software Foundation; either version 2.1 of the License, or (at your
7// option) any later version.
8//
9// This software is distributed in the hope that it will be useful, but
10// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12// License for more details.
13//
14// You should have received a copy of the GNU Lesser General Public
15// License along with this library; if not, write to the Free Software
16// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17//
18// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19// You can contact The HDF Group, Inc. at 410 E University Ave,
20// Suite 200, Champaign, IL 61820
21
36
37#include "HDF5CF.h"
38#include "HDF5RequestHandler.h"
39#include "h5cfdaputil.h"
40#include "BESDebug.h"
41
42using namespace std;
43using namespace libdap;
44using namespace HDF5CF;
45
46// A constructor of EOS5CVar
47EOS5CVar::EOS5CVar(const Var*var)
48{
49
50 newname = var->newname;
51 name = var->name;
52 fullpath = var->fullpath;
53 rank = var->rank;
54 total_elems = var->total_elems;
55 zero_storage_size = var->zero_storage_size;
56 dtype = var->dtype;
57 unsupported_attr_dtype = var->unsupported_attr_dtype;
58 unsupported_dspace = var->unsupported_dspace;
59 coord_attr_add_path = false;
60
61 for (const auto &vattr:var->attrs) {
62 auto attr_unique = make_unique<Attribute>();
63 auto attr = attr_unique.release();
64 attr->name = vattr->name;
65 attr->newname = vattr->newname;
66 attr->dtype = vattr->dtype;
67 attr->count = vattr->count;
68 attr->strsize = vattr->strsize;
69 attr->fstrsize = vattr->fstrsize;
70 attr->value = vattr->value;
71 attrs.push_back(attr);
72 }
73
74 for (const auto &vdim:var->dims) {
75 auto dim_unique = make_unique<Dimension>(vdim->size);
76 auto dim = dim_unique.release();
77 dim->name = vdim->name;
78 dim->newname = vdim->newname;
79 dim->unlimited_dim = vdim->unlimited_dim;
80 dims.push_back(dim);
81 }
82
83 // For the coordinate variable specific fields, we just fill in the default one in the ctr
84 // If needed, the caller of this function should fill in those information after calling this function.
85 eos_type = OTHERVARS;
86 is_2dlatlon = false;
87 point_lower = 0.0;
88 point_upper = 0.0;
89 point_left = 0.0;
90 point_right = 0.0;
91 xdimsize = 0;
92 ydimsize = 0;
93 eos5_pixelreg = HE5_HDFE_CENTER;
94 eos5_origin = HE5_HDFE_GD_UL;
95 eos5_projcode = HE5_GCTP_GEO;
96 zone = -1;
97 sphere = 0;
98 std::fill_n(param, 13, 0);
99
100}
101
102//This method will effectively remove any dimnames like
103// ???/XDim or ???/YDim from the dimension name set.
104// Use this function in caution.
105void EOS5CFGrid::Update_Dimnamelist()
106{
107
108 BESDEBUG("h5", "coming to Update_Dimnamelist" <<endl);
109
110 // If I put both "XDim" and "YDim" into one for loop, Mac g++ compiler
111 // gives segmentation fault, which doesn't make sense.
112 // I simply split them into two loops. It doesn't affect performance much.
113 // KY 2012-2-14
114 for (auto it = this->vardimnames.begin(); it != this->vardimnames.end(); ++it) {
115 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash(*it);
116 if ("XDim" == xydimname_candidate) {
117 this->vardimnames.erase(*it);
118 break;
119 }
120 }
121
122 for (auto it = this->vardimnames.begin(); it != this->vardimnames.end(); ++it) {
123 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash(*it);
124 if ("YDim" == xydimname_candidate) {
125 this->vardimnames.erase(*it);
126 break;
127 }
128 }
129
130}
131
132// A destructor of EOS5File
133EOS5File::~EOS5File()
134{
135 for (vector<EOS5CVar *>::const_iterator i = this->cvars.begin(); i != this->cvars.end(); ++i)
136 delete *i;
137
138 for (vector<EOS5CFGrid *>::const_iterator i = this->eos5cfgrids.begin(); i != this->eos5cfgrids.end(); ++i)
139 delete *i;
140
141 for (vector<EOS5CFSwath *>::const_iterator i = this->eos5cfswaths.begin(); i != this->eos5cfswaths.end(); ++i)
142 delete *i;
143
144 for (vector<EOS5CFZa *>::const_iterator i = this->eos5cfzas.begin(); i != this->eos5cfzas.end(); ++i)
145 delete *i;
146
147}
148
149// Helper function to make the name follow the CF conventions.
150string EOS5File::get_CF_string(string s)
151{
152
153 // We need to remove the first "/" from the full name.
154 if (s[0] != '/')
155 return File::get_CF_string(s);
156 else {
157 s.erase(0, 1);
158 return File::get_CF_string(s);
159 }
160}
161
162// Retrieve the HDF5 information for HDF-EOS5
163void EOS5File::Retrieve_H5_Info(const char *file_fullpath, hid_t file_id, bool /*include_attr*/)
164{
165 // Since we need to check the attribute info in order to determine if the file is augmented to netCDF-4,
166 // we need to retrieve the attribute info also.
167 File::Retrieve_H5_Info(file_fullpath, file_id, true);
168}
169
171{
172
173 for (const auto &cvar:this->cvars) {
174
175 // When the coordinate variables exist in the file, retrieve the attribute values.
176 if ((CV_EXIST == cvar->cvartype) || (CV_MODIFY == cvar->cvartype)) {
177 for (const auto &attr:cvar->attrs)
178 Retrieve_H5_Attr_Value(attr, cvar->fullpath);
179
180 }
181 }
182
183}
184
185// Retrieve the attribute values for the HDF-EOS5
187{
188
190 for (const auto &cvar:this->cvars) {
191
192 // When the coordinate variables exist in the file, retrieve the attribute values.
193 if ((CV_EXIST == cvar->cvartype) || (CV_MODIFY == cvar->cvartype)) {
194 for (const auto &attr:cvar->attrs)
195 Retrieve_H5_Attr_Value(attr, cvar->fullpath);
196 }
197 }
198}
199
200// Adjust attribute value
201void EOS5File::Adjust_H5_Attr_Value(const Attribute* ) const
202{
203 // For future usage.
204
205}
206
207// Handle unsupported datatype
209{
210
211 if (true == check_ignored) {
212 Gen_Unsupported_Dtype_Info(include_attr);
213 }
214
215 File::Handle_Unsupported_Dtype(include_attr);
216 Handle_EOS5_Unsupported_Dtype(include_attr);
217}
218
219// Handle EOS5 unsupported datatype,add EOS5 coordinate variables
220void EOS5File::Handle_EOS5_Unsupported_Dtype(bool include_attr)
221{
222
223 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();) {
224 if (true == include_attr) {
225 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();) {
226 H5DataType temp_dtype = (*ira)->getType();
227 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
228 delete (*ira);
229 ira = (*ircv)->attrs.erase(ira);
230 }
231 else {
232 ++ira;
233
234 }
235 }
236 }
237
238 H5DataType temp_dtype = (*ircv)->getType();
239 if (!HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
240 delete (*ircv);
241 ircv = this->cvars.erase(ircv);
242 }
243 else {
244 ++ircv;
245 }
246 }
247}
248
249// Generate unsupported datatype information
250void EOS5File::Gen_Unsupported_Dtype_Info(bool include_attr)
251{
252
253 if (true == include_attr) {
254
255 File::Gen_Group_Unsupported_Dtype_Info();
256 File::Gen_Var_Unsupported_Dtype_Info();
257 Gen_VarAttr_Unsupported_Dtype_Info();
258
259 }
260
261}
262
263// Generate variable attribute datatype info.
264void EOS5File::Gen_VarAttr_Unsupported_Dtype_Info()
265{
266
267 // Dimension scale info for general variables
268 Gen_DimScale_VarAttr_Unsupported_Dtype_Info();
269
270 // HDF-EOS5 variable attribute unsupported datatype
271 Gen_EOS5_VarAttr_Unsupported_Dtype_Info();
272
273}
274
275void EOS5File::Gen_EOS5_VarAttr_Unsupported_Dtype_Info()
276{
277
278 for (const auto &cvar:this->cvars) {
279 // If the attribute REFERENCE_LIST comes with the attribute CLASS, the
280 // attribute REFERENCE_LIST is okay to ignore. No need to report.
281 bool is_ignored = ignored_dimscale_ref_list(cvar);
282 if (false == cvar->attrs.empty()) {
283 for (const auto &attr:cvar->attrs) {
284 H5DataType temp_dtype = attr->getType();
285 // TODO: check why 64-bit integer is included.
286 if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || (temp_dtype == H5INT64) ||(temp_dtype == H5UINT64)) {
287 // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
288 // is okay to ignore if the variable has another attribute
289 // CLASS="DIMENSION_SCALE"
290 if (("DIMENSION_LIST" != attr->name)
291 && ("REFERENCE_LIST" != attr->name || true == is_ignored))
292 this->add_ignored_info_attrs(false, cvar->fullpath, attr->name);
293 }
294 }
295 }
296 }
297}
298
299// Handle unsupported data space.
301{
302
303 // Generate unsupported info.
304 if (true == check_ignored) {
305 Gen_Unsupported_Dspace_Info();
306 }
307
309 Handle_EOS5_Unsupported_Dspace(include_attr);
310
311}
312
313// Handle EOS5 unsupported data space.
314void EOS5File::Handle_EOS5_Unsupported_Dspace(bool include_attr)
315{
316
317 if (true == this->unsupported_var_dspace) {
318 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();) {
319 if (true == (*ircv)->unsupported_dspace) {
320 delete (*ircv);
321 ircv = this->cvars.erase(ircv);
322 }
323 else {
324 ++ircv;
325 }
326 }
327 }
328
329 if (true == include_attr) {
330 if (true == this->unsupported_var_attr_dspace) {
331 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
332 if (false == (*ircv)->attrs.empty()) {
333 if (true == (*ircv)->unsupported_attr_dspace) {
334 for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();) {
335 if (0 == (*ira)->count) {
336 delete (*ira);
337 ira = (*ircv)->attrs.erase(ira);
338 }
339 else {
340 ++ira;
341 }
342 }
343 }
344 }
345 }
346 }
347 }
348}
349
350// Generating unsupported data space.
351void EOS5File::Gen_Unsupported_Dspace_Info()
352{
353
354 File::Gen_Unsupported_Dspace_Info();
355
356}
357
358// Handle other unsupported EOS5 information
360{
361
362 remove_netCDF_internal_attributes(include_attr);
363#if 0
364 if(true == include_attr) {
365 for (auto irv = this->vars.begin();
366 irv != this->vars.end(); ++irv) {
367 for (auto ira = (*irv)->attrs.begin();
368 ira != (*irv)->attrs.end();) {
369 if((*ira)->name == "CLASS") {
370 string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
371
372 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
373 // "DIMENSION_SCALE", which is 15.
374 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
375 delete((*ira));
376 ira = (*irv)->attrs.erase(ira);
377 }
378#if 0
379 else if(1) {// Add a BES key,also delete
380
381 }
382#endif
383 else {
384 ++ira;
385 }
386 }
387 //else if((*ira)->name == "NAME" && 1) {// Add a BES Key later if necessary
388 else if((*ira)->name == "NAME") {// Add a BES Key
389 string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
390 if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
391 delete((*ira));
392 ira =(*irv)->attrs.erase(ira);
393 }
394 else {
395 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
396 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
397 delete((*ira));
398 ira =(*irv)->attrs.erase(ira);
399 }
400 else {
401 ++ira;
402 }
403 }
404
405 }
406 else if((*ira)->name == "_Netcdf4Dimid") {
407 delete((*ira));
408 ira =(*irv)->attrs.erase(ira);
409 }
410
411 else {
412 ++ira;
413 }
414 }
415 }
416#endif
417 if(true == include_attr) {
418 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
419 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
420 if((*ira)->name == "CLASS") {
421 string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
422
423 // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
424 // "DIMENSION_SCALE", which is 15.
425 if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
426 delete(*ira);
427 ira = (*irv)->attrs.erase(ira);
428 // Add another block to set a key
429 }
430 else {
431 ++ira;
432 }
433 }
434 else if((*ira)->name == "NAME") {// Add a BES Key later
435 delete(*ira);
436 ira=(*irv)->attrs.erase(ira);
437 //"NAME" attribute causes the file netCDF-4 failed.
438#if 0
439 string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
440 if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
441 delete(*ira);
442 ira =(*irv)->attrs.erase(ira);
443 }
444 else {
445 string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
446 if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
447 delete(*ira);
448 ira =(*irv)->attrs.erase(ira);
449 }
450 else {
451 ++ira;
452 }
453 }
454#endif
455 }
456 else if((*ira)->name == "_Netcdf4Dimid") {
457 delete(*ira);
458 ira =(*irv)->attrs.erase(ira);
459 }
460
461 else {
462 ++ira;
463 }
464 }
465 }
466 }
467
468
469 // We cannot use the general routine from the base class since
470 // the information of ignored ECS metadata variables is transferred
471 // to DAS. The ignored ECS metadata variables should not be reported.
472 //File::Handle_Unsupported_Others(include_attr);
473 if (true == this->check_ignored && true == include_attr) {
474
475 // netCDF Java lifts the string size restriction for attributes. So comment out for the time being. KY 2018/08/10
476 if (true == HDF5RequestHandler::get_drop_long_string()) {
477#if 0
478 for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
479 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
480 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
481 this->add_ignored_droplongstr_hdr();
482 this->add_ignored_grp_longstr_info("/", (*ira)->name);
483 }
484 }
485 }
486
487 for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
488 for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
489 if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
490 if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
491 this->add_ignored_droplongstr_hdr();
492 this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
493 }
494 }
495
496 }
497 }
498#endif
499 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
500 if (true == Check_DropLongStr((*irv), nullptr)) {
501 string ecsmeta_grp = "/HDFEOS INFORMATION";
502 // Ignored ECS metadata should not be reported.
503 if ((*irv)->fullpath.find(ecsmeta_grp) != 0
504 || ((*irv)->fullpath.rfind("/") != ecsmeta_grp.size())) {
505 this->add_ignored_droplongstr_hdr();
506 this->add_ignored_var_longstr_info((*irv), nullptr);
507 }
508 }
509#if 0
510 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
511 if (true == Check_DropLongStr((*irv), (*ira))) {
512 this->add_ignored_droplongstr_hdr();
513 this->add_ignored_var_longstr_info((*irv), (*ira));
514 }
515 }
516#endif
517 }
518#if 0
519 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
520 for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
521 if (true == Check_DropLongStr((*irv), (*ira))) {
522 this->add_ignored_droplongstr_hdr();
523 this->add_ignored_var_longstr_info((*irv), (*ira));
524 }
525 }
526 }
527#endif
528 }
529 }
530
531 if (false == this->have_ignored) this->add_no_ignored_info();
532
533}
534
535// Adjust HDF-EOS5 dimension info.
537{
538
539 BESDEBUG("h5", "coming to Adjust_EOS5Dim_Info" <<endl);
540
541 // Condense redundant XDim, YDim in the grid/swath/za dimension list
542 for (unsigned int i = 0; i < strmeta_info->swath_list.size(); ++i) {
543 HE5Swath& he5s = strmeta_info->swath_list.at(i);
544
545 Adjust_EOS5Dim_List(he5s.dim_list);
546
547 // Correct the possible wrong dimension size,this only happens for the unlimited dimension,
548 // WE JUST NEED TO CORRECT the EOS group dimension size.
549 // STEPS:
550 // 1. Merge SWATH data_var_list and geo_var_list
551 // Function parameters will be the object dim. list(he5s.dim_list), EOS5Type(SWATH,GRID...) and varlist
552 // Need to use Obtain_Var_EOS5Type_GroupName to find var's group name and Get_Var_EOS5_Type(var) to find
553 // Var's EOS5Type.
554 // After checking group and type, check "if(he5v.name == var->name)" and change the he5v dim. size to var size.
555 if(this->have_udim == true) {
556 vector<HE5Var> svlist = he5s.geo_var_list;
557 svlist.insert(svlist.end(),he5s.data_var_list.begin(),he5s.data_var_list.end());
558 // Only apply when the unlimited dimension is found!! So we don't have to go over this for every file.
559 Adjust_EOS5DimSize_List(he5s.dim_list,svlist,SWATH,he5s.name);
560 }
561
562 for (unsigned int j = 0; j < he5s.geo_var_list.size(); ++j) {
563 Adjust_EOS5VarDim_Info((he5s.geo_var_list)[j].dim_list, he5s.dim_list, he5s.name, SWATH);
564 }
565 for (unsigned int j = 0; j < he5s.data_var_list.size(); ++j) {
566 Adjust_EOS5VarDim_Info((he5s.data_var_list)[j].dim_list, he5s.dim_list, he5s.name, SWATH);
567 }
568 }
569
570 for (unsigned int i = 0; i < strmeta_info->grid_list.size(); ++i) {
571
572 HE5Grid& he5g = strmeta_info->grid_list.at(i);
573
574 Adjust_EOS5Dim_List(he5g.dim_list);
575
576 // Correct possible wrong dimension size in the eosdim list.
577 if(this->have_udim == true) {
578 // Only apply when the unlimited dimension is found!! So we don't have to go over this for every file.
579 Adjust_EOS5DimSize_List(he5g.dim_list,he5g.data_var_list,GRID,he5g.name);
580 }
581
582 for (unsigned int j = 0; j < he5g.data_var_list.size(); ++j) {
583 Adjust_EOS5VarDim_Info((he5g.data_var_list)[j].dim_list, he5g.dim_list, he5g.name, GRID);
584 }
585 }
586
587 for (unsigned int i = 0; i < strmeta_info->za_list.size(); ++i) {
588 HE5Za& he5z = strmeta_info->za_list.at(i);
589
590 Adjust_EOS5Dim_List(he5z.dim_list);
591
592 // Correct possible wrong dimension size in the eosdim list.
593 if(this->have_udim == true) {
594 // Only apply when the unlimited dimension is found!! So we don't have to go over this for every file.
595 Adjust_EOS5DimSize_List(he5z.dim_list,he5z.data_var_list,ZA,he5z.name);
596 }
597
598 for (unsigned int j = 0; j < he5z.data_var_list.size(); ++j) {
599 Adjust_EOS5VarDim_Info((he5z.data_var_list)[j].dim_list, he5z.dim_list, he5z.name, ZA);
600 }
601 }
602}
603
604// Adjust HDF-EOS5 dimension list.
605void EOS5File::Adjust_EOS5Dim_List(vector<HE5Dim>& groupdimlist) const
606{
607
608 BESDEBUG("h5", "Coming to Adjust_EOS5Dim_List"<<endl);
609
610 // The negative dimension sizes are found in some HDF-EOS5 files.
611 // We need to remove them.
612 Remove_NegativeSizeDims(groupdimlist);
613
614 // Condense redundant XDim, YDim in the grid/swath/za dimension list
615 Condense_EOS5Dim_List(groupdimlist);
616
617}
618
619// The negative dimension sizes are found in some HDF-EOS5 files.
620// We need to remove them.
621void EOS5File::Remove_NegativeSizeDims(vector<HE5Dim>& groupdimlist) const
622{
623
624 BESDEBUG("h5", "Coming to Remove_NegativeSizeDims" <<endl);
625
626 // We find one product has dimension with name: Unlimited, size: -1; this dimension
627 // will not be used by any variables. The "Unlimited" dimension is useful for extended
628 // datasets when data is written. It is not useful for data accessing as far as I know.
629 // So we will remove it from the list.
630 // This algorithm will also remove any dimension with size <=0. KY 2011-1-14
631 // Note: Unlimited dimension is supported by the handler but not by using this "Unlimited" name.
632 // For the unlimited dimension support, check class Dimension and function Retrieve_H5_VarDim.
633 for (auto id = groupdimlist.begin(); id != groupdimlist.end();) {
634 if ((*id).size <= 0) {
635 id = groupdimlist.erase(id);
636 }
637 else {
638 ++id;
639 }
640 }
641}
642
643// Condense redundant XDim, YDim in the grid/swath/za dimension list
644// Some products use Xdim rather XDim, Ydim rather than Ydim.
645// This is significant for grids. We need to make them "XDim" and "YDim".
646// See comments of function Adjust_EOS5VarDim_Info for the reason.
647void EOS5File::Condense_EOS5Dim_List(vector<HE5Dim>& groupdimlist) const
648{
649
650 BESDEBUG("h5", "Coming to Condense_EOS5Dim_List"<<endl);
651 set<int> xdimsizes;
652 set<int> ydimsizes;
653 pair<set<int>::iterator, bool> setret;
654 vector<HE5Dim>::iterator id;
655
656 for (id = groupdimlist.begin(); id != groupdimlist.end();) {
657 if ("XDim" == (*id).name || "Xdim" == (*id).name) {
658 setret = xdimsizes.insert((*id).size);
659 if (false == setret.second) {
660 id = groupdimlist.erase(id);
661 }
662 else if ("Xdim" == (*id).name) {
663 (*id).name = "XDim";
664 ++id;
665 }
666 else {
667 ++id;
668 }
669
670 }
671 else {
672 ++id;
673 }
674 }
675
676 for (id = groupdimlist.begin(); id != groupdimlist.end();) {
677 if ("YDim" == (*id).name || "Ydim" == (*id).name) {
678 setret = ydimsizes.insert((*id).size);
679 if (false == setret.second) {
680 id = groupdimlist.erase(id);
681 }
682 else if ("Ydim" == (*id).name) {
683 (*id).name = "YDim";
684 ++id;
685 }
686 else {
687 ++id;
688 }
689 }
690 else {
691 ++id;
692 }
693 }
694}
695
696void EOS5File:: Adjust_EOS5DimSize_List(vector<HE5Dim>& eos5objdimlist,const vector<HE5Var> & eos5objvarlist,
697 const EOS5Type eos5type, const string & eos5objname) const
698{
699
700 set<string>updated_dimlist;
701 pair<set<string>::iterator,bool> set_insert_ret;
702
703 for(unsigned int i = 0; i<eos5objvarlist.size();i++) {
704 HE5Var he5v = eos5objvarlist.at(i);
705 for(unsigned int j = 0; j<he5v.dim_list.size();j++) {
706 HE5Dim he5d = he5v.dim_list.at(j);
707 set_insert_ret = updated_dimlist.insert(he5d.name);
708 if(set_insert_ret.second == true) {
709 // Find out the index of this dimension in eos5objdimlist
710 unsigned int objdimlist_index = 9999;
711 bool has_objdimlist_index = false;
712 for(unsigned int k = 0; k <eos5objdimlist.size();k++) {
713 if(eos5objdimlist[k].name == he5d.name) {
714 objdimlist_index = k;
715 has_objdimlist_index = true;
716 break;
717 }
718 }
719 if(has_objdimlist_index == false)
720 throw2("Cannot find the dimension in the EOS5 object dimension list for the dimension ", he5d.name);
721 for (const auto &var:this->vars) {
722
723 EOS5Type vartype = Get_Var_EOS5_Type(var);
724 // Compare the EOS5 object type: SWATH,GRID or ZA
725 // eos5objvarlist only stores the variable name, not the path. So we have to ensure the path matches.
726 if(vartype == eos5type) {
727 string var_eos5gname = Obtain_Var_EOS5Type_GroupName(var,vartype);
728 // Compare the EOS5 object name
729 // Now we need to match the var name from eos5objvarlist with the var name.
730 if(var_eos5gname == eos5objname) {
731 if(var->name == he5v.name) {
732 if (he5v.dim_list.size() != var->dims.size())
733 throw2("Number of dimensions don't match with the structmetadata for variable ", var->name);
734 // Change dimension size
735 (eos5objdimlist[objdimlist_index]).size = (var->dims[j])->size;
736 break;
737 }
738
739 }
740 }
741 }
742 }
743
744 }
745 // Don't need to go over every var, just find enough.
746 if(updated_dimlist.size() == eos5objdimlist.size())// Finish updating the eos5objdimlist
747 break;
748 }
749#if 0
750for(unsigned int k = 0; k <eos5objdimlist.size();k++) {
751 cerr<<"eos5 obj dim name is "<<eos5objdimlist[k].name << " Size is "<< eos5objdimlist[k].size << endl;
752}
753#endif
754}
755
756
757// Adjust HDF-EOS5 Variable,dimension information.
758void EOS5File::Adjust_EOS5VarDim_Info(vector<HE5Dim>& vardimlist, vector<HE5Dim>& groupdimlist,
759 const string & eos5_obj_name, EOS5Type eos5type)
760{
761
762 BESDEBUG("h5", "Coming to Adjust_EOS5VarDim_Info"<<endl);
763 set<string> dimnamelist;
764 pair<set<string>::iterator, bool> setret;
765
766 // For EOS5 Grids: Dimension names XDim and YDim are predefined.
767 // Even the data producers make a mistake to define "xdim", "ydim" etc in the grid
768 // dimension name list, the variable will still pick up "XDim" and "YDim" as their
769 // dimension names So we assume that 'xdim", "ydim" etc will never appear in the
770 // variable name list.
771 for (unsigned int i = 0; i < vardimlist.size(); ++i) {
772
773 HE5Dim& he5d = vardimlist.at(i);
774 bool dim_in_groupdimlist = false;
775 for (unsigned int j = 0; j < groupdimlist.size(); ++j) {
776 HE5Dim he5gd = groupdimlist.at(j);
777 if (he5gd.name == he5d.name) {
778 he5d.size = he5gd.size;
779 dim_in_groupdimlist = true;
780 break;
781 }
782 }
783
784 if (false == dim_in_groupdimlist)
785 throw2("The EOS5 group dimension name list doesn't include the dimension ", he5d.name);
786
787 // Some variables have data like float foo[nlevel= 10][nlevel= 10],need to make the dimname unique
788 // to ensure the coordinate variables to be generated correctly.
789 //
790 setret = dimnamelist.insert(he5d.name);
791 if (false == setret.second) {
792 int clash_index = 1;
793 string temp_clashname = he5d.name + '_';
794 HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
795
796 string ori_dimname = he5d.name;
797
798 he5d.name = temp_clashname;
799
800 // We have to add this dim. to this dim. list if this dim doesn't exist in the dim. list.
801 bool dim_exist = false;
802 for (unsigned int j = 0; j < groupdimlist.size(); ++j) {
803 if (he5d.name == groupdimlist[j].name && he5d.size == groupdimlist[j].size) {
804 dim_exist = true;
805 break;
806 }
807 }
808
809 // Add the new dim. to the dim. list
810 if (false == dim_exist) {
811 ori_dimname = eos5_obj_name + "/" + ori_dimname;
812 string dup_dimname = eos5_obj_name + "/" + he5d.name;
813 if (GRID == eos5type) {
814 ori_dimname = "/GRIDS/" + ori_dimname;
815 dup_dimname = "/GRIDS/" + dup_dimname;
816 }
817 else if (SWATH == eos5type) {
818 ori_dimname = "/SWATHS/" + ori_dimname;
819 dup_dimname = "/SWATHS/" + dup_dimname;
820 }
821 else if (ZA == eos5type) {
822 ori_dimname = "/ZAS/" + ori_dimname;
823 dup_dimname = "/ZAS/" + dup_dimname;
824 }
825
826 // Need to remember the dimname and dupdimname relation in case the situation happens at other variables.
827 dimname_to_dupdimnamelist.insert(pair<string, string>(ori_dimname, dup_dimname));
828 groupdimlist.push_back(he5d);
829 }
830
831 } //end of if(false == setret.second)
832 } // end of for (unsigned int i = 0; i <vardimlist.size(); ++i)
833
834}
835
836// Add EOS5 FIle information
837void EOS5File::Add_EOS5File_Info(HE5Parser * strmeta_info, bool grids_mllcv)
838{
839
840 BESDEBUG("h5", "Coming to Add_EOS5File_Info"<<endl);
841 string fslash_str = "/";
842 string grid_str = "/GRIDS/";
843 string swath_str = "/SWATHS/";
844 string za_str = "/ZAS/";
845
846 // Assign the original number of grids. These number will be useful
847 // to generate the final DAP object names for grids/swaths/zas that don't have coordinate
848 // variables. For example, OMI level 2G product has latitude and longitude with 3-D arrays.
849 // There is no way to make the lat/lon become CF coordinate variables. To still follow the
850 // HDF-EOS5 object name conventions, the original number of grid is expected.
851 // Since this happens only for grids, we just keep the original number for grids now.
852 this->orig_num_grids = (int)(strmeta_info->grid_list.size());
853
854 //
855 for (unsigned int i = 0; i < strmeta_info->grid_list.size(); i++) {
856 HE5Grid he5g = strmeta_info->grid_list.at(i);
857 auto eos5grid_unique = make_unique<EOS5CFGrid>();
858 auto eos5grid = eos5grid_unique.release();
859 eos5grid->name = he5g.name;
860 eos5grid->dimnames.resize(he5g.dim_list.size());
861
862 for (unsigned int j = 0; j < he5g.dim_list.size(); j++) {
863
864 HE5Dim he5d = he5g.dim_list.at(j);
865 if ("XDim" == he5d.name) eos5grid->xdimsize = he5d.size;
866 if ("YDim" == he5d.name) eos5grid->ydimsize = he5d.size;
867
868 // Here we add the grid name connecting with "/" to
869 // adjust the dim names to assure the uniqueness of
870 // the dimension names for multiple grids.
871 // For single grid, we don't have to do that.
872 // However, considering the rare case that one
873 // can have one grid, one swath and one za, the dimnames
874 // without using the group names may cause the name clashing.
875 // so still add the group path.
876 string unique_dimname = grid_str + he5g.name + fslash_str + he5d.name;
877
878 (eos5grid->dimnames)[j] = unique_dimname;
879
880 pair<map<hsize_t, string>::iterator, bool> mapret1;
881 mapret1 = eos5grid->dimsizes_to_dimnames.insert(pair<hsize_t, string>((hsize_t) he5d.size, unique_dimname));
882
883 // Create the dimname to dimsize map. This will be used to create the missing coordinate
884 // variables. Based on our understanding, dimension names should be unique for
885 // grid/swath/zonal average. We will throw an error if we find the same dimension name used.
886 pair<map<string, hsize_t>::iterator, bool> mapret2;
887 mapret2 = eos5grid->dimnames_to_dimsizes.insert(pair<string, hsize_t>(unique_dimname, (hsize_t) he5d.size));
888 if (false == mapret2.second)
889 throw5("The dimension name ", unique_dimname, " with the dimension size ", he5d.size, "is not unique");
890
891 } // "for (int j=0; j <he5g.dim_list.size(); j++)"
892
893 // Check if having Latitude/Longitude. We will use those Latitude and Longitude as CVs if possible.
894 EOS5SwathGrid_Set_LatLon_Flags(eos5grid, he5g.data_var_list);
895
896 // Using map for possible the third-D CVs.
897 map<string, string> dnames_to_1dvnames;
898 EOS5Handle_nonlatlon_dimcvars(he5g.data_var_list, GRID, he5g.name, dnames_to_1dvnames);
899 eos5grid->dnames_to_1dvnames = dnames_to_1dvnames;
900 eos5grid->point_lower = he5g.point_lower;
901 eos5grid->point_upper = he5g.point_upper;
902 eos5grid->point_left = he5g.point_left;
903 eos5grid->point_right = he5g.point_right;
904
905 eos5grid->eos5_pixelreg = he5g.pixelregistration;
906 eos5grid->eos5_origin = he5g.gridorigin;
907 eos5grid->eos5_projcode = he5g.projection;
908
909 for (unsigned int k = 0; k < 13; k++)
910 eos5grid->param[k] = he5g.param[k];
911 eos5grid->zone = he5g.zone;
912 eos5grid->sphere = he5g.sphere;
913
914 this->eos5cfgrids.push_back(eos5grid);
915
916 } // "for(int i=0; i < strmeta_info->grid_list.size(); i++)"
917
918 // Adding this here seems a hack.
919 this->grids_multi_latloncvs = grids_mllcv;
920
921 // Second Swath
922 for (unsigned int i = 0; i < strmeta_info->swath_list.size(); i++) {
923
924 HE5Swath he5s = strmeta_info->swath_list.at(i);
925 auto eos5swath_unique = make_unique<EOS5CFSwath>();
926 auto eos5swath = eos5swath_unique.release();
927 eos5swath->name = he5s.name;
928 eos5swath->dimnames.resize(he5s.dim_list.size());
929
930 for (unsigned int j = 0; j < he5s.dim_list.size(); j++) {
931
932 HE5Dim he5d = he5s.dim_list.at(j);
933
934 // Here we add the swath name connecting with "/" to
935 // adjust the dim names to assure the uniqueness of
936 // the dimension names for multiple swaths.
937 // For single swath, we don't have to do that.
938 // However, considering the rare case that one
939 // can have one grid, one swath and one za, the dimnames
940 // without using the group names may cause the name clashing.
941 // so still add the group path.
942 string unique_dimname = swath_str + he5s.name + fslash_str + he5d.name;
943 (eos5swath->dimnames)[j] = unique_dimname;
944
945 // Create the dimsize to dimname map for those variables missing dimension names.
946 // Note: For different dimnames sharing the same dimsizes, we only pick up the first one.
947 pair<map<hsize_t, string>::iterator, bool> mapret1;
948 mapret1 = eos5swath->dimsizes_to_dimnames.insert(
949 pair<hsize_t, string>((hsize_t) he5d.size, unique_dimname));
950
951 // Create the dimname to dimsize map. This will be used to create the missing coordinate
952 // variables. Based on our understanding, dimension names should be unique for
953 // grid/swath/zonal average. We will throw an error if we find the same dimension name used.
954 pair<map<string, hsize_t>::iterator, bool> mapret2;
955 mapret2 = eos5swath->dimnames_to_dimsizes.insert(
956 pair<string, hsize_t>(unique_dimname, (hsize_t) he5d.size));
957 if (false == mapret2.second)
958 throw5("The dimension name ", unique_dimname, " with the dimension size ", he5d.size, "is not unique");
959
960 } // "for (int j=0; j <he5s.dim_list.size(); j++)"
961
962 // Check if having Latitude/Longitude.
963 EOS5SwathGrid_Set_LatLon_Flags(eos5swath, he5s.geo_var_list);
964
965 // Using map for possible the third-D CVs.
966 map<string, string> dnames_to_geo1dvnames;
967 EOS5Handle_nonlatlon_dimcvars(he5s.geo_var_list, SWATH, he5s.name, dnames_to_geo1dvnames);
968 eos5swath->dnames_to_geo1dvnames = dnames_to_geo1dvnames;
969 this->eos5cfswaths.push_back(eos5swath);
970 } // "for (int i=0; i < strmeta_info->swath_list.size(); i++)"
971
972 // Third Zonal average
973 for (unsigned int i = 0; i < strmeta_info->za_list.size(); i++) {
974
975 HE5Za he5z = strmeta_info->za_list.at(i);
976
977 auto eos5za_unique = make_unique<EOS5CFZa>();
978 auto eos5za = eos5za_unique.release();
979 eos5za->name = he5z.name;
980 eos5za->dimnames.resize(he5z.dim_list.size());
981
982 for (unsigned int j = 0; j < he5z.dim_list.size(); j++) {
983
984 HE5Dim he5d = he5z.dim_list.at(j);
985
986 // Here we add the grid name connecting with "/" to
987 // adjust the dim names to assure the uniqueness of
988 // the dimension names for multiple grids.
989 // For single grid, we don't have to do that.
990 string unique_dimname = za_str + he5z.name + fslash_str + he5d.name;
991 (eos5za->dimnames)[j] = unique_dimname;
992 pair<map<hsize_t, string>::iterator, bool> mapret1;
993 mapret1 = eos5za->dimsizes_to_dimnames.insert(pair<hsize_t, string>((hsize_t) he5d.size, unique_dimname));
994
995 // Create the dimname to dimsize map. This will be used to create the missing coordinate
996 // variables. Based on our understanding, dimension names should be unique for
997 // grid/swath/zonal average. We will throw an error if we find the same dimension name used.
998 pair<map<string, hsize_t>::iterator, bool> mapret2;
999 mapret2 = eos5za->dimnames_to_dimsizes.insert(pair<string, hsize_t>(unique_dimname, (hsize_t) he5d.size));
1000 if (false == mapret2.second)
1001 throw5("The dimension name ", unique_dimname, " with the dimension size ", he5d.size, "is not unique");
1002
1003 } // "for (int j=0; j <he5z.dim_list.size(); j++) "
1004
1005 // Using map for possible the third-D CVs.
1006 map<string, string> dnames_to_1dvnames;
1007 EOS5Handle_nonlatlon_dimcvars(he5z.data_var_list, ZA, he5z.name, dnames_to_1dvnames);
1008 eos5za->dnames_to_1dvnames = dnames_to_1dvnames;
1009 this->eos5cfzas.push_back(eos5za);
1010 } // "for(int i=0; i < strmeta_info->za_list.size(); i++)"
1011
1012// Debugging info,leave it here. They are very useful.
1013#if 0
1014 for (auto irg = this->eos5cfgrids.begin();
1015 irg != this->eos5cfgrids.end(); ++irg) {
1016
1017 cerr<<"grid name "<<(*irg)->name <<endl;
1018 cerr<<"eos5_pixelreg"<<(*irg)->eos5_pixelreg <<endl;
1019 cerr<<"eos5_origin"<<(*irg)->eos5_pixelreg <<endl;
1020 cerr<<"point_lower "<<(*irg)->point_lower <<endl;
1021 cerr<<"xdimsize "<<(*irg)->xdimsize <<endl;
1022
1023 if((*irg)->has_g2dlatlon) cerr<<"has g2dlatlon"<<endl;
1024 if((*irg)->has_2dlatlon) cerr<<"has 2dlatlon"<<endl;
1025 if((*irg)->has_1dlatlon) cerr<<"has 1dlatlon"<<endl;
1026 if((*irg)->has_nolatlon) cerr<<"has no latlon" <<endl;
1027 if(this->grids_multi_latloncvs) cerr<<"having multiple lat/lon from structmeta" <<endl;
1028 else cerr<<"no multiple lat/lon from structmeta" <<endl;
1029
1030// Dimension names
1031 "h5","number of dimensions "<<(*irg)->dimnames.size() <<endl;
1032 for (auto irv = (*irg)->dimnames.begin();
1033 irv != (*irg)->dimnames.end(); ++irv)
1034 cerr<<"dim names" <<*irv <<endl;
1035
1036// mapping size to name
1037 for (auto im1 = (*irg)->dimsizes_to_dimnames.begin();
1038 im1 !=(*irg)->dimsizes_to_dimnames.end();++im1) {
1039 cerr<<"size to name "<< (int)((*im1).first) <<"=> "<<(*im1).second <<endl;
1040 }
1041
1042// mapping dime names to 1d varname
1043 for (auto im2 = (*irg)->dnames_to_1dvnames.begin();
1044 im2 !=(*irg)->dnames_to_1dvnames.end();++im2) {
1045 cerr<<"dimanme to 1d var name "<< (*im2).first <<"=> "<<(*im2).second <<endl;
1046 }
1047 }
1048
1049//Swath
1050 for (auto irg = this->eos5cfswaths.begin();
1051 irg != this->eos5cfswaths.end(); ++irg) {
1052
1053 cerr<<"swath name "<<(*irg)->name <<endl;
1054 if((*irg)->has_nolatlon) cerr<<"has no latlon" <<endl;
1055 if((*irg)->has_1dlatlon) cerr<<"has 1dlatlon"<<endl;
1056 if((*irg)->has_2dlatlon) cerr<<"has 2dlatlon"<<endl;
1057
1058// Dimension names
1059 for (auto irv = (*irg)->dimnames.begin();
1060 irv != (*irg)->dimnames.end(); ++irv)
1061 cerr<<"dim names" <<*irv <<endl;
1062
1063// mapping size to name
1064 for (auto im1 = (*irg)->dimsizes_to_dimnames.begin();
1065 im1 !=(*irg)->dimsizes_to_dimnames.end();++im1) {
1066 cerr<<"size to name "<< (int)((*im1).first) <<"=> "<<(*im1).second <<endl;
1067 }
1068
1069// mapping dime names to 1d varname
1070 for (auto im2 = (*irg)->dnames_to_geo1dvnames.begin();
1071 im2 !=(*irg)->dnames_to_geo1dvnames.end();++im2) {
1072 cerr<<"dimname to 1d varname "<< (*im2).first <<"=> "<<(*im2).second <<endl;
1073 }
1074 }
1075
1076 for (auto irg = this->eos5cfzas.begin();
1077 irg != this->eos5cfzas.end(); ++irg) {
1078
1079 cerr<<"za name now"<<(*irg)->name <<endl;
1080
1081// Dimension names
1082 for (auto irv = (*irg)->dimnames.begin();
1083 irv != (*irg)->dimnames.end(); ++irv)
1084 cerr<<"dim names" <<*irv <<endl;
1085
1086// mapping size to name
1087 for (auto im1 = (*irg)->dimsizes_to_dimnames.begin();
1088 im1 !=(*irg)->dimsizes_to_dimnames.end();++im1) {
1089 cerr<<"size to name "<< (int)((*im1).first) <<"=> "<<(*im1).second <<endl;
1090 }
1091
1092// mapping dime names to 1d varname
1093 for (auto im2 = (*irg)->dnames_to_1dvnames.begin();
1094 im2 !=(*irg)->dnames_to_1dvnames.end();++im2) {
1095 cerr<<"dimname to 1d varname "<< (*im2).first <<"=> "<<(*im2).second <<endl;
1096 }
1097 }
1098#endif
1099
1100}
1101
1102// Check if EOS5 Swath and Grid hold Latitude and Longitude fields.
1103template<class T>
1104void EOS5File::EOS5SwathGrid_Set_LatLon_Flags(T* eos5gridswath, vector<HE5Var> &eos5varlist) const
1105{
1106
1107 BESDEBUG("h5", "Coming to EOS5SwathGrid_Set_LatLon_Flags"<<endl);
1108 bool find_lat = false;
1109 bool find_lon = false;
1110 bool has_1dlat = false;
1111 bool has_1dlon = false;
1112 bool has_2dlat = false;
1113 string lat_xdimname;
1114 string lat_ydimname;
1115 string lon_xdimname;
1116 string lon_ydimname;
1117 bool has_2dlon = false;
1118 bool has_g2dlat = false;
1119 bool has_g2dlon = false;
1120
1121 for (unsigned int i = 0; i < eos5varlist.size(); ++i) {
1122 HE5Var he5v = eos5varlist.at(i);
1123 if ("Latitude" == he5v.name) {
1124 find_lat = true;
1125 auto num_dims = (int)(he5v.dim_list.size());
1126 if (1 == num_dims)
1127 has_1dlat = true;
1128 else if (2 == num_dims) {
1129 lat_ydimname = (he5v.dim_list)[0].name;
1130 lat_xdimname = (he5v.dim_list)[1].name;
1131 has_2dlat = true;
1132 }
1133 else if (num_dims > 2)
1134 has_g2dlat = true;
1135 else
1136 throw1("The number of dimension should not be 0 for grids or swaths");
1137 } // "if ("Latitude" == he5v.name)"
1138
1139 if ("Longitude" == he5v.name) {
1140 find_lon = true;
1141 auto num_dims = (int)(he5v.dim_list.size());
1142 if (1 == num_dims)
1143 has_1dlon = true;
1144 else if (2 == num_dims) {
1145 lon_ydimname = (he5v.dim_list)[0].name;
1146 lon_xdimname = (he5v.dim_list)[1].name;
1147 has_2dlon = true;
1148 }
1149 else if (num_dims > 2)
1150 has_g2dlon = true;
1151 else
1152 throw1("The number of dimension should not be 0 for grids or swaths");
1153 } // "if ("Longitude" == he5v.name)"
1154
1155 if (true == find_lat && true == find_lon) {
1156 if (true == has_1dlat && true == has_1dlon) eos5gridswath->has_1dlatlon = true;
1157
1158 // Make sure we have lat[YDIM][XDIM] and lon[YDIM][XDIM]
1159 if (true == has_2dlat && true == has_2dlon && lat_ydimname == lon_ydimname && lat_xdimname == lon_xdimname)
1160 eos5gridswath->has_2dlatlon = true;
1161
1162 if (true == has_g2dlat && true == has_g2dlon) eos5gridswath->has_g2dlatlon = true;
1163
1164 eos5gridswath->has_nolatlon = false;
1165 break;
1166 } // "if (true == find_lat && true == find_lon) "
1167 } // "for (unsigned int i = 0; i < eos5varlist.size(); ++i)"
1168}
1169
1170// This function builds up the map from dimension names to coordinate variables
1171// for non-latitude and longitude fields.
1172void EOS5File::EOS5Handle_nonlatlon_dimcvars(vector<HE5Var> & eos5varlist, EOS5Type eos5type, const string &groupname,
1173 map<string, string>& dnamesgeo1dvnames) const
1174{
1175
1176 BESDEBUG("h5", "Coming to EOS5Handle_nonlatlon_dimcvars"<<endl);
1177
1178 set<string> nocvdimnames;
1179 string grid_str = "/GRIDS/";
1180 string xdim_str = "XDim";
1181 string ydim_str = "YDim";
1182 string fslash_str = "/";
1183 string eos5typestr;
1184
1185 if (GRID == eos5type) {
1186 string xdimname = grid_str + groupname + fslash_str + xdim_str;
1187 nocvdimnames.insert(xdimname);
1188 string ydimname = grid_str + groupname + fslash_str + ydim_str;
1189 nocvdimnames.insert(ydimname);
1190 eos5typestr = "/GRIDS/";
1191 }
1192 else if (SWATH == eos5type)
1193 eos5typestr = "/SWATHS/";
1194 else if (ZA == eos5type)
1195 eos5typestr = "/ZAS/";
1196 else
1197 throw1("Unsupported HDF-EOS5 type, this type is not swath, grid or zonal average");
1198
1199 // This assumption is pretty bold. It says: Any 1-D var that has a unique dim. name
1200 // in the var list is a 3rd-dim cv. We need to review this as time goes on. KY 2017-10-19
1201 pair<map<string, string>::iterator, bool> mapret;
1202 for (unsigned int i = 0; i < eos5varlist.size(); ++i) {
1203 HE5Var he5v = eos5varlist.at(i);
1204 if (1 == he5v.dim_list.size()) {
1205 HE5Dim he5d = he5v.dim_list.at(0);
1206 string dimname;
1207 dimname = eos5typestr + groupname + fslash_str + he5d.name;
1208 string varname; // using the new var name format
1209 varname = eos5typestr + groupname + fslash_str + he5v.name;
1210 mapret = dnamesgeo1dvnames.insert(pair<string, string>(dimname, varname));
1211
1212 // If another geo field already shares the same dimname, we need to
1213 // disqualify this geofield as the coordinate variable since it is not
1214 // unique anymore.
1215 if (false == mapret.second) nocvdimnames.insert(dimname);
1216 }
1217 }
1218
1219 // Manage the coordinate variables. We only want to leave fields that uniquely hold
1220 // the dimension name to be the possible cv candidate.
1221 for (auto itset = nocvdimnames.begin(); itset != nocvdimnames.end(); ++itset)
1222 dnamesgeo1dvnames.erase(*itset);
1223}
1224
1225// Adjust variable names after obtain the parsing information.
1227{
1228
1229 BESDEBUG("h5", "Coming to Adjust_Var_NewName_After_Parsing"<<endl);
1230 for (const auto &var:this->vars) {
1231 Obtain_Var_NewName(var);
1232 }
1233}
1234
1235void EOS5File::Obtain_Var_NewName(Var *var) const
1236{
1237
1238 BESDEBUG("h5", "Coming to Obtain_Var_NewName"<<endl);
1239 string fslash_str = "/";
1240 string eos5typestr;
1241
1242 EOS5Type vartype = Get_Var_EOS5_Type(var);
1243
1244 // Actually the newname is used to check if we have the existing
1245 // third dimension coordinate variable. To avoid the check of
1246 // fullpath again, we will make newname to have the unique information
1247 // in the path to identify the objects(Essentially "HDFEOS" is removed).
1248 switch (vartype) {
1249 case GRID: {
1250 eos5typestr = "/GRIDS/";
1251 string eos5_groupname = Obtain_Var_EOS5Type_GroupName(var, vartype);
1252#if 0
1253 // var->newname = ((1 == num_grids)?var->name:
1254 // eos5typestr + eos5_groupname + fslash_str + var->name);
1255#endif
1256 var->newname = eos5typestr + eos5_groupname + fslash_str + var->name;
1257 }
1258 break;
1259
1260 case SWATH: {
1261 eos5typestr = "/SWATHS/";
1262 string eos5_groupname = Obtain_Var_EOS5Type_GroupName(var, vartype);
1263#if 0
1264 // var->newname = ((1 == num_swaths)?var->name:
1265 // eos5typestr + eos5_groupname + fslash_str + var->name);
1266#endif
1267 var->newname = eos5typestr + eos5_groupname + fslash_str + var->name;
1268 }
1269 break;
1270 case ZA: {
1271 eos5typestr = "/ZAS/";
1272 string eos5_groupname = Obtain_Var_EOS5Type_GroupName(var, vartype);
1273#if 0
1274 // var->newname = ((1 == num_zas)?var->name:
1275 // eos5typestr + eos5_groupname + fslash_str + var->name);
1276#endif
1277 var->newname = eos5typestr + eos5_groupname + fslash_str + var->name;
1278 }
1279 break;
1280 case OTHERVARS: {
1281 string eos5infopath = "/HDFEOS INFORMATION";
1282 if (var->fullpath.size() > eos5infopath.size()) {
1283 if (eos5infopath == var->fullpath.substr(0, eos5infopath.size())) var->newname = var->name;
1284 }
1285 else
1286 var->newname = var->fullpath;
1287 }
1288 break;
1289 default:
1290 throw1("Non-supported EOS type");
1291 }
1292}
1293
1294// Get the HDF-EOS5 type: The type is either grids, swaths or zonal average
1295EOS5Type EOS5File::Get_Var_EOS5_Type(const Var* var) const
1296{
1297
1298 BESDEBUG("h5", "Coming to Get_Var_EOS5_Type"<<endl);
1299
1300 string EOS5GRIDPATH = "/HDFEOS/GRIDS";
1301 string EOS5SWATHPATH = "/HDFEOS/SWATHS";
1302 string EOS5ZAPATH = "/HDFEOS/ZAS";
1303
1304 if (var->fullpath.size() >= EOS5GRIDPATH.size()) {
1305 if (EOS5GRIDPATH == var->fullpath.substr(0, EOS5GRIDPATH.size())) return GRID;
1306 }
1307 if (var->fullpath.size() >= EOS5SWATHPATH.size()) {
1308 if (EOS5SWATHPATH == var->fullpath.substr(0, EOS5SWATHPATH.size())) return SWATH;
1309 }
1310 if (var->fullpath.size() >= EOS5ZAPATH.size()) {
1311 if (EOS5ZAPATH == var->fullpath.substr(0, EOS5ZAPATH.size())) return ZA;
1312 }
1313 return OTHERVARS;
1314
1315}
1316
1317// Add dimension information from the parseing info.
1319{
1320
1321 BESDEBUG("h5", "Coming to Add_Dim_Name"<<endl);
1322 for (const auto &var:this->vars) {
1323 Obtain_Var_Dims(var, strmeta_info);
1324#if 0
1325 for (auto ird = (*irv)->dims.begin();
1326 ird != (*irv)->dims.end();++ird) {
1327 cerr<<"dim name right after change "<<(*ird)->newname <<endl;
1328 }
1329#endif
1330
1331 }
1332}
1333
1334// CHECK if finding the same variables from the parser.
1335bool EOS5File::Obtain_Var_Dims(const Var *var, HE5Parser * strmeta_info)
1336{
1337
1338 BESDEBUG("h5", "Coming to Obtain_Var_Dims"<<endl);
1339 string varname_from_parser;
1340 EOS5Type vartype = Get_Var_EOS5_Type(var);
1341
1342 if (GRID == vartype) {
1343
1344 auto num_grids = (int)(strmeta_info->grid_list.size());
1345
1346 for (int i = 0; i < num_grids; ++i) {
1347 HE5Grid he5g = strmeta_info->grid_list.at(i);
1348 if (he5g.name == Obtain_Var_EOS5Type_GroupName(var, vartype)) {
1349 EOS5CFGrid *eos5cfgrid = (this->eos5cfgrids)[i];
1350 bool var_is_parsed = Set_Var_Dims(eos5cfgrid, var, he5g.data_var_list, he5g.name, num_grids, GRID);
1351 if (false == var_is_parsed) {
1352 map<hsize_t, string> dimsizes_to_dimnames = eos5cfgrid->dimsizes_to_dimnames;
1353 // Check if this grid includes data fields(variables) that don't have any dimension names.
1354 // This rarely happens. But we do find one NASA Aura product that has this problem. Although
1355 // this has been fixed, we should anticipiate that the similar problem may happen in the future.
1356 // So check here to avoid the potential problems. KY 2012-1-9
1357 Set_NonParse_Var_Dims(eos5cfgrid, var, dimsizes_to_dimnames, num_grids, vartype);
1358 }
1359 }
1360 }
1361
1362 }
1363 else if (SWATH == vartype) {
1364
1365 auto num_swaths = (int)(strmeta_info->swath_list.size());
1366
1367 for (int i = 0; i < num_swaths; ++i) {
1368
1369 HE5Swath he5s = strmeta_info->swath_list.at(i);
1370
1371 if (he5s.name == Obtain_Var_EOS5Type_GroupName(var, vartype)) {
1372
1373 EOS5CFSwath *eos5cfswath = (this->eos5cfswaths)[i];
1374
1375 bool var_is_parsed = true;
1376 int swath_fieldtype_flag = Check_EOS5Swath_FieldType(var);
1377 if (1 == swath_fieldtype_flag)
1378 var_is_parsed = Set_Var_Dims(eos5cfswath, var, he5s.geo_var_list, he5s.name, num_swaths, SWATH);
1379 else if (0 == swath_fieldtype_flag)
1380 var_is_parsed = Set_Var_Dims(eos5cfswath, var, he5s.data_var_list, he5s.name, num_swaths, SWATH);
1381 else
1382 // Neither Geo nor Data(For example, added by the augmentation tool)
1383 var_is_parsed = false;
1384
1385 if (false == var_is_parsed) {
1386 map<hsize_t, string> dimsizes_to_dimnames = eos5cfswath->dimsizes_to_dimnames;
1387 Set_NonParse_Var_Dims(eos5cfswath, var, dimsizes_to_dimnames, num_swaths, vartype);
1388 }
1389 } // end of inner if
1390 } // end of for
1391 } // end of else if
1392
1393 else if (ZA == vartype) {
1394
1395 auto num_zas = (int)(strmeta_info->za_list.size());
1396
1397 for (int i = 0; i < num_zas; ++i) {
1398 HE5Za he5z = strmeta_info->za_list.at(i);
1399 if (he5z.name == Obtain_Var_EOS5Type_GroupName(var, vartype)) {
1400 EOS5CFZa *eos5cfza = (this->eos5cfzas)[i];
1401 bool var_is_parsed = Set_Var_Dims(eos5cfza, var, he5z.data_var_list, he5z.name, num_zas, ZA);
1402 if (false == var_is_parsed) {
1403 map<hsize_t, string> dimsizes_to_dimnames = eos5cfza->dimsizes_to_dimnames;
1404 Set_NonParse_Var_Dims(eos5cfza, var, dimsizes_to_dimnames, num_zas, vartype);
1405 }
1406 }
1407 }
1408 }
1409 return false;
1410}
1411
1412// Set dimension info.(dimension names and sizes) to variables.
1413template<class T>
1414bool EOS5File::Set_Var_Dims(T* eos5data, const Var *var, vector<HE5Var> &he5var, const string& groupname, int num_groups,
1415 EOS5Type eos5type)
1416{
1417
1418 BESDEBUG("h5", "Coming to Set_Var_Dims"<<endl);
1419
1420 bool is_parsed = false;
1421 string eos5typestr;
1422 string fslash_str = "/";
1423
1424 if (GRID == eos5type)
1425 eos5typestr = "/GRIDS/";
1426 else if (SWATH == eos5type)
1427 eos5typestr = "/SWATHS/";
1428 else if (ZA == eos5type)
1429 eos5typestr = "/ZAS/";
1430 else
1431 throw1("Unsupported HDF-EOS5 type, this type is not swath, grid or zonal average");
1432
1433 for (unsigned int i = 0; i < he5var.size(); i++) {
1434
1435 HE5Var he5v = he5var.at(i);
1436
1437 if (he5v.name == var->name) {
1438 if (he5v.dim_list.size() != var->dims.size())
1439 throw2("Number of dimensions don't match with the structmetadata for variable ", var->name);
1440 is_parsed = true;
1441
1442 // Some variables have the same dim. names shared. For examples, we
1443 // see variables that have int foo[nlevels][nlevels]. To generate the CVs,
1444 // we have to make the dimension name unique for one variable. So we will
1445 // change the dimension names. The variable for the same example will be
1446 // int foo[nlevels][nlevels_1]. Note this is not required by CF conventions.
1447 // This is simply due to the missing of the third coordinate variable for some
1448 // NASA products. Another way is to totally ignore this kind of variables which
1449 // we will wait for users' responses.
1450
1451 // Here is the killer, if different dim. names share the same size,
1452 // Currently there are no ways to know which dimension name is corresponding to
1453 // which size. HDF-EOS model gives too much freedom to users. The DimList in
1454 // the StructMetadata doesn't reflect the order at all. See two example files
1455 // CH4 in TES-Aura_L3-CH4_r0000010410_F01_07.he5 and NO2DayColumn in
1456 // HIRDLS-Aura_L3SCOL_v06-00-00-c02_2005d022-2008d077.he5.
1457 // Fortunately it seems that it doesn't matter for us to make the mapping from
1458 // dimension names to coordinate variables.
1459 // KY 2012-1-10
1460
1461 // Dimension list of some OMI level 2 products doesn't include all dimension name and size
1462 // pairs. For example, Latitude[1644][60]. We have no way to find the dimension name of
1463 // the dimension with the size of 1644. The dimension name list of the variable also
1464 // includes the wrong dimension name. In this case, a dimension with the dimension size =1
1465 // is allocated in the latitude's dimension list. The latest version still has this bug.
1466 // To serve this kind of files, we create a fakedim name for the unmatched size.
1467 // KY 2012-1-13
1468
1469 set<hsize_t> dimsize_have_name_set;
1470 pair<set<hsize_t>::iterator, bool> setret1;
1471 set<string> thisvar_dimname_set;
1472 pair<set<string>::iterator, bool> setret2;
1473
1474 for (unsigned int j = 0; j < he5v.dim_list.size(); j++) {
1475 HE5Dim he5d = he5v.dim_list.at(j);
1476 for (const auto &dim:var->dims) {
1477
1478 if ((hsize_t) (he5d.size) == dim->size) {
1479 // This will assure that the same size dims be assigned to different dims
1480 if ("" == dim->name) {
1481 string dimname_candidate = eos5typestr + groupname + fslash_str + he5d.name;
1482 setret2 = thisvar_dimname_set.insert(dimname_candidate);
1483 if (true == setret2.second) {
1484 dim->name = dimname_candidate;
1485 // Should check in the future if the newname may cause potential inconsistency. KY:2012-3-9
1486 dim->newname = (num_groups == 1) ? he5d.name : dim->name;
1487 eos5data->vardimnames.insert(dim->name);
1488 // Since there is no way to figure out the unlimited dimension info. of an individual variable
1489 // from the dimension list. Here we just provide the dimnames to unlimited dimension mapping
1490 // based on the variable mapping. KY 2016-02-18
1491 eos5data->dimnames_to_unlimited[dim->name] = dim->unlimited_dim;
1492 }
1493 }
1494 }
1495 }
1496 } // for (unsigned int j=0; j<he5v.dim_list.size();j++)
1497
1498 // We have to go through the dimension list of this variable again to assure that every dimension has a name.
1499 // This is how that FakeDim is added. We still need it just in case. KY 2017-10-19
1500 for (const auto &dim:var->dims) {
1501 if ("" == dim->name)
1502 Create_Unique_DimName(eos5data, thisvar_dimname_set, dim, num_groups, eos5type);
1503 }
1504 } // "if (he5v.name == var->name) "
1505 } // "for (unsigned int i = 0; i < he5var.size(); i++)"
1506 return is_parsed;
1507}
1508
1509// Create unique dimension names. Se the comments below.
1510template<class T>
1511void EOS5File::Create_Unique_DimName(T*eos5data, set<string>& thisvar_dimname_set, Dimension *dim, int num_groups,
1512 EOS5Type eos5type)
1513{
1514
1515 BESDEBUG("h5", "Coming to Create_Unique_DimName"<<endl);
1516 map<hsize_t, string>::iterator itmap1;
1517 map<string, hsize_t>::iterator itmap2;
1518 pair<set<string>::iterator, bool> setret2;
1519 itmap1 = (eos5data->dimsizes_to_dimnames).find(dim->size);
1520
1521 // Even if we find this dimension matches the dimsizes_to_dimnames map, we have to check if the dimension
1522 // name has been used for this size. This is to make sure each dimension has a unique name in a variable.
1523 // For example, float foo[100][100] can be float foo[nlevels = 100][nlevels_1 = 100].
1524 // Step 1: Check if there is a dimension name that matches the size
1525
1526 if (itmap1 != (eos5data->dimsizes_to_dimnames).end()) {
1527 string dimname_candidate = (eos5data->dimsizes_to_dimnames)[dim->size];
1528
1529 // First check local var dimname set
1530 setret2 = thisvar_dimname_set.insert(dimname_candidate);
1531
1532 if (false == setret2.second) {
1533
1534 // Will see if other dimension names have this size
1535 bool match_some_dimname = Check_All_DimNames(eos5data, dimname_candidate, dim->size);
1536
1537 if (false == match_some_dimname) {
1538
1539 // dimname_candidate is updated.
1540 Get_Unique_Name(eos5data->vardimnames, dimname_candidate);
1541 thisvar_dimname_set.insert(dimname_candidate);
1542
1543 // Finally generate a new dimension(new dim. name with a size);Update all information
1544 Insert_One_NameSizeMap_Element2(eos5data->dimnames_to_dimsizes, eos5data->dimnames_to_unlimited,
1545 dimname_candidate, dim->size, dim->unlimited_dim);
1546 eos5data->dimsizes_to_dimnames.insert(pair<hsize_t, string>(dim->size, dimname_candidate));
1547 eos5data->dimnames.push_back(dimname_candidate);
1548 }
1549 }
1550
1551 // The final dimname_candidate(perhaps updated) should be assigned to the name of this dimension
1552 dim->name = dimname_candidate;
1553 if (num_groups > 1)
1554 dim->newname = dim->name;
1555 else {
1556 string dname = HDF5CFUtil::obtain_string_after_lastslash(dim->name);
1557 if ("" == dname)
1558 throw3("The dimension name ", dim->name, " of the variable is not right");
1559 else
1560 dim->newname = dname;
1561 }
1562 }
1563
1564 else { // No dimension names match or close to march this dimension name, we will create a fakedim.
1565 // Check Add_One_FakeDim_Name in HDF5CF.cc Fakedimname must be as a string reference.
1566 string Fakedimname = Create_Unique_FakeDimName(eos5data, eos5type);
1567 thisvar_dimname_set.insert(Fakedimname);
1568
1569 // Finally generate a new dimension(new dim. name with a size);Update all information
1570 Insert_One_NameSizeMap_Element2(eos5data->dimnames_to_dimsizes, eos5data->dimnames_to_unlimited, Fakedimname,
1571 dim->size, dim->unlimited_dim);
1572 eos5data->dimsizes_to_dimnames.insert(pair<hsize_t, string>(dim->size, Fakedimname));
1573 eos5data->dimnames.push_back(Fakedimname);
1574 dim->name = Fakedimname;
1575 if (num_groups > 1)
1576 dim->newname = dim->name;
1577 else {
1578 string dname = HDF5CFUtil::obtain_string_after_lastslash(dim->name);
1579 if ("" == dname)
1580 throw3("The dimension name ", dim->name, " of the variable is not right");
1581 else
1582 dim->newname = dname;
1583 }
1584 }
1585}
1586
1587// Check all dim. names to see if this dim. size is used by another dim. name.
1588template<class T>
1589bool EOS5File::Check_All_DimNames(T* eos5data, string& dimname, hsize_t dimsize) const
1590{
1591
1592 BESDEBUG("h5", "Coming to Check_All_DimNames"<<endl);
1593 bool ret_flag = false;
1594 for (map<string, hsize_t>::iterator im = eos5data->dimnames_to_dimsizes.begin();
1595 im != eos5data->dimnames_to_dimsizes.end(); ++im) {
1596 // dimname must not be the same one since the same one is rejected.
1597 if (dimsize == (*im).second && dimname != (*im).first) {
1598 dimname = (*im).first;
1599 ret_flag = true;
1600 break;
1601 }
1602 }
1603 return ret_flag;
1604}
1605
1606// Get a unique name.
1607void EOS5File::Get_Unique_Name(set<string> & nameset, string& dimname_candidate) const
1608{
1609
1610 BESDEBUG("h5", "Coming to Get_Unique_Name"<<endl);
1611 int clash_index = 1;
1612 string temp_clashname = dimname_candidate + '_';
1613 HDF5CFUtil::gen_unique_name(temp_clashname, nameset, clash_index);
1614 dimname_candidate = temp_clashname;
1615}
1616
1617// We may need to generate a unique "fake" dim. name for dimensions that don't have any dimension names.
1618template<class T>
1619string EOS5File::Create_Unique_FakeDimName(T*eos5data, EOS5Type eos5type) const
1620{
1621
1622 BESDEBUG("h5", "Coming to Create_Unique_FakeDimName"<<endl);
1623 string fslash_str = "/";
1624 string eos5typestr;
1625 if (GRID == eos5type)
1626 eos5typestr = "/GRIDS/";
1627 else if (SWATH == eos5type)
1628 eos5typestr = "/SWATHS/";
1629 else if (ZA == eos5type)
1630 eos5typestr = "/ZAS/";
1631 else
1632 throw1("Unsupported HDF-EOS5 type, this type is not swath, grid or zonal average");
1633
1634 stringstream sfakedimindex;
1635 sfakedimindex << eos5data->addeddimindex;
1636 string fakedimstr = "FakeDim";
1637 string added_dimname = eos5typestr + eos5data->name + fslash_str + fakedimstr + sfakedimindex.str();
1638
1639 pair<set<string>::iterator, bool> setret;
1640 setret = eos5data->vardimnames.insert(added_dimname);
1641 if (false == setret.second) Get_Unique_Name(eos5data->vardimnames, added_dimname);
1642 eos5data->addeddimindex = eos5data->addeddimindex + 1;
1643 return added_dimname;
1644}
1645
1646// Obtain the group name this variable belongs.
1647string EOS5File::Obtain_Var_EOS5Type_GroupName(const Var*var, EOS5Type eos5type) const
1648{
1649
1650 BESDEBUG("h5", "Coming to Obtain_Var_EOS5Type_GroupName"<<endl);
1651 string EOS5GRIDPATH = "/HDFEOS/GRIDS";
1652 string EOS5SWATHPATH = "/HDFEOS/SWATHS";
1653 string EOS5ZAPATH = "/HDFEOS/ZAS";
1654 size_t eostypename_start_pos = 0;
1655 size_t eostypename_end_pos;
1656 string groupname;
1657
1658 // The fullpath is like "HDFEOS/GRIDS/Temp/Data Fields/etc."
1659 // To get "Temp", we obtain the position of "T" and the position of "p"
1660 // and then generate a substr.
1661
1662 if (GRID == eos5type)
1663 eostypename_start_pos = EOS5GRIDPATH.size() + 1;
1664 else if (SWATH == eos5type)
1665 eostypename_start_pos = EOS5SWATHPATH.size() + 1;
1666 else if (ZA == eos5type)
1667 eostypename_start_pos = EOS5ZAPATH.size() + 1;
1668 else
1669 throw2("Non supported eos5 type for var ", var->fullpath);
1670
1671 eostypename_end_pos = var->fullpath.find('/', eostypename_start_pos) - 1;
1672 groupname = var->fullpath.substr(eostypename_start_pos, eostypename_end_pos - eostypename_start_pos + 1);
1673
1674 BESDEBUG("h5", "In Obtain_Var_EOS5Type_GroupName(), the groupname is "<<groupname << endl);
1675
1676 return groupname;
1677}
1678
1679// Check whether this field belongs to "Geolocation Fields" or "Data Fields"
1680int EOS5File::Check_EOS5Swath_FieldType(const Var*var) const
1681{
1682
1683 string geofield_relative_path = "/Geolocation Fields/" + var->name;
1684 string datafield_relative_path = "/Data Fields/" + var->name;
1685
1686 int tempflag = -1;
1687
1688 if (var->fullpath.size() > datafield_relative_path.size()) {
1689 size_t field_pos_in_full_path = var->fullpath.size() - datafield_relative_path.size();
1690 if (var->fullpath.rfind(datafield_relative_path, field_pos_in_full_path) != string::npos) tempflag = 0;
1691 }
1692
1693 if (tempflag != 0 && (var->fullpath.size() > geofield_relative_path.size())) {
1694 size_t field_pos_in_full_path = var->fullpath.size() - geofield_relative_path.size();
1695 if (var->fullpath.rfind(geofield_relative_path, field_pos_in_full_path) != string::npos) tempflag = 1;
1696 }
1697 return tempflag;
1698}
1699
1700// An error will be thrown if we find a dimension size that doesn't match any dimension name
1701// in this EOS5 group.
1702template<class T>
1703void EOS5File::Set_NonParse_Var_Dims(T*eos5data, const Var* var, const map<hsize_t, string>& /*dimsizes_to_dimnames*/,
1704 int num_groups, EOS5Type eos5type)
1705{
1706
1707 BESDEBUG("h5", "Coming to Set_NonParse_Var_Dims"<<endl);
1708 map<hsize_t, string>::iterator itmap;
1709 set<string> thisvar_dimname_set;
1710
1711 for (const auto &vdim:var->dims) {
1712 if ("" == vdim->name)
1713 Create_Unique_DimName(eos5data, thisvar_dimname_set, vdim, num_groups, eos5type);
1714 else
1715 throw5("The dimension name ", vdim->name, " of the variable ", var->name, " is not right");
1716 }
1717}
1718
1719// Aura files don't use the CF attribute names for the bunch of attributes. We need to make it right.
1721{
1722
1723 BESDEBUG("h5", "Coming to Check_Aura_Product_Status"<<endl);
1724 // Aura files will put an attribute called InStrumentName under /HDFEOS/ADDITIONAL/FILE_ATTRIBUTES
1725 // We just need to check that attribute.
1726 string eos5_fattr_group_name = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES";
1727 string instrument_attr_name = "InstrumentName";
1728
1729 // Check if this file is an aura file
1730 for (const auto &grp:this->groups) {
1731 if (eos5_fattr_group_name == grp->path) {
1732 for (const auto &attr:grp->attrs) {
1733 if (instrument_attr_name == attr->name) {
1734 Retrieve_H5_Attr_Value(attr, grp->path);
1735 string attr_value(attr->value.begin(), attr->value.end());
1736 if ("OMI" == attr_value) {
1737 this->isaura = true;
1738 this->aura_name = OMI;
1739 }
1740 else if ("MLS Aura" == attr_value) {
1741 this->isaura = true;
1742 this->aura_name = MLS;
1743 }
1744 else if ("TES" == attr_value) {
1745 this->isaura = true;
1746 this->aura_name = TES;
1747 }
1748 else if ("HIRDLS" == attr_value) {
1749 this->isaura = true;
1750 this->aura_name = HIRDLS;
1751 }
1752 break;
1753 }
1754 }
1755 }
1756 }
1757
1758 // Assign EOS5 to CF MAP values for Aura files
1759 if (true == this->isaura) {
1760 eos5_to_cf_attr_map["FillValue"] = "_FillValue";
1761 eos5_to_cf_attr_map["MissingValue"] = "missing_value";
1762 eos5_to_cf_attr_map["Units"] = "units";
1763 eos5_to_cf_attr_map["Offset"] = "add_offset";
1764 eos5_to_cf_attr_map["ScaleFactor"] = "scale_factor";
1765 eos5_to_cf_attr_map["ValidRange"] = "valid_range";
1766 eos5_to_cf_attr_map["Title"] = "title";
1767 }
1768
1769}
1770
1771// Handle Coordinate variables
1773{
1774
1775 BESDEBUG("h5", "Coming to Handle_CVar()"<<endl);
1776
1777 // If this file is augmented.
1778 bool is_augmented = Check_Augmentation_Status();
1779
1780#if 0
1781 if(is_augmented) cerr<<"The file is augmented "<<endl;
1782 else cerr<<"The file is not augmented "<<endl;
1783#endif
1784
1785 // Handle coordinate variables for grids.
1786 if (this->eos5cfgrids.empty() == false)
1787 Handle_Grid_CVar(is_augmented);
1788 if (this->eos5cfswaths.empty() == false)
1789 Handle_Swath_CVar(is_augmented);
1790 if (this->eos5cfzas.empty() == false)
1791 Handle_Za_CVar(is_augmented);
1792
1793#if 0
1794 for (auto irv = this->cvars.begin();
1795 irv != this->cvars.end(); irv++) {
1796 cerr<<"EOS5CVar name "<<(*irv)->name <<endl;
1797 cerr<<"EOS5CVar dimension name "<< (*irv)->cfdimname <<endl;
1798 cerr<<"EOS5CVar new name "<<(*irv)->newname <<endl;
1799 cerr<<"EOS5CVar type is "<<(*irv)->cvartype <<endl;
1800//cerr<<"EOS5CVar dtype is "<<(*irv)->dtype <<endl;
1801 }
1802#endif
1803
1804}
1805
1806// Handle Grid Coordinate variables
1807void EOS5File::Handle_Grid_CVar(bool is_augmented)
1808{
1809
1810 BESDEBUG("h5", "Coming to Handle_Grid_CVar"<<endl);
1811 if (true == is_augmented) {
1812 // Create latitude/longitude based on the first XDim and YDim
1813 Handle_Augmented_Grid_CVar();
1814 }
1815 else {
1816 Remove_MultiDim_LatLon_EOS5CFGrid();
1817 // If the grid size is 0, it must be a Grid file that cannot be handled
1818 // with the CF option, simply return with handling any coordinate variables.
1819 if (this->eos5cfgrids.empty()) return;
1820 if (1 == this->eos5cfgrids.size())
1821 Handle_Single_Nonaugment_Grid_CVar((this->eos5cfgrids)[0]);
1822 else
1823 Handle_Multi_Nonaugment_Grid_CVar();
1824 }
1825}
1826
1827// Check if this file is augmented. The current augmentation tool will
1828// add extra variables for every EOS5 object. This function will check
1829// if that is the case.
1830bool EOS5File::Check_Augmentation_Status() const
1831{
1832
1833 BESDEBUG("h5", "Coming to Check_Augmentation_Status()"<<endl);
1834 bool aug_status = false;
1835 int num_aug_eos5grp = 0;
1836
1837 for (const auto &cfgrid:this->eos5cfgrids) {
1838 for (const auto &var:this->vars) {
1839 bool is_augmented = Check_Augmented_Var_Candidate(cfgrid, var, GRID);
1840 if (true == is_augmented) {
1841 num_aug_eos5grp++;
1842 break;
1843 }
1844 }
1845 }
1846
1847 for (const auto &cfswath:this->eos5cfswaths) {
1848 for (const auto &var:this->vars) {
1849 bool is_augmented = Check_Augmented_Var_Candidate(cfswath, var, SWATH);
1850 if (true == is_augmented) {
1851 num_aug_eos5grp++;
1852 break;
1853 }
1854
1855 }
1856 }
1857
1858 for (const auto &cfza:this->eos5cfzas) {
1859 for (const auto &var:this->vars) {
1860 bool is_augmented = Check_Augmented_Var_Candidate(cfza, var, ZA);
1861 if (true == is_augmented) {
1862 num_aug_eos5grp++;
1863 break;
1864 }
1865 }
1866 }
1867
1868 int total_num_eos5grp = this->eos5cfgrids.size() + this->eos5cfswaths.size() + this->eos5cfzas.size();
1869
1870#if 0
1871//cerr<< "total_num_eos5grp "<<total_num_eos5grp <<endl;
1872//"h5","num_aug_eos5grp "<< num_aug_eos5grp <<endl;
1873#endif
1874
1875 if (num_aug_eos5grp == total_num_eos5grp) aug_status = true;
1876 return aug_status;
1877
1878}
1879
1880// This method is not used. Still keep it now since it may be useful in the future. KY 2012-3-09
1881// Don't remove the #if 0 #endif block.
1882#if 0
1883bool EOS5File::Check_Augmented_Var_Attrs(Var *var) {
1884
1885 // We will check whether the attribute "CLASS" and the attribute "REFERENCE_LIST" exist.
1886 // For the attribute "CLASS", we would like to check if the value is "DIMENSION_SCALE".
1887 bool has_dimscale_class = false;
1888 bool has_reflist = false;
1889 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
1890 if ("CLASS" == (*ira)->name) {
1891 Retrieve_H5_Attr_Value(*ira,var->fullpath);
1892 string class_value((*ira)->value.begin(),(*ira)->value.end());
1893 if ("DIMENSION_SCALE"==class_value)
1894 has_dimscale_class = true;
1895 }
1896
1897 if ("REFERENCE_LIST" == (*ira)->name)
1898 has_reflist = true;
1899 if (true == has_reflist && true == has_dimscale_class)
1900 break;
1901 }
1902
1903 if (true == has_reflist && true == has_dimscale_class)
1904 return true;
1905 else
1906 return false;
1907
1908}
1909#endif
1910
1911// Check if the variable candidate exists for the augmented case.
1912// The augmented variables have path like /HDFEOS/GRIDS/HIRDLS/nTimes
1913// The general HDF-EOS5 variables have path like /HDFEOS/GRIDS/HIRDLS/Data Fields/Times.
1914// So if we find the var name is the same as the string stripped from /HDFEOS/GRIDS/HIRDLS,
1915// then this file is augmented.
1916// Hope that no other hybrid-HDFEOS5 files fall to this category.
1917template<class T>
1918bool EOS5File::Check_Augmented_Var_Candidate(T *eos5data, const Var *var, EOS5Type eos5type) const
1919{
1920
1921 BESDEBUG("h5", "Coming to Check_Augmented_Var_Candidate"<<endl);
1922 bool augmented_var = false;
1923
1924 string EOS5DATAPATH;
1925 if (GRID == eos5type)
1926 EOS5DATAPATH = "/HDFEOS/GRIDS/";
1927 else if (ZA == eos5type)
1928 EOS5DATAPATH = "/HDFEOS/ZAS/";
1929 else if (SWATH == eos5type)
1930 EOS5DATAPATH = "/HDFEOS/SWATHS/";
1931 else
1932 throw1("Non supported EOS5 type");
1933
1934 string fslash_str = "/";
1935 string THIS_EOS5DATAPATH = EOS5DATAPATH + eos5data->name + fslash_str;
1936
1937 // Match the EOS5 type
1938 if (eos5type == Get_Var_EOS5_Type(var)) {
1939 string var_eos5data_name = Obtain_Var_EOS5Type_GroupName(var, eos5type);
1940 // Match the EOS5 group name
1941 if (var_eos5data_name == eos5data->name) {
1942 if (var->fullpath.size() > THIS_EOS5DATAPATH.size()) {
1943 // Obtain the var name from the full path
1944 string var_path_after_eos5dataname = var->fullpath.substr(THIS_EOS5DATAPATH.size());
1945 // Match the variable name
1946 if (var_path_after_eos5dataname == var->name) augmented_var = true;
1947 }
1948 }
1949 }
1950
1951 return augmented_var;
1952
1953}
1954
1955// Handle augmented grid coordinate variables.
1956void EOS5File::Handle_Augmented_Grid_CVar()
1957{
1958 BESDEBUG("h5", "Coming to Handle_Augmented_Grid_CVar()"<<endl);
1959 for (const auto &cfgrid:this->eos5cfgrids)
1960 Handle_Single_Augment_CVar(cfgrid, GRID);
1961}
1962
1963// Handle the coordinate variables for the single HDF-EOS5 objects(grid,swath,zonal average) for an augmented file
1964template<class T>
1965void EOS5File::Handle_Single_Augment_CVar(T* cfeos5data, EOS5Type eos5type)
1966{
1967
1968 BESDEBUG("h5", "Coming to Handle_Single_Augment_CVar()"<<endl);
1969 set<string> tempvardimnamelist;
1970 tempvardimnamelist = cfeos5data->vardimnames;
1971 set<string>::iterator its;
1972 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
1973 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
1974
1975 bool is_augmented = Check_Augmented_Var_Candidate(cfeos5data, *irv, eos5type);
1976
1977 if (true == is_augmented) {
1978
1979 // Since we have already checked if this file is augmented or not, we can safely
1980 // compare the dimension name with the var name now.
1981 string tempdimname = HDF5CFUtil::obtain_string_after_lastslash(*its);
1982
1983 // The added variable name is always the same as the dimension name.
1984 if (tempdimname == (*irv)->name) {
1985
1986 //Find it, create a coordinate variable.
1987 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
1988 auto EOS5cvar = EOS5cvar_unique.release();
1989
1990 // Still keep the original dimension name to avoid the name clashing when
1991 // one grid and one swath and one za occur in the same file
1992 EOS5cvar->cfdimname = *its;
1993 EOS5cvar->cvartype = CV_EXIST;
1994 EOS5cvar->eos_type = eos5type;
1995
1996 // Save this cv to the cv vector
1997 this->cvars.push_back(EOS5cvar);
1998
1999 // Remove this var from the var vector since it becomes a cv.
2000 delete (*irv);
2001 irv = this->vars.erase(irv);
2002 }
2003 else {
2004 ++irv;
2005 }
2006 } // "if (true == is_augmented)"
2007 else {
2008 ++irv;
2009 }
2010 } // end of for (auto irv = this->vars.begin();....
2011 } // end of for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2012
2013 for (const auto &cvar:this->cvars) {
2014 its = tempvardimnamelist.find(cvar->cfdimname);
2015 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2016 }
2017
2018 if (false == tempvardimnamelist.empty())
2019 throw1("Augmented files still need to provide more coordinate variables");
2020}
2021
2022//Currently we remove HDF-EOS5 grid if we find the latitude/longitude is >2D. This is a big question mark
2023// given some data producers just don't follow the HDF-EOS5 specification to generate the latitude/longitude.
2024// KY 2016-07-12
2025void EOS5File::Remove_MultiDim_LatLon_EOS5CFGrid()
2026{
2027
2028 BESDEBUG("h5", "Coming to Remove_MultiDim_LatLon_EOS5CFGrid()"<<endl);
2029 for (auto irg = this->eos5cfgrids.begin(); irg != this->eos5cfgrids.end();) {
2030
2031 // If number of dimension latitude/longitude is >=2, no coordinate variables will be generated.
2032 // We will simply remove this grid from the vector eos5cfgrids.
2033 // In the future, we may consider supporting 2D latlon. KY 2012-1-17
2034 // I just find that new OMI level 3 data provide 2D lat/lon for geographic projection data.
2035 // The 2D lat/lon can be condensed to 1D lat/lon, which is the same calculated by the calculation of
2036 // the projection. So I don't remove this OMI grid from the grid list. KY 2012-2-9
2037 // However, I do remove the "Longitude" and "Latitude" fields since "Latitude" and "Longitude"
2038 // can be calculated.
2039
2040 bool irg_erase = false;
2041
2042 if (true == (*irg)->has_2dlatlon) {
2043
2044 if ((true == this->isaura) && (OMI == this->aura_name) && (HE5_GCTP_GEO == (*irg)->eos5_projcode))
2045
2046 { // We need to remove the redundant latitude and longitude fields
2047
2048 string EOS5GRIDPATH = "/HDFEOS/GRIDS/";
2049 string fslash_str = "/";
2050 string THIS_EOS5GRIDPATH = EOS5GRIDPATH + (*irg)->name + fslash_str;
2051 int catch_latlon = 0;
2052
2053 for (auto irv = this->vars.begin(); (irv != this->vars.end()) && (catch_latlon != 2);
2054 ) {
2055 if (GRID == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5GRIDPATH.size())) {
2056
2057 string var_grid_name = Obtain_Var_EOS5Type_GroupName(*irv, GRID);
2058 if (var_grid_name == (*irg)->name) {
2059 if (("Longitude" == (*irv)->name) || ("Latitude" == (*irv)->name)) {
2060 catch_latlon++;
2061 // Remove this var from the var vector since it becomes a cv.
2062 delete (*irv);
2063 irv = this->vars.erase(irv);
2064 }
2065 else {
2066 ++irv;
2067 }
2068 }
2069 else {
2070 ++irv;
2071 }
2072 }
2073 else {
2074 ++irv;
2075 }
2076 } // "for (auto irv = this->vars.begin() ..."
2077 if (2 == catch_latlon) {
2078 (*irg)->has_nolatlon = true;
2079 (*irg)->has_2dlatlon = false;
2080 }
2081
2082 } // "if ((true == this->isaura) ..."
2083 else { // remove this grid from the eos5cfgrids list.
2084 delete (*irg);
2085 irg = this->eos5cfgrids.erase(irg);
2086 irg_erase = true;
2087 }
2088 } // "if (true == (*irg) ..."
2089
2090 if (false == irg_erase) {
2091 ++irg;
2092 }
2093
2094 } // "for (vector <EOS5CFGrid *>::iterator irg = this->eos5cfgrids.begin() ..."
2095
2096 // Also remove >2d latlon grids.
2097 for (auto irg = this->eos5cfgrids.begin(); irg != this->eos5cfgrids.end();) {
2098
2099 if (true == (*irg)->has_g2dlatlon) {
2100 delete (*irg);
2101 irg = this->eos5cfgrids.erase(irg);
2102 }
2103 else {
2104 ++irg;
2105 }
2106 }
2107}
2108
2109// Handle single nonaugmented grid coordinate variables.
2110void EOS5File::Handle_Single_Nonaugment_Grid_CVar(EOS5CFGrid* cfgrid)
2111{
2112
2113 BESDEBUG("h5", "Coming to Handle_Single_Nonaugment_Grid_CVar()"<<endl);
2114 set<string> tempvardimnamelist;
2115 tempvardimnamelist = cfgrid->vardimnames;
2116
2117 // Handle Latitude and longitude
2118 bool use_own_latlon = false;
2119 if (true == cfgrid->has_1dlatlon)
2120 use_own_latlon = Handle_Single_Nonaugment_Grid_CVar_OwnLatLon(cfgrid, tempvardimnamelist);
2121#if 0
2122 if(use_own_latlon) "h5","using 1D latlon"<<endl;
2123 else "h5","use_own_latlon is false "<<endl;
2124#endif
2125
2126 if (false == use_own_latlon) {
2127 bool use_eos5_latlon = false;
2128 use_eos5_latlon = Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon(cfgrid, tempvardimnamelist);
2129
2130 // If we cannot obtain lat/lon from the HDF-EOS5 library, no need to create other CVs. Simply return.
2131 if (false == use_eos5_latlon) return;
2132 }
2133
2134 // Else handling non-latlon grids
2135 Handle_NonLatLon_Grid_CVar(cfgrid, tempvardimnamelist);
2136
2137}
2138
2139// Handle single nonaugmented grid coordinate variables with its own lat/lon
2140bool EOS5File::Handle_Single_Nonaugment_Grid_CVar_OwnLatLon(const EOS5CFGrid *cfgrid, set<string>& tempvardimnamelist)
2141
2142{
2143
2144 BESDEBUG("h5", "Coming to Handle_Single_Nonaugment_Grid_CVar_OwnLatLon()"<<endl);
2145 set<string>::iterator its;
2146 string EOS5GRIDPATH = "/HDFEOS/GRIDS/";
2147 string fslash_str = "/";
2148 string THIS_EOS5GRIDPATH = EOS5GRIDPATH + cfgrid->name + fslash_str;
2149
2150 // Handle latitude and longitude
2151 bool find_latydim = false;
2152 bool find_lonxdim = false;
2153
2154 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2155 if (GRID == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5GRIDPATH.size())) {
2156
2157 string var_grid_name = Obtain_Var_EOS5Type_GroupName(*irv, GRID);
2158 if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Latitude")) {
2159
2160 string tempdimname = (((*irv)->dims)[0])->name;
2161
2162 if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname)) {
2163 //Find it, create a coordinate variable.
2164 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
2165 auto EOS5cvar = EOS5cvar_unique.release();
2166
2167 // Still keep the original dimension name to avoid the nameclashing when
2168 // one grid and one swath and one za occur in the same file
2169 EOS5cvar->cfdimname = tempdimname;
2170 EOS5cvar->cvartype = CV_EXIST;
2171 EOS5cvar->eos_type = GRID;
2172
2173 // Save this cv to the cv vector
2174 this->cvars.push_back(EOS5cvar);
2175
2176 // Remove this var from the var vector since it becomes a cv.
2177 delete (*irv);
2178 this->vars.erase(irv);
2179
2180 // No need to remove back the iterator since it will go out of the loop.
2181 find_latydim = true;
2182 break;
2183 } // <if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname))>
2184 } // <if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Latitude"))>
2185 } // <if (GRID == Get_Var_EOS5_Type(*irv) ...>
2186 } // <for (auto irv = this->vars.begin() ...>
2187
2188 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2189
2190 if (GRID == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5GRIDPATH.size())) {
2191
2192 string var_grid_name = Obtain_Var_EOS5Type_GroupName(*irv, GRID);
2193
2194 if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Longitude")) {
2195
2196 string tempdimname = (((*irv)->dims)[0])->name;
2197
2198 if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname)) {
2199 //Find it, create a coordinate variable.
2200 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
2201 auto EOS5cvar = EOS5cvar_unique.release();
2202
2203 // Still keep the original dimension name to avoid the name clashing when
2204 // one grid and one swath and one za occur in the same file
2205 EOS5cvar->cfdimname = tempdimname;
2206 EOS5cvar->cvartype = CV_EXIST;
2207 EOS5cvar->eos_type = GRID;
2208
2209 // Save this cv to the cv vector
2210 this->cvars.push_back(EOS5cvar);
2211
2212 // Remove this var from the var vector since it becomes a cv.
2213 delete (*irv);
2214 this->vars.erase(irv);
2215 find_lonxdim = true;
2216 break;
2217 } // <if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(tempdimname))>
2218 } // "if ((var_grid_name == cfgrid->name) && ((*irv)->name == "Longitude"))"
2219 } // <if (GRID == Get_Var_EOS5_Type(*irv) ...>
2220 } // for (auto irv = this->vars.begin() ...
2221
2222 for (const auto &cvar:this->cvars) {
2223
2224 its = tempvardimnamelist.find(cvar->cfdimname);
2225 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2226
2227 }
2228
2229#if 0
2230 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2231 "h5","tempvardim "<<*its <<endl;
2232#endif
2233
2234 return (find_latydim == true && find_lonxdim == true);
2235}
2236
2237// Handle single non-augmented grid latitude/longitude coordinate variables.
2238bool EOS5File::Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon(const EOS5CFGrid *cfgrid, set<string>& tempvardimnamelist)
2239
2240{
2241
2242 BESDEBUG("h5", "Coming to Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon()"<<endl);
2243
2244 // Handle latitude and longitude
2245 bool find_ydim = false;
2246 bool find_xdim = false;
2247 set<string>::iterator its;
2248
2249#if 0
2250 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2251 cerr<<"dim names "<<(*its) <<endl;
2252#endif
2253
2254 string ydim_full_path;
2255 string xdim_full_path;
2256
2257 if (cfgrid->eos5_projcode != HE5_GCTP_GEO) {
2258
2259 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2260 if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(*its))
2261 ydim_full_path = *its;
2262 else if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(*its)) xdim_full_path = *its;
2263 }
2264 }
2265
2266
2267 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end();) {
2268 if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(*its)) {
2269
2270 // Create EOS5 Latitude CV
2271 auto EOS5cvar_unique = make_unique<EOS5CVar>();
2272 auto EOS5cvar = EOS5cvar_unique.release();
2273 EOS5cvar->name = "lat";
2274 Create_Added_Var_NewName_FullPath(GRID, cfgrid->name, EOS5cvar->name, EOS5cvar->newname,
2275 EOS5cvar->fullpath);
2276
2277 if (cfgrid->eos5_projcode != HE5_GCTP_GEO) {
2278 EOS5cvar->rank = 2;
2279 EOS5cvar->dtype = H5FLOAT64;
2280 }
2281 else {
2282 EOS5cvar->rank = 1;
2283 EOS5cvar->dtype = H5FLOAT32;
2284 }
2285
2286 auto eos5cvar_dim_unique = make_unique<Dimension>((hsize_t) cfgrid->ydimsize);
2287 auto eos5cvar_dim = eos5cvar_dim_unique.release();
2288 eos5cvar_dim->name = *its;
2289 eos5cvar_dim->newname = (this->eos5cfgrids.size() == 1) ? "YDim" : *its;
2290 EOS5cvar->dims.push_back(eos5cvar_dim);
2291 EOS5cvar->cfdimname = eos5cvar_dim->name;
2292
2293 if (EOS5cvar->rank == 2) {
2294
2295 auto eos5cvar_dim2_unique = make_unique<Dimension>((hsize_t) cfgrid->xdimsize);
2296 auto eos5cvar_dim2 = eos5cvar_dim2_unique.release();
2297 eos5cvar_dim2->name = xdim_full_path;
2298 eos5cvar_dim2->newname = (this->eos5cfgrids.size() == 1) ? "XDim" : xdim_full_path;
2299 EOS5cvar->dims.push_back(eos5cvar_dim2);
2300
2301 }
2302 EOS5cvar->cvartype = CV_LAT_MISS;
2303 EOS5cvar->eos_type = GRID;
2304 EOS5cvar->xdimsize = cfgrid->xdimsize;
2305 EOS5cvar->ydimsize = cfgrid->ydimsize;
2306
2307 //Special parameters for EOS5 Grid
2308 EOS5cvar->point_lower = cfgrid->point_lower;
2309 EOS5cvar->point_upper = cfgrid->point_upper;
2310 EOS5cvar->point_left = cfgrid->point_left;
2311 EOS5cvar->point_right = cfgrid->point_right;
2312 EOS5cvar->eos5_pixelreg = cfgrid->eos5_pixelreg;
2313 EOS5cvar->eos5_origin = cfgrid->eos5_origin;
2314 EOS5cvar->eos5_projcode = cfgrid->eos5_projcode;
2315
2316 for (unsigned int k = 0; k < 13; k++)
2317 EOS5cvar->param[k] = cfgrid->param[k];
2318
2319 EOS5cvar->zone = cfgrid->zone;
2320 EOS5cvar->sphere = cfgrid->sphere;
2321
2322 // Save this cv to the cv vector
2323 this->cvars.push_back(EOS5cvar);
2324 // erase the dimension name from the dimension name set
2325
2326 // This is the right way to make its platform-independent.
2327 tempvardimnamelist.erase(its++);
2328 find_ydim = true;
2329
2330 } // <if ("YDim" == HDF5CFUtil::obtain_string_after_lastslash(*its))>
2331 else if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(*its)) {
2332
2333 // Create EOS5 Latitude CV
2334 auto EOS5cvar_unique = make_unique<EOS5CVar>();
2335 auto EOS5cvar = EOS5cvar_unique.release();
2336 EOS5cvar->name = "lon";
2337 Create_Added_Var_NewName_FullPath(GRID, cfgrid->name, EOS5cvar->name, EOS5cvar->newname,
2338 EOS5cvar->fullpath);
2339
2340 if (cfgrid->eos5_projcode != HE5_GCTP_GEO) {
2341 EOS5cvar->rank = 2;
2342 EOS5cvar->dtype = H5FLOAT64;
2343 }
2344 else {
2345 EOS5cvar->rank = 1;
2346 EOS5cvar->dtype = H5FLOAT32;
2347 }
2348
2349
2350 if (EOS5cvar->rank == 2) {
2351 auto eos5cvar_dim_unique = make_unique<Dimension>((hsize_t) cfgrid->ydimsize);
2352 auto eos5cvar_dim = eos5cvar_dim_unique.release();
2353 eos5cvar_dim->name = ydim_full_path;
2354 eos5cvar_dim->newname = (this->eos5cfgrids.size() == 1) ? "YDim" : ydim_full_path;
2355 EOS5cvar->dims.push_back(eos5cvar_dim);
2356 }
2357
2358 auto eos5cvar_dim2_unique = make_unique<Dimension>((hsize_t) cfgrid->xdimsize);
2359 auto eos5cvar_dim2 = eos5cvar_dim2_unique.release();
2360
2361 eos5cvar_dim2->name = *its;
2362 eos5cvar_dim2->newname = (this->eos5cfgrids.size() == 1) ? "XDim" : *its;
2363 EOS5cvar->dims.push_back(eos5cvar_dim2);
2364 EOS5cvar->cfdimname = eos5cvar_dim2->name;
2365
2366 EOS5cvar->cvartype = CV_LON_MISS;
2367 EOS5cvar->eos_type = GRID;
2368 EOS5cvar->xdimsize = cfgrid->xdimsize;
2369 EOS5cvar->ydimsize = cfgrid->ydimsize;
2370
2371 //Special parameters for EOS5 Grid
2372 EOS5cvar->point_lower = cfgrid->point_lower;
2373 EOS5cvar->point_upper = cfgrid->point_upper;
2374 EOS5cvar->point_left = cfgrid->point_left;
2375 EOS5cvar->point_right = cfgrid->point_right;
2376 EOS5cvar->eos5_pixelreg = cfgrid->eos5_pixelreg;
2377 EOS5cvar->eos5_origin = cfgrid->eos5_origin;
2378 EOS5cvar->eos5_projcode = cfgrid->eos5_projcode;
2379 for (unsigned int k = 0; k < 13; k++)
2380 EOS5cvar->param[k] = cfgrid->param[k];
2381 EOS5cvar->zone = cfgrid->zone;
2382 EOS5cvar->sphere = cfgrid->sphere;
2383
2384 // Save this cv to the cv vector
2385 this->cvars.push_back(EOS5cvar);
2386
2387 // erase the dimension name from the dimension name set,platform independent way.
2388 tempvardimnamelist.erase(its++);
2389 find_xdim = true;
2390
2391 } // "else if ("XDim" == HDF5CFUtil::obtain_string_after_lastslash(*its))"
2392 else
2393 ++its;
2394 if (true == find_xdim && true == find_ydim) break;
2395 } // <for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)>
2396
2397 return (true == find_xdim && true == find_ydim);
2398}
2399
2400// Handle non-latitude/longitude grid coordinate variables.
2401void EOS5File::Handle_NonLatLon_Grid_CVar(EOS5CFGrid *cfgrid, set<string>& tempvardimnamelist)
2402{
2403
2404 // First check if we have existing coordinate variable
2405 set<string>::iterator its;
2406 auto num_dimnames = (int)(tempvardimnamelist.size());
2407 bool has_dimnames = true;
2408
2409 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2410 if (cfgrid->dnames_to_1dvnames.find(*its) != cfgrid->dnames_to_1dvnames.end()) {
2411 for (auto irv = this->vars.begin(); has_dimnames && (irv != this->vars.end());) {
2412 // We need to check if this var is a grid and use "newname"
2413 // of var to check the dnames_to_1dvnames since it is
2414 // possible to have name clashing for the "name" of a var.
2415 if (GRID == Get_Var_EOS5_Type(*irv) && (*irv)->newname == (cfgrid->dnames_to_1dvnames)[*its]) {
2416
2417 //Find it, create a coordinate variable.
2418 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
2419 auto EOS5cvar = EOS5cvar_unique.release();
2420
2421 // Still keep the original dimension name to avoid the name clashing when
2422 // one grid and one swath and one za occur in the same file
2423 EOS5cvar->cfdimname = *its;
2424 EOS5cvar->cvartype = CV_EXIST;
2425 EOS5cvar->eos_type = GRID;
2426
2427 // Save this cv to the cv vector
2428 this->cvars.push_back(EOS5cvar);
2429
2430 // Remove this var from the var vector since it becomes a cv.
2431 delete (*irv);
2432 irv = this->vars.erase(irv);
2433 num_dimnames--;
2434 if (0 == num_dimnames) has_dimnames = false;
2435 } // if (GRID == Get_Var_EOS5_Type(*irv) ...
2436 else {
2437 ++irv;
2438 }
2439 } // for (auto irv = this->vars.begin(); ...
2440 } // if (cfgrid->dnames_to_1dvnames.find(*its) !=cfgrid->dnames_to_1dvnames.end())
2441 } // for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2442
2443 // Remove the dimension name that finds the cooresponding variables from the tempvardimlist.
2444 for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
2445 its = tempvardimnamelist.find((*irv)->cfdimname);
2446 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2447 }
2448
2449 // Second: Some dimension names still need to find CVs, create the missing CVs
2450 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2451
2452 auto EOS5cvar_unique = make_unique<EOS5CVar>();
2453 auto EOS5cvar = EOS5cvar_unique.release();
2454 Create_Missing_CV(cfgrid, EOS5cvar, *its, GRID, this->eos5cfgrids.size());
2455 this->cvars.push_back(EOS5cvar);
2456
2457 }
2458}
2459
2460// Handle none-augmented grid coordinate variables for multiple grids.
2461void EOS5File::Handle_Multi_Nonaugment_Grid_CVar()
2462{
2463
2464 BESDEBUG("h5", "Coming to Handle_Multi_nonaugment_Grid_CVar()"<<endl);
2465
2466 // If the multiple grids don't share the same lat/lon according to the parameters
2467 // We then assume that each single grid has its own lat/lon, just loop through each grid.
2468 if (true == this->grids_multi_latloncvs) {
2469 for (const auto &cfgrid:this->eos5cfgrids)
2470 Handle_Single_Nonaugment_Grid_CVar(cfgrid);
2471 }
2472
2473 // We would like to check if lat/lon pairs provide for all grids
2474 // If lat/lon pairs are provided for all grids, then we ASSUME that
2475 // all grids share the same lat/lon values. This is what happened with
2476 // Aura grids. We only apply this to Aura files.They provide a lat/lon pair for each grid. We will observe
2477 // if this assumption is true for the future products.
2478 // If lat/lon pairs are not provided for all grids, we assume that each grid
2479 // may still have its unique lat/lon.
2480 else {
2481 int num_1dlatlon_pairs = 0;
2482 for (auto irv = this->eos5cfgrids.begin(); irv != this->eos5cfgrids.end(); ++irv)
2483 if (true == (*irv)->has_1dlatlon) num_1dlatlon_pairs++;
2484
2485 bool use_eos5_latlon = false;
2486 if ((0 == num_1dlatlon_pairs)
2487 || ((num_1dlatlon_pairs == (int) (this->eos5cfgrids.size())) && (true == this->isaura))) {
2488 set<string> tempvardimnamelist = ((this->eos5cfgrids)[0])->vardimnames;
2489 if (0 == num_1dlatlon_pairs) {
2490 use_eos5_latlon = Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon((this->eos5cfgrids)[0],
2491 tempvardimnamelist);
2492
2493 if (false == use_eos5_latlon) return;
2494 }
2495
2496 else {
2497 // One lat/lon for all grids
2498 bool use_own_latlon = false;
2499 use_own_latlon = Handle_Single_Nonaugment_Grid_CVar_OwnLatLon((this->eos5cfgrids)[0],
2500 tempvardimnamelist);
2501 if (false == use_own_latlon) {
2502 use_eos5_latlon = Handle_Single_Nonaugment_Grid_CVar_EOS5LatLon((this->eos5cfgrids)[0],
2503 tempvardimnamelist);
2504 if (false == use_eos5_latlon) return;
2505 }
2506 }
2507
2508 // We need to handle the first grid differently since it will include "XDim" and "YDim".
2509 Handle_NonLatLon_Grid_CVar((this->eos5cfgrids)[0], tempvardimnamelist);
2510
2511 // Updating the dimension name sets for other grids
2512 for (unsigned j = 1; j < this->eos5cfgrids.size(); j++)
2513 (this->eos5cfgrids)[j]->Update_Dimnamelist();
2514
2515 // Adjusting the XDim and YDim dimension names for all vars
2516 Adjust_EOS5GridDimNames((this->eos5cfgrids)[0]);
2517
2518 // Now we can safely handle the rest grids
2519 for (unsigned j = 1; j < this->eos5cfgrids.size(); j++) {
2520 tempvardimnamelist = (this->eos5cfgrids)[j]->vardimnames;
2521 Handle_NonLatLon_Grid_CVar((this->eos5cfgrids)[j], tempvardimnamelist);
2522 tempvardimnamelist.clear();
2523 }
2524 } // if (( 0 == num_1dlatlon_pairs) || .....
2525 // No unique lat/lon, just loop through.
2526 else {
2527
2528 this->grids_multi_latloncvs = true;
2529 for (const auto &cfgrid:this->eos5cfgrids)
2530 Handle_Single_Nonaugment_Grid_CVar(cfgrid);
2531 }
2532 }
2533}
2534
2535// Adjust the HDF-EOS5 grid dimension names for XDim and YDim, we need to remember the grid path
2536// Note this function is used under the assumption that only one lat/lon pair is used for all grids.
2537// This is the case for Aura.
2538void EOS5File::Adjust_EOS5GridDimNames(const EOS5CFGrid *cfgrid) const
2539{
2540
2541 BESDEBUG("h5", "Coming to Adjust_EOS5GridDimNames()"<<endl);
2542 string xdimname;
2543 string ydimname;
2544 bool find_xdim = false;
2545 bool find_ydim = false;
2546
2547 for (const auto &vardimname:cfgrid->vardimnames) {
2548 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash(vardimname);
2549 if ("XDim" == xydimname_candidate) {
2550 find_xdim = true;
2551 xdimname = vardimname;
2552 }
2553 else if ("YDim" == xydimname_candidate) {
2554 find_ydim = true;
2555 ydimname = vardimname;
2556 }
2557 if (find_xdim && find_ydim) break;
2558 } // for (auto it = cfgrid->vardimnames.begin() ...
2559
2560 if (false == find_xdim || false == find_ydim)
2561 throw2("Cannot find Dimension name that includes XDim or YDim in the grid ", cfgrid->name);
2562
2563 for (const auto &var:this->vars) {
2564 if (GRID == Get_Var_EOS5_Type(var)) {
2565 for (const auto &dim:var->dims) {
2566 string xydimname_candidate = HDF5CFUtil::obtain_string_after_lastslash(dim->name);
2567 if ("XDim" == xydimname_candidate)
2568 dim->name = xdimname;
2569 else if ("YDim" == xydimname_candidate) dim->name = ydimname;
2570 }
2571 }
2572 }
2573}
2574
2575// Handle Swath Coordinate variables.
2576void EOS5File::Handle_Swath_CVar(bool isaugmented)
2577{
2578
2579 BESDEBUG("h5", "Coming to Handle_Swath_CVar()"<<endl);
2580 // In this version, we will not use the augmented option for coordinate variables of swath
2581 // since MLS products don't use the recent version of the augmentation tool to allocate their
2582 // coordinate variables.
2583 for (auto irs = this->eos5cfswaths.begin(); irs != this->eos5cfswaths.end();) {
2584 if ((*irs)->has_1dlatlon) {
2585 Handle_Single_1DLatLon_Swath_CVar(*irs, isaugmented);
2586 ++irs;
2587 }
2588 else if ((*irs)->has_2dlatlon) {
2589 Handle_Single_2DLatLon_Swath_CVar(*irs, isaugmented);
2590 ++irs;
2591 }
2592 // If number of dimension latitude/longitude is >2 or no lat/lon,
2593 // no cooridnate variables will be generated.
2594 // We will simply remove this swath from the vector eos5cfswaths.
2595 // In the future, we may consider supporting non "Latitude", "Longitude" naming swaths.
2596 // KY 2011-1-20
2597 else {
2598 delete (*irs);
2599 irs = this->eos5cfswaths.erase(irs);
2600 }
2601 }
2602}
2603
2604// Handle single 1D LatLon Swath Coordinate variables.
2605void EOS5File::Handle_Single_1DLatLon_Swath_CVar(EOS5CFSwath *cfswath, bool is_augmented)
2606{
2607
2608 BESDEBUG("h5", "Coming to Handle_Single_1DLatLon_Swath_CVar"<<endl);
2609 // For 1DLatLon, we will use latitude as the coordinate variable
2610 set<string> tempvardimnamelist = cfswath->vardimnames;
2611 string EOS5SWATHPATH = "/HDFEOS/SWATHS/";
2612 string fslash_str = "/";
2613 string THIS_EOS5SWATHPATH = EOS5SWATHPATH + cfswath->name + fslash_str;
2614#if 0
2615 for (auto its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2616 cerr<<"Dimension name befor latitude " << *its << endl;
2617#endif
2618
2619 // Find latitude and assign to the coordinate variable
2620 // (*irv)->fullpath.size() > THIS_EOS5SWATHPATH.size() is necessary to handle the augmented variables.
2621 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2622 if (SWATH == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5SWATHPATH.size())) {
2623
2624 string var_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2625 if ((var_swath_name == cfswath->name) && ((*irv)->name == "Latitude")) {
2626
2627 //Find it, create a coordinate variable.
2628 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
2629 auto EOS5cvar = EOS5cvar_unique.release();
2630
2631 // Still keep the original dimension name to avoid the name clashing when
2632 // one grid and one swath and one za occur in the same file
2633 EOS5cvar->cfdimname = ((*irv)->dims)[0]->name;
2634 EOS5cvar->cvartype = CV_EXIST;
2635 EOS5cvar->eos_type = SWATH;
2636
2637 // Save this cv to the cv vector
2638 this->cvars.push_back(EOS5cvar);
2639
2640 // Remove this var from the var vector since it becomes a cv.
2641 delete (*irv);
2642 this->vars.erase(irv);
2643 break;
2644 } // if ((var_swath_name == cfswath->name) && ...
2645 } // if SWATH
2646 } // for (auto irv = this->vars.begin() ...
2647
2648 // Finish this variable, remove it from the list.
2649
2650 bool find_lat_dim = false;
2651 for (auto its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2652
2653 for (const auto &cvar:this->cvars) {
2654 if ((cvar->name == "Latitude") && cvar->cfdimname == (*its)) {
2655 tempvardimnamelist.erase(its);
2656 find_lat_dim = true;
2657 break;
2658 }
2659 }
2660
2661 if (true == find_lat_dim) break;
2662 }
2663
2664#if 0
2665 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2666 cerr<<"Dimension name afte latitude " << *its << endl;
2667#endif
2668
2669 Handle_NonLatLon_Swath_CVar(cfswath, tempvardimnamelist);
2670
2671 // Remove the added variables during the augmentation process
2672 if (true == is_augmented) {
2673 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
2674
2675 if (SWATH == Get_Var_EOS5_Type(*irv)) {
2676#if 0
2677 string my_swath_short_path = (*irv)->fullpath.substr(EOS5SWATHPATH.size());
2678 size_t first_fslash_pos = my_swath_short_path.find_first_of("/");
2679 string my_swath_name = my_swath_short_path.substr(0,first_fslash_pos);
2680#endif
2681 // Need to find the swath for this variable
2682 string my_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2683
2684 if (my_swath_name == cfswath->name) {
2685 string var_path_after_swathname = (*irv)->fullpath.substr(THIS_EOS5SWATHPATH.size());
2686 if (var_path_after_swathname == (*irv)->name) {
2687 delete (*irv);
2688 irv = this->vars.erase(irv);
2689 }
2690 else {
2691 ++irv;
2692 }
2693 }
2694 else {
2695 ++irv;
2696 }
2697 }
2698 else {
2699 ++irv;
2700 }
2701 } // end of for loop
2702 } // if (true == is_augmented)
2703}
2704
2705// Handle Single 2D lat/lon Coordinate variables for Swath
2706void EOS5File::Handle_Single_2DLatLon_Swath_CVar(EOS5CFSwath *cfswath, bool is_augmented)
2707{
2708
2709 BESDEBUG("h5", "Coming to Handle_Single_2DLatLon_Swath_CVar()"<<endl);
2710 // For 2DLatLon, we will use both latitude and longitude as the coordinate variables
2711 set<string>::iterator its;
2712 set<string> tempvardimnamelist = cfswath->vardimnames;
2713 string EOS5SWATHPATH = "/HDFEOS/SWATHS/";
2714 string fslash_str = "/";
2715 string THIS_EOS5SWATHPATH = EOS5SWATHPATH + cfswath->name + fslash_str;
2716 bool find_lat = false;
2717 bool find_lon = false;
2718
2719#if 0
2720 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2721 cerr<<"Dimension name befor latitude " << *its << endl;
2722#endif
2723
2724 // Find latitude and assign to the coordinate variable
2725 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
2726 if (SWATH == Get_Var_EOS5_Type(*irv) && ((*irv)->fullpath.size() > THIS_EOS5SWATHPATH.size())) {
2727 string var_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2728 if ((var_swath_name == cfswath->name) && ((*irv)->name == "Latitude")) {
2729
2730 //Find it, create a coordinate variable.
2731 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
2732 auto EOS5cvar = EOS5cvar_unique.release();
2733
2734 // Still keep the original dimension name to avoid the name clashing when
2735 // one grid and one swath and one za occur in the same file
2736 EOS5cvar->cfdimname = ((*irv)->dims)[0]->name;
2737 EOS5cvar->cvartype = CV_EXIST;
2738 EOS5cvar->eos_type = SWATH;
2739 EOS5cvar->is_2dlatlon = true;
2740
2741 // Save this cv to the cv vector
2742 this->cvars.push_back(EOS5cvar);
2743
2744 // Remove this var from the var vector since it becomes a cv.
2745 delete (*irv);
2746 irv = this->vars.erase(irv);
2747 find_lat = true;
2748 }
2749 else if ((var_swath_name == cfswath->name) && ((*irv)->name == "Longitude")) {
2750
2751 //Find it, create a coordinate variable.
2752 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
2753 auto EOS5cvar = EOS5cvar_unique.release();
2754
2755 // Still keep the original dimension name to avoid the name clashing when
2756 // one grid and one swath and one za occur in the same file
2757 EOS5cvar->cfdimname = ((*irv)->dims)[1]->name;
2758 EOS5cvar->cvartype = CV_EXIST;
2759 EOS5cvar->eos_type = SWATH;
2760 EOS5cvar->is_2dlatlon = true;
2761
2762 // Save this cv to the cv vector
2763 this->cvars.push_back(EOS5cvar);
2764
2765 // Remove this var from the var vector since it becomes a cv.
2766 delete (*irv);
2767 irv = this->vars.erase(irv);
2768 find_lon = true;
2769
2770 }
2771 else {
2772 ++irv;
2773 }
2774 } // if (SWATH == Get_Var_EOS5_Type(*irv) && ...
2775 else {
2776 ++irv;
2777 }
2778
2779 if (true == find_lat && true == find_lon) break;
2780 } // for (auto irv = this->vars.begin();
2781
2782 // Remove the dim. of latitude
2783 find_lat = false;
2784 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2785 for (const auto &cvar:this->cvars) {
2786 if ((cvar->name == "Latitude") && cvar->cfdimname == (*its)) {
2787 tempvardimnamelist.erase(its);
2788 find_lat = true;
2789 break;
2790 }
2791 }
2792
2793 if (true == find_lat) break;
2794 }
2795
2796 // Remove the dim. of longitude
2797 find_lon = false;
2798 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2799
2800 for (const auto &cvar:this->cvars) {
2801
2802 if ((cvar->name == "Longitude") && cvar->cfdimname == (*its)) {
2803 tempvardimnamelist.erase(its);
2804 find_lon = true;
2805 break;
2806 }
2807 }
2808
2809 if (true == find_lon) break;
2810 }
2811
2812#if 0
2813 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its)
2814 cerr<<"Dimension name afte latitude " << *its << endl;
2815#endif
2816
2817 Handle_NonLatLon_Swath_CVar(cfswath, tempvardimnamelist);
2818
2819 // Remove the added variables during the augmentation process
2820 // For Swath, we don't want to keep the augmented files. This is because
2821 // some aura files assign the dimensional scale as zero.
2822 // We will actively check the new NASA HDF-EOS5 products and will
2823 // revise the following section as needed. KY 2012-03-09
2824 if (true == is_augmented) {
2825 for (auto irv = this->vars.begin(); irv != this->vars.end();) {
2826
2827 if (SWATH == Get_Var_EOS5_Type(*irv)) {
2828
2829 string my_swath_name = Obtain_Var_EOS5Type_GroupName(*irv, SWATH);
2830 if (my_swath_name == cfswath->name) {
2831 string var_path_after_swathname = (*irv)->fullpath.substr(THIS_EOS5SWATHPATH.size());
2832 if (var_path_after_swathname == (*irv)->name) {
2833 delete (*irv);
2834 irv = this->vars.erase(irv);
2835 }
2836 else {
2837 ++irv;
2838 }
2839 }
2840 else {
2841 ++irv;
2842 }
2843 }
2844 else {
2845 ++irv;
2846 }
2847 }
2848 }
2849}
2850
2851// Handle non-lat/lon Swath coordinate variables.
2852void EOS5File::Handle_NonLatLon_Swath_CVar(EOS5CFSwath *cfswath, set<string>& tempvardimnamelist)
2853{
2854
2855 BESDEBUG("h5", "Coming to Handle_NonLatLon_Swath_CVar()"<<endl);
2856 // First check if we have existing coordinate variable
2857 set<string>::iterator its;
2858 auto num_dimnames = (int)(tempvardimnamelist.size());
2859 bool has_dimnames = true;
2860 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2861 if (cfswath->dnames_to_geo1dvnames.find(*its) != cfswath->dnames_to_geo1dvnames.end()) {
2862 for (auto irv = this->vars.begin(); has_dimnames && (irv != this->vars.end());) {
2863
2864 // We need to check if this var is a swath and use "newname"
2865 // of var to check the dnames_to_1dvnames since it is
2866 // possible to have name clashing for the "name" of a var.
2867 if (SWATH == Get_Var_EOS5_Type(*irv) && (*irv)->newname == (cfswath->dnames_to_geo1dvnames)[*its]) {
2868
2869 //Find it, create a coordinate variable.
2870 auto EOS5cvar_unique = make_unique<EOS5CVar>(*irv);
2871 auto EOS5cvar = EOS5cvar_unique.release();
2872
2873 // Still keep the original dimension name to avoid the name clashing when
2874 // one grid and one swath and one za occur in the same file
2875 EOS5cvar->cfdimname = *its;
2876 EOS5cvar->cvartype = CV_EXIST;
2877 EOS5cvar->eos_type = SWATH;
2878
2879 // Save this cv to the cv vector
2880 this->cvars.push_back(EOS5cvar);
2881
2882 // Remove this var from the var vector since it becomes a cv.
2883 delete (*irv);
2884 irv = this->vars.erase(irv);
2885 num_dimnames--;
2886 if (0 == num_dimnames) has_dimnames = false;
2887 }
2888 else {
2889 ++irv;
2890 }
2891 } // for (auto irv = this->vars.begin(); ...
2892 } // if (cfswath->dnames_to_geo1dvnames.find(*its) ....
2893 } // for (its = tempvardimnamelist.begin()...
2894
2895 // Remove the dimension name that finds the cooresponding variables from the tempvardimlist.
2896 for (const auto &cvar:this->cvars) {
2897 its = tempvardimnamelist.find(cvar->cfdimname);
2898 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2899 }
2900
2901 // Check if some attributes have CV information for some special products
2902 // Currently TES needs to be handled carefully
2903 Handle_Special_NonLatLon_Swath_CVar(cfswath, tempvardimnamelist);
2904
2905 // Remove the dimension name that finds the cooresponding variables from the tempvardimlist.
2906 for (const auto &cvar:this->cvars) {
2907 its = tempvardimnamelist.find(cvar->cfdimname);
2908 if (its != tempvardimnamelist.end()) tempvardimnamelist.erase(its);
2909 }
2910
2911 // Second: Some dimension names still need to find CVs, create the missing CVs
2912 for (its = tempvardimnamelist.begin(); its != tempvardimnamelist.end(); ++its) {
2913
2914 auto EOS5cvar_unique = make_unique<EOS5CVar>();
2915 auto EOS5cvar = EOS5cvar_unique.release();
2916 Create_Missing_CV(cfswath, EOS5cvar, *its, SWATH, this->eos5cfswaths.size());
2917 this->cvars.push_back(EOS5cvar);
2918
2919 }
2920}
2921
2922// Handle special non-lat/lon coordinate variables for swath.
2923void EOS5File::Handle_Special_NonLatLon_Swath_CVar(EOS5CFSwath *cfswath, const set<string>& tempvardimnamelist)
2924
2925{
2926
2927 BESDEBUG("h5", "Handle_Special_NonLatLon_Swath_CVar()"<<endl);
2928 // We have no choice but hard-code this one.
2929 // TES swath puts "Pressure" as the VerticalCoordinate but doesn't provide "Pressure" values.
2930 // Moreover, the number of pressure level(66) is one less than the total number of corresponding dimension size(67)
2931 // most probably due to the missing pressure level on the ground. To make the handler visualize some
2932 // TES variables and to follow the general physical sense. We have to add a pressure level by linear interpolation.
2933 // KY 2012-1-27
2934 if (true == this->isaura && TES == this->aura_name) {
2935
2936 string eos5_swath_group_name = "/HDFEOS/SWATHS/" + cfswath->name;
2937 string eos5_vc_attr_name = "VerticalCoordinate";
2938 string eos5_pre_attr_name = "Pressure";
2939 bool has_vc_attr = false;
2940 const Group *vc_group = nullptr;
2941
2942 // 1. Check if having the "VerticalCoordinate" attribute in this swath and the attribute is "Pressure".
2943 for (const auto &grp:this->groups) {
2944 if (eos5_swath_group_name == grp->path) {
2945 for (const auto &attr:grp->attrs) {
2946 if (eos5_vc_attr_name == attr->name) {
2947 Retrieve_H5_Attr_Value(attr, grp->path);
2948 string attr_value(attr->value.begin(), attr->value.end());
2949 if (eos5_pre_attr_name == attr_value) {
2950 has_vc_attr = true;
2951 vc_group = grp;
2952 break;
2953 }
2954 }
2955 } // for (vector<Attribute *>:: iterator ira =grp->attrs.begin(); ...
2956 if (true == has_vc_attr) break;
2957 } // if (eos5_swath_group_name ==grp->path)
2958 } // for (const auto &grp: ...
2959
2960 // 2. Check if having the "Pressure" attribute and if the attribute size is 1 less than
2961 // the dimension size of "nLevels". If yes,
2962 // add one pressure value by using the nearest neighbor value. This value should be the first value
2963 // of the "Pressure" attribute.
2964 // Another special part of the TES file is that dimension name nLevels is used twice in some variables
2965 // float foo[...][nLevels][nLevels]. To make the variable visualized by tools, the dimension name
2966 // needs to be changed and the coordinate variable needs to separately created. Note this is not
2967 // against CF conventions. However, the popular tools are not happy with the duplicate dimension names
2968 // in a variable.
2969 // Though may not cover 100% cases, searching the string after the last forward slash and see if
2970 // it contains nLevels should catch 99% memebers of the "nLevels" family. We will then create the
2971 // corresponding coordinate variables.
2972
2973 // 2.1. Check if we have the dimension name called "nLevels" for this swath
2974 if (true == has_vc_attr) {
2975 string dimname_candidate = "/SWATHS/" + cfswath->name + "/nLevels";
2976 for (auto it = tempvardimnamelist.begin(); it != tempvardimnamelist.end(); ++it) {
2977 if ((*it).find(dimname_candidate) != string::npos) {
2978 hsize_t dimsize_candidate = 0;
2979 if ((cfswath->dimnames_to_dimsizes).find(*it) != (cfswath->dimnames_to_dimsizes).end())
2980 dimsize_candidate = cfswath->dimnames_to_dimsizes[*it];
2981 else
2982 throw2("Cannot find the dimension size of the dimension name ", *it);
2983
2984 // Note: we don't have to use two loops to create the coordinate variables.
2985 // However, there are only 3-4 attributes for this group and so far TES has only
2986 // one additional nLevels.
2987 // So essentially the following loop doesn't hurt the performance.
2988 // KY 2012-2-1
2989 for (const auto &attr:vc_group->attrs) {
2990 if ((eos5_pre_attr_name == attr->name) && (attr->count == (dimsize_candidate - 1))) {
2991
2992 // Should change the attr_value from char type to float type when reading the data
2993 // Here just adding a coordinate variable by using this name.
2994 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(*it);
2995 string orig_dimname = "nLevels";
2996 auto EOS5cvar_unique = make_unique<EOS5CVar>();
2997 auto EOS5cvar = EOS5cvar_unique.release();
2998 if ("nLevels" == reduced_dimname)
2999 EOS5cvar->name = eos5_pre_attr_name + "_CV";
3000 else
3001 // the dimname will be ..._CV_1 etc.
3002 EOS5cvar->name = eos5_pre_attr_name + "_CV"
3003 + reduced_dimname.substr(orig_dimname.size());
3004 Create_Added_Var_NewName_FullPath(SWATH, cfswath->name, EOS5cvar->name, EOS5cvar->newname,
3005 EOS5cvar->fullpath);
3006 EOS5cvar->rank = 1;
3007 EOS5cvar->dtype = attr->dtype;
3008 auto eos5cvar_dim_unique = make_unique<Dimension>(dimsize_candidate);
3009 auto eos5cvar_dim = eos5cvar_dim_unique.release();
3010 eos5cvar_dim->name = *it;
3011 if (1 == this->eos5cfswaths.size())
3012 eos5cvar_dim->newname = reduced_dimname;
3013 else
3014 eos5cvar_dim->newname = eos5cvar_dim->name;
3015
3016 EOS5cvar->dims.push_back(eos5cvar_dim);
3017 EOS5cvar->cvartype = CV_SPECIAL;
3018 EOS5cvar->cfdimname = eos5cvar_dim->name;
3019 EOS5cvar->eos_type = SWATH;
3020
3021 // Save this cv to the cv vector
3022 this->cvars.push_back(EOS5cvar);
3023 } // if ((eos5_pre_attr_name == attr->name) && ...
3024 } // for auto attr
3025 } // if ((*it).find(dimname_candidate) != string::npos)
3026 } // for (it = tempvardimnamelist.begin(); ...
3027 } // if (true == has_vc_attr) ...
3028 } // if (true == this->isaura && ...
3029}
3030
3031// Handle Zonal average coordinate variables.
3032void EOS5File::Handle_Za_CVar(bool isaugmented)
3033{
3034
3035 BESDEBUG("h5", "Coming to Handle_Za_CVar()"<<endl);
3036 // We are not supporting non-augmented zonal average HDF-EOS5 product now. KY:2012-1-20
3037 if (false == isaugmented) return;
3038
3039 for (const auto &cfza:this->eos5cfzas)
3040 Handle_Single_Augment_CVar(cfza, ZA);
3041
3042}
3043
3044// Adjust the newname(final names appeared at DDS) for variable and dimensions before flattening.
3046{
3047
3048 BESDEBUG("h5", "Coming to Adjust_Var_Dim_NewName_Before_Flattening()"<<endl);
3049 auto num_grids = (int)(this->eos5cfgrids.size());
3050 auto num_swaths = (int)(this->eos5cfswaths.size());
3051 auto num_zas = (int)(this->eos5cfzas.size());
3052
3053 bool mixed_eos5typefile = false;
3054
3055 // Check if this file mixes grid,swath and zonal average
3056 if (((num_grids > 0) && (num_swaths > 0)) || ((num_grids > 0) && (num_zas > 0))
3057 || ((num_swaths > 0) && (num_zas > 0))) mixed_eos5typefile = true;
3058
3059 // This file doesn't mix swath, grid and zonal average
3060 for (const auto &var:this->vars)
3061 Adjust_Per_Var_Dim_NewName_Before_Flattening(var, mixed_eos5typefile, num_grids, num_swaths, num_zas);
3062
3063 for (const auto &cvar:this->cvars)
3064 Adjust_Per_Var_Dim_NewName_Before_Flattening(cvar, mixed_eos5typefile, num_grids, num_swaths, num_zas);
3065#if 0
3066 for (auto irv = this->cvars.begin();
3067 irv != this->cvars.end(); ++irv) {
3068 cerr<<"eos5svar var new name "<<(*irv)->newname <<endl;
3069 for (auto ird = (*irv)->dims.begin();
3070 ird !=(*irv)->dims.end(); ++ird) {
3071 cerr<<"eos5svar dimension new name "<<(*ird)->newname <<endl;
3072 }
3073 }
3074#endif
3075 // If (lat,lon) is shared for grids, more consideration for the names
3076 Adjust_SharedLatLon_Grid_Var_Dim_Name();
3077
3078}
3079
3080// Adjust the final name of one variable or dim. before flattening the names
3081template<class T>
3082void EOS5File::Adjust_Per_Var_Dim_NewName_Before_Flattening(T* var, bool mixed_eos5type, int num_grids, int num_swaths,
3083 int num_zas) const
3084{
3085
3086 BESDEBUG("h5", "Coming to Adjust_Per_Var_Dim_NewName_Before_Flattening()"<<endl);
3087
3088 string eos5typestr;
3089 EOS5Type vartype = Get_Var_EOS5_Type(var);
3090 switch (vartype) {
3091
3092 case GRID: {
3093 eos5typestr = "/GRIDS/";
3094 if (false == mixed_eos5type) {
3095 if (0 == num_grids)
3096 var->newname = ((1 == this->orig_num_grids) ? var->name : var->newname.substr(eos5typestr.size()));
3097 else
3098 var->newname = ((1 == num_grids) ? var->name : var->newname.substr(eos5typestr.size()));
3099 // Dimension newname is unlike Var newname, when num_grids is equal to 1, the
3100 // newname is Dimension name already. So we don't need to do anything with
3101 // the dimension newname when the num_grids is 1. The main reason we handle
3102 // the var newname and the dimension newname differently is that the variable name is
3103 // more critical for users to pick up the meanings of that variable. So we would like
3104 // to work hard to keep the original form. However, the dimension name is mainly used to
3105 // generate the coordinate variables. So the different usage makes us relax the dimension
3106 // name a bit. This is an example of end-user priority driven implementation.
3107 // KY 2012-1-24
3108 // Just receive a user request: the dimension name is also very important.
3109 // So a bunch of code has been updated. For number of grid/swath/za = 1, I still maintain
3110 // the newname to be the same as the last part of the dim name. Hopefully this
3111 // will handle the current HDF-EOS5 products. Improvement for complicate HDF-EOS5 products
3112 // will be supported as demanded in the future. KY 2012-1-26
3113 if (num_grids > 1) {
3114 for (auto &dim:var->dims) {
3115 if (dim->newname.size() <= eos5typestr.size())
3116 throw5("The size of the dimension new name ", dim->newname, "of variable ", var->newname,
3117 " is too small");
3118 dim->newname = dim->newname.substr(eos5typestr.size());
3119 }
3120 }
3121 } // if(false == mixed_eos5type)
3122 else {
3123 // No need to set the dimension newname for the reason listed above.
3124 var->newname = ((1 == num_grids) ? (eos5typestr + var->name) : var->newname);
3125 }
3126 }
3127 break;
3128
3129 case SWATH: {
3130 eos5typestr = "/SWATHS/";
3131 if (false == mixed_eos5type) {
3132 var->newname = ((1 == num_swaths) ? var->name : var->newname.substr(eos5typestr.size()));
3133 if (num_swaths > 1) {
3134 for (auto &dim:var->dims) {
3135 if (dim->newname.size() <= eos5typestr.size())
3136 throw5("The size of the dimension new name ", dim->newname, "of variable ", var->newname,
3137 " is too small");
3138 dim->newname = dim->newname.substr(eos5typestr.size());
3139 }
3140 }
3141 }
3142 else {
3143 var->newname = ((1 == num_swaths) ? (eos5typestr + var->name) : var->newname);
3144 }
3145 }
3146 break;
3147
3148 case ZA: {
3149 eos5typestr = "/ZAS/";
3150 if (false == mixed_eos5type) {
3151 var->newname = ((1 == num_zas) ? var->name : var->newname.substr(eos5typestr.size()));
3152 if (num_zas > 1) {
3153 for (auto &dim:var->dims) {
3154 if (dim->newname.size() <= eos5typestr.size())
3155 throw5("The size of the dimension new name ", dim->newname, "of variable ", var->newname,
3156 " is too small");
3157 dim->newname = dim->newname.substr(eos5typestr.size());
3158 }
3159 }
3160 }
3161 else {
3162 var->newname = ((1 == num_zas) ? (eos5typestr + var->name) : var->newname);
3163 }
3164 }
3165 break;
3166 case OTHERVARS:
3167 break;
3168 default:
3169 throw1("Non-supported EOS type");
3170 }
3171
3172}
3173
3174// Adjust shared var and dim names for shared lat/lon grid case.
3175void EOS5File::Adjust_SharedLatLon_Grid_Var_Dim_Name() const
3176{
3177
3178 BESDEBUG("h5", "Adjust_SharedLatLon_Grid_Var_Dim_Name()"<<endl);
3179 // Remove the EOS5 type string("GRIDS") and the GRID Name from
3180 // the variable newname and the dimension newname
3181 // This case won't happen for the current version, but may occur
3182 // if curviliner grid exists in the file. KY 2012-1-26
3183 if ((this->eos5cfgrids.size() > 1) && (true == this->eos5cfswaths.empty()) && (true == this->eos5cfzas.empty())
3184 && (false == this->grids_multi_latloncvs)) {
3185
3186 // We would like to condense the dimension name and the coordinate variable name for lat/lon.
3187 string lat_dimname;
3188 string lat_dimnewname;
3189 string lon_dimname;
3190 string lon_dimnewname;
3191 for (const auto &cvar:this->cvars) {
3192 if ("lat" == cvar->name || "Latitude" == cvar->name) {
3193 cvar->newname = cvar->name;
3194 lat_dimnewname = ((cvar->dims)[0])->newname;
3195 lat_dimnewname = HDF5CFUtil::obtain_string_after_lastslash(lat_dimnewname);
3196 if ("" == lat_dimnewname)
3197 throw2("/ is not included in the dimension new name ", ((cvar->dims)[0])->newname);
3198 ((cvar->dims)[0])->newname = lat_dimnewname;
3199 lat_dimname = cvar->cfdimname;
3200 }
3201 else if ("lon" == cvar->name || "Longitude" == cvar->name) {
3202 cvar->newname = cvar->name;
3203 lon_dimnewname = ((cvar->dims)[0])->newname;
3204 lon_dimnewname = HDF5CFUtil::obtain_string_after_lastslash(lon_dimnewname);
3205 if ("" == lon_dimnewname)
3206 throw2("/ is not included in the dimension new name ", ((cvar->dims)[0])->newname);
3207 ((cvar->dims)[0])->newname = lon_dimnewname;
3208 lon_dimname = cvar->cfdimname;
3209 }
3210 } // for
3211
3212 for (auto &var:this->vars) {
3213 for (auto &dim:var->dims) {
3214 if (dim->name == lat_dimname)
3215 dim->newname = lat_dimnewname;
3216 else if (dim->name == lon_dimname) dim->newname = lon_dimnewname;
3217 }
3218 }
3219 } // if ((this->eos5cfgrids.size() > 1) && ...
3220}
3221
3222// Flatten the object names.
3223void EOS5File::Flatten_Obj_Name(bool include_attr)
3224{
3225
3226 BESDEBUG("h5", "Coming to Flatten_Obj_Name()"<<endl);
3227 File::Flatten_Obj_Name(include_attr);
3228
3229 for (auto &cvar:this->cvars) {
3230 cvar->newname = get_CF_string(cvar->newname);
3231
3232 for (auto &dim:cvar->dims)
3233 dim->newname = get_CF_string(dim->newname);
3234
3235 if (true == include_attr) {
3236 for (auto &attr:cvar->attrs) {
3237 attr->newname = File::get_CF_string(attr->newname);
3238 }
3239 }
3240 } // end for
3241}
3242
3243// Handle Object Name clashing
3245{
3246
3247 BESDEBUG("h5", "Coming to Handle_Obj_NameClashing()"<<endl);
3248 // objnameset will be filled with all object names that we are going to check the name clashing.
3249 // For example, we want to see if there are any name clashing for all variable names in this file.
3250 // objnameset will include all variable names. If a name clashing occurs, we can figure out from the set operation immediately.
3251 set<string> objnameset;
3252 Handle_EOS5CVar_NameClashing(objnameset);
3253 File::Handle_GeneralObj_NameClashing(include_attr, objnameset);
3254 if (true == include_attr) {
3255 Handle_EOS5CVar_AttrNameClashing();
3256 }
3257#if 0
3258 //if (this->cvars.size() >0)
3259 // Handle_DimNameClashing();
3260#endif
3261}
3262
3263// Handle EOS5 coordinate variable name clashing
3264void EOS5File::Handle_EOS5CVar_NameClashing(set<string> &objnameset)
3265{
3266
3267 BESDEBUG("h5", "Coming to Handle_EOS5CVar_NameClashing()"<<endl);
3268 EOS5Handle_General_NameClashing(objnameset, this->cvars);
3269}
3270
3271// Handle EOS5 coordinate variable attribute name clashing
3272void EOS5File::Handle_EOS5CVar_AttrNameClashing()
3273{
3274
3275 BESDEBUG("h5", "Coming to Handle_EOS5CVar_AttrNameClashing()"<<endl);
3276 set<string> objnameset;
3277
3278 for (const auto &cvar:this->cvars) {
3279 Handle_General_NameClashing(objnameset, cvar->attrs);
3280 objnameset.clear();
3281 }
3282}
3283// The routine to handle general name clashing
3284//class T must have member string newname
3285template<class T> void EOS5File::EOS5Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
3286
3287{
3288
3289 BESDEBUG("h5", "Coming to EOS5Handle_General_NameClashing()"<<endl);
3290 pair<set<string>::iterator, bool> setret;
3291 set<string>::iterator iss;
3292
3293 vector<string> clashnamelist;
3294
3295 map<int, int> cl_to_ol;
3296 int ol_index = 0;
3297 int cl_index = 0;
3298
3299#if 0
3300 typename vector<T*>::iterator irv;
3301#endif
3302
3303 for (auto irv = objvec.begin(); irv != objvec.end(); ++irv) {
3304
3305 setret = objnameset.insert((*irv)->newname);
3306 if (!setret.second) {
3307 clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
3308 cl_to_ol[cl_index] = ol_index;
3309 cl_index++;
3310 }
3311 ol_index++;
3312 }
3313
3314 // Now change the clashed elements to unique elements;
3315 // Generate the set which has the same size as the original vector.
3316 for (auto &clashname:clashnamelist) {
3317 int clash_index = 1;
3318 string temp_clashname = clashname + '_';
3319 HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
3320 clashname = temp_clashname;
3321 }
3322
3323 // Now go back to the original vector, make it unique.
3324 for (unsigned int i = 0; i < clashnamelist.size(); i++)
3325 objvec[cl_to_ol[i]]->newname = clashnamelist[i];
3326
3327}
3328
3329// Handle Dimension name clashing
3331{
3332
3333 BESDEBUG("h5", "Coming to Handle_DimNameClashing()"<<endl);
3334 map<string, string> dimname_to_dimnewname;
3335 pair<map<string, string>::iterator, bool> mapret;
3336 set<string> dimnameset;
3337 vector<Dimension*> vdims;
3338 set<string> dimnewnameset;
3339 pair<set<string>::iterator, bool> setret;
3340
3341 // First: Generate the dimset/dimvar based on coordinate variables.
3342 for (const auto &cvar:this->cvars) {
3343 for (const auto &dim:cvar->dims) {
3344#if 0
3345 //setret = dimnameset.insert((*ird)->newname);
3346#endif
3347 setret = dimnameset.insert(dim->name);
3348 if (setret.second) vdims.push_back(dim);
3349 }
3350 }
3351
3352 // For some cases, dimension names are provided but there are no corresponding coordinate
3353 // variables. For now, we will assume no such cases.
3354 // Actually, we find such a case in our fake testsuite. So we need to fix it.
3355
3356 for (const auto &var:this->vars) {
3357 for (const auto &dim:var->dims) {
3358#if 0
3359 //setret = dimnameset.insert((*ird)->newname);
3360#endif
3361 setret = dimnameset.insert(dim->name);
3362 if (setret.second) vdims.push_back(dim);
3363 }
3364 }
3365
3366#if 0
3367 for (auto ird=vdims.begin();ird!=vdims.end();++ird)
3368 cerr<<"dimension name "<<(*ird)->name <<endl;
3369#endif
3370
3371 // For some cases, dimension names are provided but there are no corresponding coordinate
3372 // variables. For now, we will assume no such cases.
3373 EOS5Handle_General_NameClashing(dimnewnameset, vdims);
3374
3375 // Third: Make dimname_to_dimnewname map
3376 for (const auto &dim:vdims) {
3377 mapret = dimname_to_dimnewname.insert(pair<string, string>(dim->name, dim->newname));
3378 if (false == mapret.second)
3379 throw4("The dimension name ", dim->name, " should map to ", dim->newname);
3380 }
3381
3382 // Fourth: Change the original dimension new names to the unique dimension new names
3383 for (auto &cvar:this->cvars)
3384 for (auto &dim:cvar->dims)
3385 dim->newname = dimname_to_dimnewname[dim->name];
3386
3387 for (auto &var:this->vars)
3388 for (auto &dim:var->dims)
3389 dim->newname = dimname_to_dimnewname[dim->name];
3390
3391}
3392
3393// Set COARDS Status, if we can follow COARDS, we should follow COARDS.
3394// http://ferret.wrc.noaa.gov/noaa_coop/coop_cdf_profile.html
3396{
3397
3398 BESDEBUG("h5", "Coming to Set_COARDS_Status()"<<endl);
3399 iscoard = true;
3400 for (const auto &cfgrid:this->eos5cfgrids) {
3401 if (false == cfgrid->has_1dlatlon) {
3402 if (false == cfgrid->has_nolatlon || (HE5_GCTP_GEO != cfgrid->eos5_projcode)) iscoard = false;
3403 break;
3404 }
3405 }
3406
3407 if (true == iscoard) {
3408 for (const auto &cfswath:this->eos5cfswaths) {
3409 if (false == cfswath->has_1dlatlon) {
3410 iscoard = false;
3411 break;
3412 }
3413 }
3414 }
3415}
3416
3417// Adjust attribute info., mostly for CF name correction of Aura files.
3419{
3420
3421 BESDEBUG("h5", "Coming to Adjust_Attr_Info()"<<endl);
3422 if (true == this->isaura) {
3423 Adjust_Aura_Attr_Name();
3424 Adjust_Aura_Attr_Value();
3425 }
3426 else {
3427 Handle_EOS5CVar_Unit_Attr();
3428 Add_EOS5_Grid_CF_Attr();
3429 }
3430}
3431
3432// Adjust Attribute Name, mostly for Aura files.
3433void EOS5File::Adjust_Aura_Attr_Name()
3434{
3435
3436 BESDEBUG("h5", "Coming to Adjust_Attr_Name() for Aura"<<endl);
3437 for (auto &var:this->vars) {
3438 for (auto &attr:var->attrs) {
3439 if (eos5_to_cf_attr_map.find(attr->name) != eos5_to_cf_attr_map.end())
3440 attr->newname = eos5_to_cf_attr_map[attr->name];
3441 }
3442 }
3443
3444 for (auto &var:this->cvars) {
3445 for (auto &attr:var->attrs) {
3446 if (eos5_to_cf_attr_map.find(attr->name) != eos5_to_cf_attr_map.end())
3447 attr->newname = eos5_to_cf_attr_map[attr->name];
3448 }
3449 }
3450}
3451
3452void EOS5File::Adjust_Aura_Attr_Value() const
3453{
3454
3455 BESDEBUG("h5", "Coming to Adjust_Attr_Value() for Aura"<<endl);
3456 // Handle Units
3457 Handle_EOS5CVar_Unit_Attr();
3458 Handle_Aura_Special_Attr();
3459
3460 // Handle Time. This is just for Aura files.
3461 // This is for speical NASA requests only for Aura.
3462 // We need to pay attention if things get changed later.
3463 string time_cf_units_value = "seconds since 1993-01-01";
3464 for (const auto &var:this->vars) {
3465 if ((var->name == "Time") || (var->name == "nTimes")) {
3466 for (const auto &attr:var->attrs) {
3467 if ("units" == attr->name) {
3468 Retrieve_H5_Attr_Value(attr, var->fullpath);
3469 string units_value(attr->value.begin(), attr->value.end());
3470 if (time_cf_units_value != units_value) {
3471
3472 units_value = time_cf_units_value;
3473 attr->value.resize(units_value.size());
3474 if (H5FSTRING == attr->dtype) attr->fstrsize = units_value.size();
3475 // strsize is used by both fixed and variable length strings.
3476 attr->strsize.resize(1);
3477 attr->strsize[0] = units_value.size();
3478
3479 copy(units_value.begin(), units_value.end(), attr->value.begin());
3480 }
3481 break;
3482 } // if ("units" == attr->name)
3483 } // for (const auto &attr
3484 } // if((var->name == "Time") || (var->name == "nTimes"))
3485 } // for (auto irv = this->vars.begin()...
3486}
3487
3488// Handle EOS5 coordinate variable special attributes.
3489void EOS5File::Handle_Aura_Special_Attr() const
3490{
3491
3492 BESDEBUG("h5", "Coming to Handle_Aura_Special_Attr()"<<endl);
3493 // Need to handle MLS aura file specially.
3494 if (true == this->isaura && MLS == this->aura_name) {
3495
3496 const string File_attr_group_path = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES";
3497 const string PCF1_attr_name = "PCF1";
3498 bool find_group = false;
3499 bool find_attr = false;
3500 for (const auto &grp:this->groups) {
3501 if (File_attr_group_path == grp->path) {
3502 find_group = true;
3503 for (const auto &attr:grp->attrs) {
3504 if (PCF1_attr_name == attr->name) {
3505 Retrieve_H5_Attr_Value(attr, grp->path);
3506 string pcf_value(attr->value.begin(), attr->value.end());
3507 HDF5CFDAPUtil::replace_double_quote(pcf_value);
3508 attr->value.resize(pcf_value.size());
3509 if (H5FSTRING == attr->dtype) attr->fstrsize = pcf_value.size();
3510 // strsize is used by both fixed and variable length strings.
3511 attr->strsize.resize(1);
3512 attr->strsize[0] = pcf_value.size();
3513
3514 copy(pcf_value.begin(), pcf_value.end(), attr->value.begin());
3515 find_attr = true;
3516 break;
3517 } // if (PCF1_attr_name == attr->name)
3518 } // for (auto attr
3519 } // if (File_attr_group_path == (*it_g)->path)
3520 if (true == find_group && true == find_attr) break;
3521 } // end of for it_g ...
3522 } // if (true == this->isaura && MLS == this->aura_name)
3523}
3524
3525// Handle coordinate variable units attribute
3526void EOS5File::Handle_EOS5CVar_Unit_Attr() const
3527{
3528
3529 BESDEBUG("h5", "Coming to Handle_EOS5CVar_Unit_Attr()"<<endl);
3530 string unit_attrname = "units";
3531 string nonll_cf_level_attrvalue = "level";
3532 string lat_cf_unit_attrvalue = "degrees_north";
3533 string lon_cf_unit_attrvalue = "degrees_east";
3534 string tes_cf_pre_attrvalue = "hPa";
3535
3536 for (const auto &cvar:this->cvars) {
3537 switch (cvar->cvartype) {
3538 case CV_EXIST:
3539 case CV_MODIFY: {
3540 for (const auto &attr:cvar->attrs) {
3541 if (attr->newname == unit_attrname) {
3542 Retrieve_H5_Attr_Value(attr, cvar->fullpath);
3543 string units_value(attr->value.begin(), attr->value.end());
3544 if ((lat_cf_unit_attrvalue != units_value)
3545 && ((cvar->name == "Latitude") || ((this->eos5cfzas.empty() == false) && (cvar->name == "nLats")))) {
3546 units_value = lat_cf_unit_attrvalue;
3547#if 0
3548//cerr<<"coming to obtain the correct units_value: "<<units_value <<endl;
3549//cerr<<"cvar name is "<<cvar->newname <<endl;
3550#endif
3551 attr->value.resize(units_value.size());
3552 if (H5FSTRING == attr->dtype) attr->fstrsize = units_value.size();
3553 // strsize is used by both fixed and variable length strings.
3554 attr->strsize.resize(1);
3555 attr->strsize[0] = units_value.size();
3556 copy(units_value.begin(), units_value.end(), attr->value.begin());
3557 }
3558 else if ((lon_cf_unit_attrvalue != units_value) && cvar->name == "Longitude") {
3559 units_value = lon_cf_unit_attrvalue;
3560 attr->value.resize(units_value.size());
3561 if (H5FSTRING == attr->dtype) attr->fstrsize = units_value.size();
3562 // strsize is used by both fixed and variable length strings.
3563 attr->strsize.resize(1);
3564 attr->strsize[0] = units_value.size();
3565
3566 copy(units_value.begin(), units_value.end(), attr->value.begin());
3567 }
3568 break;
3569 } // if (attr->newname ==unit_attrname)
3570 }
3571 }
3572 break;
3573
3574 case CV_LAT_MISS: {
3575 auto attr_unique = make_unique<Attribute>();
3576 auto attr = attr_unique.release();
3577 Add_Str_Attr(attr, unit_attrname, lat_cf_unit_attrvalue);
3578 cvar->attrs.push_back(attr);
3579 }
3580 break;
3581
3582 case CV_LON_MISS: {
3583 auto attr_unique = make_unique<Attribute>();
3584 auto attr = attr_unique.release();
3585 Add_Str_Attr(attr, unit_attrname, lon_cf_unit_attrvalue);
3586 cvar->attrs.push_back(attr);
3587 }
3588 break;
3589
3590 case CV_NONLATLON_MISS: {
3591 auto attr_unique = make_unique<Attribute>();
3592 auto attr = attr_unique.release();
3593 Add_Str_Attr(attr, unit_attrname, nonll_cf_level_attrvalue);
3594 cvar->attrs.push_back(attr);
3595 }
3596 break;
3597 case CV_SPECIAL: {
3598 if (true == this->isaura && TES == this->aura_name) {
3599 auto attr_unique = make_unique<Attribute>();
3600 auto attr = attr_unique.release();
3601 Add_Str_Attr(attr, unit_attrname, tes_cf_pre_attrvalue);
3602 cvar->attrs.push_back(attr);
3603 }
3604 }
3605 break;
3606 default:
3607 throw1("Non-supported Coordinate Variable Type.");
3608 }
3609 } // for (auto cvar
3610}
3611
3612void EOS5File::Add_EOS5_Grid_CF_Attr()
3613{
3614 BESDEBUG("h5", "Coming to Add_EOS5_Grid_CF_Attr()"<<endl);
3615
3616 bool has_eos5_grid_nongeo_proj = false;
3617
3618 // Check if we have EOS5 grids that are not using the geographic projection.
3619 for (const auto &cvar:this->cvars) {
3620 if (cvar->cvartype == CV_LAT_MISS) {
3621 if(cvar->eos5_projcode !=HE5_GCTP_GEO) {
3622 has_eos5_grid_nongeo_proj = true;
3623 break;
3624 }
3625 }
3626 }
3627
3628 // We would like to add the CF conventions mark if the mark is not there.
3629 if(true == has_eos5_grid_nongeo_proj) {
3630 string conventions_attrname = "Conventions";
3631 string conventions_attrvalue = "CF-1.7";
3632 bool has_conventions_attr=false;
3633 for (const auto &root_attr:this->root_attrs) {
3634 if(root_attr->name==conventions_attrname){
3635 has_conventions_attr = true;
3636 break;
3637 }
3638
3639 }
3640 if(false==has_conventions_attr) {
3641 auto attr_unique = make_unique<Attribute>();
3642 auto attr = attr_unique.release();
3643 Add_Str_Attr(attr,conventions_attrname,conventions_attrvalue);
3644 this->root_attrs.push_back(attr);
3645 }
3646 }
3647
3648}
3649
3650
3651
3652// Adjust Dimension name
3654{
3655
3656 BESDEBUG("h5", "Coming to Adjust_Dim_Name()"<<endl);
3657 // No need if this is following COARDS.
3658 if (false == this->iscoard)
3659 return;
3660 else {
3661 for (const auto &cvar:this->cvars) {
3662 if (cvar->dims.size() != 1)
3663 throw3("Coard coordinate variable ", cvar->name, "is not 1D");
3664 if (cvar->newname != ((cvar->dims)[0]->newname)) {
3665 (cvar->dims)[0]->newname = cvar->newname;
3666
3667 // For all variables that have this dimension,the dimension newname should also change.
3668 for (const auto &var:this->vars) {
3669 for (const auto &dim:var->dims) {
3670 // This is the key, the dimension name of this dimension
3671 // should be equal to the dimension name of the coordinate variable.
3672 // Then the dimension name matches and the dimension name should be changed to
3673 // the new dimension name.
3674 if (dim->name == (cvar->dims)[0]->name) dim->newname = (cvar->dims)[0]->newname;
3675 }
3676 }
3677 } // if (cvar->newname != ((cvar->dims)[0]->newname))
3678 } // for (auto irv = this->cvars.begin();
3679 } // else
3680}
3681
3682// Add supplemental attributes such as origname and fullpath.
3684{
3685
3686 BESDEBUG("h5", "Coming to Add_Supplement_Attrs()"<<endl);
3687 if (true == add_path) {
3688
3690
3691 // Adding variable original name(origname) and full path(fullpath)
3692 for (const auto &cvar:this->cvars) {
3693 if ((cvar->cvartype == CV_EXIST) || (cvar->cvartype == CV_MODIFY)) {
3694 const string varname = cvar->name;
3695 const string attrname = "origname";
3696 auto attr_unique = make_unique<Attribute>();
3697 auto attr = attr_unique.release();
3698 Add_Str_Attr(attr, attrname, varname);
3699 cvar->attrs.push_back(attr);
3700 }
3701 }
3702
3703 for (const auto &cvar:this->cvars) {
3704 if ((cvar->cvartype == CV_EXIST) || (cvar->cvartype == CV_MODIFY)) {
3705 // Turn off the fullnamepath attribute when zero_storage_size is 0.
3706 // Use the BES key since quite a few testing cases will be affected.
3707 // KY 2020-03-23
3708 if(cvar->zero_storage_size==false
3709 || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
3710 const string varname = cvar->fullpath;
3711 const string attrname = "fullnamepath";
3712 auto attr_unique = make_unique<Attribute>();
3713 auto attr = attr_unique.release();
3714 Add_Str_Attr(attr, attrname, varname);
3715 cvar->attrs.push_back(attr);
3716 }
3717 }
3718 }
3719 } // if(true == add_path)
3720
3721 if (true == this->iscoard) {
3722 for (const auto &cvar:this->cvars) {
3723 if ((cvar->cvartype == CV_EXIST) || (cvar->cvartype == CV_MODIFY)) {
3724 const string attrname = "orig_dimname";
3725 string orig_dimname = ((cvar->dims)[0])->name;
3726 orig_dimname = HDF5CFUtil::obtain_string_after_lastslash(orig_dimname);
3727 if ("" == orig_dimname)
3728 throw2("wrong dimension name ", orig_dimname);
3729 if (orig_dimname.find("FakeDim") != string::npos) orig_dimname = "";
3730 auto attr_unique = make_unique<Attribute>();
3731 auto attr = attr_unique.release();
3732 Add_Str_Attr(attr, attrname, orig_dimname);
3733 cvar->attrs.push_back(attr);
3734 }
3735 } // for (auto irv = this->cvars.begin()
3736
3737 for (const auto &var:this->vars) {
3738
3739 if (var->dims.empty() == false) {
3740 auto attr_unique = make_unique<Attribute>();
3741 auto attr = attr_unique.release();
3742 if (1 == var->dims.size()) {
3743 const string attrname = "orig_dimname";
3744 string orig_dimname = ((var->dims)[0])->name;
3745 if ("" == orig_dimname)
3746 orig_dimname = "NoDimName";
3747 else
3748 orig_dimname = HDF5CFUtil::obtain_string_after_lastslash(orig_dimname);
3749 if (orig_dimname.find("FakeDim") != string::npos) orig_dimname = "NoDimName";
3750 Add_Str_Attr(attr, attrname, orig_dimname);
3751 }
3752 else {
3753 const string attrname = "orig_dimname_list";
3754 string orig_dimname_list;
3755 for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
3756 string orig_dimname = (*ird)->name;
3757 if ("" == orig_dimname)
3758 orig_dimname = "NoDimName";
3759 else
3760 orig_dimname = HDF5CFUtil::obtain_string_after_lastslash((*ird)->name);
3761 if (orig_dimname.find("FakeDim") != string::npos) orig_dimname = "NoDimName";
3762 if ("" == orig_dimname_list)
3763 orig_dimname_list = orig_dimname;
3764 else
3765 orig_dimname_list = orig_dimname_list + " " + orig_dimname;
3766#if 0
3767// orig_dimname_list = orig_dimname_list + " ";
3768#endif
3769 }
3770 Add_Str_Attr(attr, attrname, orig_dimname_list);
3771 }
3772 var->attrs.push_back(attr);
3773 }
3774 } // for (auto irv = this->vars.begin();
3775 } // if(true == this->iscoard )
3776
3777}
3778
3779// Handle coordinate attributes.
3781{
3782
3783 BESDEBUG("h5", "Coming to Handle_Coor_Attr()"<<endl);
3784 string co_attrname = "coordinates";
3785 string co_attrvalue;
3786
3787 if (iscoard) return;
3788
3789 for (const auto &var:this->vars) {
3790
3791 for (const auto &dim:var->dims) {
3792 for (const auto &cvar:this->cvars) {
3793 if (dim->name == cvar->cfdimname)
3794 co_attrvalue = (co_attrvalue.empty()) ? cvar->newname : co_attrvalue + " " + cvar->newname;
3795 }
3796 }
3797 if (false == co_attrvalue.empty()) {
3798 auto attr_unique = make_unique<Attribute>();
3799 auto attr = attr_unique.release();
3800 Add_Str_Attr(attr, co_attrname, co_attrvalue);
3801 var->attrs.push_back(attr);
3802 }
3803 co_attrvalue.clear();
3804 } // for (auto irv = this->vars.begin(); ...
3805
3806 // We will check if 2dlatlon coordinate variables exist
3807 bool has_2dlatlon_cv = false;
3808 for (const auto &cvar:this->cvars) {
3809 if (true == cvar->is_2dlatlon) {
3810 has_2dlatlon_cv = true;
3811 break;
3812 }
3813 }
3814
3815 if (true == has_2dlatlon_cv) {
3816
3817 string dimname1;
3818 string dimname2;
3819 for (const auto &cvar:this->cvars) {
3820 if (true == cvar->is_2dlatlon) {
3821 dimname1 = ((cvar->dims)[0])->name;
3822 dimname2 = ((cvar->dims)[1])->name;
3823 break;
3824 }
3825 }
3826
3827 int num_latlondims = 0;
3828
3829 for (const auto &var:this->vars) {
3830 for (const auto &dim:var->dims) {
3831 if (dimname1 == dim->name) num_latlondims++;
3832 if (dimname2 == dim->name) num_latlondims++;
3833 }
3834 if ((num_latlondims != 0) && (num_latlondims != 2)) {
3835 // need to remove the coordinates attribute.
3836 for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
3837 if (co_attrname == (*ira)->name) {
3838 delete (*ira);
3839 var->attrs.erase(ira);
3840 break;
3841 }
3842 }
3843 }
3844 num_latlondims = 0;
3845 } // for (auto irv = this->vars.begin();
3846 } // if (true == has_2dlatlon_cv)
3847}
3848
3849// This function is from the original requirement of NASA, then
3850// NASA changes the requirment. Still leave it here for future usage.
3851#if 0
3852void EOS5File::Adjust_Special_EOS5CVar_Name() {
3853
3854 int num_grids =this->eos5cfgrids.size();
3855 int num_swaths = this->eos5cfswaths.size();
3856 int num_zas = this->eos5cfzas.size();
3857
3858 bool mixed_eos5typefile = false;
3859
3860 // Check if this file mixes grid,swath and zonal average
3861 if (((num_grids > 0) && (num_swaths > 0)) ||
3862 ((num_grids > 0) && (num_zas > 0)) ||
3863 ((num_swaths >0) && (num_zas > 0)))
3864 mixed_eos5typefile = true;
3865
3866 if (false == mixed_eos5typefile) {
3867
3868 // Grid is very special since all grids may share the same lat/lon.
3869 // so we also consider this case.
3870
3871 if ((1 == num_swaths) || ( 1 == num_zas) ||
3872 (1 == num_grids) || ((num_grids >1) && (this->grids_multi_latloncvs))) {
3873
3874 string unit_attrname = "units";
3875 string nonll_cf_level_attralue ="level";
3876 string lat_cf_unit_attrvalue ="degrees_north";
3877 string lon_cf_unit_attrvalue ="degrees_east";
3878
3879 for (auto irv = this->cvars.begin();
3880 irv != this->cvars.end(); irv++) {
3881 switch((*irv)->eos_type) {
3882 case CV_EXIST:
3883 case CV_MODIFY:
3884 case CV_LAT_MISS:
3885 case CV_LON_MISS:
3886 {
3887 for(vector <Attribute*>::iterator ira = (*irv)->attrs.begin();
3888 ira != (*irv)->attrs.end(); ira++) {
3889 if ((*ira)->name ==unit_attrname) {
3890 if ((*ira)->value.size() > 0) {
3891 string units_value((*ira)->value.begin(),(*ira)->value.end());
3892 if (lat_cf_unit_attrvalue ==units_value) (*irv)->newname = "lat";
3893 if (lon_cf_unit_attrvalue ==units_value) (*irv)->newname = "lon";
3894 }
3895 }
3896 }
3897 }
3898 break;
3899 case CV_NONLATLON_MISS:
3900 {
3901 for(vector <Attribute*>::iterator ira = (*irv)->attrs.begin();
3902 ira != (*irv)->attrs.end(); ira++) {
3903 if ((*ira)->name ==unit_attrname) {
3904 if ((*ira)->value.size() > 0) {
3905 string units_value((*ira)->value.begin(),(*ira)->value.end());
3906 if (nonll_cf_level_attralue ==units_value) {
3907 (*irv)->newname = "lev";
3908 break;
3909 }
3910 }
3911 }
3912 }
3913 }
3914 break;
3915 default:
3916 throw1("Non-supported coordinate variable type");
3917 }
3918 }
3919 }
3920 }
3921}
3922#endif
3923
3924// Create missing coordinate variables. Some NASA files don't provide coordinate
3925// variables for some dimensions. To make the visualization tools plot the data,
3926// we provide index number to be the coordinate variable values for these missing coordinate variables.
3927// These missing coordinate variables are all 1-D.
3928template<class T>
3929void EOS5File::Create_Missing_CV(T* eos5data, EOS5CVar *EOS5cvar, const string& dimname, EOS5Type eos5type,
3930 int num_eos5data) const
3931{
3932
3933 BESDEBUG("h5", "Coming to Create_Missing_CV()"<<endl);
3934 string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(dimname);
3935 if ("" == reduced_dimname) throw2("wrong dimension name ", dimname);
3936 EOS5cvar->name = reduced_dimname;
3937 Create_Added_Var_NewName_FullPath(eos5type, eos5data->name, EOS5cvar->name, EOS5cvar->newname, EOS5cvar->fullpath);
3938 EOS5cvar->rank = 1;
3939 EOS5cvar->dtype = H5INT32;
3940 hsize_t eos5cvar_dimsize = (eos5data->dimnames_to_dimsizes)[dimname];
3941 auto eos5cvar_dim_unique = make_unique<Dimension>(eos5cvar_dimsize);
3942 auto eos5cvar_dim = eos5cvar_dim_unique.release();
3943 eos5cvar_dim->name = dimname;
3944 eos5cvar_dim->unlimited_dim = (eos5data->dimnames_to_unlimited)[dimname];
3945 if (1 == num_eos5data)
3946 eos5cvar_dim->newname = reduced_dimname;
3947 else
3948 eos5cvar_dim->newname = dimname;
3949
3950 EOS5cvar->dims.push_back(eos5cvar_dim);
3951 EOS5cvar->cfdimname = dimname;
3952 EOS5cvar->cvartype = CV_NONLATLON_MISS;
3953 EOS5cvar->eos_type = eos5type;
3954}
3955
3956// Helper function for Create_Missing_CV
3957void EOS5File::Create_Added_Var_NewName_FullPath(EOS5Type eos5type, const string& eos5_groupname, const string& varname,
3958 string &var_newname, string &var_fullpath) const
3959{
3960
3961 BESDEBUG("h5", "Coming to Create_Added_Var_NewName_FullPath()"<<endl);
3962 string fslash_str = "/";
3963 string eos5typestr;
3964 string top_eos5_groupname = "/HDFEOS";
3965
3966 switch (eos5type) {
3967 case GRID: {
3968 eos5typestr = "/GRIDS/";
3969 var_newname = eos5typestr + eos5_groupname + fslash_str + varname;
3970 var_fullpath = top_eos5_groupname + eos5typestr + eos5_groupname + fslash_str + varname;
3971 }
3972 break;
3973
3974 case SWATH: {
3975 eos5typestr = "/SWATHS/";
3976 var_newname = eos5typestr + eos5_groupname + fslash_str + varname;
3977 var_fullpath = top_eos5_groupname + eos5typestr + eos5_groupname + fslash_str + varname;
3978
3979 }
3980 break;
3981
3982 case ZA: {
3983 eos5typestr = "/ZAS/";
3984 var_newname = eos5typestr + eos5_groupname + fslash_str + varname;
3985 var_fullpath = top_eos5_groupname + eos5typestr + eos5_groupname + fslash_str + varname;
3986
3987 }
3988 break;
3989 case OTHERVARS:
3990 default:
3991 throw1("Non-supported EOS type");
3992 }
3993}
3994
3995// Handle special variables, various speical cases are handled here.
3997{
3998
3999 BESDEBUG("h5", "Coming to Handle_SpVar()"<<endl);
4000 if (true == this->isaura && TES == this->aura_name) {
4001 const string ProHist_full_path = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/ProductionHistory";
4002 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4003 if (ProHist_full_path == (*irv)->fullpath) {
4004 delete (*irv);
4005 this->vars.erase(irv);
4006 break;
4007 }
4008 }
4009 }
4010
4011 // First, if the duplicate dimension exists,
4012 if (dimname_to_dupdimnamelist.empty() == false) {
4013 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
4014 if ((*ircv)->cvartype == CV_EXIST) {
4015 pair<multimap<string, string>::iterator, multimap<string, string>::iterator> mm_er_ret;
4016 multimap<string, string>::iterator itmm;
4017 for (itmm = dimname_to_dupdimnamelist.begin(); itmm != dimname_to_dupdimnamelist.end(); ++itmm) {
4018
4019 // Find the original dimension(the coordinate variable)
4020 if ((*ircv)->cfdimname == (*itmm).first) {
4021
4022 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4023 for (auto irv2 = this->cvars.begin(); irv2 != this->cvars.end();
4024 irv2++) {
4025 if ((*irv2)->cvartype == CV_NONLATLON_MISS) {
4026#if 0
4027 //cerr<<"the duplicate cf dimension name "<<(*irv2)->cfdimname <<endl;
4028 //if((*irv2)->cfdimname == (*ircv)->cfdimname) {
4029#endif
4030 // Obtain the fake CV that has the duplicate dimension.
4031 if ((*irv2)->cfdimname == (*itmm).second) {
4032
4033 //find the duplicate dimension name
4034 string dup_var_name = (*irv2)->newname;
4035 Replace_Var_Info_EOS((*ircv), (*irv2));
4036 // The following two lines are key to make sure duplicate CV
4037 // using a different name but keep all other info.
4038 (*irv2)->newname = dup_var_name;
4039 (*irv2)->getDimensions()[0]->newname = dup_var_name;
4040 }
4041 }
4042 }
4043 }
4044 }
4045 }
4046 }
4047 }
4048
4049 // No need to loop through the variables. We just need to loop through the coordinate variables and check cfdimname.
4050#if 0
4051 // For the EOS case, Loop through every variable that has a >=2 rank,
4052 for (auto irv = this->vars.begin();
4053 irv != this->vars.end(); ++irv) {
4054
4055 // Check if having the duplicate dimensions.
4056 if((*irv)->rank >=2) {
4057 // Loop through the dimensions
4058 for (auto ird = (*irv)->dims.begin();
4059 ird != (*irv)->dims.end(); ++ ird) {
4060 pair<multimap<string,string>::iterator,multimap<string,string>::iterator> mm_er_ret;
4061 multimap<string,string>::iterator itmm;
4062 for (itmm = dimname_to_dupdimnamelist.begin(); itmm!=dimname_to_dupdimnamelist.end();++itmm) {
4063//cerr<<"the original dim. name is "<<(*itmm).first <<endl;
4064//cerr<<"the duplicate dim. name is "<<(*itmm).second <<endl;
4065//if((*irv)->name == "RetrievalAveragingKernelMatrixDay")
4066 cerr<<"duplicate dimension name of a variable is "<<(*ird)->name <<endl;
4067 // Find the duplicated dim name in the dimname_to_dupdimnamelist,
4068 // Now retrieve the dim. name and loop through all CV_EXIST variable to see if
4069 // one CV_EXIST variable has a dimension of which name is the dim. name.
4070 // If yes, loop through all CV_NONLLMISS variables and find the CV variable that has the
4071 // duplicate dim. name. If found, replace this variable's information(except name and newname) with the
4072 // fullpath of the CV_EXIST variable. In this way, the duplicate CV variable will read
4073 // correctly the existing CV values and other information. This is the most complicate process.
4074
4075// if((*itmm).second == HDF5CFUtil::obtain_string_after_lastslash((*ird)->name)) {
4076 if((*itmm).second == (*ird)->name) {
4077 cerr<<"coming to find the duplicate dim. name "<<endl;
4078 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
4079 if((*ircv)->cvartype == CV_EXIST) {
4080 cerr<<"cf dim. name is "<<(*ircv)->cfdimname <<endl;
4081 // Find the original dimension(the coordinate variable)
4082 if((*ircv)->cfdimname == (*itmm).first) {
4083 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4084 for (auto irv2 = this->cvars.begin();
4085 irv2 != this->cvars.end(); irv2++) {
4086 if((*irv2)->cvartype == CV_NONLATLON_MISS) {
4087 // Obtain the fake CV that has the duplicate dimension.
4088 if((*irv2)->cfdimname == (*itmm).second) {
4089 string dup_var_name = (*irv2)->newname;
4090 Replace_Var_Info_EOS((*ircv),(*irv2));
4091 (*irv2)->newname = dup_var_name;
4092 (*irv2)->getDimensions()[0]->newname = dup_var_name;
4093 }
4094
4095 }
4096 }
4097
4098 }
4099
4100 }
4101
4102 }
4103
4104 }
4105
4106 }
4107
4108 }
4109
4110 }
4111 }
4112
4113}
4114#endif
4115}
4116
4117// Handle special variable attributes
4119{
4120
4121 BESDEBUG("h5", "Coming to Handle_SpVar_Attr()"<<endl);
4122
4123 // First, if the duplicate dimension exists,
4124 if (dimname_to_dupdimnamelist.empty() == false) {
4125
4126 pair<multimap<string, string>::iterator, multimap<string, string>::iterator> mm_er_ret;
4127 multimap<string, string>::iterator itmm;
4128 for (itmm = dimname_to_dupdimnamelist.begin(); itmm != dimname_to_dupdimnamelist.end(); ++itmm) {
4129 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
4130 // The duplicated CV must share with an existing coordinate variable
4131 if ((*ircv)->cvartype == CV_EXIST) {
4132
4133 // Find the original dimension(the coordinate variable)
4134 if ((*ircv)->cfdimname == (*itmm).first) {
4135
4136 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4137 // The duplciated CV must be CV_NONLATLON_MISS.
4138 for (auto irv2 = this->cvars.begin(); irv2 != this->cvars.end();
4139 irv2++) {
4140 if ((*irv2)->cvartype == CV_NONLATLON_MISS) {
4141
4142 // Obtain the fake CV that has the duplicate dimension.
4143#if 0
4144 //if((*irv2)->cfdimname == (*ircv)->cfdimname)
4145#endif
4146 if ((*irv2)->cfdimname == (*itmm).second) Replace_Var_Attrs_EOS((*ircv), (*irv2));
4147
4148 }
4149 }
4150 } // if((*ircv)->cfdimname == (*itmm).first)
4151 } // if((*ircv)->cvartype == CV_EXIST)
4152 } // for (auto ircv = this->cvars.begin()
4153 } // for (itmm = dimname_to_dupdimnamelist.begin();
4154 } // if(dimname_to_dupdimnamelist.size() > 0)
4155}
4156
4157// Handle special variables, various special cases are handled here.
4159{
4160
4161 BESDEBUG("h5", "Coming to Handle_SpVar_DMR()"<<endl);
4162 if (true == this->isaura && TES == this->aura_name) {
4163 const string ProHist_full_path = "/HDFEOS/ADDITIONAL/FILE_ATTRIBUTES/ProductionHistory";
4164 for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4165 if (ProHist_full_path == (*irv)->fullpath) {
4166 delete (*irv);
4167 this->vars.erase(irv);
4168 break;
4169 }
4170 }
4171 }
4172
4173 // First, if the duplicate dimension exists,
4174 if (dimname_to_dupdimnamelist.empty() == false) {
4175 for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ircv++) {
4176 if ((*ircv)->cvartype == CV_EXIST) {
4177 pair<multimap<string, string>::iterator, multimap<string, string>::iterator> mm_er_ret;
4178 multimap<string, string>::iterator itmm;
4179 for (itmm = dimname_to_dupdimnamelist.begin(); itmm != dimname_to_dupdimnamelist.end(); ++itmm) {
4180
4181 // Find the original dimension(the coordinate variable)
4182 if ((*ircv)->cfdimname == (*itmm).first) {
4183
4184 // Loop through the cv again,this time just check CV_NONLATLON_MISS
4185 for (auto irv2 = this->cvars.begin(); irv2 != this->cvars.end();
4186 irv2++) {
4187 if ((*irv2)->cvartype == CV_NONLATLON_MISS) {
4188#if 0
4189 //cerr<<"the duplicate cf dimension name "<<(*irv2)->cfdimname <<endl;
4190 //if((*irv2)->cfdimname == (*ircv)->cfdimname) {
4191#endif
4192 // Obtain the fake CV that has the duplicate dimension.
4193 if ((*irv2)->cfdimname == (*itmm).second) {
4194
4195 Replace_Var_Attrs_EOS((*ircv), (*irv2));
4196 //find the duplicate dimension name
4197 string dup_var_name = (*irv2)->newname;
4198 Replace_Var_Info_EOS((*ircv), (*irv2));
4199
4200 // The following two lines are key to make sure duplicate CV
4201 // using a different name but keep all other info.
4202 (*irv2)->newname = dup_var_name;
4203 (*irv2)->getDimensions()[0]->newname = dup_var_name;
4204 }
4205 }
4206 }
4207 }
4208 }
4209 }
4210 }
4211 }
4212
4213}
4214
4215
4217{
4218 //Intentionally unimplemented, may have use cases for the future.
4219}
4220
4227
4228
4229// Sometimes need to replace information of a variable with the information of another variable.
4230void EOS5File::Replace_Var_Info_EOS(const EOS5CVar *src, EOS5CVar*target)
4231{
4232
4233 BESDEBUG("h5", "Coming to Replace_Var_Info_EOS()"<<endl);
4234 File::Replace_Var_Info(src, target);
4235 target->cfdimname = src->cfdimname;
4236 target->cvartype = src->cvartype;
4237 target->eos_type = src->eos_type;
4238 target->total_elems = src->total_elems;
4239
4240}
4241
4242//Sometimes the attributes of a variable need to replace with the attribute of another variable.
4243void EOS5File::Replace_Var_Attrs_EOS(const EOS5CVar *src, EOS5CVar*target)
4244{
4245
4246 BESDEBUG("h5", "Coming to Replace_Var_Attrs_EOS()"<<endl);
4247 File::Replace_Var_Attrs(src, target);
4248
4249}
4250
4251#if 0
4252void
4253EOS5File:: add_ignored_info_attrs(bool is_grp,bool is_first) {
4254
4255}
4256void
4257EOS5File:: add_ignored_info_objs(bool is_dim_related, bool is_first) {
4258
4259}
4260#endif
4261
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
include the entry functions to execute the handlers
This class represents one attribute.
Definition HDF5CF.h:185
This class simulates an HDF-EOS5 Grid. Currently only geographic projection is supported.
Definition HDF5CF.h:1074
This class is a derived class of CVar. It represents a coordinate variable for HDF-EOS5 files.
Definition HDF5CF.h:427
void Adjust_Obj_Name() override
This method is a no-op operation. Leave here since the method in the base class is pure virtual.
void Add_EOS5File_Info(HE5Parser *, bool)
Add HDF-EOS5 dimension and coordinate variable related info. to EOS5Grid,EOS5Swath etc.
Definition HDFEOS5CF.cc:837
void Handle_Grid_Mapping_Vars() override
Handle Grid Mapping Vars.
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr) override
Retrieve DDS information from the HDF5 file; a real implementation for HDF-EOS5 products.
Definition HDFEOS5CF.cc:163
void Adjust_Var_NewName_After_Parsing() const
Adjust variable names for HDF-EOS5 files.
void Add_Supplement_Attrs(bool) override
Add the supplemental attributes for HDF-EOS5 products.
void Set_COARDS_Status()
Set COARDS flag.
void Handle_CVar() override
Handle coordinate variable for HDF-EOS5 files.
void Adjust_Attr_Info()
Adjust the attribute info for HDF-EOS5 products.
void Handle_SpVar_Attr() override
Handle special variables for HDF-EOS5 files.
void Handle_Obj_NameClashing(bool)
Handle the object name clashing for HDF-EOS5 products.
void Handle_Unsupported_Dtype(bool) override
Handle unsupported HDF5 datatypes for HDF-EOS5 products.
Definition HDFEOS5CF.cc:208
bool Have_Grid_Mapping_Attrs() override
Check if having Grid Mapping Attrs.
void Handle_DimNameClashing() override
void Handle_Unsupported_Others(bool) override
Handle other unmapped objects/attributes for HDF-EOS5 products.
Definition HDFEOS5CF.cc:359
void Retrieve_H5_CVar_Supported_Attr_Values() override
Retrieve coordinate variable attributes.
Definition HDFEOS5CF.cc:170
void Handle_SpVar() override
Handle special variables for HDF-EOS5 files.
void Retrieve_H5_Supported_Attr_Values() override
Retrieve attribute values for the supported HDF5 datatypes for HDF-EOS5 products.
Definition HDFEOS5CF.cc:186
void Add_Dim_Name(HE5Parser *)
Add the dimension name for HDF-EOS5 files.
void Adjust_Dim_Name() override
Adjust the dimension name for HDF-EOS5 products.
void Handle_SpVar_DMR() override
Handle special variables and attributes for HDF-EOS5 files(for DMR)
void Handle_Unsupported_Dspace(bool) override
Handle unsupported HDF5 dataspaces for HDF-EOS5 products.
Definition HDFEOS5CF.cc:300
void Adjust_Var_Dim_NewName_Before_Flattening() const
Adjust variable dimension names before the flattening for HDF-EOS5 files.
void Flatten_Obj_Name(bool include_attr) override
Flatten the object name for HDF-EOS5 files.
void Check_Aura_Product_Status()
Check if the HDF-EOS5 file is an Aura file. Special CF operations need to be used.
void Adjust_EOS5Dim_Info(HE5Parser *strmeta_info)
Adjust HDF-EOS5 dimension information.
Definition HDFEOS5CF.cc:536
void Handle_Coor_Attr() override
Handle the coordinates attribute for HDF-EOS5 products.
std::vector< Group * > groups
Non-root group vectors.
Definition HDF5CF.h:791
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition HDF5CF.cc:1279
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition HDF5CF.cc:2208
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition HDF5CF.cc:735
std::vector< Var * > vars
Var vectors.
Definition HDF5CF.h:785
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition HDF5CF.cc:2027
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition HDF5CF.cc:170
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition HDF5CF.h:788
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition HDF5CF.cc:922
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition HDF5CF.cc:1376
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition HDF5CF.cc:2191
This class represents one HDF5 dataset(CF variable)
Definition HDF5CF.h:252
Helper functions for generating DAS attributes and a function to check BES Key.
Definition HE5Dim.h:7
double point_right
The rightmost coordinate value of a Grid.
Definition HE5Grid.h:25
double point_upper
The top coordinate value of a Grid.
Definition HE5Grid.h:21
double point_left
The leftmost coordinate value of a Grid.
Definition HE5Grid.h:23
double point_lower
The bottom coordinate value of a Grid.
Definition HE5Grid.h:19
Definition HE5Var.h:8
Definition HE5Za.h:6