bes Updated for version 3.21.1
The Backend Server (BES) is the lower two tiers of the Hyrax data server
CmrApi.cc
1// -*- mode: c++; c-basic-offset:4 -*-
2
3// This file is part of cmr_module, A C++ MODULE that can be loaded in to
4// the OPeNDAP Back-End Server (BES) and is able to handle remote requests.
5
6// Copyright (c) 2022 OPeNDAP, Inc.
7// Author: Nathan Potter <ndp@opendap.org>
8//
9// This library is free software; you can redistribute it and/or
10// modify it under the terms of the GNU Lesser General Public
11// License as published by the Free Software Foundation; either
12// version 2.1 of the License, or (at your option) any later version.
13//
14// This library is distributed in the hope that it will be useful,
15// but WITHOUT ANY WARRANTY; without even the implied warranty of
16// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17// Lesser General Public License for more details.
18//
19// You should have received a copy of the GNU Lesser General Public
20// License along with this library; if not, write to the Free Software
21// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22//
23// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24
25/*
26 * CmrApi.cc
27 *
28 * Created on: July, 13 2018
29 * Author: ndp
30 */
31#include <memory>
32#include <cstdio>
33#include <cstring>
34#include <iostream>
35#include <sstream>
36
37#include "nlohmann/json.hpp"
38
39#include <libdap/util.h>
40#include <libdap/debug.h>
41
42#include "HttpUtils.h"
43
44#include "BESError.h"
45#include "BESInternalError.h"
46#include "BESStopWatch.h"
47#include "BESNotFoundError.h"
48#include "BESDebug.h"
49#include "BESUtil.h"
50#include "TheBESKeys.h"
51#include "BESLog.h"
52#include "CmrApi.h"
53#include "CmrNames.h"
54#include "CmrInternalError.h"
55#include "CmrNotFoundError.h"
56#include "JsonUtils.h"
57
58using std::string;
59using json = nlohmann::json;
60
61#define prolog string("CmrApi::").append(__func__).append("() - ")
62
63namespace cmr {
64
65std::string truth(bool t){ if(t){return "true";} return "false"; }
66
67
68CmrApi::CmrApi() {
69 bool found;
70 string cmr_endpoint_url;
71 TheBESKeys::TheKeys()->get_value(CMR_HOST_URL_KEY, cmr_endpoint_url, found);
72 if (found) {
73 d_cmr_endpoint_url = cmr_endpoint_url;
74 }
75 BESDEBUG(MODULE, prolog << "d_cmr_endpoint_url: " << d_cmr_endpoint_url << endl);
76
77 d_cmr_providers_search_endpoint_url = BESUtil::assemblePath(d_cmr_endpoint_url,
78 CMR_PROVIDERS_SEARCH_ENDPOINT);
79 BESDEBUG(MODULE,
80 prolog << "d_cmr_providers_search_endpoint_url: " << d_cmr_providers_search_endpoint_url << endl);
81
82 d_cmr_collections_search_endpoint_url = BESUtil::assemblePath(d_cmr_endpoint_url,
83 CMR_COLLECTIONS_SEARCH_API_ENDPOINT);
84 BESDEBUG(MODULE,
85 prolog << "d_cmr_collections_search_endpoint_url: " << d_cmr_collections_search_endpoint_url << endl);
86
87 d_cmr_granules_search_endpoint_url = BESUtil::assemblePath(d_cmr_endpoint_url,
88 CMR_GRANULES_SEARCH_API_ENDPOINT);
89 BESDEBUG(MODULE,
90 prolog << "d_cmr_granules_search_endpoint_url: " << d_cmr_granules_search_endpoint_url << endl);
91
92 d_cmr_granules_umm_search_endpoint_url = BESUtil::assemblePath(d_cmr_endpoint_url,
93 CMR_GRANULES_SEARCH_API_UMM_ENDPOINT);
94 BESDEBUG(MODULE,
95 prolog << "d_cmr_granules_umm_search_endpoint_url: " << d_cmr_granules_umm_search_endpoint_url << endl);
96}
97
101const nlohmann::json& CmrApi::get_related_urls_array(const nlohmann::json& json_obj) const
102{
103 JsonUtils json;
104 return json.qc_get_array(CMR_UMM_RELATED_URLS_KEY,json_obj);
105}
106
107
113const nlohmann::json &CmrApi::get_children(const nlohmann::json &jobj) const
114{
115 JsonUtils json;
116 BESDEBUG(MODULE, prolog << json.probe_json(jobj) << endl);
117 bool result = jobj.is_null();
118 if(result){
119 string msg;
120 msg.append("ERROR: Json document is NULL. json: ").append(jobj.dump());
121 BESDEBUG(MODULE, prolog << msg << "\n");
122 throw CmrInternalError(msg, __FILE__, __LINE__);
123 }
124
125 result = jobj.is_object();
126 if(!result){
127 string msg;
128 msg.append("ERROR: Json document is NOT an object. json: ").append(jobj.dump());
129 BESDEBUG(MODULE, prolog << msg << "\n");
130 throw CmrInternalError(msg, __FILE__, __LINE__);
131 }
132
133 const auto &has_children_j = jobj[CMR_V2_HAS_CHILDREN_KEY];
134 if(!has_children_j.get<bool>()){
135 string msg(prolog);
136 msg.append("This json object does not have a child property of type ").append(CMR_V2_HAS_CHILDREN_KEY);
137 msg.append(". json: ").append(jobj.dump());
138 BESDEBUG(MODULE, msg << "\n");
139 INFO_LOG(msg);
140 }
141
142 return json.qc_get_array(CMR_V2_CHILDREN_KEY,jobj);
143}
144
150const nlohmann::json &CmrApi::get_feed(const nlohmann::json &cmr_doc) const
151{
152 JsonUtils json;
153 return json.qc_get_object(CMR_V2_FEED_KEY, cmr_doc);
154}
155
161const json& CmrApi::get_entries(const json &cmr_doc) const
162{
163
164 JsonUtils json;
165 BESDEBUG(MODULE, prolog << "cmr_doc" << endl << cmr_doc.dump(2) << endl);
166 const auto &feed = get_feed(cmr_doc);
167 return json.qc_get_array(CMR_V2_ENTRY_KEY,feed);
168}
169
170
176const json& CmrApi::get_items(const json &cmr_doc) const
177{
178 JsonUtils json;
179 BESDEBUG(MODULE, prolog << "cmr_doc" << endl << cmr_doc.dump(2) << endl);
180 return json.qc_get_array(CMR_UMM_ITEMS_KEY,cmr_doc);
181}
182
183
189const nlohmann::json &CmrApi::get_temporal_group(const nlohmann::json &cmr_doc) const
190{
191 JsonUtils json;
192 const auto &feed = get_feed(cmr_doc);
193
194 const auto &facets_obj = json.qc_get_object(CMR_V2_FACETS_KEY, feed);
195
196 const auto &facets_array = get_children(facets_obj);
197 for(const auto &facet:facets_array){
198 string facet_title = facet[CMR_V2_TITLE_KEY].get<string>();
199 if(facet_title == CMR_V2_TEMPORAL_TITLE_VALUE){
200 string msg = prolog + "Found Temporal object.";
201 BESDEBUG(MODULE, msg << endl);
202 return facet;
203 }
204 else {
205 string msg = prolog + "The child of 'facets' with title '"+facet_title+"' does not match "+ CMR_V2_TEMPORAL_TITLE_VALUE;
206 BESDEBUG(MODULE, msg << endl);
207 }
208
209 }
210 stringstream msg;
211 msg << "Failed to locate the Temporal facet in : " << endl << cmr_doc.dump(2) << endl;
212 BESDEBUG(MODULE, prolog << msg.str() << endl);
213 throw BESNotFoundError(msg.str(), __FILE__, __LINE__);
214} // CmrApi::get_temporal_group()
215
216
222const nlohmann::json &CmrApi::get_year_group(const nlohmann::json &cmr_doc) const
223{
224
225 const auto &temporal_group = get_temporal_group(cmr_doc);
226 const auto &temporal_children = get_children(temporal_group);
227 for(const auto &temporal_child : temporal_children){
228 string temporal_child_title = temporal_child[CMR_V2_TITLE_KEY].get<string>();
229 if ( temporal_child_title == CMR_V2_YEAR_TITLE_VALUE ){
230 string msg = prolog + "Found Year object.";
231 BESDEBUG(MODULE, msg << endl);
232 return temporal_child;
233 }
234 else {
235 string msg = prolog + "The child of 'Temporal' with title '"+temporal_child_title+"' does not match 'Year'";
236 BESDEBUG(MODULE, msg << endl);
237 }
238 }
239 string msg = prolog + "Failed to locate the Year group.";
240 BESDEBUG(MODULE, msg << endl);
241 throw CmrInternalError(msg, __FILE__, __LINE__);
242}
243
249const nlohmann::json &CmrApi::get_years(const nlohmann::json &cmr_doc) const
250{
251 auto &year_group = get_year_group(cmr_doc);
252 if (year_group[CMR_V2_HAS_CHILDREN_KEY]) {
253 return get_children(year_group);
254 }
255 throw CmrNotFoundError("The Year object had no children.", __FILE__, __LINE__);
256}
257
264const nlohmann::json &CmrApi::get_year(const std::string &target_year, const nlohmann::json &cmr_doc) const
265{
266 const auto &years = get_years(cmr_doc);
267 for( auto &year:years) {
268 string year_title = year[CMR_V2_TITLE_KEY].get<string>();
269 if (target_year == year_title) {
270 BESDEBUG(MODULE, prolog + "Found matching Year object. target_year: " << target_year << endl);
271 return year;
272 }
273 BESDEBUG(MODULE, prolog + "The current year: " << year_title <<
274 " does not match the target_year of: " << target_year << endl);
275 }
276 throw CmrNotFoundError("The list of years did not contain on the matched: "+target_year, __FILE__, __LINE__);
277}
278
285const nlohmann::json &CmrApi::get_month_group(const std::string &target_year, const nlohmann::json &cmr_doc) const
286{
287
288
289 auto &year = get_year(target_year,cmr_doc);
290 auto &months = get_children(year);
291 for(auto &month: months){
292 string title = month[CMR_V2_TITLE_KEY];
293 if( title == CMR_V2_MONTH_TITLE_VALUE ){
294 BESDEBUG(MODULE, prolog + "Found Month object." << endl);
295 return month;
296 }
297 else {
298 stringstream msg;
299 msg << prolog << "The child of '" << CMR_V2_YEAR_TITLE_VALUE << "' with title '"+title+"' does not match 'Month'";
300 BESDEBUG(MODULE, msg.str() << endl);
301 }
302 }
303
304#if 0
305 auto &year_group = get_year_group(cmr_doc);
306 if(year_group[CMR_V2_HAS_CHILDREN_KEY]){
307 auto &years = get_children(year_group);
308 for( auto &year:years){
309 string year_title = year[CMR_V2_TITLE_KEY].get<string>();
310 if(target_year == year_title){
311 BESDEBUG(MODULE, prolog + "Found matching Year object. target_year: " << target_year << endl);
312 auto &months = get_children(year);
313 for(auto &month: months){
314 string title = month[CMR_V2_TITLE_KEY];
315 if( title == CMR_V2_MONTH_TITLE_VALUE ){
316 BESDEBUG(MODULE, prolog + "Found Month object." << endl);
317 return month;
318 }
319 else {
320 stringstream msg;
321 msg << prolog << "The child of '" << CMR_V2_YEAR_TITLE_VALUE << "' with title '"+title+"' does not match 'Month'";
322 BESDEBUG(MODULE, msg.str() << endl);
323 }
324 }
325 }
326 }
327 }
328#endif
329
330 string msg = prolog + "Failed to locate the Month group.";
331 BESDEBUG(MODULE, msg << endl);
332 throw CmrInternalError(msg, __FILE__, __LINE__);
333}
334
335
336
345const nlohmann::json &
346CmrApi::get_month(const std::string &target_month,
347 const std::string &target_year,
348 const nlohmann::json &cmr_doc)
349 const {
350
351
352 auto &month_group = get_month_group(target_year, cmr_doc);
353 auto &months = get_children(month_group);
354 for (auto &month: months){
355 string month_id = month[CMR_V2_TITLE_KEY].get<string>();
356 if(month_id == target_month){
357 stringstream msg;
358 msg << prolog << "Located requested month ("<< target_month << ")";
359 BESDEBUG(MODULE, msg.str() << endl);
360 return month;
361 }
362 else {
363 stringstream msg;
364 msg << prolog << "The month titled '"<<month_id << "' does not match the requested month ("<< target_month <<")";
365 BESDEBUG(MODULE, msg.str() << endl);
366 }
367 }
368 stringstream msg;
369 msg << prolog << "Failed to locate request Year/Month.";
370 BESDEBUG(MODULE, msg.str() << endl);
371 throw CmrInternalError(msg.str(), __FILE__, __LINE__);
372
373}
374
375
376
377
378
386const nlohmann::json &
387CmrApi::get_day_group(const std::string &target_month,
388 const std::string &target_year,
389 const nlohmann::json &cmr_doc)
390const {
391
392 const auto &month = get_month( target_month, target_year,cmr_doc);
393 const auto &m_kids = get_children(month);
394 for ( const auto &m_kid : m_kids ) {
395 string title = m_kid[CMR_V2_TITLE_KEY].get<string>();
396 if(title == CMR_V2_DAY_TITLE_VALUE){
397 stringstream msg;
398 msg << prolog << "Located Day group for year: " << target_year << " month: "<< target_month;
399 BESDEBUG(MODULE, msg.str() << endl);
400 return m_kid;
401 }
402 }
403 stringstream msg;
404 msg << prolog << "Failed to locate requested Day, year: " << target_year << " month: "<< target_month;
405 BESDEBUG(MODULE, msg.str() << endl);
406 throw CmrInternalError(msg.str(), __FILE__, __LINE__);
407}
408
409
410
417void CmrApi::get_years(const string &collection_name, vector<string> &years_result) const
418{
419 JsonUtils json;
420 stringstream cmr_query_url;
421
422 cmr_query_url << d_cmr_granules_search_endpoint_url;
423 cmr_query_url << "?concept_id=" + collection_name << "&";
424 cmr_query_url << "include_facets=v2&page_size="<<CMR_MAX_PAGE_SIZE;
425
426 BESDEBUG(MODULE, prolog << "CMR Query URL: "<< cmr_query_url.str() << endl);
427
428 const auto &cmr_doc = json.get_as_json(cmr_query_url.str());
429
430 const auto &year_group = get_year_group(cmr_doc);
431 if(year_group[CMR_V2_HAS_CHILDREN_KEY].get<bool>()) {
432 for (const auto &year_obj: year_group[CMR_V2_CHILDREN_KEY]) {
433 years_result.emplace_back(year_obj[CMR_V2_TITLE_KEY].get<string>());
434 }
435 }
436} // CmrApi::get_years()
437
438
439
448void
449CmrApi::get_months(const string &collection_name,
450 const string &r_year,
451 vector<string> &months_result) const{
452 JsonUtils json;
453 stringstream msg;
454 stringstream cmr_query_url;
455
456 cmr_query_url << d_cmr_granules_search_endpoint_url << "?" ;
457 cmr_query_url << "concept_id=" << collection_name << "&";
458 cmr_query_url << "include_facets=v2&";
459 cmr_query_url << http::url_encode("temporal_facet[0][year]") << "=" << r_year;
460 BESDEBUG(MODULE, prolog << "CMR Query URL: "<< cmr_query_url.str() << endl);
461
462 const auto &cmr_doc = json.get_as_json(cmr_query_url.str());
463 BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << cmr_doc.dump(2) << endl);
464
465 const auto &year_group = get_year_group(cmr_doc);
466 const auto &years = get_children(year_group);
467 if(years.size() != 1){
468 msg.str("");
469 msg << prolog << "We expected to get back one year (" << r_year << ") but we got back " << years.size();
470 BESDEBUG(MODULE, msg.str() << endl);
471 throw CmrInternalError(msg.str(), __FILE__, __LINE__);
472 }
473
474 const auto &year = years[0];
475
476 string year_title = year[CMR_V2_TITLE_KEY].get<string>();
477 if(year_title != r_year){
478 msg.str("");
479 msg << prolog << "The returned year (" << year_title << ") does not match the requested year ("<< r_year << ")";
480 BESDEBUG(MODULE, msg.str() << endl);
481 throw CmrInternalError(msg.str(), __FILE__, __LINE__);
482 }
483
484 const auto &year_children = get_children(year);
485 if(year_children.size() != 1){
486 msg.str("");
487 msg << prolog << "We expected to get back one child for the year (" << r_year << ") but we got back " << years.size();
488 BESDEBUG(MODULE, msg.str() << endl);
489 throw CmrInternalError(msg.str(), __FILE__, __LINE__);
490 }
491
492 const auto &month_group = year_children[0];
493 string month_title = month_group[CMR_V2_TITLE_KEY].get<string>();
494 if(month_title != CMR_V2_MONTH_TITLE_VALUE){
495 msg.str("");
496 msg << prolog << "We expected to get back a Month object, but we did not.";
497 BESDEBUG(MODULE, msg.str() << endl);
498 throw CmrInternalError(msg.str(), __FILE__, __LINE__);
499 }
500
501 const auto &months = get_children(month_group);
502 for (const auto &month: months) { // Uses SizeType instead of size_t
503 months_result.push_back(month[CMR_V2_TITLE_KEY].get<string>());
504 }
505
506} // CmrApi::get_months()
507
508
509
510
518void CmrApi::get_days(const string &collection_concept_id,
519 const string& r_year,
520 const string &r_month,
521 vector<string> &days_result) const
522{
523 stringstream cmr_query_url;
524 cmr_query_url << d_cmr_granules_search_endpoint_url << "?";
525
526 stringstream cmr_query_string;
527 cmr_query_string << "concept_id=" << collection_concept_id << "&";
528 cmr_query_string << "include_facets=v2" << "&";
529 cmr_query_string << http::url_encode("temporal_facet[0][year]") << "=" << r_year << "&";
530 cmr_query_string << http::url_encode("temporal_facet[0][month]") << "=" << r_month;
531 cmr_query_url << cmr_query_string.str();
532 BESDEBUG(MODULE, prolog << "CMR Query URL: " << cmr_query_url.str() << endl);
533
534 JsonUtils json;
535 const auto &cmr_doc = json.get_as_json(cmr_query_url.str());
536
537 const auto &day_group = get_day_group(r_month, r_year, cmr_doc);
538 const auto &days = get_children(day_group);
539 for ( const auto &day : days ){
540 string day_title = json.get_str_if_present(CMR_V2_TITLE_KEY,day);
541 if(day_title.empty())
542 day_title = "MISSING DAY TITLE";
543
544 days_result.push_back(day_title);
545 }
546}// CmrApi::get_days()
547
548
549
550
551
552
561void CmrApi::get_granule_ids(const std::string& collection_name,
562 const std::string& r_year,
563 const std::string &r_month,
564 const std::string &r_day,
565 std::vector<std::string> &granule_ids) const
566{
567 json cmr_doc;
568
569 granule_search(collection_name, r_year, r_month, r_day, cmr_doc);
570
571 const auto &granules = get_entries(cmr_doc);
572 for( const auto &granule : granules){
573 string day_id = granule[CMR_GRANULE_ID_KEY].get<string>();
574 granule_ids.push_back(day_id);
575 }
576}
577
578
579
580
589unsigned long CmrApi::granule_count(const string &collection_name,
590 const string &r_year,
591 const string &r_month,
592 const string &r_day) const
593{
594 stringstream msg;
595 json cmr_doc;
596 granule_search(collection_name, r_year, r_month, r_day, cmr_doc);
597 const auto &entries = get_entries(cmr_doc);
598 return entries.size();
599}
600
601
602
603
608void CmrApi::granule_search(const std::string &collection_name,
609 const std::string &r_year,
610 const std::string &r_month,
611 const std::string &r_day,
612 nlohmann::json &cmr_doc) const
613{
614
615 JsonUtils json;
616
617 stringstream cmr_query_url;
618 cmr_query_url << d_cmr_granules_search_endpoint_url << "?";
619 cmr_query_url << "concept_id=" << collection_name << "&";
620 cmr_query_url << "include_facets=v2&";
621 cmr_query_url << "page_size=" << CMR_MAX_PAGE_SIZE << "&";
622
623 if(!r_year.empty()) {
624 cmr_query_url << http::url_encode("temporal_facet[0][year]") << "=" << r_year<< "&";
625 }
626 if(!r_month.empty()) {
627 cmr_query_url << http::url_encode("temporal_facet[0][month]") << "=" << r_month<< "&";
628 }
629 if(!r_day.empty()) {
630 cmr_query_url << http::url_encode("temporal_facet[0][day]") << "=" << r_day;
631 }
632
633 BESDEBUG(MODULE, prolog << "CMR Granule Search Request Url: " << cmr_query_url.str() << endl);
634
635 cmr_doc = json.get_as_json(cmr_query_url.str());
636 BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << cmr_doc.dump(4) << endl);
637}
638
639
640
645void CmrApi::granule_umm_search(const std::string &collection_name,
646 const std::string &r_year,
647 const std::string &r_month,
648 const std::string &r_day,
649 nlohmann::json &cmr_doc)
650const {
651
652 JsonUtils json;
653
654 stringstream cmr_query_url;
655 cmr_query_url << d_cmr_granules_umm_search_endpoint_url << "?";
656 cmr_query_url << "concept_id=" << collection_name << "&";
657 cmr_query_url << "page_size=" << CMR_MAX_PAGE_SIZE << "&";
658
659 if(!r_year.empty()) {
660 cmr_query_url << http::url_encode("temporal_facet[0][year]") << "=" << r_year<< "&";
661 }
662 if(!r_month.empty()) {
663 cmr_query_url << http::url_encode("temporal_facet[0][month]") << "=" << r_month<< "&";
664 }
665 if(!r_day.empty()) {
666 cmr_query_url << http::url_encode("temporal_facet[0][day]") << "=" << r_day;
667 }
668
669 BESDEBUG(MODULE, prolog << "CMR Granule Search Request Url: " << cmr_query_url.str() << endl);
670
671 cmr_doc = json.get_as_json(cmr_query_url.str());
672 BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << cmr_doc.dump(4) << endl);
673}
674
675
676
680void CmrApi::get_granules(const std::string& collection_name,
681 const std::string &r_year,
682 const std::string &r_month,
683 const std::string &r_day,
684 std::vector<unique_ptr<cmr::Granule>> &granule_objs)
685const {
686 stringstream msg;
687 json cmr_doc;
688
689 granule_search(collection_name, r_year, r_month, r_day, cmr_doc);
690
691 const auto& granules = get_entries(cmr_doc);
692 for ( auto &granule : granules){
693 auto g = unique_ptr<Granule>(new Granule(granule));
694 granule_objs.emplace_back(std::move(g));
695 }
696}
697
698
702void CmrApi::get_granules_umm(const std::string& collection_name,
703 const std::string &r_year,
704 const std::string &r_month,
705 const std::string &r_day,
706 std::vector<unique_ptr<cmr::GranuleUMM>> &granule_objs) const
707{
708 stringstream msg;
709 json cmr_doc;
710
711 granule_umm_search(collection_name, r_year, r_month, r_day, cmr_doc);
712 const auto& granules = get_items(cmr_doc);
713 for ( auto &granule : granules){
714 auto g = unique_ptr<GranuleUMM>(new GranuleUMM(granule));
715 granule_objs.emplace_back(std::move(g));
716 }
717}
718
719
720
721
722
723void CmrApi::get_collection_ids(std::vector<std::string> &collection_ids) const
724{
725 bool found = false;
726 TheBESKeys::TheKeys()->get_values(CMR_COLLECTIONS_KEY, collection_ids, found);
727 if(!found){
728 throw BESInternalError(string("The '") + CMR_COLLECTIONS_KEY
729 + "' field has not been configured.", __FILE__, __LINE__);
730 }
731}
732
733
745unique_ptr<Granule> CmrApi::get_granule(const string& collection_name,
746 const string& r_year,
747 const string& r_month,
748 const string& r_day,
749 const string& r_granule_ur)
750const {
751 // @TODO If this code is supposed to get a single granule, and it has the granule_concept_id
752 // this should be making a direct cmr query for just that granule.
753 std::vector<unique_ptr<cmr::Granule>> granules;
754 unique_ptr<Granule> result;
755
756 get_granules(collection_name, r_year, r_month, r_day, granules);
757 for(auto & granule : granules){
758 string id = granule->getName();
759 BESDEBUG(MODULE, prolog << "Comparing granule_ur: '" << r_granule_ur << "' to collection member id: " << id << endl);
760 if( id == r_granule_ur){
761 result = std::move(granule);
762 }
763 }
764 return result;
765}
766
777void CmrApi::get_providers(vector<unique_ptr<cmr::Provider> > &providers) const
778{
779 JsonUtils json;
780 BES_STOPWATCH_START(MODULE, prolog + "Timing");
781
782 const auto &cmr_doc = json.get_as_json(d_cmr_providers_search_endpoint_url);
783 unsigned int hits = cmr_doc["hits"];
784 BESDEBUG(MODULE, prolog << "hits: " << hits << endl);
785 if (hits == 0){
786 return;
787 }
788 for (const auto &provider_json : cmr_doc["items"]) {
789 if(provider_json.type() != nlohmann::detail::value_t::null) {
790 auto prvdr = std::make_unique<Provider>(provider_json);
791 if(prvdr!=nullptr)
792 providers.emplace_back(std::move(prvdr));
793 }
794 }
795}
796
797
803void CmrApi::get_opendap_providers(map<string, unique_ptr<cmr::Provider>> &opendap_providers) const
804{
805 vector<unique_ptr<cmr::Provider>> all_providers;
806 BES_STOPWATCH_START(MODULE, prolog + "Timing");
807 get_providers(all_providers);
808 for(auto &provider: all_providers){
809 BESDEBUG(MODULE, prolog << "Processing PROVIDER: " << provider->id() << endl);
810 auto hits = get_opendap_collections_count(provider->id());
811 if (hits > 0){
812 provider->set_opendap_collection_count(hits);
813 opendap_providers.emplace(provider->id(), std::move(provider));
814 }
815 }
816}
817
825unsigned long int CmrApi::get_opendap_collections_count(const string &provider_id) const
826{
827 JsonUtils json;
828 stringstream cmr_query_url;
829 cmr_query_url << d_cmr_collections_search_endpoint_url;
830 cmr_query_url << "?has_opendap_url=true&page_size=0";
831 cmr_query_url << "&provider=" << provider_id;
832 BESDEBUG(MODULE, prolog << "cmr_query_url: " << cmr_query_url.str() << endl);
833 const auto &cmr_doc = json.get_as_json(cmr_query_url.str());
834
835 unsigned long int hits = json.qc_integer(CMR_HITS_KEY,cmr_doc);
836 BESDEBUG(MODULE, prolog << CMR_HITS_KEY << ": " << hits << endl);
837 return hits;
838}
839
840
841
842void CmrApi::get_collections_worker( const std::string &provider_id,
843 std::map<std::string, std::unique_ptr<cmr::Collection>> &collections,
844 unsigned int page_size,
845 bool just_opendap)
846const {
847 unsigned int page_num=1;
848 JsonUtils json;
849 string cmr_query_url_base;
850 cmr_query_url_base = d_cmr_collections_search_endpoint_url + "?";
851 cmr_query_url_base += "provider=" + provider_id + "&";
852 if(just_opendap){
853 cmr_query_url_base += "has_opendap_url=true&";
854 }
855 cmr_query_url_base += "page_size=" + to_string(page_size) + "&";
856 BESDEBUG(MODULE, prolog << "cmr_query_url_base: " << cmr_query_url_base << endl);
857
858 string cmr_query_url = cmr_query_url_base + "page_num=" + to_string(page_num);
859 BESDEBUG(MODULE, prolog << "cmr_query_url: " << cmr_query_url << endl);
860
861 const auto &cmr_doc = json.get_as_json(cmr_query_url);
862
863 unsigned int hits = cmr_doc["hits"];
864 BESDEBUG(MODULE, prolog << "hits: " << hits << endl);
865 if (hits == 0){
866 return;
867 }
868 for (const auto &collection_json : cmr_doc["items"]) {
869 auto collection = unique_ptr<Collection>(new Collection(collection_json));
870 collections.emplace(collection->id(), std::move(collection));
871 }
872 BESDEBUG(MODULE, prolog << "collections.size(): " << collections.size() << endl);
873
874 while (collections.size() < hits){
875 page_num++;
876 cmr_query_url = cmr_query_url_base + "page_num=" + to_string(page_num);
877 BESDEBUG(MODULE, prolog << "cmr_query_url: " << cmr_query_url << endl);
878 const auto &cmr_collection_doc = json.get_as_json(cmr_query_url);
879
880 for (const auto &collection_json : cmr_collection_doc["items"]) {
881 auto collection = unique_ptr<Collection>(new Collection(collection_json));
882 collections.emplace(collection->id(), std::move(collection));
883 }
884 BESDEBUG(MODULE, prolog << "collections.size(): " << collections.size() << endl);
885 }
886
887}
888
889
890
891void CmrApi::get_opendap_collections(const std::string &provider_id,
892 std::map<std::string,std::unique_ptr<cmr::Collection>> &collections) const{
893 get_collections_worker(provider_id,collections, CMR_MAX_PAGE_SIZE, true);
894}
895void CmrApi::get_collections(const std::string &provider_id,
896 std::map<std::string,std::unique_ptr<cmr::Collection>> &collections) const{
897 get_collections_worker(provider_id,collections, CMR_MAX_PAGE_SIZE, false);
898}
899
900
901
902
903
904
905} // namespace cmr
906
BESInternalError
exception thrown if internal error encountered
Definition BESInternalError.h:43
BESUtil::assemblePath
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition BESUtil.cc:804
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition TheBESKeys.cc:288
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Access to the singleton.
Definition TheBESKeys.cc:85
TheBESKeys::get_values
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
Definition TheBESKeys.cc:318
cmr::CmrApi::granule_count
unsigned long granule_count(const std::string &collection_name, const std::string &r_year, const std::string &r_month, const std::string &r_day) const
Definition CmrApi.cc:589
cmr::CmrApi::get_granules_umm
void get_granules_umm(const std::string &collection_name, const std::string &r_year, const std::string &r_month, const std::string &r_day, std::vector< std::unique_ptr< cmr::GranuleUMM > > &granule_objs) const
Definition CmrApi.cc:702
cmr::CmrApi::get_related_urls_array
const nlohmann::json & get_related_urls_array(const nlohmann::json &go) const
Definition CmrApi.cc:101
cmr::CmrApi::get_opendap_collections_count
unsigned long int get_opendap_collections_count(const std::string &provider_id) const
Definition CmrApi.cc:825
cmr::CmrApi::get_providers
void get_providers(std::vector< std::unique_ptr< cmr::Provider > > &providers) const
Definition CmrApi.cc:777
cmr::CmrApi::get_opendap_providers
void get_opendap_providers(std::map< std::string, std::unique_ptr< cmr::Provider > > &providers) const
Definition CmrApi.cc:803
cmr::CmrApi::get_granules
void get_granules(const std::string &collection_name, const std::string &r_year, const std::string &r_month, const std::string &r_day, std::vector< std::unique_ptr< cmr::Granule > > &granule_objs) const
Definition CmrApi.cc:680
cmr::CmrApi::get_days
void get_days(const std::string &collection_name, const std::string &r_year, const std::string &r_month, std::vector< std::string > &days_result) const
Definition CmrApi.cc:518
cmr::CmrApi::get_granule_ids
void get_granule_ids(const std::string &collection_name, const std::string &r_year, const std::string &r_month, const std::string &r_day, std::vector< std::string > &granule_ids) const
Definition CmrApi.cc:561
cmr::CmrApi::get_months
void get_months(const std::string &collection_name, const std::string &year, std::vector< std::string > &months_result) const
Definition CmrApi.cc:449
cmr::JsonUtils::get_as_json
nlohmann::json get_as_json(const std::string &url) const
Definition JsonUtils.cc:58
cmr::JsonUtils::qc_get_array
const nlohmann::json & qc_get_array(const std::string &key, const nlohmann::json &json_obj) const
Definition JsonUtils.cc:288
map
STL class.
stringstream
STL class.
unique_ptr
STL class.
vector
STL class.
http::url_encode
string url_encode(const string &s)
Definition HttpUtils.cc:215