FoInstanceJsonTransform.cc

// -*- mode: c++; c-basic-offset:4 -*-
//
// FoInstanceJsonTransform.cc
//
// This file is part of BES JSON File Out Module
//
// Copyright (c) 2014 OPeNDAP, Inc.
// Author: Nathan Potter <ndp@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
// (c) COPYRIGHT URI/MIT 1995-1999
// Please read the full copyright statement in the file COPYRIGHT_URI.
//
 
#include "config.h"
 
#include <cassert>
 
#include <sstream>
#include <iostream>
#include <fstream>
#include <stddef.h>
#include <string>
#include <typeinfo>
 
#include <libdap/DDS.h>
#include <libdap/Structure.h>
#include <libdap/Constructor.h>
#include <libdap/Array.h>
#include <libdap/Grid.h>
#include <libdap/Sequence.h>
#include <libdap/Str.h>
#include <libdap/Url.h>
 
#include <BESDebug.h>
#include "BESUtil.h"
#include <BESInternalError.h>
#include "RequestServiceTimer.h"
 
#include "FoInstanceJsonTransform.h"
#include "fojson_utils.h"
 
using namespace std;
 
#define MODULE "bes"
#define prolog string("FoInstanceJsonTransform::").append(__func__).append("() - ")
 
#define ATTRIBUTE_SEPARATOR "."
#define JSON_ORIGINAL_NAME "json_original_name"
 
#define FoInstanceJsonTransform_debug_key "fojson"
const int int_64_precision = 15; // See also in FODapJsonTransform.cc. jhrg 9/14/15

template<typename T>
unsigned int FoInstanceJsonTransform::json_simple_type_array_worker(std::ostream *strm,
    const std::vector<T> &values, unsigned int indx, const std::vector<unsigned int> &shape, unsigned int currentDim)
{
    *strm << "[";
 
    unsigned int currentDimSize = shape.at(currentDim);        // at is slower than [] but safe
 
    for (unsigned int i = 0; i < currentDimSize; i++) {
        if (currentDim < shape.size() - 1) {
            BESDEBUG(FoInstanceJsonTransform_debug_key,
                "json_simple_type_array_worker() - Recursing! indx:  " << indx << " currentDim: " << currentDim << " currentDimSize: " << currentDimSize << endl);
 
            indx = json_simple_type_array_worker<T>(strm, values, indx, shape, currentDim + 1);
            if (i + 1 != currentDimSize) *strm << ", ";
        }
        else {
            if (i) *strm << ", ";
            *strm << values[indx++];
        }
    }
 
    *strm << "]";
 
    return indx;
}

template<typename T> void FoInstanceJsonTransform::json_simple_type_array(std::ostream *strm, libdap::Array *a,
    std::string indent, bool sendData)
{
    std::string name = a->name();
    *strm << indent << "\"" << fojson::escape_for_json(name) + "\":  ";
 
    if (sendData) { // send data
        std::vector<unsigned int> shape(a->dimensions(true));
        long length = fojson::computeConstrainedShape(a, &shape);
 
        vector<T> src(length);
        a->value(src.data());
 
        unsigned int indx = 0;
 
        if (typeid(T) == typeid(libdap::dods_float64)) {
            streamsize prec = strm->precision(int_64_precision);
            try {
                indx = json_simple_type_array_worker(strm, src, 0, shape, 0);
                strm->precision(prec);
            }
            catch (...) {
                strm->precision(prec);
                throw;
            }
        }
        else {
            indx = json_simple_type_array_worker(strm, src, 0, shape, 0);
        }
 
        // make this an assert?
        assert(length == indx);
#if 0
        if (length != indx)
        BESDEBUG(FoInstanceJsonTransform_debug_key,
            "json_simple_type_array() - indx NOT equal to content length! indx:  " << indx << "  length: " << length << endl);
#endif
    }
    else { // otherwise send metadata
        *strm << "{" << endl;
        //Attributes
        transform(strm, a->get_attr_table(), indent + _indent_increment);
        *strm << endl << indent << "}";
    }
}

void FoInstanceJsonTransform::json_string_array(std::ostream *strm, libdap::Array *a, std::string indent, bool sendData)
{
    std::string name = a->name();
    *strm << indent << "\"" << fojson::escape_for_json(name) + "\":  ";
 
    if (sendData) { // send data
        std::vector<unsigned int> shape(a->dimensions(true));
        long length = fojson::computeConstrainedShape(a, &shape);
 
        // The string type utilizes a specialized version of libdap::Array::value()
        std::vector<std::string> sourceValues;
        a->value(sourceValues);
 
        unsigned int indx = json_simple_type_array_worker(strm, sourceValues, 0, shape, 0);
 
        // make this an assert?
        if (length != indx)
            BESDEBUG(FoInstanceJsonTransform_debug_key,
                "json_string_array() - indx NOT equal to content length! indx:  " << indx << "  length: " << length << endl);
    }
    else { // otherwise send metadata
        *strm << "{" << endl;
        //Attributes
        transform(strm, a->get_attr_table(), indent + _indent_increment);
        *strm << endl << indent << "}";
    }
}

#if 0
FoInstanceJsonTransform::FoInstanceJsonTransform(libdap::DDS *dds, BESDataHandlerInterface &/*dhi*/,
    const string &localfile) :
    _dds(dds), _localfile(localfile), _indent_increment(" "), _ostrm(0)
{
    // I'd make these asserts - ctors generally shoulf not throw exceptions if it can be helped
    // jhrg 3/11/5
    if (!_dds) throw BESInternalError("File out JSON, null DDS passed to constructor", __FILE__, __LINE__);
    if (_localfile.empty())
        throw BESInternalError("File out JSON, empty local file name passed to constructor", __FILE__, __LINE__);
}
#endif
FoInstanceJsonTransform::FoInstanceJsonTransform(libdap::DDS *dds):  _dds(dds), _indent_increment(" ")
{
    if (!_dds) throw BESInternalError("File out JSON, null DDS passed to constructor", __FILE__, __LINE__);
}

void FoInstanceJsonTransform::dump(std::ostream &strm) const
{
    strm << BESIndent::LMarg << "FoInstanceJsonTransform::dump - (" << (void *) this << ")" << endl;
    BESIndent::Indent();
    if (_dds != 0) {
        _dds->print(strm);
    }
    BESIndent::UnIndent();
}

void FoInstanceJsonTransform::transform(ostream &ostrm, bool sendData)
{
    transform(&ostrm, _dds, "", sendData);
}

void FoInstanceJsonTransform::transform(std::ostream *strm, libdap::DDS *dds, string indent, bool sendData)
{
    bool sentSomething = false;
 
    // Verify the request hasn't exceeded bes_timeout.
    RequestServiceTimer::TheTimer()->throw_if_timeout_expired( prolog + "ERROR: bes-timeout expired before transmit", __FILE__, __LINE__);
 
    // Now that we are ready to start streaming the response data we
    // cancel any pending timeout alarm according to the configuration.
    BESUtil::conditional_timeout_cancel();
 
    // Open returned JSON object
    *strm << "{" << endl;
 
    // Name object
    std::string name = dds->get_dataset_name();
    *strm << indent + _indent_increment << "\"name\": \"" << fojson::escape_for_json(name) << "\"," << endl;
 
    if (!sendData) {
        // Send metadata if we aren't sending data
 
        //Attributes
        transform(strm, dds->get_attr_table(), indent);
        if (dds->get_attr_table().get_size() > 0) *strm << ",";
        *strm << endl;
    }
 
    // Process the variables in the DDS
    if (dds->num_var() > 0) {
 
        libdap::DDS::Vars_iter vi = dds->var_begin();
        libdap::DDS::Vars_iter ve = dds->var_end();
        for (; vi != ve; vi++) {
            if ((*vi)->send_p()) {
 
                libdap::BaseType *v = *vi;
                BESDEBUG(FoInstanceJsonTransform_debug_key, "Processing top level variable: " << v->name() << endl);
 
                RequestServiceTimer::TheTimer()->throw_if_timeout_expired(prolog + "ERROR: bes-timeout expired before transmitting " + v->name(), __FILE__, __LINE__);
 
                if (sentSomething) {
                    *strm << ",";
                    *strm << endl;
                }
                transform(strm, v, indent + _indent_increment, sendData);
 
                sentSomething = true;
            }
        }
    }
 
    // Close the JSON object
    *strm << endl << "}" << endl;
}

void FoInstanceJsonTransform::transform(std::ostream *strm, libdap::BaseType *bt, string indent, bool sendData)
{
    switch (bt->type()) {
    // Handle the atomic types - that's easy!
    case libdap::dods_byte_c:
    case libdap::dods_int16_c:
    case libdap::dods_uint16_c:
    case libdap::dods_int32_c:
    case libdap::dods_uint32_c:
    case libdap::dods_float32_c:
    case libdap::dods_float64_c:
    case libdap::dods_str_c:
    case libdap::dods_url_c:
        transformAtomic(strm, bt, indent, sendData);
        break;
 
    case libdap::dods_structure_c:
        transform(strm, (libdap::Structure *) bt, indent, sendData);
        break;
 
    case libdap::dods_grid_c:
        transform(strm, (libdap::Grid *) bt, indent, sendData);
        break;
 
    case libdap::dods_sequence_c:
        transform(strm, (libdap::Sequence *) bt, indent, sendData);
        break;
 
    case libdap::dods_array_c:
        transform(strm, (libdap::Array *) bt, indent, sendData);
        break;
 
    case libdap::dods_int8_c:
    case libdap::dods_uint8_c:
    case libdap::dods_int64_c:
    case libdap::dods_uint64_c:
        // Removed from DAP4? jhrg 1/7/15
        // case libdap::dods_url4_c:
    case libdap::dods_enum_c:
    case libdap::dods_group_c: {
        string s = (string) "File out JSON, " + "DAP4 types not yet supported.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
 
    default: {
        string s = (string) "File out JSON, " + "Unrecognized type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
 
    }
 
}

void FoInstanceJsonTransform::transformAtomic(std::ostream *strm, libdap::BaseType *b, string indent, bool sendData)
{
    std::string name = b->name();
    *strm << indent << "\"" << fojson::escape_for_json(name) << "\": ";
 
    if (sendData) {
 
        if (b->type() == libdap::dods_str_c || b->type() == libdap::dods_url_c) {
            libdap::Str *strVar = (libdap::Str *) b;
            std::string tmpString = strVar->value();
            *strm << "\"" << fojson::escape_for_json(tmpString) << "\"";
        }
        else {
            b->print_val(*strm, "", false);
        }
 
    }
    else {
        transform(strm, b->get_attr_table(), indent);
    }
}

void FoInstanceJsonTransform::transform(std::ostream *strm, libdap::Structure *b, string indent, bool sendData)
{
 
    // Open object with name of the structure
    std::string name = b->name();
    *strm << indent << "\"" << fojson::escape_for_json(name) << "\": {" << endl;
 
    // Process the variables.
    if (b->width(true) > 0) {
 
        libdap::Structure::Vars_iter vi = b->var_begin();
        libdap::Structure::Vars_iter ve = b->var_end();
        for (; vi != ve; vi++) {
 
            // If the variable is projected, send (transform) it.
            if ((*vi)->send_p()) {
                libdap::BaseType *v = *vi;
                BESDEBUG(FoInstanceJsonTransform_debug_key,
                    "FoInstanceJsonTransform::transform() - Processing structure variable: " << v->name() << endl);
                transform(strm, v, indent + _indent_increment, sendData);
                if ((vi + 1) != ve) {
                    *strm << ",";
                }
                *strm << endl;
            }
        }
    }
    *strm << indent << "}";
}

void FoInstanceJsonTransform::transform(std::ostream *strm, libdap::Grid *g, string indent, bool sendData)
{
 
    // Open JSON property object with name of the grid
    std::string name = g->name();
    *strm << indent << "\"" << fojson::escape_for_json(name) << "\": {" << endl;
 
    BESDEBUG(FoInstanceJsonTransform_debug_key,
        "FoInstanceJsonTransform::transform() - Processing Grid data Array: " << g->get_array()->name() << endl);
 
    // Process the data array
    transform(strm, g->get_array(), indent + _indent_increment, sendData);
    *strm << "," << endl;
 
    // Process the MAP arrays
    for (libdap::Grid::Map_iter mapi = g->map_begin(); mapi < g->map_end(); mapi++) {
        BESDEBUG(FoInstanceJsonTransform_debug_key,
            "FoInstanceJsonTransform::transform() - Processing Grid Map Array: " << (*mapi)->name() << endl);
        if (mapi != g->map_begin()) {
            *strm << "," << endl;
        }
        transform(strm, *mapi, indent + _indent_increment, sendData);
    }
    // Close the JSON property object
    *strm << endl << indent << "}";
 
}

void FoInstanceJsonTransform::transform(std::ostream *strm, libdap::Sequence *s, string indent, bool sendData)
{
 
    // Open JSON property object with name of the sequence
    std::string name = s->name();
    *strm << indent << "\"" << fojson::escape_for_json(name) << "\": {" << endl;
 
    string child_indent = indent + _indent_increment;
 
#if 0
    the erdap way
    *strm << indent << "\"table\": {" << endl;
 
    string child_indent = indent + _indent_increment;
 
    *strm << child_indent << "\"name\": \"" << s->name() << "\"," << endl;
 
#endif
 
    *strm << child_indent << "\"columnNames\": [";
    for (libdap::Constructor::Vars_iter v = s->var_begin(); v < s->var_end(); v++) {
        if (v != s->var_begin()) *strm << ",";
        std::string name = (*v)->name();
        *strm << "\"" << fojson::escape_for_json(name) << "\"";
    }
    *strm << "]," << endl;
 
    *strm << child_indent << "\"columnTypes\": [";
    for (libdap::Constructor::Vars_iter v = s->var_begin(); v < s->var_end(); v++) {
        if (v != s->var_begin()) *strm << ",";
        *strm << "\"" << (*v)->type_name() << "\"";
    }
    *strm << "]," << endl;
 
    bool first = true;
    *strm << child_indent << "\"rows\": [";
    while (s->read()) {
        if (!first) *strm << ", ";
        *strm << endl << child_indent << "[";
        for (libdap::Constructor::Vars_iter v = s->var_begin(); v < s->var_end(); v++) {
            if (v != s->var_begin()) *strm << child_indent << ",";
            transform(strm, (*v), child_indent + _indent_increment, sendData);
        }
        *strm << child_indent << "]";
        first = false;
    }
    *strm << endl << child_indent << "]" << endl;
 
    // Close the JSON property object
    *strm << indent << "}" << endl;
}

void FoInstanceJsonTransform::transform(std::ostream *strm, libdap::Array *a, string indent, bool sendData)
{
 
    BESDEBUG(FoInstanceJsonTransform_debug_key,
        "FoInstanceJsonTransform::transform() - Processing Array. " << " a->type(): " << a->type() << " a->var()->type(): " << a->var()->type() << endl);
 
    switch (a->var()->type()) {
    // Handle the atomic types - that's easy!
    case libdap::dods_byte_c:
        json_simple_type_array<libdap::dods_byte>(strm, a, indent, sendData);
        break;
 
    case libdap::dods_int16_c:
        json_simple_type_array<libdap::dods_int16>(strm, a, indent, sendData);
        break;
 
    case libdap::dods_uint16_c:
        json_simple_type_array<libdap::dods_uint16>(strm, a, indent, sendData);
        break;
 
    case libdap::dods_int32_c:
        json_simple_type_array<libdap::dods_int32>(strm, a, indent, sendData);
        break;
 
    case libdap::dods_uint32_c:
        json_simple_type_array<libdap::dods_uint32>(strm, a, indent, sendData);
        break;
 
    case libdap::dods_float32_c:
        json_simple_type_array<libdap::dods_float32>(strm, a, indent, sendData);
        break;
 
    case libdap::dods_float64_c:
        json_simple_type_array<libdap::dods_float64>(strm, a, indent, sendData);
        break;
 
    case libdap::dods_str_c: {
        json_string_array(strm, a, indent, sendData);
        break;
#if 0
        //json_simple_type_array<Str>(strm,a,indent);
        string s = (string) "File out JSON, " + "Arrays of Strings are not yet a supported return type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
#endif
 
    }
 
    case libdap::dods_url_c: {
        json_string_array(strm, a, indent, sendData);
        break;
#if 0
        //json_simple_type_array<Url>(strm,a,indent);
        string s = (string) "File out JSON, " + "Arrays of URLs are not yet a supported return type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
#endif
 
    }
 
    case libdap::dods_structure_c: {
        string s = (string) "File out JSON, " + "Arrays of Structure objects not a supported return type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
    case libdap::dods_grid_c: {
        string s = (string) "File out JSON, " + "Arrays of Grid objects not a supported return type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
 
    case libdap::dods_sequence_c: {
        string s = (string) "File out JSON, " + "Arrays of Sequence objects not a supported return type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
 
    case libdap::dods_array_c: {
        string s = (string) "File out JSON, " + "Arrays of Array objects not a supported return type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
    case libdap::dods_int8_c:
    case libdap::dods_uint8_c:
    case libdap::dods_int64_c:
    case libdap::dods_uint64_c:
        // case libdap::dods_url4_c:
    case libdap::dods_enum_c:
    case libdap::dods_group_c: {
        string s = (string) "File out JSON, " + "DAP4 types not yet supported.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
 
    default: {
        string s = (string) "File out JSON, " + "Unrecognized type.";
        throw BESInternalError(s, __FILE__, __LINE__);
        break;
    }
 
    }
}

void FoInstanceJsonTransform::transform(std::ostream *strm, libdap::AttrTable &attr_table, string indent)
{
    /*
     * Since Attributes get promoted to JSON "properties" of their parent object (either a
     * BaseType variable or a DDS derived JSON object) this method does not open a new JSON
     * object, but rather continues to add content to the currently open object.
     */
    string child_indent = indent + _indent_increment;
 
    // Are there any attributes?
    if (attr_table.get_size() != 0) {
        //*strm << endl;
        libdap::AttrTable::Attr_iter begin = attr_table.attr_begin();
        libdap::AttrTable::Attr_iter end = attr_table.attr_end();
 
        // Process each Attribute
        for (libdap::AttrTable::Attr_iter at_iter = begin; at_iter != end; at_iter++) {
 
            switch (attr_table.get_attr_type(at_iter)) {
 
            case libdap::Attr_container:    // If it's a container attribute
            {
                libdap::AttrTable *atbl = attr_table.get_attr_table(at_iter);
 
                if (at_iter != begin) *strm << "," << endl;
 
                // Open a JSON property with the name of the Attribute Table and
                std::string name = atbl->get_name();
                *strm << child_indent << "\"" << fojson::escape_for_json(name) << "\": {" << endl;
 
                // Process the Attribute Table.
                transform(strm, *atbl, child_indent + _indent_increment);
 
                // Close JSON property object
                *strm << endl << child_indent << "}";
 
                break;
 
            }
            default: // so it's not an Attribute Table. woot. time to print
                // First?
                if (at_iter != begin) *strm << "," << endl;
 
                // Name of property
                std::string name = attr_table.get_name(at_iter);
                *strm << child_indent << "\"" << fojson::escape_for_json(name) << "\": ";
 
                // Open values array
                *strm << "[";
                // Process value(s)
                std::vector<std::string> *values = attr_table.get_attr_vector(at_iter);
                for (std::vector<std::string>::size_type i = 0; i < values->size(); i++) {
                    if (i > 0) *strm << ",";
                    if (attr_table.get_attr_type(at_iter) == libdap::Attr_string
                        || attr_table.get_attr_type(at_iter) == libdap::Attr_url) {
                        *strm << "\"";
 
                        string value = (*values)[i];
                        *strm << fojson::escape_for_json(value);
                        *strm << "\"";
                    }
                    else {
                        *strm << (*values)[i];
                    }
 
                }
                // Close values array
                *strm << "]";
                break;
            }
        }
    }
}