DODSFilter.cc

// -*- mode: c++; c-basic-offset:4 -*-

// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.

// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.

// (c) COPYRIGHT URI/MIT 1997-1999
// Please read the full copyright statement in the file COPYRIGHT_URI.
//
// Authors:
//      jhrg,jimg       James Gallagher <jgallagher@gso.uri.edu>

// Implementation of the DODSFilter class. This class is used to build dods
// filter programs which, along with a CGI program, comprise OPeNDAP servers.
// jhrg 8/26/97


#include "config.h"

#include <signal.h>

#ifndef WIN32
#include <unistd.h>   // for getopt
#include <sys/wait.h>
#else
#include <io.h>
#include <fcntl.h>
#include <process.h>
#endif

#include <iostream>
#include <sstream>
#include <string>
#include <algorithm>
#include <cstdlib>
#include <cstring>

#ifdef HAVE_UUID_UUID_H
#include <uuid/uuid.h>  // used to build CID header value for data ddx
#elif defined(HAVE_UUID_H)
#include <uuid.h>
#else
#error "Could not find UUID library header"
#endif

#include <GetOpt.h>

#include "DAS.h"
#include "DDS.h"
#include "debug.h"
#include "mime_util.h"
#include "Ancillary.h"
#include "util.h"
#include "escaping.h"
#include "DODSFilter.h"
#include "XDRStreamMarshaller.h"
#include "InternalErr.h"

#ifndef WIN32
#include "SignalHandler.h"
#include "EventHandler.h"
#include "AlarmHandler.h"
#endif

#define CRLF "\r\n"             // Change here, expr-test.cc and DODSFilter.cc

using namespace std;

namespace libdap {

const string usage =
    "Usage: <handler name> -o <response> -u <url> [options ...] [data set]\n\
    \n\
    options: -o <response>: DAS, DDS, DataDDS, DDX, BLOB or Version (Required)\n\
    -u <url>: The complete URL minus the CE (required for DDX)\n\
    -c: Compress the response using the deflate algorithm.\n\
    -e <expr>: When returning a DataDDS, use <expr> as the constraint.\n\
    -v <version>: Use <version> as the version number\n\
    -d <dir>: Look for ancillary file in <dir> (deprecated).\n\
    -f <file>: Look for ancillary data in <file> (deprecated).\n\
    -r <dir>: Use <dir> as a cache directory\n\
    -l <time>: Conditional request; if data source is unchanged since\n\
    <time>, return an HTTP 304 response.\n\
    -t <seconds>: Timeout the handler after <seconds>.\n\
    -h: This message.";

DODSFilter::DODSFilter(int argc, char *argv[]) throw(Error)
{
    initialize(argc, argv);

    DBG(cerr << "d_comp: " << d_comp << endl);
    DBG(cerr << "d_dap2ce: " << d_dap2ce << endl);
    DBG(cerr << "d_cgi_ver: " << d_cgi_ver << endl);
    DBG(cerr << "d_response: " << d_response << endl);
    DBG(cerr << "d_anc_dir: " << d_anc_dir << endl);
    DBG(cerr << "d_anc_file: " << d_anc_file << endl);
    DBG(cerr << "d_cache_dir: " << d_cache_dir << endl);
    DBG(cerr << "d_conditional_request: " << d_conditional_request << endl);
    DBG(cerr << "d_if_modified_since: " << d_if_modified_since << endl);
    DBG(cerr << "d_url: " << d_url << endl);
    DBG(cerr << "d_timeout: " << d_timeout << endl);
}

DODSFilter::~DODSFilter()
{
}

void
DODSFilter::initialize()
{
    // Set default values. Don't use the C++ constructor initialization so
    // that a subclass can have more control over this process.
    d_comp = false;
    d_bad_options = false;
    d_conditional_request = false;
    d_dataset = "";
    d_dap2ce = "";
    d_cgi_ver = "";
    d_anc_dir = "";
    d_anc_file = "";
    d_cache_dir = "";
    d_response = Unknown_Response;;
    d_anc_das_lmt = 0;
    d_anc_dds_lmt = 0;
    d_if_modified_since = -1;
    d_url = "";
    d_program_name = "Unknown";
    d_timeout = 0;

#ifdef WIN32
    //  We want serving from win32 to behave in a manner
    //  similar to the UNIX way - no CR->NL terminated lines
    //  in files. Hence stdout goes to binary mode.
    _setmode(_fileno(stdout), _O_BINARY);
#endif
}

void
DODSFilter::initialize(int argc, char *argv[])
{
    initialize();

    d_program_name = argv[0];

    // This should be specialized by a subclass. This may throw Error.
    int next_arg = process_options(argc, argv);

    // Look at what's left after processing the command line options. Either
    // there MUST be a dataset name OR the caller is asking for version
    // information. If neither is true, then the options are bad.
    if (next_arg < argc) {
        d_dataset = argv[next_arg];
        d_dataset = www2id(d_dataset, "%", "%20");
    }
    else if (get_response() != Version_Response)
        print_usage();   // Throws Error
}

int
DODSFilter::process_options(int argc, char *argv[])
{
    DBG(cerr << "Entering process_options... ");

    int option_char;
    GetOpt getopt (argc, argv, "ce: v: d: f: r: l: o: u: t: ");

    while ((option_char = getopt()) != -1) {
        switch (option_char) {
        case 'c': d_comp = true; break;
        case 'e': set_ce(getopt.optarg); break;
        case 'v': set_cgi_version(getopt.optarg); break;
        case 'd': d_anc_dir = getopt.optarg; break;
        case 'f': d_anc_file = getopt.optarg; break;
        case 'r': d_cache_dir = getopt.optarg; break;
        case 'o': set_response(getopt.optarg); break;
        case 'u': set_URL(getopt.optarg); break;
        case 't': d_timeout = atoi(getopt.optarg); break;
        case 'l':
            d_conditional_request = true;
            d_if_modified_since
            = static_cast<time_t>(strtol(getopt.optarg, NULL, 10));
            break;
        case 'h': print_usage();
            break;
                                 // exit(1);
                                 // Removed 12/29/2011; exit should
                                 // not be called by a library. NB:
                                 // print_usage() throws Error.
        default: print_usage();  // Throws Error
            break;
        }
    }

    DBGN(cerr << "exiting." << endl);

    return getopt.optind; // return the index of the next argument
}

bool
DODSFilter::is_conditional() const
{
    return d_conditional_request;
}

void
DODSFilter::set_cgi_version(string version)
{
    d_cgi_ver = version;
}

string
DODSFilter::get_cgi_version() const
{
    return d_cgi_ver;
}

string
DODSFilter::get_ce() const
{
    return d_dap2ce;
}

void
DODSFilter::set_ce(string _ce)
{
    d_dap2ce = www2id(_ce, "%", "%20");
}

string
DODSFilter::get_dataset_name() const
{
    return d_dataset;
}

void
DODSFilter::set_dataset_name(const string ds)
{
    d_dataset = www2id(ds, "%", "%20");
}

string
DODSFilter::get_URL() const
{
    return d_url;
}

void
DODSFilter::set_URL(const string &url)
{
    if (url.find('?') != url.npos)
        print_usage();  // Throws Error

    d_url = url;
}

string
DODSFilter::get_dataset_version() const
{
    return "";
}

void DODSFilter::set_response(const string &r)
{
    if (r == "DAS" || r == "das") {
    d_response = DAS_Response;
    d_action = "das" ;
    }
    else if (r == "DDS" || r == "dds") {
    d_response = DDS_Response;
    d_action = "dds" ;
    }
    else if (r == "DataDDS" || r == "dods") {
    d_response = DataDDS_Response;
    d_action = "dods" ;
    }
    else if (r == "DDX" || r == "ddx") {
    d_response = DDX_Response;
    d_action = "ddx" ;
    }
    else if (r == "DataDDX" || r == "dataddx") {
    d_response = DataDDX_Response;
    d_action = "dataddx" ;
    }
    else if (r == "Version") {
    d_response = Version_Response;
    d_action = "version" ;
    }
    else
    print_usage();   // Throws Error
}

DODSFilter::Response
DODSFilter::get_response() const
{
    return d_response;
}

string DODSFilter::get_action() const
{
    return d_action;
}

time_t
DODSFilter::get_dataset_last_modified_time() const
{
    return last_modified_time(d_dataset);
}

time_t
DODSFilter::get_das_last_modified_time(const string &anc_location) const
{
    DBG(cerr << "DODSFilter::get_das_last_modified_time(anc_location="
        << anc_location << "call faf(das) d_dataset=" << d_dataset
        << " d_anc_file=" << d_anc_file << endl);

    string name
    = Ancillary::find_ancillary_file(d_dataset, "das",
                          (anc_location == "") ? d_anc_dir : anc_location,
                          d_anc_file);

    return max((name != "") ? last_modified_time(name) : 0,
               get_dataset_last_modified_time());
}

time_t
DODSFilter::get_dds_last_modified_time(const string &anc_location) const
{
    DBG(cerr << "DODSFilter::get_das_last_modified_time(anc_location="
        << anc_location << "call faf(dds) d_dataset=" << d_dataset
        << " d_anc_file=" << d_anc_file << endl);

    string name
    = Ancillary::find_ancillary_file(d_dataset, "dds",
                          (anc_location == "") ? d_anc_dir : anc_location,
                          d_anc_file);

    return max((name != "") ? last_modified_time(name) : 0,
               get_dataset_last_modified_time());
}

time_t
DODSFilter::get_data_last_modified_time(const string &anc_location) const
{
    DBG(cerr << "DODSFilter::get_das_last_modified_time(anc_location="
        << anc_location << "call faf(both) d_dataset=" << d_dataset
        << " d_anc_file=" << d_anc_file << endl);

    string dds_name
    = Ancillary::find_ancillary_file(d_dataset, "dds",
                          (anc_location == "") ? d_anc_dir : anc_location,
                          d_anc_file);
    string das_name
    = Ancillary::find_ancillary_file(d_dataset, "das",
                          (anc_location == "") ? d_anc_dir : anc_location,
                          d_anc_file);

    time_t m = max((das_name != "") ? last_modified_time(das_name) : (time_t)0,
                   (dds_name != "") ? last_modified_time(dds_name) : (time_t)0);
    // Note that this is a call to get_dataset_... not get_data_...
    time_t n = get_dataset_last_modified_time();

    return max(m, n);
}

time_t
DODSFilter::get_request_if_modified_since() const
{
    return d_if_modified_since;
}

string
DODSFilter::get_cache_dir() const
{
    return d_cache_dir;
}

void
DODSFilter::set_timeout(int t)
{
    d_timeout = t;
}

int
DODSFilter::get_timeout() const
{
    return d_timeout;
}

void
DODSFilter::establish_timeout(FILE *stream) const
{
#ifndef WIN32
    if (d_timeout > 0) {
        SignalHandler *sh = SignalHandler::instance();
        EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler(stream));
        delete old_eh;
        alarm(d_timeout);
    }
#endif
}

void
DODSFilter::establish_timeout(ostream &stream) const
{
#ifndef WIN32
    if (d_timeout > 0) {
        SignalHandler *sh = SignalHandler::instance();
        EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler(stream));
        delete old_eh;
        alarm(d_timeout);
    }
#endif
}

static const char *emessage = "DODS internal server error; usage error. Please report this to the dataset maintainer, or to the opendap-tech@opendap.org mailing list.";

void
DODSFilter::print_usage() const
{
    // Write a message to the WWW server error log file.
    ErrMsgT(usage.c_str());

    throw Error(emessage);
}

void
DODSFilter::send_version_info() const
{
    do_version(d_cgi_ver, get_dataset_version());
}

void
DODSFilter::send_das(FILE *out, DAS &das, const string &anc_location,
                     bool with_mime_headers) const
{
    ostringstream oss;
    send_das(oss, das, anc_location, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

void
DODSFilter::send_das(ostream &out, DAS &das, const string &anc_location,
                     bool with_mime_headers) const
{
    time_t das_lmt = get_das_last_modified_time(anc_location);
    if (is_conditional()
        && das_lmt <= get_request_if_modified_since()
        && with_mime_headers) {
        set_mime_not_modified(out);
    }
    else {
        if (with_mime_headers)
            set_mime_text(out, dods_das, d_cgi_ver, x_plain, das_lmt);
        das.print(out);
    }
    out << flush ;
}

void
DODSFilter::send_das(DAS &das, const string &anc_location,
                     bool with_mime_headers) const
{
    send_das(cout, das, anc_location, with_mime_headers);
}

void
DODSFilter::send_dds(FILE *out, DDS &dds, ConstraintEvaluator &eval,
                     bool constrained,
                     const string &anc_location,
                     bool with_mime_headers) const
{
    ostringstream oss;
    send_dds(oss, dds, eval, constrained, anc_location, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

void
DODSFilter::send_dds(ostream &out, DDS &dds, ConstraintEvaluator &eval,
                     bool constrained,
                     const string &anc_location,
                     bool with_mime_headers) const
{
    // If constrained, parse the constraint. Throws Error or InternalErr.
    if (constrained)
        eval.parse_constraint(d_dap2ce, dds);

    if (eval.functional_expression())
        throw Error("Function calls can only be used with data requests. To see the structure of the underlying data source, reissue the URL without the function.");

    time_t dds_lmt = get_dds_last_modified_time(anc_location);
    if (is_conditional()
        && dds_lmt <= get_request_if_modified_since()
        && with_mime_headers) {
        set_mime_not_modified(out);
    }
    else {
        if (with_mime_headers)
            set_mime_text(out, dods_dds, d_cgi_ver, x_plain, dds_lmt);
        if (constrained)
            dds.print_constrained(out);
        else
            dds.print(out);
    }

    out << flush ;
}

void
DODSFilter::send_dds(DDS &dds, ConstraintEvaluator &eval,
                     bool constrained, const string &anc_location,
                     bool with_mime_headers) const
{
    send_dds(cout, dds, eval, constrained, anc_location, with_mime_headers);
}

// 'lmt' unused. Should it be used to supply a LMT or removed from the
// method? jhrg 8/9/05
void
DODSFilter::functional_constraint(BaseType &var, DDS &dds,
                                  ConstraintEvaluator &eval, FILE *out) const
{
    ostringstream oss;
    functional_constraint(var, dds, eval, oss);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

// 'lmt' unused. Should it be used to supply a LMT or removed from the
// method? jhrg 8/9/05
void
DODSFilter::functional_constraint(BaseType &var, DDS &dds,
                                  ConstraintEvaluator &eval, ostream &out) const
{
    out << "Dataset {\n" ;
    var.print_decl(out, "    ", true, false, true);
    out << "} function_value;\n" ;
    out << "Data:\n" ;

    out << flush ;

    // Grab a stream that encodes using XDR.
    XDRStreamMarshaller m( out ) ;

    try {
        // In the following call to serialize, suppress CE evaluation.
        var.serialize(eval, dds, m, false);
    }
    catch (Error &e) {
        throw;
    }
}

void
DODSFilter::dataset_constraint(DDS & dds, ConstraintEvaluator & eval,
                               FILE * out, bool ce_eval) const
{
    ostringstream oss;
    dataset_constraint(dds, eval, oss, ce_eval);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

void
DODSFilter::dataset_constraint(DDS & dds, ConstraintEvaluator & eval,
                               ostream &out, bool ce_eval) const
{
    // send constrained DDS
    dds.print_constrained(out);
    out << "Data:\n" ;
    out << flush ;

    // Grab a stream that encodes using XDR.
    XDRStreamMarshaller m( out ) ;

    try {
        // Send all variables in the current projection (send_p())
        for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++)
            if ((*i)->send_p()) {
                DBG(cerr << "Sending " << (*i)->name() << endl);
                (*i)->serialize(eval, dds, m, ce_eval);
            }
    }
    catch (Error & e) {
        throw;
    }
}

void
DODSFilter::dataset_constraint_ddx(DDS & dds, ConstraintEvaluator & eval,
                               ostream &out, const string &boundary,
                               const string &start, bool ce_eval) const
{
    // Write the MPM headers for the DDX (text/xml) part of the response
    set_mime_ddx_boundary(out, boundary, start, dods_ddx);

    // Make cid
    uuid_t uu;
    uuid_generate(uu);
    char uuid[37];
    uuid_unparse(uu, &uuid[0]);
    char domain[256];
    if (getdomainname(domain, 255) != 0 || strlen(domain) == 0)
    strncpy(domain, "opendap.org", 255);

    string cid = string(&uuid[0]) + "@" + string(&domain[0]);

    // Send constrained DDX with a data blob reference
    dds.print_xml_writer(out, true, cid);

    // Write the MPM headers for the data part of the response.
    set_mime_data_boundary(out, boundary, cid, dap4_data, binary);

    // Grab a stream that encodes using XDR.
    XDRStreamMarshaller m( out ) ;

    try {
        // Send all variables in the current projection (send_p())
        for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++)
            if ((*i)->send_p()) {
                DBG(cerr << "Sending " << (*i)->name() << endl);
                (*i)->serialize(eval, dds, m, ce_eval);
            }
    }
    catch (Error & e) {
        throw;
    }
}

void
DODSFilter::send_data(DDS & dds, ConstraintEvaluator & eval,
                      FILE * data_stream, const string & anc_location,
                      bool with_mime_headers) const
{
    ostringstream oss;
    send_data(dds, eval, oss, anc_location, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), data_stream);
}

void
DODSFilter::send_data(DDS & dds, ConstraintEvaluator & eval,
                      ostream & data_stream, const string & anc_location,
                      bool with_mime_headers) const
{
    // If this is a conditional request and the server should send a 304
    // response, do that and exit. Otherwise, continue on and send the full
    // response.
    time_t data_lmt = get_data_last_modified_time(anc_location);
    if (is_conditional()
        && data_lmt <= get_request_if_modified_since()
        && with_mime_headers) {
        set_mime_not_modified(data_stream);
        return;
    }
    // Set up the alarm.
    establish_timeout(data_stream);
    dds.set_timeout(d_timeout);

    eval.parse_constraint(d_dap2ce, dds);   // Throws Error if the ce doesn't
                    // parse.

    dds.tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.

    // Start sending the response...

    // Handle *functional* constraint expressions specially
#if 0
    if (eval.functional_expression()) {
        // Get the result and then start sending the headers. This provides a
        // way to send errors back to the client w/o colliding with the
        // normal response headers. There's some duplication of code with this
        // and the else-clause.
        BaseType *var = eval.eval_function(dds, d_dataset);
        if (!var)
            throw Error(unknown_error, "Error calling the CE function.");

       if (with_mime_headers)
            set_mime_binary(data_stream, dods_data, d_cgi_ver, x_plain, data_lmt);

    data_stream << flush ;

        functional_constraint(*var, dds, eval, data_stream);
        delete var;
        var = 0;
    }
#endif
    if (eval.function_clauses()) {
    DDS *fdds = eval.eval_function_clauses(dds);
        if (with_mime_headers)
            set_mime_binary(data_stream, dods_data, d_cgi_ver, x_plain, data_lmt);

        dataset_constraint(*fdds, eval, data_stream, false);
    delete fdds;
    }
    else {
        if (with_mime_headers)
            set_mime_binary(data_stream, dods_data, d_cgi_ver, x_plain, data_lmt);

        dataset_constraint(dds, eval, data_stream);
    }

    data_stream << flush ;
}

void
DODSFilter::send_ddx(DDS &dds, ConstraintEvaluator &eval, FILE *out,
                     bool with_mime_headers) const
{
    ostringstream oss;
    send_ddx(dds, eval, oss, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

void
DODSFilter::send_ddx(DDS &dds, ConstraintEvaluator &eval, ostream &out,
                     bool with_mime_headers) const
{
    // If constrained, parse the constraint. Throws Error or InternalErr.
    if (!d_dap2ce.empty())
        eval.parse_constraint(d_dap2ce, dds);

    if (eval.functional_expression())
        throw Error("Function calls can only be used with data requests. To see the structure of the underlying data source, reissue the URL without the function.");

    time_t dds_lmt = get_dds_last_modified_time(d_anc_dir);

    // If this is a conditional request and the server should send a 304
    // response, do that and exit. Otherwise, continue on and send the full
    // response.
    if (is_conditional() && dds_lmt <= get_request_if_modified_since()
        && with_mime_headers) {
        set_mime_not_modified(out);
        return;
    }
    else {
        if (with_mime_headers)
            set_mime_text(out, dods_ddx, d_cgi_ver, x_plain, dds_lmt);
        dds.print_xml_writer(out, !d_dap2ce.empty(), "");
    }
}

void
DODSFilter::send_data_ddx(DDS & dds, ConstraintEvaluator & eval,
                      ostream & data_stream, const string &start,
                      const string &boundary, const string & anc_location,
                      bool with_mime_headers) const
{
    // If this is a conditional request and the server should send a 304
    // response, do that and exit. Otherwise, continue on and send the full
    // response.
    time_t data_lmt = get_data_last_modified_time(anc_location);
    if (is_conditional()
        && data_lmt <= get_request_if_modified_since()
        && with_mime_headers) {
        set_mime_not_modified(data_stream);
        return;
    }
    // Set up the alarm.
    establish_timeout(data_stream);
    dds.set_timeout(d_timeout);

    eval.parse_constraint(d_dap2ce, dds);   // Throws Error if the ce doesn't
                    // parse.

    dds.tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.

    // Start sending the response...

    // Handle *functional* constraint expressions specially
#if 0
    if (eval.functional_expression()) {
        BaseType *var = eval.eval_function(dds, d_dataset);
        if (!var)
            throw Error(unknown_error, "Error calling the CE function.");

        if (with_mime_headers)
            set_mime_multipart(data_stream, boundary, start, dods_data_ddx,
        d_cgi_ver, x_plain, data_lmt);
    data_stream << flush ;
    BaseTypeFactory btf;
    DDS var_dds(&btf, var->name());
    var->set_send_p(true);
    var_dds.add_var(var);
        serialize_dap2_data_ddx(var_dds, eval, data_stream, boundary, start);

        // functional_constraint_ddx(*var, dds, eval, data_stream, boundary);
        delete var;
        var = 0;
    }
#endif
    if (eval.function_clauses()) {
        DDS *fdds = eval.eval_function_clauses(dds);
        if (with_mime_headers)
            set_mime_multipart(data_stream, boundary, start, dods_data_ddx,
                d_cgi_ver, x_plain, data_lmt);
        data_stream << flush ;
        dataset_constraint(*fdds, eval, data_stream, false);
        delete fdds;
    }
    else {
        if (with_mime_headers)
            set_mime_multipart(data_stream, boundary, start, dods_data_ddx,
                d_cgi_ver, x_plain, data_lmt);
        data_stream << flush ;
        dataset_constraint_ddx(dds, eval, data_stream, boundary, start);
    }

    data_stream << flush ;

    if (with_mime_headers)
    data_stream << CRLF << "--" << boundary << "--" << CRLF;
}

} // namespace libdap