DODSFilter.cc

 
// -*- mode: c++; c-basic-offset:4 -*-
 
// This file is part of libdap, A C++ implementation of the OPeNDAP Data
// Access Protocol.
 
// Copyright (c) 2002,2003 OPeNDAP, Inc.
// Author: James Gallagher <jgallagher@opendap.org>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//
// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
 
// (c) COPYRIGHT URI/MIT 1997-1999
// Please read the full copyright statement in the file COPYRIGHT_URI.
//
// Authors:
//      jhrg,jimg       James Gallagher <jgallagher@gso.uri.edu>
 
// Implementation of the DODSFilter class. This class is used to build dods
// filter programs which, along with a CGI program, comprise OPeNDAP servers.
// jhrg 8/26/97
 
#include "config.h"
 
#include <signal.h>
 
#ifndef WIN32
#include <sys/wait.h>
#include <unistd.h> // for getopt
#else
#include <fcntl.h>
#include <io.h>
#include <process.h>
#endif
 
#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <sstream>
#include <string>
 
#ifdef HAVE_UUID_UUID_H
#include <uuid/uuid.h> // used to build CID header value for data ddx
#elif defined(HAVE_UUID_H)
#include <uuid.h>
#else
#error "Could not find UUID library header"
#endif
 
#include <GetOpt.h>
 
#include "Ancillary.h"
#include "DAS.h"
#include "DDS.h"
#include "DODSFilter.h"
#include "InternalErr.h"
#include "XDRStreamMarshaller.h"
#include "debug.h"
#include "escaping.h"
#include "mime_util.h"
#include "util.h"
 
#ifndef WIN32
#include "AlarmHandler.h"
#include "EventHandler.h"
#include "SignalHandler.h"
#endif
 
#define CRLF "\r\n" // Change here, expr-test.cc and DODSFilter.cc
 
using namespace std;
 
namespace libdap {
 
const string usage = "Usage: <handler name> -o <response> -u <url> [options ...] [data set]\n\
    \n\
    options: -o <response>: DAS, DDS, DataDDS, DDX, BLOB or Version (Required)\n\
    -u <url>: The complete URL minus the CE (required for DDX)\n\
    -c: Compress the response using the deflate algorithm.\n\
    -e <expr>: When returning a DataDDS, use <expr> as the constraint.\n\
    -v <version>: Use <version> as the version number\n\
    -d <dir>: Look for ancillary file in <dir> (deprecated).\n\
    -f <file>: Look for ancillary data in <file> (deprecated).\n\
    -r <dir>: Use <dir> as a cache directory\n\
    -l <time>: Conditional request; if data source is unchanged since\n\
    <time>, return an HTTP 304 response.\n\
    -t <seconds>: Timeout the handler after <seconds>.\n\
    -h: This message.";

 
DODSFilter::DODSFilter(int argc, char *argv[]) throw(Error) {
    initialize(argc, argv);
 
    DBG(cerr << "d_comp: " << d_comp << endl);
    DBG(cerr << "d_dap2ce: " << d_dap2ce << endl);
    DBG(cerr << "d_cgi_ver: " << d_cgi_ver << endl);
    DBG(cerr << "d_response: " << d_response << endl);
    DBG(cerr << "d_anc_dir: " << d_anc_dir << endl);
    DBG(cerr << "d_anc_file: " << d_anc_file << endl);
    DBG(cerr << "d_cache_dir: " << d_cache_dir << endl);
    DBG(cerr << "d_conditional_request: " << d_conditional_request << endl);
    DBG(cerr << "d_if_modified_since: " << d_if_modified_since << endl);
    DBG(cerr << "d_url: " << d_url << endl);
    DBG(cerr << "d_timeout: " << d_timeout << endl);
}
 
DODSFilter::~DODSFilter() {}

void DODSFilter::initialize() {
    // Set default values. Don't use the C++ constructor initialization so
    // that a subclass can have more control over this process.
    d_comp = false;
    d_bad_options = false;
    d_conditional_request = false;
    d_dataset = "";
    d_dap2ce = "";
    d_cgi_ver = "";
    d_anc_dir = "";
    d_anc_file = "";
    d_cache_dir = "";
    d_response = Unknown_Response;
    ;
    d_anc_das_lmt = 0;
    d_anc_dds_lmt = 0;
    d_if_modified_since = -1;
    d_url = "";
    d_program_name = "Unknown";
    d_timeout = 0;
 
#ifdef WIN32
    //  We want serving from win32 to behave in a manner
    //  similar to the UNIX way - no CR->NL terminated lines
    //  in files. Hence stdout goes to binary mode.
    _setmode(_fileno(stdout), _O_BINARY);
#endif
}

void DODSFilter::initialize(int argc, char *argv[]) {
    initialize();
 
    d_program_name = argv[0];
 
    // This should be specialized by a subclass. This may throw Error.
    int next_arg = process_options(argc, argv);
 
    // Look at what's left after processing the command line options. Either
    // there MUST be a dataset name OR the caller is asking for version
    // information. If neither is true, then the options are bad.
    if (next_arg < argc) {
        d_dataset = argv[next_arg];
        d_dataset = www2id(d_dataset, "%", "%20");
    } else if (get_response() != Version_Response)
        print_usage(); // Throws Error
}

int DODSFilter::process_options(int argc, char *argv[]) {
    DBG(cerr << "Entering process_options... ");
 
    int option_char;
    GetOpt getopt(argc, argv, "ce: v: d: f: r: l: o: u: t: ");
 
    while ((option_char = getopt()) != -1) {
        switch (option_char) {
        case 'c':
            d_comp = true;
            break;
        case 'e':
            set_ce(getopt.optarg);
            break;
        case 'v':
            set_cgi_version(getopt.optarg);
            break;
        case 'd':
            d_anc_dir = getopt.optarg;
            break;
        case 'f':
            d_anc_file = getopt.optarg;
            break;
        case 'r':
            d_cache_dir = getopt.optarg;
            break;
        case 'o':
            set_response(getopt.optarg);
            break;
        case 'u':
            set_URL(getopt.optarg);
            break;
        case 't':
            d_timeout = atoi(getopt.optarg);
            break;
        case 'l':
            d_conditional_request = true;
            d_if_modified_since = static_cast<time_t>(strtol(getopt.optarg, NULL, 10));
            break;
        case 'h':
            print_usage();
            break;
            // exit(1);
            // Removed 12/29/2011; exit should
            // not be called by a library. NB:
            // print_usage() throws Error.
        default:
            print_usage(); // Throws Error
            break;
        }
    }
 
    DBGN(cerr << "exiting." << endl);
 
    return getopt.optind; // return the index of the next argument
}

bool DODSFilter::is_conditional() const { return d_conditional_request; }

void DODSFilter::set_cgi_version(string version) { d_cgi_ver = version; }

string DODSFilter::get_cgi_version() const { return d_cgi_ver; }

string DODSFilter::get_ce() const { return d_dap2ce; }
 
void DODSFilter::set_ce(string _ce) { d_dap2ce = www2id(_ce, "%", "%20"); }

string DODSFilter::get_dataset_name() const { return d_dataset; }
 
void DODSFilter::set_dataset_name(const string ds) { d_dataset = www2id(ds, "%", "%20"); }

string DODSFilter::get_URL() const { return d_url; }

void DODSFilter::set_URL(const string &url) {
    if (url.find('?') != url.npos)
        print_usage(); // Throws Error
 
    d_url = url;
}

string DODSFilter::get_dataset_version() const { return ""; }

void DODSFilter::set_response(const string &r) {
    if (r == "DAS" || r == "das") {
        d_response = DAS_Response;
        d_action = "das";
    } else if (r == "DDS" || r == "dds") {
        d_response = DDS_Response;
        d_action = "dds";
    } else if (r == "DataDDS" || r == "dods") {
        d_response = DataDDS_Response;
        d_action = "dods";
    } else if (r == "DDX" || r == "ddx") {
        d_response = DDX_Response;
        d_action = "ddx";
    } else if (r == "DataDDX" || r == "dataddx") {
        d_response = DataDDX_Response;
        d_action = "dataddx";
    } else if (r == "Version") {
        d_response = Version_Response;
        d_action = "version";
    } else
        print_usage(); // Throws Error
}

DODSFilter::Response DODSFilter::get_response() const { return d_response; }

string DODSFilter::get_action() const { return d_action; }

time_t DODSFilter::get_dataset_last_modified_time() const { return last_modified_time(d_dataset); }

time_t DODSFilter::get_das_last_modified_time(const string &anc_location) const {
    DBG(cerr << "DODSFilter::get_das_last_modified_time(anc_location=" << anc_location
             << "call faf(das) d_dataset=" << d_dataset << " d_anc_file=" << d_anc_file << endl);
 
    string name =
        Ancillary::find_ancillary_file(d_dataset, "das", (anc_location == "") ? d_anc_dir : anc_location, d_anc_file);
 
    return max((name != "") ? last_modified_time(name) : 0, get_dataset_last_modified_time());
}

time_t DODSFilter::get_dds_last_modified_time(const string &anc_location) const {
    DBG(cerr << "DODSFilter::get_das_last_modified_time(anc_location=" << anc_location
             << "call faf(dds) d_dataset=" << d_dataset << " d_anc_file=" << d_anc_file << endl);
 
    string name =
        Ancillary::find_ancillary_file(d_dataset, "dds", (anc_location == "") ? d_anc_dir : anc_location, d_anc_file);
 
    return max((name != "") ? last_modified_time(name) : 0, get_dataset_last_modified_time());
}

time_t DODSFilter::get_data_last_modified_time(const string &anc_location) const {
    DBG(cerr << "DODSFilter::get_das_last_modified_time(anc_location=" << anc_location
             << "call faf(both) d_dataset=" << d_dataset << " d_anc_file=" << d_anc_file << endl);
 
    string dds_name =
        Ancillary::find_ancillary_file(d_dataset, "dds", (anc_location == "") ? d_anc_dir : anc_location, d_anc_file);
    string das_name =
        Ancillary::find_ancillary_file(d_dataset, "das", (anc_location == "") ? d_anc_dir : anc_location, d_anc_file);
 
    time_t m = max((das_name != "") ? last_modified_time(das_name) : (time_t)0,
                   (dds_name != "") ? last_modified_time(dds_name) : (time_t)0);
    // Note that this is a call to get_dataset_... not get_data_...
    time_t n = get_dataset_last_modified_time();
 
    return max(m, n);
}

time_t DODSFilter::get_request_if_modified_since() const { return d_if_modified_since; }

string DODSFilter::get_cache_dir() const { return d_cache_dir; }

void DODSFilter::set_timeout(int t) { d_timeout = t; }

int DODSFilter::get_timeout() const { return d_timeout; }

 
void DODSFilter::establish_timeout(FILE *stream) const {
#ifndef WIN32
    if (d_timeout > 0) {
        SignalHandler *sh = SignalHandler::instance();
        EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler(stream));
        delete old_eh;
        alarm(d_timeout);
    }
#endif
}
 
void DODSFilter::establish_timeout(ostream &stream) const {
#ifndef WIN32
    if (d_timeout > 0) {
        SignalHandler *sh = SignalHandler::instance();
        EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler(stream));
        delete old_eh;
        alarm(d_timeout);
    }
#endif
}
 
static const char *emessage = "DODS internal server error; usage error. Please report this to the dataset maintainer, "
                              "or to the opendap-tech@opendap.org mailing list.";

void DODSFilter::print_usage() const {
    // Write a message to the WWW server error log file.
    ErrMsgT(usage.c_str());
 
    throw Error(emessage);
}

void DODSFilter::send_version_info() const { do_version(d_cgi_ver, get_dataset_version()); }

void DODSFilter::send_das(FILE *out, DAS &das, const string &anc_location, bool with_mime_headers) const {
    ostringstream oss;
    send_das(oss, das, anc_location, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

void DODSFilter::send_das(ostream &out, DAS &das, const string &anc_location, bool with_mime_headers) const {
    time_t das_lmt = get_das_last_modified_time(anc_location);
    if (is_conditional() && das_lmt <= get_request_if_modified_since() && with_mime_headers) {
        set_mime_not_modified(out);
    } else {
        if (with_mime_headers)
            set_mime_text(out, dods_das, d_cgi_ver, x_plain, das_lmt);
        das.print(out);
    }
    out << flush;
}
 
void DODSFilter::send_das(DAS &das, const string &anc_location, bool with_mime_headers) const {
    send_das(cout, das, anc_location, with_mime_headers);
}

void DODSFilter::send_dds(FILE *out, DDS &dds, ConstraintEvaluator &eval, bool constrained, const string &anc_location,
                          bool with_mime_headers) const {
    ostringstream oss;
    send_dds(oss, dds, eval, constrained, anc_location, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

void DODSFilter::send_dds(ostream &out, DDS &dds, ConstraintEvaluator &eval, bool constrained,
                          const string &anc_location, bool with_mime_headers) const {
    // If constrained, parse the constraint. Throws Error or InternalErr.
    if (constrained)
        eval.parse_constraint(d_dap2ce, dds);
 
    if (eval.functional_expression())
        throw Error("Function calls can only be used with data requests. To see the structure of the underlying data "
                    "source, reissue the URL without the function.");
 
    time_t dds_lmt = get_dds_last_modified_time(anc_location);
    if (is_conditional() && dds_lmt <= get_request_if_modified_since() && with_mime_headers) {
        set_mime_not_modified(out);
    } else {
        if (with_mime_headers)
            set_mime_text(out, dods_dds, d_cgi_ver, x_plain, dds_lmt);
        if (constrained)
            dds.print_constrained(out);
        else
            dds.print(out);
    }
 
    out << flush;
}
 
void DODSFilter::send_dds(DDS &dds, ConstraintEvaluator &eval, bool constrained, const string &anc_location,
                          bool with_mime_headers) const {
    send_dds(cout, dds, eval, constrained, anc_location, with_mime_headers);
}
 
// 'lmt' unused. Should it be used to supply a LMT or removed from the
// method? jhrg 8/9/05
void DODSFilter::functional_constraint(BaseType &var, DDS &dds, ConstraintEvaluator &eval, FILE *out) const {
    ostringstream oss;
    functional_constraint(var, dds, eval, oss);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}
 
// 'lmt' unused. Should it be used to supply a LMT or removed from the
// method? jhrg 8/9/05
void DODSFilter::functional_constraint(BaseType &var, DDS &dds, ConstraintEvaluator &eval, ostream &out) const {
    out << "Dataset {\n";
    var.print_decl(out, "    ", true, false, true);
    out << "} function_value;\n";
    out << "Data:\n";
 
    out << flush;
 
    // Grab a stream that encodes using XDR.
    XDRStreamMarshaller m(out);
 
    try {
        // In the following call to serialize, suppress CE evaluation.
        var.serialize(eval, dds, m, false);
    } catch (Error &e) {
        throw;
    }
}
 
void DODSFilter::dataset_constraint(DDS &dds, ConstraintEvaluator &eval, FILE *out, bool ce_eval) const {
    ostringstream oss;
    dataset_constraint(dds, eval, oss, ce_eval);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}
 
void DODSFilter::dataset_constraint(DDS &dds, ConstraintEvaluator &eval, ostream &out, bool ce_eval) const {
    // send constrained DDS
    dds.print_constrained(out);
    out << "Data:\n";
    out << flush;
 
    // Grab a stream that encodes using XDR.
    XDRStreamMarshaller m(out);
 
    try {
        // Send all variables in the current projection (send_p())
        for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++)
            if ((*i)->send_p()) {
                DBG(cerr << "Sending " << (*i)->name() << endl);
                (*i)->serialize(eval, dds, m, ce_eval);
            }
    } catch (Error &e) {
        throw;
    }
}
 
void DODSFilter::dataset_constraint_ddx(DDS &dds, ConstraintEvaluator &eval, ostream &out, const string &boundary,
                                        const string &start, bool ce_eval) const {
    // Write the MPM headers for the DDX (text/xml) part of the response
    set_mime_ddx_boundary(out, boundary, start, dods_ddx);
 
    // Make cid
    uuid_t uu;
    uuid_generate(uu);
    char uuid[37];
    uuid_unparse(uu, uuid);
    char domain[256];
    if (getdomainname(domain, 255) != 0 || strlen(domain) == 0)
        strncpy(domain, "opendap.org", 255);
 
    string cid = string(uuid) + "@" + string(domain);
 
    // Send constrained DDX with a data blob reference
    dds.print_xml_writer(out, true, cid);
 
    // Write the MPM headers for the data part of the response.
    set_mime_data_boundary(out, boundary, cid, dap4_data, binary);
 
    // Grab a stream that encodes using XDR.
    XDRStreamMarshaller m(out);
 
    try {
        // Send all variables in the current projection (send_p())
        for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++)
            if ((*i)->send_p()) {
                DBG(cerr << "Sending " << (*i)->name() << endl);
                (*i)->serialize(eval, dds, m, ce_eval);
            }
    } catch (Error &e) {
        throw;
    }
}

void DODSFilter::send_data(DDS &dds, ConstraintEvaluator &eval, FILE *data_stream, const string &anc_location,
                           bool with_mime_headers) const {
    ostringstream oss;
    send_data(dds, eval, oss, anc_location, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), data_stream);
}

void DODSFilter::send_data(DDS &dds, ConstraintEvaluator &eval, ostream &data_stream, const string &anc_location,
                           bool with_mime_headers) const {
    // If this is a conditional request and the server should send a 304
    // response, do that and exit. Otherwise, continue on and send the full
    // response.
    time_t data_lmt = get_data_last_modified_time(anc_location);
    if (is_conditional() && data_lmt <= get_request_if_modified_since() && with_mime_headers) {
        set_mime_not_modified(data_stream);
        return;
    }
    // Set up the alarm.
    establish_timeout(data_stream);
    dds.set_timeout(d_timeout);
 
    eval.parse_constraint(d_dap2ce, dds); // Throws Error if the ce doesn't
                                          // parse.
 
    dds.tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
 
    // Start sending the response...
 
    // Handle *functional* constraint expressions specially
#if 0
    if (eval.functional_expression()) {
        // Get the result and then start sending the headers. This provides a
        // way to send errors back to the client w/o colliding with the
        // normal response headers. There's some duplication of code with this
        // and the else-clause.
        BaseType *var = eval.eval_function(dds, d_dataset);
        if (!var)
            throw Error(unknown_error, "Error calling the CE function.");
 
       if (with_mime_headers)
            set_mime_binary(data_stream, dods_data, d_cgi_ver, x_plain, data_lmt);
 
    data_stream << flush ;
 
        functional_constraint(*var, dds, eval, data_stream);
        delete var;
        var = 0;
    }
#endif
    if (eval.function_clauses()) {
        DDS *fdds = eval.eval_function_clauses(dds);
        if (with_mime_headers)
            set_mime_binary(data_stream, dods_data, d_cgi_ver, x_plain, data_lmt);
 
        dataset_constraint(*fdds, eval, data_stream, false);
        delete fdds;
    } else {
        if (with_mime_headers)
            set_mime_binary(data_stream, dods_data, d_cgi_ver, x_plain, data_lmt);
 
        dataset_constraint(dds, eval, data_stream);
    }
 
    data_stream << flush;
}

void DODSFilter::send_ddx(DDS &dds, ConstraintEvaluator &eval, FILE *out, bool with_mime_headers) const {
    ostringstream oss;
    send_ddx(dds, eval, oss, with_mime_headers);
    fwrite(oss.str().data(), sizeof(char), oss.str().length(), out);
}

void DODSFilter::send_ddx(DDS &dds, ConstraintEvaluator &eval, ostream &out, bool with_mime_headers) const {
    // If constrained, parse the constraint. Throws Error or InternalErr.
    if (!d_dap2ce.empty())
        eval.parse_constraint(d_dap2ce, dds);
 
    if (eval.functional_expression())
        throw Error("Function calls can only be used with data requests. To see the structure of the underlying data "
                    "source, reissue the URL without the function.");
 
    time_t dds_lmt = get_dds_last_modified_time(d_anc_dir);
 
    // If this is a conditional request and the server should send a 304
    // response, do that and exit. Otherwise, continue on and send the full
    // response.
    if (is_conditional() && dds_lmt <= get_request_if_modified_since() && with_mime_headers) {
        set_mime_not_modified(out);
        return;
    } else {
        if (with_mime_headers)
            set_mime_text(out, dods_ddx, d_cgi_ver, x_plain, dds_lmt);
        dds.print_xml_writer(out, !d_dap2ce.empty(), "");
    }
}

void DODSFilter::send_data_ddx(DDS &dds, ConstraintEvaluator &eval, ostream &data_stream, const string &start,
                               const string &boundary, const string &anc_location, bool with_mime_headers) const {
    // If this is a conditional request and the server should send a 304
    // response, do that and exit. Otherwise, continue on and send the full
    // response.
    time_t data_lmt = get_data_last_modified_time(anc_location);
    if (is_conditional() && data_lmt <= get_request_if_modified_since() && with_mime_headers) {
        set_mime_not_modified(data_stream);
        return;
    }
    // Set up the alarm.
    establish_timeout(data_stream);
    dds.set_timeout(d_timeout);
 
    eval.parse_constraint(d_dap2ce, dds); // Throws Error if the ce doesn't
                                          // parse.
 
    dds.tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
 
    // Start sending the response...
 
    // Handle *functional* constraint expressions specially
#if 0
    if (eval.functional_expression()) {
        BaseType *var = eval.eval_function(dds, d_dataset);
        if (!var)
            throw Error(unknown_error, "Error calling the CE function.");
 
        if (with_mime_headers)
            set_mime_multipart(data_stream, boundary, start, dods_data_ddx,
        d_cgi_ver, x_plain, data_lmt);
    data_stream << flush ;
    BaseTypeFactory btf;
    DDS var_dds(&btf, var->name());
    var->set_send_p(true);
    var_dds.add_var(var);
        serialize_dap2_data_ddx(var_dds, eval, data_stream, boundary, start);
 
        // functional_constraint_ddx(*var, dds, eval, data_stream, boundary);
        delete var;
        var = 0;
    }
#endif
    if (eval.function_clauses()) {
        DDS *fdds = eval.eval_function_clauses(dds);
        if (with_mime_headers)
            set_mime_multipart(data_stream, boundary, start, dods_data_ddx, d_cgi_ver, x_plain, data_lmt);
        data_stream << flush;
        dataset_constraint(*fdds, eval, data_stream, false);
        delete fdds;
    } else {
        if (with_mime_headers)
            set_mime_multipart(data_stream, boundary, start, dods_data_ddx, d_cgi_ver, x_plain, data_lmt);
        data_stream << flush;
        dataset_constraint_ddx(dds, eval, data_stream, boundary, start);
    }
 
    data_stream << flush;
 
    if (with_mime_headers)
        data_stream << CRLF << "--" << boundary << "--" << CRLF;
}
 
} // namespace libdap