metkitlib 1.16.0.9__cp312-cp312-manylinux_2_28_aarch64.whl

metkitlib/include/metkit/mars/MarsLocation.h

@@ -0,0 +1,100 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Tiago Quintino
+
+/// @date Dec 2015
+
+#ifndef metkit_MarsLocation_H
+#define metkit_MarsLocation_H
+
+
+#include "eckit/config/Configuration.h"
+#include "metkit/mars/MarsRequest.h"
+
+namespace eckit {
+class JSON;
+class Stream;
+}  // namespace eckit
+
+namespace metkit {
+namespace mars {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+/// MarsLocation represents a MarsRequest associated with the hostname and port.
+/// From this location, the data can be directly retrieved without going through a queueing system.
+/// It assumes all the data identified by the request can be got from the same location.
+/// This is useful to identify and retrieve data directly from memory.
+///
+/// If we consider that a MarsRequest is analogue to a URI, then a MarsLocation is an analogue to a URL
+
+class MarsLocation {
+
+public:  // methods
+
+    // - Constructors
+
+    MarsLocation(const MarsRequest& r, const std::string& hostname, int port);
+    MarsLocation(eckit::Stream&);
+    MarsLocation(const eckit::Configuration&);
+
+    // -- Destructor
+
+    ~MarsLocation();
+
+    // -- Operators
+
+    operator eckit::Value() const;
+
+    void json(eckit::JSON&) const;
+
+    const MarsRequest& request() const;
+
+    std::string hostname() const;
+
+    int port() const;
+
+
+private:  // members
+
+    MarsRequest request_;
+    std::string hostname_;
+    int port_;
+
+private:  // methods
+
+    void print(std::ostream&) const;
+    void encode(eckit::Stream&) const;
+
+    // -- Class members
+
+
+    friend std::ostream& operator<<(std::ostream& s, const MarsLocation& r) {
+        r.print(s);
+        return s;
+    }
+
+    friend eckit::JSON& operator<<(eckit::JSON& s, const MarsLocation& r) {
+        r.json(s);
+        return s;
+    }
+
+    friend eckit::Stream& operator<<(eckit::Stream& s, const MarsLocation& r) {
+        r.encode(s);
+        return s;
+    }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+}  // namespace mars
+}  // namespace metkit
+
+#endif

metkitlib/include/metkit/mars/MarsParsedRequest.h

@@ -0,0 +1,44 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Manuel Fuentes
+/// @author Baudouin Raoult
+/// @author Tiago Quintino
+
+/// @date Sep 96
+
+#pragma once
+
+#include "metkit/mars/MarsRequest.h"
+
+namespace metkit::mars {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class MarsParsedRequest : public MarsRequest {
+public:
+
+    MarsParsedRequest(const std::string& verb, size_t line);
+    virtual ~MarsParsedRequest() = default;
+
+    virtual void info(std::ostream&) const;
+    friend std::ostream& operator<<(std::ostream& s, const MarsParsedRequest& r) {
+        r.info(s);
+        return s;
+    }
+
+private:
+
+    std::size_t line_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace metkit::mars

metkitlib/include/metkit/mars/MarsParser.h

@@ -0,0 +1,70 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Baudouin Raoult
+/// @date Jun 2012
+
+#pragma once
+
+#include "eckit/parser/StreamParser.h"
+#include "eckit/types/Types.h"
+#include "metkit/mars/MarsParsedRequest.h"
+
+namespace metkit::mars {
+
+class MarsExpandContext;
+class MarsRequest;
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+
+class MarsParserCallback {
+public:
+
+    virtual void operator()(const MarsExpandContext&, const MarsRequest&) = 0;
+
+protected:
+
+    virtual ~MarsParserCallback() = default;
+};
+
+
+//----------------------------------------------------------------------------------------------------------------------
+
+
+class MarsParser : public eckit::StreamParser {
+
+public:  // methods
+
+    MarsParser(std::istream& in);
+
+    std::vector<MarsParsedRequest> parse();
+
+    void parse(MarsParserCallback& cb);
+
+    static void quoted(std::ostream& out, const std::string& value);
+
+private:  // methods
+
+    MarsParsedRequest parseRequest();
+    std::string parseVerb();
+    std::string parseKeyword();
+    std::vector<std::string> parseValues();
+    std::string parseValue();
+    std::string parseIndent();
+    std::string parseIndents();
+
+    std::string parseString(char c);
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace metkit::mars

metkitlib/include/metkit/mars/MarsRequest.h

@@ -0,0 +1,198 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Manuel Fuentes
+/// @author Baudouin Raoult
+/// @author Tiago Quintino
+
+/// @date Sep 96
+
+#ifndef metkit_MarsRequest_H
+#define metkit_MarsRequest_H
+
+#include <optional>
+#include "eckit/value/Value.h"
+#include "metkit/mars/Parameter.h"
+
+namespace eckit {
+class MD5;
+namespace message {
+class Message;
+}
+}  // namespace eckit
+
+struct metkit_marsrequest_t;
+
+namespace metkit::mars {
+
+class Type;
+class MarsRequest;
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class MarsRequest {
+public:  // methods
+
+    MarsRequest();
+
+    explicit MarsRequest(const std::string&);
+    explicit MarsRequest(eckit::Stream&, bool lowercase = false);
+
+    MarsRequest(const std::string&, const std::map<std::string, std::string>&);
+    MarsRequest(const std::string&, const eckit::Value&);
+
+    explicit MarsRequest(const eckit::message::Message&);
+
+    ~MarsRequest() = default;
+
+    bool operator<(const MarsRequest& other) const;
+
+    // eckit::Value&        operator[](const std::string&);
+    const std::string& operator[](const std::string&) const;
+
+    operator eckit::Value() const;
+
+    const std::string& verb() const;
+
+    size_t countValues(const std::string&) const;
+    bool has(const std::string&) const;
+
+    bool is(const std::string& param, const std::string& value) const;
+
+    const std::vector<std::string>& values(const std::string&, bool emptyOk = false) const;
+
+    // Returns reference to values or nullopt if not found
+    std::optional<std::reference_wrapper<const std::vector<std::string>>> get(const std::string& keyword) const;
+
+    template <class T>
+    size_t getValues(const std::string& name, std::vector<T>& v, bool emptyOk = false) const;
+
+    void getParams(std::vector<std::string>&) const;
+    std::vector<std::string> params() const;
+
+    std::list<Parameter>& parameters() { return params_; }
+
+    const std::list<Parameter>& parameters() const { return params_; }
+
+    void verb(const std::string&);
+
+    void values(const std::string&, const std::vector<std::string>&);
+
+    template <class T>
+    void setValue(const std::string& name, const T& value);
+
+    void setValue(const std::string& name, const char* value);
+
+    void unsetValues(const std::string&);
+
+    /// Splits a MARS request into multiple requests along the provided key
+    std::vector<MarsRequest> split(const std::string& keys) const;
+
+    /// Splits a MARS request into multiple requests along the indicated keys
+    std::vector<MarsRequest> split(const std::vector<std::string>& keys) const;
+
+    /// Merges one MarsRequest into another
+    /// @todo Improve performance -- uses O(N^2) search / merge in std::list's
+    void merge(const MarsRequest& other);
+
+    /// Create a new MarsRequest from this one with only the given set of keys
+    MarsRequest subset(const std::set<std::string>&) const;
+
+    void json(eckit::JSON&, bool array = false) const;
+
+    void md5(eckit::MD5&) const;
+
+    void dump(std::ostream&, const char* cr = "\n", const char* tab = "\t", bool verb = true) const;
+
+    void setValuesTyped(Type*, const std::vector<std::string>&);
+
+    bool filter(const MarsRequest& filter);
+    bool matches(const MarsRequest& filter) const;
+    bool empty() const;
+
+    size_t count() const;
+
+    MarsRequest extract(const std::string& category) const;
+
+    void erase(const std::string& param);
+
+    std::string asString() const;
+
+public:  // static methods
+
+    static MarsRequest parse(const std::string& s, bool strict = false);
+    static std::vector<MarsRequest> parse(std::istream&, bool strict = false);
+
+    /// Implementation in api/metkit_c.cc
+    static const MarsRequest& fromOpaque(const metkit_marsrequest_t* request);
+
+private:  // members
+
+    std::string verb_;
+    std::list<Parameter> params_;
+
+private:  // methods
+
+    void print(std::ostream&) const;
+    void encode(eckit::Stream&) const;
+
+    std::list<Parameter>::const_iterator find(const std::string&) const;
+    std::list<Parameter>::iterator find(const std::string&);
+
+    // -- Class members
+
+    static eckit::ClassSpec classSpec_;
+    static eckit::Reanimator<MarsRequest> reanimator_;
+
+    friend std::ostream& operator<<(std::ostream& s, const MarsRequest& r) {
+        r.print(s);
+        return s;
+    }
+
+    friend eckit::JSON& operator<<(eckit::JSON& s, const MarsRequest& r) {
+        r.json(s);
+        return s;
+    }
+
+    friend eckit::Stream& operator<<(eckit::Stream& s, const MarsRequest& r) {
+        r.encode(s);
+        return s;
+    }
+};
+
+
+template <class T>
+size_t MarsRequest::getValues(const std::string& name, std::vector<T>& v, bool emptyOk) const {
+    const std::vector<std::string>& s = values(name, emptyOk);
+
+    eckit::Translator<std::string, T> t;
+
+    v.clear();
+
+    for (std::vector<std::string>::const_iterator j = s.begin(); j != s.end(); ++j) {
+        v.push_back(t(*j));
+    }
+
+    return v.size();
+}
+
+
+template <class T>
+void MarsRequest::setValue(const std::string& name, const T& value) {
+    eckit::Translator<T, std::string> t;
+    std::vector<std::string> v(1, t(value));
+    values(name, v);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace metkit::mars
+
+#endif

metkitlib/include/metkit/mars/MarsRequestHandle.h

@@ -0,0 +1,79 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+// File MarsRequestHandle.h
+// Baudouin Raoult - (c) ECMWF Feb 12
+
+#ifndef MarsRequestHandle_H
+#define MarsRequestHandle_H
+
+#include <memory>
+
+#include "eckit/exception/Exceptions.h"
+#include "eckit/io/DataHandle.h"
+
+#include "metkit/mars/BaseProtocol.h"
+#include "metkit/mars/MarsRequest.h"
+
+namespace metkit {
+namespace mars {
+
+class MarsRequestHandle : public eckit::DataHandle {
+public:
+
+    class RetryTransfer : public eckit::Exception {
+        virtual bool retryOnClient() const { return true; }
+
+    public:
+
+        RetryTransfer(const std::string& what) : eckit::Exception(what) {}
+    };
+
+public:
+
+    MarsRequestHandle(eckit::Stream&);
+
+    MarsRequestHandle(const metkit::mars::MarsRequest& request, const eckit::Configuration& database);
+
+    MarsRequestHandle(const metkit::mars::MarsRequest& request, metkit::mars::BaseProtocol* protocol);
+
+    ~MarsRequestHandle();
+
+    // -- Overridden methods (from Streamable)
+    virtual std::string className() const override { return "MarsRequestHandle"; }
+    virtual const eckit::ReanimatorBase& reanimator() const override;
+    static const eckit::ClassSpec& classSpec();
+
+private:  // members
+
+    metkit::mars::MarsRequest request_;
+
+    std::unique_ptr<BaseProtocol> protocol_;
+
+    bool opened_;
+
+private:  // members
+
+    void print(std::ostream&) const override;
+    void encode(eckit::Stream&) const override;
+
+    eckit::Length openForRead() override;
+    void openForWrite(const eckit::Length&) override;
+    void openForAppend(const eckit::Length&) override;
+    long read(void*, long) override;
+    long write(const void*, long) override;
+    void close() override;
+    bool canSeek() const override;
+};
+
+}  // namespace mars
+}  // namespace metkit
+
+#endif

metkitlib/include/metkit/mars/Matcher.h

@@ -0,0 +1,88 @@
+/*
+ * (C) Copyright 2025- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Chris Bradley
+/// @date Sep 2025
+
+#pragma once
+
+#include <functional>
+#include <map>
+#include <string>
+#include <variant>
+#include "eckit/utils/Regex.h"
+#include "metkit/mars/MarsRequest.h"
+
+namespace metkit::mars {
+
+class RequestLike;
+
+/// Parse a match expression into a map of key:regex pairs
+/// @param expr is a series of comma separated key:regex pairs e.g.
+///   "expver=(0001|o[0-9a-z]{3}),dataset=^climate-dt$"
+std::map<std::string, eckit::Regex> parseMatchString(const std::string& expr);
+
+// -------------------------------------------------------------------------------------------------------------
+
+/// For matching requests against a set of key:regex conditions. For constructing select/exclude filters.
+
+class Matcher {
+
+public:
+
+    /// Policy for handling requests with multiple values for a given key.
+    enum class Policy {
+        All,  ///< Require all values to match
+        Any   ///< Require at least one value to match
+    };
+
+    /// Policy for handling keys in the matcher that are absent in the request.
+    enum class MatchMissingPolicy {
+        MatchOnMissing,
+        DontMatchOnMissing
+    };
+    static constexpr auto MatchOnMissing     = MatchMissingPolicy::MatchOnMissing;
+    static constexpr auto DontMatchOnMissing = MatchMissingPolicy::DontMatchOnMissing;
+
+
+public:
+
+    /// Construct a matcher from a map of key:regex pairs
+    Matcher(std::map<std::string, eckit::Regex> regexMap, Policy policy);
+
+    Matcher(const std::string& expr, Policy policy);
+
+    bool match(const RequestLike& request, MatchMissingPolicy matchOnMissing = MatchOnMissing) const;
+    bool match(const MarsRequest& request, MatchMissingPolicy matchOnMissing = MatchOnMissing) const;
+
+private:
+
+    std::map<std::string, eckit::Regex> regexMap_;
+    Policy policy_;
+};
+
+// ----------------------------------------------------------------------------------------------------------------------
+
+/// Interface to access keyword / value(s) pairs across similar types.
+/// Designed to allow uniform access to keywords / values for MarsRequest / fdb5::Key
+class RequestLike {
+public:
+
+    using values_t = std::variant<std::reference_wrapper<const std::string>,              ///< single value
+                                  std::reference_wrapper<const std::vector<std::string>>  ///< multiple values
+                                  >;
+
+    virtual ~RequestLike() = default;
+
+    /// Get a reference_wrapper to the value(s) for the given keyword, or nullopt if not found
+    virtual std::optional<values_t> get(const std::string& keyword) const = 0;
+};
+
+}  // namespace metkit::mars

metkitlib/include/metkit/mars/Param.b

@@ -0,0 +1,43 @@
+
+Param(eckit::Bless& b):
+	table_(b(&table_)),
+	value_(b(&value_))
+{
+}
+
+Param(eckit::Evolve b):
+	table_(b("Param","table_")),
+	value_(b("Param","value_"))
+{
+}
+
+static const char* specName()      { return "Param"; }
+static void isa(TypeInfo* t)  { eckit::Isa::add(t,specName()); }
+static eckit::Isa* isa()             { return eckit::Isa::get(specName());  }
+
+static void schema(eckit::Schema& s)
+{
+	s.start(specName(),sizeof(Param));
+	s.member("table_",member_size(Param,table_),member_offset(Param,table_),"long");
+	s.member("value_",member_size(Param,value_),member_offset(Param,value_),"long");
+	s.end(specName());
+}
+
+
+void describe(std::ostream& s,int depth = 0) const {
+	eckit::_startClass(s,depth,specName());
+	eckit::_describe(s,depth+1,"table_",table_);
+	eckit::_describe(s,depth+1,"value_",value_);
+	eckit::_endClass(s,depth,specName());
+}
+
+
+
+void _export(eckit::Exporter& h) const {
+	eckit::_startClass(h,"Param");
+	eckit::_export(h,"table_",table_);
+	eckit::_export(h,"value_",value_);
+	eckit::_endClass(h,"Param");
+}
+
+