CMIP7-data-request-api 1.1.2__py3-none-any.whl

data_request_api/stable/content/dreq_api/mapping_table.py

@@ -0,0 +1,335 @@
+"""
+Mapping Table
+
+The mapping_table dictionary defines how to map the three-base structure to the one-base structure.
+Each entry in the dictionary represents a table in the one-base structure and includes the information
+how to obtain it from the three-base structure.
+
+Explanation of the dictionary keys:
+
+Base ("source_base"):
+   The base containing the table to be selected.
+
+Table ("source_table"):
+    The table to be selected from the "source_base".
+
+Internal Mapping of record attributes ("internal_mapping"):
+    Record attributes may point to records of other tables.
+    However, there is no cross-linkage between the three bases,
+    so these links need to be mapped as well.
+    "internal_mapping" is a dictionary with the key corresponding
+    to the record attributes to be mapped and the values containing
+    the actual mapping information.
+
+    The mapping information is again a dictionary with the following keys:
+    - base_copy_of_table:
+        If a copy of table corresponding to the record attribute exists in the current base,
+        provide the name; otherwise, set to False.
+    - base:
+        The base containing the original table the record attribute points to.
+    - table:
+        The original table the record attribute points to.
+    - operation:
+        The operation to perform on the attribute value (either "split" or "", if it is
+        already provided as list or a string without comma separated values).
+    - map_by_key:
+        A list of keys to map by.
+    - entry_type:
+        The type of entry (either "record_id" or "name").
+
+(Internal) Filters of record attributes ("internal_filters"):
+    Not all records of the raw export shall be included since they may be
+    labeled as junk or not be approved by the community. The filters are applied on all records
+    and also internally on links to other records. "internal_filters" is a dictionary
+    with the key corresponding to the record attributes used for filtering and the value
+    another dictionary with the following possible keys:
+    - operator: Can be one of "nonempty", "in", "not in"
+    - values:  A list of values, not necessary for "nonempty" operator.
+
+Example Configuration
+
+Suppose we want to map the "CMIP7 Variable Groups" key in the "Variables" table of the "Data Request Variables (Public)"
+base to a list of record IDs of "Variable Group" records in the "Data Request Opportunities (Public)" base.
+
+We would define the mapping_table as follows:
+mapping_table = {
+      "Variables": {
+                "base": "Data Request Variables (Public)",
+                "source_table": "Variables",
+                "internal_mapping": {
+                    "CMIP7 Variable Groups": {
+                        "base_copy_of_table": False,
+                        "base": "Data Request Opportunities (Public)",
+                        "table": "Variable Group",
+                        "operation": "split",
+                        "map_by_key": ["Name"],
+                        "entry_type": "name",
+                    },
+                },
+                "internal_filters": {
+                    "Status": {"operator": "not in", "values": ["Junk"]},
+                }
+      },
+}
+"""
+
+mapping_table = {
+    "CF Standard Names": {
+        "source_base": "Data Request Physical Parameters (Public)",
+        "source_table": "CF Standard Name",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Physical Parameters", "Physical Parameters 2"],
+    },
+    "Cell Measures": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Cell Measures",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variables"],
+    },
+    "Cell Methods": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Cell Methods",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Structures", "Variables"],
+    },
+    "Coordinates and Dimensions": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Coordinate or Dimension",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Structure", "Variables"],
+    },
+    "Data Request Themes": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Data Request Themes",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": [],
+    },
+    "Docs for Opportunities": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Docs for Opportunities",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": [],
+    },
+    "ESM-BCV 1.3": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "ESM-BCV 1.3",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": [],
+    },
+    "Experiment Group": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Experiment Group",
+        "internal_mapping": {},
+        "internal_filters": {
+            "Status": {"operator": "not in", "values": ["Junk"]},
+            "Status (from Opportunity)": {
+                "operator": "in",
+                "values": ["New", "Under review", "Accepted"],
+            },
+        },
+        "rm_keys": ["Opportunity"],
+    },
+    "Experiments": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Experiment",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Experiment Group"],
+    },
+    "CMIP6 Frequency (legacy)": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "CMIP6 Frequency (legacy)",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": [],
+    },
+    "CMIP7 Frequency": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "CMIP7 Frequency",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Table Identifiers", "Variables"],
+    },
+    "Glossary": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Glossary",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Opportunity"],
+    },
+    "MIPs": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "MIP",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variable Group"],
+    },
+    "Modelling Realm": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Modelling Realm",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variables"],
+    },
+    "Opportunity": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Opportunity",
+        "internal_mapping": {},
+        "internal_filters": {
+            "Status": {"operator": "any", "values": ["Under review", "Accepted"]},
+        },
+        "rm_keys": [],
+    },
+    "Opportunity/Variable Group Comments": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Comment",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Experiment Groups", "Opportunities", "Theme", "Variable Groups"],
+    },
+    "Physical Parameter Comments": {
+        "source_base": "Data Request Physical Parameters (Public)",
+        "source_table": "Comment",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Physical parameters"],
+    },
+    "Physical Parameters": {
+        "source_base": "Data Request Physical Parameters (Public)",
+        "source_table": "Physical Parameter",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variables"],
+    },
+    "Priority Level": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Priority Level",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variable Group"],
+    },
+    "Ranking": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Ranking",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": [],
+    },
+    "Spatial Shape": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Spatial Shape",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Dimensions", "Structure", "Variables"],
+    },
+    "Structure": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Structure",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variables"],
+    },
+    "Table Identifiers": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Table Identifiers",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variables"],
+    },
+    "Temporal Shape": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Temporal Shape",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Dimensions", "Structure", "Variables"],
+    },
+    "Time Slice": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Time Slice",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": [],
+    },
+    "Variable Comments": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Comment",
+        "internal_mapping": {},
+        "internal_filters": {},
+        "rm_keys": ["Variables"],
+    },
+    "Variable Group": {
+        "source_base": "Data Request Opportunities (Public)",
+        "source_table": "Variable Group",
+        "internal_mapping": {
+            "Variables": {
+                "base_copy_of_table": "Variables",
+                "base": "Data Request Variables (Public)",
+                "table": "Variable",
+                "operation": "",
+                "map_by_key": ["UID", "Compound Name"],
+                "entry_type": "record_id",
+            }
+        },
+        "internal_filters": {
+            "Final Opportunity selection": {"operator": "nonempty"},
+            "Status (from Final Opportunity selection)": {
+                "operator": "in",
+                "values": ["Under review", "Accepted"],
+            },
+        },
+        "rm_keys": ["Opportunity", "Theme"],
+    },
+    "Variables": {
+        "source_base": "Data Request Variables (Public)",
+        "source_table": "Variable",
+        "internal_mapping": {
+            "CMIP7 Variable Groups": {
+                "base_copy_of_table": False,
+                "base": "Data Request Opportunities (Public)",
+                "table": "Variable Group",
+                "operation": "split",
+                "map_by_key": ["Name"],
+                "entry_type": "name",
+            },
+            "Physical Parameter": {
+                "base_copy_of_table": "Physical Parameter",
+                "base": "Data Request Physical Parameters (Public)",
+                "table": "Physical Parameter",
+                "operation": "",
+                "map_by_key": ["UID", "Name"],
+                "entry_type": "record_id",
+            },
+            "CF Standard Name (from MIP Variables)": {
+                "base_copy_of_table": False,
+                "base": "Data Request Physical Parameters (Public)",
+                "table": "CF Standard Name",
+                "operation": "",
+                "map_by_key": ["name"],
+                "entry_type": "name",
+            },
+        },
+        "internal_filters": {
+            "CMIP7 Variable Groups": {"operator": "nonempty"},
+            "Opportunity Status (from CMIP7 Variable Groups)": {
+                "operator": "in",
+                "values": ["Under review", "Accepted"],
+            },
+        },
+        "rm_keys": ["CMIP7 Variable Groups"],
+    },
+}
+
+# Renaming of certain tables dependent on the release version
+#  version : {table_name_old : table_name_new}
+version_consistency = {
+    "v1.0alpha": {
+        "Frequency": "CMIP7 Frequency",
+    },
+}

data_request_api/stable/content/dreq_api/test_dreq_content.py

@@ -0,0 +1,194 @@
+import os
+
+from data_request_api.stable.content.dreq_api import dreq_content as dc
+import pytest
+
+
+def test_parse_version():
+    "Test the _parse_version function with different version strings."
+    assert dc._parse_version("v1.0.0") == (1, 0, 0, "", 0)
+    assert dc._parse_version("v1.0alpha2") == (1, 0, 0, "a", 2)
+    assert dc._parse_version("1.0.0a3") == (1, 0, 0, "a", 3)
+    assert dc._parse_version("1.0.0beta") == (1, 0, 0, "b", 0)
+    assert dc._parse_version("something") == (0, 0, 0, "", 0)
+    with pytest.raises(TypeError):
+        dc._parse_version(None)
+
+
+def test_get_versions():
+    "Test the get_versions function."
+    versions = dc.get_versions()
+    assert "dev" in versions
+    assert "v1.0alpha" in versions
+    assert "v1.0beta" in versions
+
+
+def test_get_versions_list_branches():
+    "Test the get_versions function for branches."
+    branches = dc.get_versions(target="branches")
+    assert "dev" not in branches
+    assert "main" not in branches
+
+
+def test_get_latest_version(monkeypatch):
+    "Test the _get_latest_version function."
+    monkeypatch.setattr(dc, "get_versions", lambda: ["1.0.0", "2.0.2b", "2.0.2a"])
+    assert dc._get_latest_version() == "1.0.0"
+    assert dc._get_latest_version(stable=False) == "2.0.2b"
+
+
+def test_get_cached(tmp_path):
+    "Test the get_cached function."
+    # Create a temporary directory with a subdirectory containing a dreq.json file
+    version_dir = tmp_path / "v1.0.0"
+    version_dir.mkdir()
+    (version_dir / dc._json_release).touch()
+
+    # Set the _dreq_res variable to the temporary directory
+    dc._dreq_res = str(tmp_path)
+
+    # Test the get_cached function
+    cached_versions = dc.get_cached()
+    assert cached_versions == ["v1.0.0"]
+
+
+def test_retrieve(tmp_path, capfd):
+    "Test the retrieval function."
+    dc._dreq_res = str(tmp_path)
+
+    # Retrieve 'dev' version
+    json_path = dc.retrieve("dev")["dev"]
+    assert os.path.isfile(json_path)
+
+    # Alter on disk (delete first line)
+    with open(json_path) as f:
+        lines = f.read().splitlines(keepends=True)
+    with open(json_path, "w") as f:
+        f.writelines(lines[1:])
+
+    # Make sure it updates
+    json_path = dc.retrieve("dev")["dev"]
+    stdout = capfd.readouterr().out.splitlines()
+    assert len(stdout) == 2
+    assert "Retrieved version 'dev'." in stdout
+    assert "Updated version 'dev'." in stdout
+    # ... and the file was replaced
+    with open(json_path) as f:
+        lines_update = f.read().splitlines(keepends=True)
+    assert lines == lines_update
+
+
+def test_retrieve_with_invalid_version(tmp_path):
+    "Test the retrieval function with an invalid version."
+    dc._dreq_res = str(tmp_path)
+    with pytest.raises(ValueError):
+        dc.retrieve(" invalid-version ")
+
+
+def test_api_and_html_request():
+    "Test the _send_api_request and _send_html_request functions."
+    tags1 = set(dc._send_api_request(dc.REPO_API_URL, "", "tags"))
+    tags2 = set(dc._send_html_request(dc.REPO_PAGE_URL, "tags"))
+    assert tags1 == tags2
+
+    branches1 = set(dc._send_api_request(dc.REPO_API_URL, "", "branches"))
+    branches2 = set(dc._send_html_request(dc.REPO_PAGE_URL, "branches"))
+    assert branches1 == branches2
+
+
+def test_load(tmp_path):
+    "Test the load function."
+    dc._dreq_res = str(tmp_path)
+
+    with pytest.raises(ValueError):
+        jsondict = dc.load(" invalid-version ")
+
+    jsondict = dc.load("dev")
+    assert isinstance(jsondict, dict)
+    assert os.path.isfile(tmp_path / "dev" / dc._json_raw)
+    assert not os.path.isfile(tmp_path / "dev" / dc._json_release)
+
+    jsondict = dc.load("dev", export="release")
+    assert isinstance(jsondict, dict)
+    assert os.path.isfile(tmp_path / "dev" / dc._json_release)
+
+
+class TestDreqContent:
+    """
+    Test various functions of the dreq_content module.
+    """
+
+    @pytest.fixture(autouse=True, scope="function")
+    def setup(self, tmp_path):
+        self.versions = ["v1.0.0", "1.0.1", "2.0.1b", "2.0.1", "2.0.2b"]
+        self.branches = ["one", "or", "another"]
+        self.dreq_res = tmp_path
+        for v in self.versions:
+            (self.dreq_res / v).mkdir()
+            (self.dreq_res / v / dc._json_release).write_text("{}")
+        for b in self.branches:
+            (self.dreq_res / b).mkdir()
+            (self.dreq_res / b / dc._json_raw).write_text("{}")
+
+    def test_get_cached(self):
+        "Test the get_cached function."
+        dc._dreq_res = self.dreq_res
+        # Basic
+        cached_versions = dc.get_cached()
+        assert set(cached_versions) == set(self.versions + self.branches)
+
+        # With export kwarg "release"
+        cached_tags = dc.get_cached(export="release")
+        assert set(cached_tags) == set(self.versions)
+
+        # With export kwarg "raw"
+        cached_branches = dc.get_cached(export="raw")
+        assert set(cached_branches) == set(self.branches)
+
+        # With invalid export kwarg
+        with pytest.warns(UserWarning, match="Unknown export type"):
+            dc.get_cached(export="invalid")
+
+    def test_delete(self, capfd):
+        "Test the delete function."
+        dc._dreq_res = self.dreq_res
+        # Delete non-existent version
+        dc.delete("notpresent")
+        stdout = capfd.readouterr().out.splitlines()
+        assert len(stdout) == 1
+        assert "No version(s) found to delete." in stdout
+
+        # Delete only branches / dryrun
+        dc.delete("all", export="raw", dryrun=True)
+        stdout = capfd.readouterr().out.splitlines()
+        assert len(stdout) == 5
+        assert "Deleting the following version(s):" in stdout
+        for b in self.branches:
+            assert (
+                f"Dryrun: would delete '{dc._dreq_res / b / dc._json_raw}'." in stdout
+            )
+
+        # Delete all but latest
+        dc.delete(keep_latest=True)
+        assert set(dc.get_cached()) == {"2.0.1", "2.0.2b"}
+        assert dc.get_cached(export="raw") == []
+
+        # Delete 2.0.1 with warning
+        with pytest.warns(UserWarning, match=" option is ignored "):
+            dc.delete("2.0.1", keep_latest=True)
+
+        # Delete all with ValueError for kwargs
+        with pytest.raises(ValueError):
+            dc.delete(export="none")
+
+        # Delete all
+        dc.delete()
+        assert dc.get_cached() == []
+
+        # Now there is no ValueError since no version is found
+        dc.delete(export="none")
+
+    # def test_load(self):
+    #    dc._dreq_res = self.tmp_dir.name
+    #    data = dc.load("1.0.0")
+    #    assert data == {}