scriptengine-tasks-ecearth 0.7.0__tar.gz → 0.8.0__tar.gz

{scriptengine_tasks_ecearth-0.7.0 → scriptengine_tasks_ecearth-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: scriptengine-tasks-ecearth
-Version: 0.7.0
+Version: 0.8.0
 Summary: ScriptEngine tasks for use with the EC-Earth climate model
 Author-email: Valentina Schueller <valentina.schueller@gmail.com>, Uwe Fladrich <uwe.fladrich@protonmail.com>
 Project-URL: Homepage, https://github.com/uwefladrich/scriptengine-tasks-ecearth
@@ -19,6 +19,8 @@ Requires-Dist: imageio>=2.18
 Requires-Dist: scitools-iris>=3.5
 Requires-Dist: cartopy>=0.20
 Requires-Dist: python-redmine
+Requires-Dist: python-gitlab
+Dynamic: license-file
 
 [![Tests](https://github.com/uwefladrich/scriptengine-tasks-ecearth/actions/workflows/pytest.yml/badge.svg)](https://github.com/uwefladrich/scriptengine-tasks-ecearth/actions)
 [![Coverage Status](https://coveralls.io/repos/github/uwefladrich/scriptengine-tasks-ecearth/badge.svg)](https://coveralls.io/github/uwefladrich/scriptengine-tasks-ecearth)

scriptengine_tasks_ecearth-0.8.0/monitoring/gitlab.py

@@ -0,0 +1,140 @@
+"""Presentation Task that uploads Data and Plots to the EC-Earth 4 Gitlab."""
+
+import re
+import urllib.parse
+from pathlib import Path
+
+import gitlab
+import jinja2
+import requests.exceptions
+from scriptengine.exceptions import ScriptEngineTaskRunError
+from scriptengine.jinja import filters as j2filters
+from scriptengine.tasks.core import Task, timed_runner
+
+from helpers.exceptions import PresentationException
+from helpers.presentation_objects import PresentationObject
+
+# Repository where monitoring results are posted
+SERVER_URL = "https://git.smhi.se"
+PROJECT_ID = 1982
+
+
+class Gitlab(Task):
+    """Gitlab Presentation Task"""
+
+    _required_arguments = (
+        "src",
+        "local_dst",
+        "subject",
+        "template",
+        "api_key",
+    )
+
+    def __init__(self, arguments=None):
+        Gitlab.check_arguments(arguments)
+        super().__init__(arguments)
+
+    @timed_runner
+    def run(self, context):
+        sources = self.getarg("src", context)
+        dst_folder = self.getarg("local_dst", context)
+        issue_subject = self.getarg("subject", context, parse_yaml=False)
+        template_path = self.getarg("template", context)
+        key = self.getarg("api_key", context)
+        self.log_info(f"Create Gitlab issue '{issue_subject}'.")
+        self.log_debug(f"Template: {template_path}, Source File(s): {sources}")
+
+        presentation_list = self.get_presentation_list(sources, dst_folder)
+        gitlab_template = self.get_template(context, template_path)
+        gitlab_template.globals["urlencode"] = urllib.parse.quote
+
+        self.log_debug("Connecting to Gitlab.")
+        project, issue = self.get_project_and_issue(key, issue_subject)
+
+        self.log_debug("Uploading attachments.")
+        issue.uploads = []
+        for item in presentation_list:
+            if item["presentation_type"] == "image":
+                file_name = Path(item["path"]).name
+                upload = project.upload(file_name, filepath=f"{dst_folder}/{file_name}")
+                item["image_link"] = upload[
+                    "markdown"
+                ]  # string of the form "![]()" generated by Gitlab
+
+        # render the template and add as description
+        self.log_debug("Updating the issue description.")
+        issue.description = gitlab_template.render(
+            presentation_list=presentation_list,
+            issue_url=issue.web_url,
+            create_anchor=create_anchor,
+        )
+        self.log_debug("Saving issue.")
+        issue.save()
+
+    def get_presentation_list(self, sources, dst_folder):
+        """create a list of presentation objects"""
+        self.log_debug("Getting list of presentation objects.")
+        presentation_list = []
+        for src in sources:
+            try:
+                try:
+                    pres_object = PresentationObject(dst_folder, **src)
+                except TypeError:
+                    pres_object = PresentationObject(dst_folder, src)
+                self.log_debug(
+                    f"Loading {pres_object.loader.diag_type} diagnostic from {pres_object.loader.path}."
+                )
+                presentation_list.append(pres_object.create_dict())
+            except PresentationException as msg:
+                self.log_warning(f"Can not present diagnostic: {msg}")
+        return presentation_list
+
+    def get_template(self, context, template):
+        """get Jinja2 template file"""
+        search_path = [".", "templates"]
+        if "_se_cmd_cwd" in context:
+            search_path.extend(
+                [
+                    context["_se_cmd_cwd"],
+                    Path(context["_se_cmd_cwd"]) / "templates",
+                ]
+            )
+        self.log_debug(f"Search path for template: {search_path}")
+
+        loader = jinja2.FileSystemLoader(search_path)
+        environment = jinja2.Environment(loader=loader)
+        for name, function in j2filters().items():
+            environment.filters[name] = function
+        return environment.get_template(template)
+
+    def get_project_and_issue(self, key, issue_subject):
+        """Connect to Gitlab server, find and return correct issue and project based on user input"""
+
+        gl = gitlab.Gitlab(SERVER_URL, private_token=key)
+        try:
+            project = gl.projects.get(PROJECT_ID)
+        except (
+            gitlab.GitlabAuthenticationError,
+            requests.exceptions.ConnectionError,
+        ) as e:
+            msg = f"Could not log in to Gitlab server ({e})"
+            self.log_error(msg)
+            raise ScriptEngineTaskRunError()
+
+        # Find issue or create if none exists
+        issues = project.issues.list()
+        issue = next((issue for issue in issues if issue.title == issue_subject), None)
+        if issue is None:
+            issue = project.issues.create({"title": issue_subject})
+            issue.save()
+        return project, issue
+
+
+def create_anchor(title: str) -> str:
+    """
+    create correct anchors for Gitlab issue URLs
+    """
+    anchor = re.sub(r"[^a-zA-Z\s-]", "", title)
+    anchor = re.sub(r"\s+(\-+\s*)?", "-", anchor)
+    anchor = anchor.lower()
+    return anchor

{scriptengine_tasks_ecearth-0.7.0 → scriptengine_tasks_ecearth-0.8.0}/monitoring/si3_hemis_sum_month_mean_timeseries.py

@@ -1,6 +1,5 @@
 """Processing Task that calculates the seasonal cycle of sea ice variables in one leg."""
 
-import warnings
 from pathlib import Path
 
 import cf_units
@@ -19,12 +18,14 @@ _meta_dict = {
         "standard_name": "sea_ice_volume",
         "var_name": "sivol",
         "new_unit": "1e3 km3",
+        "old_unit": "m3",
     },
     "siconc": {
         "long_name": "Sea-Ice Area",
         "standard_name": "sea_ice_area",
         "var_name": "siarea",
         "new_unit": "1e6 km2",
+        "old_unit": "m2",
     },
 }
 
@@ -75,7 +76,7 @@ class Si3HemisSumMonthMeanTimeseries(Timeseries):
             self.log_warning(
                 (
                     f"Invalid varname '{varname}', must be one of {_meta_dict.keys()}; "
-                    "diagnostic will not be ignored."
+                    "diagnostic will be ignored."
                 )
             )
             return
@@ -84,7 +85,7 @@ class Si3HemisSumMonthMeanTimeseries(Timeseries):
             self.log_warning(
                 (
                     f"Invalid hemisphere '{hemisphere}', must be 'north' or 'south'; "
-                    "diagnostic will not be ignored."
+                    "diagnostic will be ignored."
                 )
             )
             return
@@ -100,6 +101,8 @@ class Si3HemisSumMonthMeanTimeseries(Timeseries):
             this_leg, domain, "T", iris.analysis.SUM
         )
 
+        # sivolu and siarea do not have units assigned, we need to add them manually
+        this_leg_summed.units = cf_units.Unit(_meta_dict[varname]["old_unit"])
         this_leg_summed.standard_name = _meta_dict[varname]["standard_name"]
         this_leg_summed.convert_units(_meta_dict[varname]["new_unit"])
         this_leg_summed.long_name = (

{scriptengine_tasks_ecearth-0.7.0 → scriptengine_tasks_ecearth-0.8.0}/pyproject.toml

@@ -7,7 +7,7 @@
 
 [project]
     name = "scriptengine-tasks-ecearth"
-    version = "0.7.0"
+    version = "0.8.0"
     authors = [
         { name = "Valentina Schueller", email = "valentina.schueller@gmail.com" },
         { name = "Uwe Fladrich", email = "uwe.fladrich@protonmail.com" },
@@ -29,6 +29,7 @@
         "scitools-iris>=3.5",
         "cartopy>=0.20",
         "python-redmine",
+        "python-gitlab",
     ]
 
     [project.urls]
@@ -54,4 +55,5 @@
         "ece.mon.oifs_global_mean_year_mean_timeseries" = "monitoring.oifs_global_mean_year_mean_timeseries:OifsGlobalMeanYearMeanTimeseries"
         "ece.mon.presentation.markdown" = "monitoring.markdown:Markdown"
         "ece.mon.presentation.redmine" = "monitoring.redmine:Redmine"
+        "ece.mon.presentation.gitlab" = "monitoring.gitlab:Gitlab"
         "ece.mon.linear_combination" = "monitoring.linear_combination:LinearCombination"

{scriptengine_tasks_ecearth-0.7.0 → scriptengine_tasks_ecearth-0.8.0}/scriptengine_tasks_ecearth.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: scriptengine-tasks-ecearth
-Version: 0.7.0
+Version: 0.8.0
 Summary: ScriptEngine tasks for use with the EC-Earth climate model
 Author-email: Valentina Schueller <valentina.schueller@gmail.com>, Uwe Fladrich <uwe.fladrich@protonmail.com>
 Project-URL: Homepage, https://github.com/uwefladrich/scriptengine-tasks-ecearth
@@ -19,6 +19,8 @@ Requires-Dist: imageio>=2.18
 Requires-Dist: scitools-iris>=3.5
 Requires-Dist: cartopy>=0.20
 Requires-Dist: python-redmine
+Requires-Dist: python-gitlab
+Dynamic: license-file
 
 [![Tests](https://github.com/uwefladrich/scriptengine-tasks-ecearth/actions/workflows/pytest.yml/badge.svg)](https://github.com/uwefladrich/scriptengine-tasks-ecearth/actions)
 [![Coverage Status](https://coveralls.io/repos/github/uwefladrich/scriptengine-tasks-ecearth/badge.svg)](https://coveralls.io/github/uwefladrich/scriptengine-tasks-ecearth)

{scriptengine_tasks_ecearth-0.7.0 → scriptengine_tasks_ecearth-0.8.0}/scriptengine_tasks_ecearth.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ helpers/map_type_handling.py
 helpers/nemo.py
 helpers/presentation_objects.py
 monitoring/diskusage_rte_scalar.py
+monitoring/gitlab.py
 monitoring/linear_combination.py
 monitoring/map.py
 monitoring/markdown.py
@@ -38,6 +39,7 @@ tests/test_cubes.py
 tests/test_dates.py
 tests/test_diskusage_rte_scalar.py
 tests/test_file_handling.py
+tests/test_gitlab.py
 tests/test_linear_combination.py
 tests/test_map.py
 tests/test_map_type_handling.py

{scriptengine_tasks_ecearth-0.7.0 → scriptengine_tasks_ecearth-0.8.0}/scriptengine_tasks_ecearth.egg-info/entry_points.txt

@@ -10,6 +10,7 @@ ece.mon.nemo_year_mean_timeseries = monitoring.nemo_timeseries:NemoYearMeanTimes
 ece.mon.oifs_all_mean_map = monitoring.oifs_all_mean_map:OifsAllMeanMap
 ece.mon.oifs_global_mean_year_mean_timeseries = monitoring.oifs_global_mean_year_mean_timeseries:OifsGlobalMeanYearMeanTimeseries
 ece.mon.oifs_year_mean_temporalmap = monitoring.oifs_year_mean_temporalmap:OifsYearMeanTemporalmap
+ece.mon.presentation.gitlab = monitoring.gitlab:Gitlab
 ece.mon.presentation.markdown = monitoring.markdown:Markdown
 ece.mon.presentation.redmine = monitoring.redmine:Redmine
 ece.mon.scalar = monitoring.scalar:Scalar

{scriptengine_tasks_ecearth-0.7.0 → scriptengine_tasks_ecearth-0.8.0}/scriptengine_tasks_ecearth.egg-info/requires.txt

@@ -6,3 +6,4 @@ imageio>=2.18
 scitools-iris>=3.5
 cartopy>=0.20
 python-redmine
+python-gitlab

scriptengine_tasks_ecearth-0.8.0/tests/test_gitlab.py

@@ -0,0 +1,112 @@
+"""Tests for monitoring/gitlab.py"""
+
+from unittest.mock import Mock, patch
+
+import gitlab
+import jinja2.exceptions
+import pytest
+import scriptengine.exceptions
+
+from monitoring.gitlab import Gitlab, create_anchor
+
+
+def test_gitlab_presentation_list(tmp_path):
+    init = {
+        "src": str(tmp_path),
+        "local_dst": str(tmp_path),
+        "template": str(tmp_path / "gitlab.txt.j2"),
+        "subject": "Test Issue",
+        "api_key": "Invalid Key",
+    }
+    redmine_task = Gitlab(init)
+    test_sources = ["path.txt", {"path": "path.txt"}, "path.yml"]
+    with patch.object(redmine_task, "log_warning") as mock:
+        result = redmine_task.get_presentation_list(test_sources, str(tmp_path))
+    mock.assert_called_with("Can not present diagnostic: File not found: path.yml")
+    assert result == []
+
+
+def test_gitlab_connection_error(tmp_path):
+    init = {
+        "src": str(tmp_path),
+        "local_dst": str(tmp_path),
+        "template": str(tmp_path / "gitlab.txt.j2"),
+        "subject": "Test Issue",
+        "api_key": "Invalid Key",
+    }
+    gitlab_task = Gitlab(init)
+    pytest.raises(
+        scriptengine.exceptions.ScriptEngineTaskRunError,
+        gitlab_task.get_project_and_issue,
+        init["api_key"],
+        init["subject"],
+    )
+
+
+def test_gitlab_auth_error(tmp_path):
+    init = {
+        "src": str(tmp_path),
+        "local_dst": str(tmp_path),
+        "template": str(tmp_path / "gitlab.txt.j2"),
+        "subject": "Test Issue",
+        "api_key": "Invalid Key",
+    }
+    gitlab_task = Gitlab(init)
+    pytest.raises(
+        scriptengine.exceptions.ScriptEngineTaskRunError,
+        gitlab_task.get_project_and_issue,
+        init["api_key"],
+        init["subject"],
+    )
+
+
+def test_gitlab_get_template(tmp_path):
+    init = {
+        "src": str(tmp_path),
+        "local_dst": str(tmp_path),
+        "template": str(tmp_path / "gitlab.txt.j2"),
+        "subject": "Test Issue",
+        "api_key": "Invalid Key",
+    }
+    gitlab_task = Gitlab(init)
+    with pytest.raises(jinja2.exceptions.TemplateNotFound):
+        gitlab_task.get_template(init, init["template"])
+
+
+class MockTemplateOrIssue:
+    def __init__(self):
+        self.globals = {}
+        self.web_url = ""
+
+    def render(self, **kwargs):
+        return str(kwargs)
+
+    def save(self):
+        return None
+
+    def upload(self, file_name, filepath=""):
+        return f"![{file_name}]({filepath})"
+
+
+def test_gitlab_run(tmp_path):
+    init = {
+        "src": str(tmp_path),
+        "local_dst": str(tmp_path),
+        "template": str(tmp_path / "redmine.txt.j2"),
+        "subject": "Test Issue",
+        "api_key": "Invalid Key",
+    }
+    gitlab_task = Gitlab(init)
+    gitlab_task.get_presentation_list = Mock(return_value=[])
+    mock_object = MockTemplateOrIssue()
+    gitlab_task.get_template = Mock(return_value=mock_object)
+    gitlab_task.get_project_and_issue = Mock(return_value=(mock_object, mock_object))
+    with patch.object(gitlab_task, "log_debug") as mock:
+        gitlab_task.run(init)
+    mock.assert_called_with("Saving issue.")
+
+
+def test_create_anchor():
+    test_string = "Precipitation - Evaporation (Annual Mean Climatology)"
+    gitlab_anchor = "precipitation-evaporation-annual-mean-climatology"
+    assert create_anchor(test_string) == gitlab_anchor