rs-dpr-service 0.2__tar.gz

rs_dpr_service-0.2/PKG-INFO

@@ -0,0 +1,52 @@
+Metadata-Version: 2.3
+Name: rs-dpr-service
+Version: 0.2
+Summary: Copernicus Sentinel rs-server, DPR as a service.
+License: Apache-2.0
+Keywords: stac,copernicus,sentinel,processing
+Author: ESA under contract with Airbus and CS
+Author-email: ops_coprs@airbus.com
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: FastAPI
+Classifier: Framework :: OpenTelemetry
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: GIS
+Classifier: Topic :: Scientific/Engineering :: Image Processing
+Requires-Dist: SQLAlchemy (>=2.0.37,<3.0.0)
+Requires-Dist: bandit (>=1.8.3,<2.0.0)
+Requires-Dist: dask-gateway (==2024.1.0)
+Requires-Dist: dask[distributed] (==2024.5.2)
+Requires-Dist: fastapi (>=0.116.1,<0.118.0)
+Requires-Dist: flake8-pyproject (>=1.2.3,<2.0.0)
+Requires-Dist: flask (>=3.1.1,<4.0.0)
+Requires-Dist: geoalchemy2 (>=0.17.1,<0.18.0)
+Requires-Dist: httpx (>=0.28.1,<0.29.0)
+Requires-Dist: jsonpatch (>=1.33,<2.0)
+Requires-Dist: mypy (>=1.15.0,<2.0.0)
+Requires-Dist: openapi-core (>=0.19.5,<0.20.0)
+Requires-Dist: opentelemetry-distro (>=0.57b0,<0.58)
+Requires-Dist: opentelemetry-exporter-otlp (>=1.36.0,<2.0.0)
+Requires-Dist: opentelemetry-instrumentation-aws-lambda (>=0.57b0,<0.58)
+Requires-Dist: opentelemetry-test-utils (>=0.57b0,<0.58)
+Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
+Requires-Dist: pygeoapi (>=0.21.dev0)
+Requires-Dist: pylint (>=3.3.6,<4.0.0)
+Requires-Dist: python-logging-loki (>=0.3.1,<0.4.0)
+Requires-Dist: s3fs (>=2025.7.0,<2026.0.0)
+Requires-Dist: tornado (>=6.5,<7.0)
+Requires-Dist: uvicorn (>=0.34.1,<0.35.0)
+Project-URL: Documentation, https://home.rs-python.eu/rs-documentation/rs-server/docs/doc/
+Project-URL: Homepage, https://home.rs-python.eu/
+Project-URL: Repository, https://github.com/RS-PYTHON/rs-dpr-service
+Description-Content-Type: text/markdown
+
+# rs-dpr-service
+A service responsible for triggering the DPR processor execution within the Dask cluster.
+

rs_dpr_service-0.2/README.md

@@ -0,0 +1,2 @@
+# rs-dpr-service
+A service responsible for triggering the DPR processor execution within the Dask cluster.

rs_dpr_service-0.2/pyproject.toml

@@ -0,0 +1,138 @@
+[project]
+name = "rs-dpr-service"
+dynamic = ["version"] # see: https://github.com/mtkennerly/poetry-dynamic-versioning
+
+[tool.poetry]
+name = "rs-dpr-service"
+version = "0.2"
+description = "Copernicus Sentinel rs-server, DPR as a service."
+license = "Apache-2.0"
+authors = ["ESA under contract with Airbus and CS <ops_coprs@airbus.com>"]
+readme = "README.md"
+homepage = "https://home.rs-python.eu/"
+repository = "https://github.com/RS-PYTHON/rs-dpr-service"
+documentation = "https://home.rs-python.eu/rs-documentation/rs-server/docs/doc/"
+keywords = ["stac", "copernicus", "sentinel", "processing"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Framework :: FastAPI",
+    "Framework :: OpenTelemetry",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Scientific/Engineering :: GIS",
+    "Topic :: Scientific/Engineering :: Image Processing"
+]
+
+packages = [
+    { include = "rs_dpr_service", from = "."},
+]
+
+# Add configuration files to the .whl package
+include = [
+    { path = "config", format = ["wheel"] }
+]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+pygeoapi = ">=0.21.dev0"
+flask = "^3.1.1" # for pygeoapi
+uvicorn = ">=0.34.1,<0.35.0"
+fastapi = ">=0.116.1,<0.118.0"
+geoalchemy2 = ">=0.17.1,<0.18.0"
+SQLAlchemy = "^2.0.37"
+jsonpatch = "^1.33"
+dask-gateway = "2024.1.0"
+# same dask version as in the cluster i.e. same as eopf
+dask = {extras = ["distributed"], version = "2024.5.2"}
+openapi-core = "^0.19.5"
+bandit = "^1.8.3"
+mypy = "^1.15.0"
+pylint = "^3.3.6"
+flake8-pyproject = "^1.2.3"
+httpx = "^0.28.1"
+psycopg2 = "^2.9.10"
+opentelemetry-distro = "^0.57b0" # then run 'poetry run opentelemetry-bootstrap -a install'
+opentelemetry-instrumentation-aws-lambda = "^0.57b0"
+opentelemetry-test-utils = "^0.57b0"
+opentelemetry-exporter-otlp = "^1.36.0"
+tornado = "^6.5" # fix trivy vulnerability
+python-logging-loki = "^0.3.1" # to avoid otl error: No module named 'logging_loki'
+s3fs = "^2025.7.0"
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+python = "^3.11"
+pygeoapi = { git = "https://github.com/geopython/pygeoapi.git", branch = "master" }
+pre-commit = ">=3.7.1,<5.0.0"
+pylint = "^3.3.2"
+pytest = "^8.3.3"
+pytest-asyncio = "^1.1.0"
+pytest-cov = ">=5,<7"
+pytest-docker = "^3.1.0"
+pytest-error-for-skips = "^2.0.2"
+pytest-mock = "^3.14.0"
+responses = "^0.25.3"
+safety = "^3.5.1"
+
+[tool.black]
+line-length = 120
+
+[tool.pytest.ini_options]
+markers = [
+    "unit", # mark a test as unittest
+    "integration", # mark a test as integration test
+]
+asyncio_mode = "auto"
+
+[tool.mypy]
+warn_redundant_casts = true
+# Force analyze of partially annotate function signatures
+check_untyped_defs = true
+ignore_missing_imports = true
+
+[tool.flake8]
+max-line-length = 120
+# per-file-ignores = [
+#     '__init__.py:F401',
+# ]
+
+[tool.pylint]
+max-line-length = 120
+disable = [
+    "logging-fstring-interpolation",  # Logging f-string interpolation
+    "R0917",  # Too many positional arguments
+    "E0401", # import-error
+    "R0902", # too-many-instance-attributes
+    "R0913", # too-many-arguments
+    "too-many-statements",
+    "not-context-manager",
+    "too-many-branches",
+]
+ignore-paths = '^docs/.*$'
+
+# Only add notes as comments that are not checked for errors.
+notes = ["XXX"]  # No errors on TODO and FIXME
+
+min-similarity-lines = 10  # Minimum lines number of a similarity
+
+[tool.bandit.assert_used]
+skips = ['*/*.py']
+
+[tool.bandit]
+skips = ['B603', 'B607', 'B404']
+
+
+[tool.poetry.requires-plugins]
+poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
+
+[tool.poetry-dynamic-versioning]
+enable = false
+metadata = false
+
+[build-system]
+requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
+build-backend = "poetry_dynamic_versioning.backend"

rs_dpr_service-0.2/rs_dpr_service/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 CS Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

rs_dpr_service-0.2/rs_dpr_service/call_dask.py

@@ -0,0 +1,358 @@
+# Copyright 2025 CS Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains the code that is related to dask and/or sent to the dask workers.
+Avoid import unnecessary dependencies here.
+"""
+import ast
+import importlib
+import json
+import logging
+import os
+import os.path as osp
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import zipfile
+from pathlib import Path
+
+import yaml
+from distributed.client import Client as DaskClient
+from opentelemetry.trace.span import SpanContext
+
+from rs_dpr_service.utils import init_opentelemetry
+
+SERVICE_NAME = "rs.dpr.dask"
+
+# Don't use rs_dpr_service.utils.logging, it's not forwarded to the client
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+def upload_this_module(dask_client: DaskClient):
+    """
+    Upload this current module from the caller environment to the dask client.
+
+    WARNING: These modules should not import other modules that are not installed in the dask
+    environment or you'll have import errors.
+
+    Args:
+        clients: list of dask clients to which upload the modules.
+    """
+    # Root of the current project
+    root = Path(__file__).parent
+
+    # Files and dirs to upload and associated name in the zip archive
+    files = {
+        root / "__init__.py": "rs_dpr_service/__init__.py",
+        root / "call_dask.py": "rs_dpr_service/call_dask.py",
+        root / "safe_to_zarr.py": "rs_dpr_service/safe_to_zarr.py",
+        root / "utils/__init__.py": "rs_dpr_service/utils/__init__.py",
+        root / "utils/init_opentelemetry.py": "rs_dpr_service/utils/init_opentelemetry.py",
+        root / "utils/logging.py": "rs_dpr_service/utils/logging.py",
+        root / "utils/utils.py": "rs_dpr_service/utils/utils.py",
+    }
+
+    # From a temp dir
+    with tempfile.TemporaryDirectory() as tmpdir:
+
+        # Create a zip with our files
+        zip_path = f"{tmpdir}/{root.name}.zip"
+        with zipfile.ZipFile(zip_path, "w") as zipped:
+
+            # Zip all files
+            for key, value in files.items():
+                zipped.write(str(key), str(value))
+
+        # Upload zip file to dask clients.
+        # This also installs the zipped modules inside the dask python interpreter.
+        try:
+            dask_client.upload_file(zip_path)
+
+        # We have this error if we scale up the number of workers.
+        # But it's OK, the zip file is automatically uploaded to them anyway.
+        except KeyError as e:
+            logger.debug(f"Ignoring error {e}")
+
+
+def copy_caller_env(caller_env: dict[str, str]):
+    """
+    Copy environment variables from the calling service environment to the dask client.
+
+    Args:
+        caller_env: os.environ coming from caller
+    """
+    local_mode = caller_env.get("RSPY_LOCAL_MODE") == "1"
+
+    # Copy env vars from the caller
+    keys = [
+        "RSPY_LOCAL_MODE",
+        "S3_ACCESSKEY",
+        "S3_SECRETKEY",
+        "S3_ENDPOINT",
+        "S3_REGION",
+        "PREFECT_BUCKET_NAME",
+        "PREFECT_BUCKET_FOLDER",
+        "DASK_GATEWAY_EOPF_ADDRESS",
+        "DASK_CLUSTER_EOPF_NAME",
+        "AWS_REQUEST_CHECKSUM_CALCULATION",
+        "AWS_RESPONSE_CHECKSUM_VALIDATION",
+        "TEMPO_ENDPOINT",
+        "OTEL_PYTHON_REQUESTS_TRACE_HEADERS",
+        "OTEL_PYTHON_REQUESTS_TRACE_BODY",
+    ]
+    if local_mode:
+        keys.extend(
+            [
+                "LOCAL_DASK_USERNAME",
+                "LOCAL_DASK_PASSWORD",
+                "access_key",
+                "bucket_location",
+                "host_base",
+                "host_bucket",
+                "secret_key",
+            ],
+        )
+    else:
+        keys.extend(["JUPYTERHUB_API_TOKEN"])
+    for key in keys:
+        if value := caller_env.get(key):
+            os.environ[key] = value
+
+
+def dpr_tasktable_task(
+    caller_env: dict[str, str],
+    flow_span_context: SpanContext,
+    use_mockup: bool,
+    module_name: str,
+    class_name: str,
+):
+    """
+    Return the DPR tasktable. This function is run from inside the dask pod.
+    """
+    # Copy env vars from the caller
+    copy_caller_env(caller_env)
+
+    # Init opentelemetry and record all task in an Opentelemetry span
+    init_opentelemetry.init_traces(None, SERVICE_NAME, logger)
+    with init_opentelemetry.start_span(__name__, "main_dask_flow", flow_span_context):
+
+        if use_mockup:
+            time.sleep(1)
+            return {}
+
+        # Load the python class
+        class_ = getattr(importlib.import_module(module_name), class_name)
+
+        # Get the tasktable for default mode. See:
+        # https://cpm.pages.eopf.copernicus.eu/eopf-cpm/main/processor-orchestration-guide/tasktables.html#tasktables
+        logger.debug(f"Available modes for {class_}: {class_.get_available_modes()}")
+        default_mode = class_.get_default_mode()
+        tasktable = class_.get_tasktable_description(default_mode)
+        return tasktable
+
+
+def dpr_processor_task(  # pylint: disable=R0914, R0917
+    caller_env: dict[str, str],
+    data: dict,
+    use_mockup: bool,
+):
+    """
+    Run the DPR processor. This function is run from inside the dask pod.
+
+    Args:
+        caller_env: env variables coming from the caller
+        data: data to send to the processor
+        use_mockup: use the mockup or real processor
+    """
+    # Copy env vars from the caller
+    copy_caller_env(caller_env)
+
+    # Mockup processor
+    if use_mockup:
+        try:
+            payload_abs_path = osp.join("/", os.getcwd(), "payload.cfg")
+            with open(payload_abs_path, "w+", encoding="utf-8") as payload:
+                payload.write(yaml.safe_dump(data))
+        except Exception as e:
+            logger.exception("Exception during payload file creation: %s", e)
+            raise
+        command = ["python3.11", "DPR_processor_mock.py", "-p", payload_abs_path]
+        working_dir = "/src/DPR"
+        log_path = "./mockup.log"  # not used
+        logger.debug(f"Working directory for subprocess: {working_dir} (type: {type(working_dir)})")
+
+    # Real processor
+    else:
+        # Read arguments
+        s3_config_dir = data["s3_config_dir"]
+        payload_subpath = data["payload_subpath"]
+        s3_report_dir = data["s3_report_dir"]
+
+        # Get S3 file handler.
+        # NOTE: eopf exists in the dask worker environment, not in the rs-dpr-service env,
+        # so we cannot import it from the top of this module.
+        from eopf.common.file_utils import (  # pylint: disable=import-outside-toplevel
+            AnyPath,
+        )
+
+        s3 = AnyPath(
+            s3_config_dir,
+            key=os.environ["S3_ACCESSKEY"],
+            secret=os.environ["S3_SECRETKEY"],
+            client_kwargs={
+                "endpoint_url": os.environ["S3_ENDPOINT"],
+                "region_name": os.environ["S3_REGION"],
+            },
+        )
+
+        logger.info("The dpr processing task started")
+
+        # Download the configuration folder from the S3 bucket into a local temp folder
+        local_config_dir = s3.get(recursive=True)
+
+        # Payload path and parent dir
+        payload_file = osp.realpath(osp.join(local_config_dir, payload_subpath))
+        payload_dir = osp.dirname(payload_file)
+
+        with open(payload_file, encoding="utf-8") as opened:
+            payload_contents = yaml.safe_load(opened)
+            logger.debug(f"Payload file contents: {payload_file!r}\n{json.dumps(payload_contents, indent=2)}")
+
+        command = ["eopf", "trigger", "local", payload_file]
+
+        # Change working directory
+        working_dir = osp.join(local_config_dir, payload_dir)
+        os.chdir(working_dir)
+
+        # Create the reports dir
+        # WARNING: fields from the payload file: dask__export_graphs, performance_report_file, ... should
+        # also use this directory: ./reports
+        local_report_dir = osp.realpath("./reports")
+        log_path = osp.join(local_report_dir, Path(payload_file).with_suffix(".processor.log").name)
+        shutil.rmtree(local_report_dir, ignore_errors=True)
+        os.makedirs(local_report_dir, exist_ok=True)
+
+    # Trigger EOPF processing, catch output
+    with subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        cwd=working_dir,
+    ) as proc:
+
+        # Log contents
+        log_str = ""
+
+        # Write output to a log file and string + redirect to the prefect logger
+        with open(log_path, "w+", encoding="utf-8") as log_file:
+            while proc.stdout and (line := proc.stdout.readline()) != "":
+
+                # The log prints password in clear e.g 'key': '<my-secret>'... hide them with a regex
+                for key in (
+                    "key",
+                    "secret",
+                    "endpoint_url",
+                    "region_name",
+                    "api_token",
+                    "password",
+                ):
+                    line = re.sub(rf"(\W{key}\W)[^,}}]*", r"\1: ***", line)
+
+                # Write to log file and string
+                log_file.write(line)
+                log_str += line
+
+                # Write to logger if not empty
+                line = line.rstrip()
+                if line:
+                    logger.info(line)
+
+        try:
+            # Wait for the execution to finish
+            status_code = proc.wait()
+
+            # Raise exception if the status code is != 0
+            if status_code:
+                raise RuntimeError(f"EOPF error, status code {status_code!r}, please see the log.")
+            logger.info(f"EOPF finished successfully with status code {status_code!r}")
+
+            # search for the JSON-like part, parse it, and ignore the rest.
+            if use_mockup:
+                match = re.search(r"(\[\s*\{.*\}\s*\])", log_str, re.DOTALL)
+                if not match:
+                    raise ValueError(f"No valid dpr_payload structure found in the output:\n{log_str}")
+
+                payload_str = match.group(1)
+
+                # Use `ast.literal_eval` to safely evaluate the structure
+                try:
+                    # payload_str is a string that looks like a JSON, extracted from the dpr mockup's raw output.
+                    # ast.literal_eval() parses that string and returns the actual Python object (not just the string).
+                    return ast.literal_eval(payload_str)
+                except Exception as e:
+                    raise ValueError(f"Failed to parse dpr_payload structure: {e}") from e
+
+            # NOTE: with the real processor, what should we return ?
+            return {}
+
+        # In all cases, upload the reports dir to the s3 bucket.
+        finally:
+            try:
+                if not use_mockup:
+                    logger.info(f"Upload reports {local_report_dir!r} to {s3_report_dir!r}")
+                    s3._fs.put(local_report_dir, s3_report_dir, recursive=True)  # pylint: disable=protected-access
+            except Exception as exception:  # pylint: disable=broad-exception-caught
+                logger.error(exception)
+
+
+def convert_safe_to_zarr(cfg):
+    """
+    Convert from legacy product (safe format) into Zarr format using EOPF in a subprocess.
+
+    This runs the rs_dpr_service.safe_to_zarr module as a subprocess, passing config as JSON string.
+    """
+
+    # Serialize the config
+    cfg_str = json.dumps(cfg)
+
+    # Find the ZIP that this code lives in
+    module_path = Path(__file__).resolve()
+    zip_path = Path(str(module_path).split(".zip", maxsplit=1)[0] + ".zip")
+    if not zip_path.is_file():
+        raise RuntimeError(f"Cannot locate rs_dpr_service.zip at {zip_path}")
+
+    # Prepare an env that lets Python import from inside the ZIP
+    env = os.environ.copy()
+    # prepend the zip onto PYTHONPATH (so zipimport will kick in)
+    env["PYTHONPATH"] = str(zip_path) + os.pathsep + env.get("PYTHONPATH", "")
+
+    # Run the converter as a module
+    cmd = [sys.executable, "-m", "rs_dpr_service.safe_to_zarr", cfg_str]
+    result = subprocess.run(
+        cmd,
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    if result.returncode != 0:
+        raise RuntimeError(f"Conversion failed: {result.stderr.strip()}")
+    return result.stdout.strip()