apheris-utils 0.6.0__py3-none-any.whl

apheris_utils/__init__.py

@@ -0,0 +1,9 @@
+from . import data
+
+try:
+    from . import extras_nvflare  # noqa: F401
+
+    __all__ = ["data", "extras_nvflare"]
+
+except ImportError:
+    __all__ = ["data"]

apheris_utils/data/__init__.py

@@ -0,0 +1,13 @@
+from apheris_utils.data._dal_client import (
+    download_all,
+    download_dataset,
+    get_asset_policies,
+    list_dataset_ids,
+)
+
+__all__ = [
+    "download_all",
+    "download_dataset",
+    "get_asset_policies",
+    "list_dataset_ids",
+]

apheris_utils/data/_dal_client.py

@@ -0,0 +1,85 @@
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+import requests
+
+from apheris_utils.data.primitives import get, list_remote_files, settings, to_folder
+
+
+def get_asset_policies(
+    dataset_ids: Union[str, List[str]],
+) -> Dict[str, Dict[str, Any]]:
+    """
+    Retrieves policy details for one or more datasets from the configured policy endpoint.
+
+    Args:
+        dataset_ids (Union[str, List[str]]): A single dataset id or a list of dataset ids for which policies are requested.
+
+    Raises:
+        RuntimeError: If the request to the policy endpoint fails.
+
+    Returns:
+        Union[Dict[str, Any], Dict[str, Dict[str, Any]]]: A dictionary containing the policy details for the requested dataset(s).
+        If called with a single dataset id, returns a dictionary with the policy details. If called with a list of dataset ids,
+        returns a dictionary of dictionaries with each dataset's policy details.
+    """
+    if isinstance(dataset_ids, str):
+        r = requests.get(
+            f"{settings.policy_endpoint}{settings.data[dataset_ids]}",
+            headers=settings.headers,
+        )
+        if r.status_code != 200:
+            raise RuntimeError(f"Failed to get policy: {r.text}")
+
+        return {dataset_ids: r.json()}
+
+    policies = {}
+    for d in dataset_ids:
+        policies.update(get_asset_policies(d))
+
+    return policies
+
+
+# Alias to load_dataset
+def download_dataset(dataset_id: str, folder: Union[str, Path]) -> Dict[str, Path]:
+    """
+    Load a dataset from the DAL and save it to a folder.
+
+    Args:
+        dataset_id (str): The ID of the dataset to load.
+        folder (Union[str, Path]): The folder where the dataset will be saved.
+
+    Returns:
+        Dict[str,str]: A dictionary mapping dataset id to their saved locations
+    """
+    return get(list_remote_files(dataset_id), to_folder(folder))
+
+
+def list_dataset_ids() -> List[str]:
+    """
+    List all available dataset ids.
+
+    Returns:
+        List[str]: A list of all available dataset ids.
+    """
+    return list(settings.data.keys())
+
+
+# Alias to download all datasets
+def download_all(folder: Union[str, Path]) -> Dict[str, Path]:
+    """
+    Downloads all datasets specified in the settings and saves them to the provided folder.
+
+    Args:
+        folder (Union[str, Path]): The path to the directory where the datasets will be stored.
+
+    Returns:
+        Dict[str, Union[str, Dict[str,str]]]: A dictionary mapping dataset IDs to their saved locations
+    """
+    # We get a list of dataset ids such as `my-data`, `my-slug`
+    dataset_ids = list_dataset_ids()
+    output = {}
+    for dataset_id in dataset_ids:
+        remote_files = list_remote_files(dataset_id)
+        output.update(get(remote_files, to_folder(folder)))
+    return output

apheris_utils/data/primitives/__init__.py

@@ -0,0 +1,17 @@
+from ._primitives import (
+    RemoteFile,
+    get,
+    list_remote_files,
+    settings,
+    to_dict,
+    to_folder,
+)
+
+__all__ = [
+    "get",
+    "list_remote_files",
+    "to_folder",
+    "to_dict",
+    "settings",
+    "RemoteFile",
+]

apheris_utils/data/primitives/_primitives.py

@@ -0,0 +1,152 @@
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import requests
+from pydantic import BaseModel, Field
+
+from apheris_utils.data.primitives._settings import settings
+
+
+class RemoteFile(BaseModel):
+    """
+    RemoteFile encapsulates the details about the data including its internal key and the path used for local processing.
+
+    Attributes:
+        _key (str): The internal key to access the data, typically a URL like s3://data/object.csv.
+        _parent (str): The parent key if the file is part of a dataset with a folder key. Example: parent = s3://data/
+        path (str): The path derived from the _key for local processing or storage. This is calculated based on prefix provided during initialization.
+    """
+
+    id: str
+    key: str = Field(..., alias="_key")
+    parent: Optional[str] = Field(None, alias="_parent")
+
+    @property
+    def path(self) -> str:
+        core_path: str
+        if self.parent:
+            # With key s3://folder/data/data.csv and parent s3://folder/ the core path is data/data.csv
+            core_path = self.key[len(self.parent) :]
+        else:  # with key file:///data.csv the core path is data.csv
+            core_path = self.key.split("/")[-1:][0]
+
+        return f"{self.id}/{core_path}"
+
+
+def list_remote_files(dataset_id: str) -> List[RemoteFile]:
+    """
+    Retrieves a list of remote files corresponding to all files contained within a specified dataset identified by its unique dataset id.
+    The unique dataset can be seen by listing datasets
+    A dataset ID is a unique identifier that can be used to access specific datasets in a data access layer (DAL).
+    Returns:
+        List[str]: A list of keys representing remote files that are accessible through the DAL.
+    """
+    key = settings.data[dataset_id]
+
+    if key.endswith("/"):
+        r = requests.get(f"{settings.dataset_endpoint}{key}", headers=settings.headers)
+
+        if r.status_code != 200:
+            raise RuntimeError(f"Failed to list datasets: {r.text}")
+
+        keys = r.json()
+        while link := r.headers.get("Link"):
+            r = requests.get(
+                f"{settings.dataset_endpoint}{link}", headers=settings.headers
+            )
+            keys += r.json()
+
+        files = [RemoteFile(id=dataset_id, _key=k, _parent=key) for k in keys]
+    else:
+        files = [RemoteFile(id=dataset_id, _key=key, _parent=None)]
+
+    return files
+
+
+def get(
+    remote_files: Union[RemoteFile, List[RemoteFile]],
+    output_func: Callable[[RemoteFile, bytes], Dict[str, Any]],
+) -> Dict[str, Any]:
+    """
+    Fetches the data specified by `remote_files` and applies a provided output function to handle the retrieved content.
+
+    Args:
+        remote_files (Union[RemoteFile, List[RemoteFile]]): A single RemoteFile or a list of RemoteFiles to retrieve.
+        output_func (Callable[[RemoteFile, bytes], Dict[str,Any]]): A function that processes the retrieved data. It accepts
+                                                                   the RemoteFile object and its content in bytes.
+
+    Returns:
+        Dict[str, Any]: A dictionary mapping keys to the processed results from the output function.
+                       For single files, returns the direct output dictionary.
+                       For multiple files, returns a merged dictionary of all results.
+    """
+    if isinstance(remote_files, RemoteFile):
+        r = requests.get(
+            f"{settings.dataset_endpoint}{remote_files.key}", headers=settings.headers
+        )
+        if r.status_code != 200:
+            raise RuntimeError(f"Failed to get key: {r.text}")
+        return output_func(remote_files, r.content)
+
+    with ThreadPoolExecutor(max_workers=settings.max_workers) as executor:
+        results = executor.map(partial(get, output_func=output_func), remote_files)
+        return {k: v for result in results for k, v in result.items()}
+
+
+def to_folder(folder: Union[str, Path]) -> Callable[[RemoteFile, bytes], Dict[str, Path]]:
+    """
+    Create a callable function for saving content to a specified folder.
+
+    Args:
+        folder (Union[str, Path]): The root directory where files will be saved.
+
+    Returns:
+        Callable[[RemoteFile, bytes], Dict[str, Any]]: A function that takes a RemoteFile and its content as bytes,
+                                                      then saves it to the destination derived from the RemoteFile's path
+                                                      and returns a dict mapping the RemoteFile's ID to the saved path.
+    """
+    if isinstance(folder, str):
+        folder = Path(folder)
+    # Make sure folder exists
+    folder.mkdir(parents=True, exist_ok=True)
+
+    def output_func(remote_file: RemoteFile, content: bytes) -> Dict[str, Path]:
+        # Parse key to extract the complete path excluding the protocol
+        # Example paths:
+        # s3://data/folder1/folder2/data.csv -> folder1/folder2/data.csv
+        # file:///container/folder1/folder2 -> container/folder1/folder2
+        full_path = folder / remote_file.path
+
+        if full_path != full_path.resolve():
+            raise ValueError("Access denied: Path traversal attempt detected")
+
+        full_path.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(full_path, "wb") as f:
+            f.write(content)
+
+        # If it's a container dataset, return the destination folder e.g. destination/slug/
+        if remote_file.parent:
+            return {remote_file.id: folder / remote_file.id}
+
+        # If not return the full path e.g. destination/slug/data.csv
+        return {remote_file.id: full_path}
+
+    return output_func
+
+
+def to_dict() -> Callable[[RemoteFile, bytes], Dict[str, Any]]:
+    """
+    Create a callable that stores content bytes in a dictionary under RemoteFile's ID.
+
+    Returns:
+        Callable[[RemoteFile, bytes], Dict[str, Any]]: A function that takes a RemoteFile and content bytes,
+        returning a dictionary mapping the RemoteFile's ID to its content.
+    """
+
+    def output_func(remote_file: RemoteFile, content: bytes):
+        return {remote_file.path: content}
+
+    return output_func

apheris_utils/data/primitives/_settings.py

@@ -0,0 +1,45 @@
+import os
+from pathlib import Path
+from typing import Dict, Union
+
+from pydantic import AnyHttpUrl, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+def get_env_file() -> Union[Path, str, None]:
+    """Supports multiple environment configurations based on the
+    APH_ENV environment variable.
+
+    The default environment will be always local.
+    """
+    # Make sure we don't load a spurious .env file by mistake
+    env_name = os.getenv("APH_ENV", "local")
+    env_file_prefix = ".env"
+    env_file_path = f"{env_file_prefix}-{env_name.lower()}"
+
+    return env_file_path if os.path.exists(env_file_path) else None
+
+
+class DALSettings(BaseSettings):
+    api_dal_url: AnyHttpUrl = AnyHttpUrl("http://test-api")
+    dal_token: str = "test-token"
+    data: Dict[str, str] = Field(default_factory=lambda: {"test": "test-dataset"})
+    max_workers: int = 50
+
+    model_config = SettingsConfigDict(env_prefix="APH_", env_file_encoding="utf-8")
+
+    @property
+    def headers(self) -> Dict[str, str]:
+        return {"Authorization": f"Bearer {self.dal_token}"}
+
+    @property
+    def dataset_endpoint(self) -> str:
+        # HttpUrl type adds a trailing slash
+        return f"{self.api_dal_url}datasets/"
+
+    @property
+    def policy_endpoint(self) -> str:
+        return f"{self.api_dal_url}policies/"
+
+
+settings = DALSettings(_env_file=get_env_file())

apheris_utils/extras_nvflare/__init__.py

@@ -0,0 +1,13 @@
+try:
+    from . import logging
+
+    __all__ = [
+        "logging",
+    ]
+
+except ImportError:
+    raise ImportError(
+        "NVFlare package not found. Please install "
+        "apheris-utils with the 'nvflare' extra to use this module. "
+        "`pip install apheris-utils[nvflare]`"
+    )

apheris_utils/extras_nvflare/logging/__init__.py

@@ -0,0 +1,11 @@
+from . import (
+    gateway_log_sender,
+    orchestrator_log_receiver,
+    util,
+)
+
+__all__ = [
+    "gateway_log_sender",
+    "orchestrator_log_receiver",
+    "util",
+]

apheris_utils/extras_nvflare/logging/gateway_log_sender.py

@@ -0,0 +1,82 @@
+import os
+import time
+
+from nvflare.apis.dxo import DXO, DataKind
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey, ReturnCode
+from nvflare.apis.fl_context import FLContext
+from nvflare.apis.shareable import make_reply
+
+from .util import get_logger_fn_for_level, sanitised_trace
+
+MESSAGE_TO_SERVER_DELAY = 0.01
+
+message_to_server_delay = os.environ.get(
+    "APH_MESSAGE_TO_SERVER_DELAY", MESSAGE_TO_SERVER_DELAY
+)
+
+
+class GatewayLogSenderMixin(FLComponent):
+
+    def _send_to_server_log(self, fl_ctx: FLContext, message: str, level="INFO") -> None:
+        """
+        Sends message to NVFlare server to log information there.
+        """
+        self.log_info(fl_ctx, "Sending message to server log")
+        get_logger_fn_for_level(component=self, level=level)(fl_ctx, message)
+        dxo = DXO(
+            DataKind.COLLECTION,
+            {
+                "message": message,
+                "level": level,
+                "client_name": fl_ctx.get_prop(FLContextKey.CLIENT_NAME),
+            },
+        )
+        msg_data = dxo.to_shareable()
+        self.fire_fed_event(
+            event_type="debug_message", event_data=msg_data, fl_ctx=fl_ctx
+        )
+        time.sleep(float(message_to_server_delay))
+
+
+def safe_error_catchall_decorator(func):
+    """
+    A decorator to wrap functions that might throw an unhandled exception, catch the
+    exception then sanitise it by removing the raw error message and just passing limited
+    details back to the user.
+
+    Will try to use an FLComponents log_xxx methods, but if unavailable, will simply
+    print to the terminal.
+
+    """
+
+    def wrapper(self, *args, **kwargs):
+        send_to_server = isinstance(self, GatewayLogSenderMixin)
+
+        fl_ctx = None
+        for a in args:
+            if isinstance(a, FLContext):
+                fl_ctx = a
+                break
+
+        if fl_ctx is None:
+            print(
+                "Could not find an FLContext in this function so can not forward"
+                " errors to server"
+            )
+
+        try:
+            return func(self, *args, **kwargs)
+        except Exception as err:
+            if fl_ctx and send_to_server:
+                self._send_to_server_log(
+                    fl_ctx,
+                    sanitised_trace(),
+                    "ERROR",
+                )
+            else:
+                # This will only print on the client, won't be passed back
+                print(str(err))
+            return make_reply(ReturnCode.ERROR)
+
+    return wrapper

apheris_utils/extras_nvflare/logging/orchestrator_log_receiver.py

@@ -0,0 +1,30 @@
+from nvflare.apis.dxo import from_shareable
+from nvflare.apis.fl_component import FLComponent
+from nvflare.apis.fl_constant import FLContextKey
+from nvflare.apis.fl_context import FLContext
+
+from .util import get_logger_fn_for_level
+
+
+class OrchestratorLogReceiver(FLComponent):
+    """
+    A component that receives messages sent to the Orchestrator from the Gateways as
+    FLARE events and writes them into the Orchestrator logs.
+    """
+
+    def handle_event(self, event_type: str, fl_ctx: FLContext):
+        super().handle_event(event_type=event_type, fl_ctx=fl_ctx)
+
+        if event_type == "debug_message":
+            shareable = fl_ctx.get_prop(key=FLContextKey.EVENT_DATA)
+            dxo = from_shareable(shareable)
+            data = dxo.data
+            client_message = data["message"]
+            client_name = data["client_name"]
+            level = data["level"]
+
+            get_logger_fn_for_level(component=self, level=level)(
+                fl_ctx,
+                f"OrchestratorLogReceiver: Message from '{client_name} [{level}]': "
+                f"{client_message}",
+            )

apheris_utils/extras_nvflare/logging/util.py

@@ -0,0 +1,63 @@
+import sys
+import traceback
+from dataclasses import dataclass
+from typing import Callable, Literal, Optional
+
+from nvflare.apis.fl_component import FLComponent
+
+LogLevel = Literal["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG"]
+
+
+def get_logger_fn_for_level(component: FLComponent, level: LogLevel) -> Callable:
+    if level == "CRITICAL":
+        return component.log_critical
+    elif level == "ERROR":
+        return component.log_error
+    elif level == "WARNING":
+        return component.log_warning
+    elif level == "DEBUG":
+        return component.log_debug
+    else:
+        return component.log_info
+
+
+@dataclass
+class ExceptionData:
+    type: str
+    line: Optional[str] = None
+    function_name: Optional[str] = None
+    filename: Optional[str] = None
+    line_number: Optional[int] = None
+
+
+def extract_exception_data() -> ExceptionData:
+    """
+    Call from inside an active exception to extract key information but hide the full
+    stacktrace. Can be used to create sanitised error messages to send from the client
+    to the server.
+    """
+
+    # We explicitly don't extract the exc_value as this can include sensitive
+    # information
+    exc_type, _, exc_traceback = sys.exc_info()
+    tb = traceback.extract_tb(exc_traceback)
+
+    return ExceptionData(
+        line=tb[-1].line,
+        function_name=tb[-1].name,
+        filename=tb[-1].filename,
+        line_number=tb[-1].lineno,
+        type=exc_type.__name__ if exc_type else "Unknown",
+    )
+
+
+def format_exception_data(d: ExceptionData) -> str:
+    return (
+        f"Exception of type '{d.type}' has occurred in"
+        f" {d.filename}:{d.line_number} (function: '{d.function_name}').\n"
+        f"The exception was caused by the line '{d.line}'"
+    )
+
+
+def sanitised_trace() -> str:
+    return format_exception_data(extract_exception_data())

apheris_utils-0.6.0.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Apheris GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

apheris_utils-0.6.0.dist-info/METADATA

@@ -0,0 +1,13 @@
+Metadata-Version: 2.2
+Name: apheris-utils
+Version: 0.6.0
+Summary: Utility functions for Apheris projects
+License-File: LICENSE
+Requires-Dist: pydantic==2.10.6
+Requires-Dist: pydantic-settings==2.8.1
+Requires-Dist: requests==2.32.3
+Provides-Extra: nvflare
+Requires-Dist: nvflare<2.6,>=2.4; extra == "nvflare"
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary

apheris_utils-0.6.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+apheris_utils/__init__.py,sha256=lT7baPkuny_CeifqXq_e54j4md7DN12v_S6fhoxpyz4,158
+apheris_utils/data/__init__.py,sha256=el_dQJgrcxpuGM7HmJjzi4pKSZ0RlN4Osw2fUeDX2FQ,242
+apheris_utils/data/_dal_client.py,sha256=Ara2ZT24Jk3P3CeuDjfFw3A_3yJ-HOSUvUunOnL_JH4,2781
+apheris_utils/data/primitives/__init__.py,sha256=XpbdwSRxDKm_zho8YuZ2XO1RTOUEX03zmAUPwncvLXY,236
+apheris_utils/data/primitives/_primitives.py,sha256=GxsDRNL5yAhfGopy20CLFZLAAjG8Z2D7FSS7BmBfYTI,6222
+apheris_utils/data/primitives/_settings.py,sha256=j8tRMjB6oc5StPmSWfBD5e2FXz9wf1LgGhtJkx_Puv4,1375
+apheris_utils/extras_nvflare/__init__.py,sha256=tQK4BdNxBEcsXi7DIKM0FH5bEWCPMpxWnD9DULuYcgE,293
+apheris_utils/extras_nvflare/logging/__init__.py,sha256=QfgtiRDKEJ044Xj6J7jFJbjdccVBWbOYLcQxfHrZDVI,169
+apheris_utils/extras_nvflare/logging/gateway_log_sender.py,sha256=pURFw3gF3griHpvs_Rv5XCVQv0cECA2JDVIIPFFkQzE,2552
+apheris_utils/extras_nvflare/logging/orchestrator_log_receiver.py,sha256=xygpE9su_IGh2xn-NisHNBm8vIYy4g5G_amlWeQ_0pg,1109
+apheris_utils/extras_nvflare/logging/util.py,sha256=qkk0v6v-sm5CViK8XQVVIw9jZq3kYPO-1joGKHdH16U,1829
+apheris_utils-0.6.0.dist-info/LICENSE,sha256=Y0GNoZQK3WCzQkhYvD1Wq1uwdCb1rifIQWZjc8gmAaI,1069
+apheris_utils-0.6.0.dist-info/METADATA,sha256=iHjPa-4XH4ZygSdxbKk7ok1Y38WM_n2DZu5PCS83Tmc,372
+apheris_utils-0.6.0.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+apheris_utils-0.6.0.dist-info/top_level.txt,sha256=86JUzsnJ8zzFZ50NgbpLCuJm9niysx5nUVawZe2F0SI,14
+apheris_utils-0.6.0.dist-info/RECORD,,

apheris_utils-0.6.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (76.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

apheris_utils-0.6.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+apheris_utils