fred-oss 0.54.0__tar.gz → 0.56.0__tar.gz

fred_oss-0.56.0/MANIFEST.in

@@ -0,0 +1,3 @@
+include requirements.txt
+include src/main/fred/version
+include src/main/fred/dao/service/_minio/policy/templates/*.json

{fred_oss-0.54.0/src/main/fred_oss.egg-info → fred_oss-0.56.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-oss
-Version: 0.54.0
+Version: 0.56.0
 Summary: FREDOSS
 Home-page: https://fred.fahera.mx
 Author: Fahera Research, Education, and Development

{fred_oss-0.54.0 → fred_oss-0.56.0}/setup.py

@@ -35,6 +35,8 @@ setup(
     package_data={
         "": [
             version_filepath,
+            "**/*.json",
+            "**/*.yaml",
         ]
     },
     entry_points={

fred_oss-0.56.0/src/main/fred/cli/main.py

@@ -0,0 +1,59 @@
+from typing import Optional
+
+from fred.version import version
+from fred.settings import logger_manager
+from fred.cli.interface import AbstractCLI
+
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+class CLIExtensionGroups:
+    """CLI Extensions providing access to various integrations by following a lazy loading pattern."""
+
+    @property
+    def databricks(self):
+        from fred.integrations.databricks.cli_ext import DatabricksExt
+        return DatabricksExt()
+    
+    @property
+    def runpod(self):
+        from fred.integrations.runpod.cli_ext import RunPodExt
+        return RunPodExt()
+    
+    @property
+    def runner_server(self):
+        from fred.worker.runner.rest.cli_ext import RunnerServerExt
+        return RunnerServerExt()
+
+
+class CLI(AbstractCLI, CLIExtensionGroups):
+
+    def version(self) -> str:
+        return version.value
+
+    def serve(
+            self,
+            classname: Optional[str] = None,
+            classpath: Optional[str] = None,
+            include_routers: Optional[list[str]] = None,
+            exclude_routers: Optional[list[str]] = None,
+            fastapi_configs: Optional[dict] = None,
+            server_configs: Optional[dict] = None,
+    ):
+        from fred.rest.server import FredServer
+
+        include_routers = include_routers or []
+        exclude_routers = exclude_routers or []
+        fastapi_configs = fastapi_configs or {}
+        server_configs = server_configs or {}
+
+        logger.info("Starting the Fred-REST Server...")
+        server = FredServer.auto(
+            include_routers=include_routers,
+            exclude_routers=exclude_routers,
+            router_classname=classname,
+            router_classpath=classpath,
+            **fastapi_configs,
+        )
+        server.start(**server_configs)

fred_oss-0.56.0/src/main/fred/dao/service/_minio/__init__.py

@@ -0,0 +1 @@
+from .service import MinioService

fred_oss-0.56.0/src/main/fred/dao/service/_minio/policy/builder.py

@@ -0,0 +1,16 @@
+import enum
+
+
+class MinioPolicyBuilderActionBucket(enum.StrEnum):
+    ALL = "s3:*"
+    LIST = "s3:ListBucket"
+    LOCATION = "s3:GetBucketLocation"
+    DELETE = "s3:DeleteBucket"
+    CREATE = "s3:CreateBucket"
+
+
+class MinioPolicyBuilderActionObject(enum.StrEnum):
+    GET = "s3:GetObject"
+    PUT = "s3:PutObject"
+    DELETE = "s3:DeleteObject"
+    ALL = "s3:*"

fred_oss-0.56.0/src/main/fred/dao/service/_minio/policy/catalog.py

@@ -0,0 +1,28 @@
+import enum
+from typing import Optional
+
+from fred.dao.service._minio.policy.loader import MinioPolicyLoader
+
+
+class MinioPolicyCatalog(enum.Enum):
+    BUCKET_PUBLIC_RO = MinioPolicyLoader(
+        title="[Bucket Policy] Public Read-Only",
+        filename="public_ro.json",
+        requires=[
+            "bucket_name",
+        ],
+    )
+    BUCKET_PUBLIC_RW = MinioPolicyLoader(
+        title="[Bucket Policy] Public Read-Write",
+        filename="public_rw.json",
+        requires=[
+            "bucket_name",
+        ],
+    )
+
+    def load(self, path: Optional[str] = None, **params) -> dict:
+        return self.value.load(path=path, **params)
+    
+    def content(self, path: Optional[str] = None, **params) -> str:
+        import json
+        return json.dumps(self.load(path=path, **params))

fred_oss-0.56.0/src/main/fred/dao/service/_minio/policy/loader.py

@@ -0,0 +1,27 @@
+import os
+import json
+from dataclasses import dataclass, field
+from typing import Optional
+def load_policy(filename: str, path: Optional[str] = None, **kwargs) -> dict:
+    path = path or os.path.join(os.path.dirname(__file__), "templates")
+    filepath = os.path.join(path, filename)
+    # Read the policy file as a string
+    with open(filepath, "r", encoding="utf-8") as file:
+        policy_str = file.read()
+    # Replace placeholders in the policy with actual values from kwargs
+    for param_key, param_val in kwargs.items():
+        param_ref = "${{param_key}}".replace("param_key", param_key)
+        policy_str = policy_str.replace(param_ref, param_val)
+    return json.loads(policy_str)
+
+
+@dataclass(frozen=True, slots=True)
+class MinioPolicyLoader:
+    title: str
+    filename: str
+    requires: list[str] = field(default_factory=list)
+
+    def load(self, path: Optional[str] = None, **kwargs) -> dict:
+        if (missing := set(self.requires) - set(kwargs.keys())):
+            raise ValueError(f"Missing required parameters for policy '{self.filename}': {missing}")
+        return load_policy(filename=self.filename, path=path, **kwargs)

fred_oss-0.56.0/src/main/fred/dao/service/_minio/policy/templates/public_ro.json

@@ -0,0 +1,24 @@
+{
+    "Version": "2012-10-17",
+    "Statement": [
+         {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "*"
+            },
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListBucket"
+            ],
+            "Resource": "arn:aws:s3:::${{bucket_name}}"
+        },
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "*"
+            },
+            "Action": "s3:GetObject",
+            "Resource": "arn:aws:s3:::${{bucket_name}}/*"
+        }
+    ]
+}

fred_oss-0.56.0/src/main/fred/dao/service/_minio/policy/templates/public_rw.json

@@ -0,0 +1,31 @@
+ {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "*"
+            },
+            "Action": [
+                "s3:GetBucketLocation",
+                "s3:ListBucket",
+                "s3:ListBucketMultipartUploads"
+            ],
+            "Resource": "arn:aws:s3:::${{bucket_name}}"
+        },
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": "*"
+            },
+            "Action": [
+                "s3:GetObject",
+                "s3:PutObject",
+                "s3:DeleteObject",
+                "s3:ListMultipartUploadParts",
+                "s3:AbortMultipartUpload"
+            ],
+            "Resource": "arn:aws:s3:::${{bucket_name}}/*"
+        }
+    ]
+}

fred_oss-0.56.0/src/main/fred/dao/service/_minio/pool.py

@@ -0,0 +1,55 @@
+from urllib3 import PoolManager, Retry
+from urllib3.util import Timeout
+
+from fred.settings import get_environ_variable, logger_manager
+from fred.dao.service.interface import ServiceConnectionPoolInterface
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+class MinioConnectionPool(ServiceConnectionPoolInterface[PoolManager]):
+
+    @classmethod
+    def _create_pool(cls, disable_cert: bool = False, **kwargs) -> PoolManager:
+        """Create a urllib3 PoolManager with the given configurations.
+
+        TODO: Consider using the inverse of 'require_cert' as the default to ensure we do have cert-check automatically.
+        For now, we keep it as is to avoid breaking changes.
+
+        Args:
+            require_cert (bool): Whether to require SSL certificate verification.
+            **kwargs: Additional keyword arguments to pass to the PoolManager constructor.
+        Returns:
+            PoolManager: A configured PoolManager instance.
+        """
+        num_pools = kwargs.pop("num_pools", 10)
+        maxsize = kwargs.pop("maxsize", 10)
+        # Default timeout of 5 minutes
+        timeout_seconds = kwargs.pop("timeout", 300)
+        timeout = Timeout(
+            connect=timeout_seconds,
+            read=timeout_seconds,
+        )
+        # Default retries of 5 with exponential backoff
+        retry = Retry(
+            total=kwargs.pop("retries", 5),
+            backoff_factor=kwargs.pop("backoff_factor", 0.25),
+            status_forcelist=[500, 502, 503, 504],
+        )
+        # Configure certificate requirements for SSL connections
+        cert_reqs = "CERT_NONE"
+        ca_certs = None
+        if not disable_cert:
+            import certifi
+            cert_reqs = "CERT_REQUIRED"
+            ca_certs = get_environ_variable("SSL_CERT_FILE") or certifi.where()
+        # Finally, create and return the PoolManager instance
+        return PoolManager(
+            num_pools=num_pools,
+            maxsize=maxsize,
+            timeout=timeout,
+            retries=retry,
+            cert_reqs=cert_reqs,
+            ca_certs=ca_certs,
+            **kwargs
+        )

fred_oss-0.54.0/src/main/fred/dao/service/_minio.py → fred_oss-0.56.0/src/main/fred/dao/service/_minio/service.py

@@ -1,62 +1,14 @@
 from minio import Minio
-from urllib3 import PoolManager, Retry
-from urllib3.util import Timeout
 
-from fred.settings import get_environ_variable, logger_manager
+from fred.settings import logger_manager
+from fred.dao.service.interface import ServiceInterface
 from fred.dao.service.utils import get_minio_from_payload
-from fred.dao.service.interface import ServiceInterface, ServiceConnectionPoolInterface
+from fred.dao.service._minio.pool import MinioConnectionPool
+from fred.dao.service._minio.policy.catalog import MinioPolicyCatalog
 
 logger = logger_manager.get_logger(name=__name__)
 
 
-class MinioConnectionPool(ServiceConnectionPoolInterface[PoolManager]):
-
-    @classmethod
-    def _create_pool(cls, disable_cert: bool = False, **kwargs) -> PoolManager:
-        """Create a urllib3 PoolManager with the given configurations.
-
-        TODO: Consider using the inverse of 'require_cert' as the default to ensure we do have cert-check automatically.
-        For now, we keep it as is to avoid breaking changes.
-
-        Args:
-            require_cert (bool): Whether to require SSL certificate verification.
-            **kwargs: Additional keyword arguments to pass to the PoolManager constructor.
-        Returns:
-            PoolManager: A configured PoolManager instance.
-        """
-        num_pools = kwargs.pop("num_pools", 10)
-        maxsize = kwargs.pop("maxsize", 10)
-        # Default timeout of 5 minutes
-        timeout_seconds = kwargs.pop("timeout", 300)
-        timeout = Timeout(
-            connect=timeout_seconds,
-            read=timeout_seconds,
-        )
-        # Default retries of 5 with exponential backoff
-        retry = Retry(
-            total=kwargs.pop("retries", 5),
-            backoff_factor=kwargs.pop("backoff_factor", 0.25),
-            status_forcelist=[500, 502, 503, 504],
-        )
-        # Configure certificate requirements for SSL connections
-        cert_reqs = "CERT_NONE"
-        ca_certs = None
-        if not disable_cert:
-            import certifi
-            cert_reqs = "CERT_REQUIRED"
-            ca_certs = get_environ_variable("SSL_CERT_FILE") or certifi.where()
-        # Finally, create and return the PoolManager instance
-        return PoolManager(
-            num_pools=num_pools,
-            maxsize=maxsize,
-            timeout=timeout,
-            retries=retry,
-            cert_reqs=cert_reqs,
-            ca_certs=ca_certs,
-            **kwargs
-        )
-
-
 class MinioService(ServiceInterface[Minio]):
     instance: Minio
     metadata: dict = {}
@@ -120,3 +72,12 @@ class MinioService(ServiceInterface[Minio]):
         except S3Error:
             logger.debug(f"Object {object_name} in bucket {bucket_name} does not exist.")
             return False
+
+    def make_bucket_public(self, bucket_name: str, readonly: bool = False):
+        """Make a bucket public with either read-only or read-write access."""
+        policy = MinioPolicyCatalog.BUCKET_PUBLIC_RO \
+            if readonly else MinioPolicyCatalog.BUCKET_PUBLIC_RW
+        self.client.set_bucket_policy(
+            bucket_name=bucket_name,
+            policy=policy.content(bucket_name=bucket_name)
+        )

fred_oss-0.56.0/src/main/fred/integrations/databricks/runtimes/16.4LTS.json

@@ -0,0 +1,10 @@
+{
+    "python_version": "3.12.3",
+    "databricks_runtime": "16.4 LTS",
+    "libraries": [
+        {
+            "name": "pydantic",
+            "version": "2.8.2"
+        }
+    ]
+}

fred_oss-0.56.0/src/main/fred/rest/__init__.py

@@ -0,0 +1,17 @@
+"""
+[Fred-REST] Fred's Simple Extensible REST-API Server
+========================================================
+This module provides a simple and extensible REST-API server implementation using FastAPI.
+It includes configurations for authentication, router management, and server startup.
+"""
+from fred.maturity import Maturity, MaturityLevel
+
+
+module_maturity = Maturity(
+    level=MaturityLevel.ALPHA,
+    reference=__name__,
+    message=(
+        "Fred-REST implementation is in early development "
+        "and therefore currently with incomplete and unstable features."
+    )
+)

fred_oss-0.56.0/src/main/fred/rest/auth.py

@@ -0,0 +1,23 @@
+from fastapi import Security, HTTPException, status
+from fastapi.security import APIKeyHeader
+
+from fred.rest.settings import FRD_RESTAPI_TOKEN
+
+
+async def verify_key(
+        api_key_header: str = Security(
+            APIKeyHeader(
+                name="X-API-Key",
+                auto_error=False
+            )
+        ),
+):
+    """
+    Verify the provided API key against the expected token.
+    Raises an HTTPException if the key is invalid or missing.
+    """
+    if api_key_header != FRD_RESTAPI_TOKEN:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or missing API Key",
+        )

fred_oss-0.56.0/src/main/fred/rest/config.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from fred.rest.router.catalog.interface import RouterCatalogInterface
+from fred.rest.settings import (
+    FRD_RESTAPI_ROUTERCATALOG_CLASSPATH,
+    FRD_RESTAPI_ROUTERCATALOG_CLASSNAME,
+)
+
+@dataclass(frozen=True, slots=True)
+class ServerRouterCatalogConfig:
+    classname: str
+    classpath: str
+
+    @classmethod
+    def auto(cls, classname: Optional[str] = None, classpath: Optional[str] = None) -> "ServerRouterCatalogConfig":
+        return cls(
+            classname=classname or FRD_RESTAPI_ROUTERCATALOG_CLASSNAME,
+            classpath=classpath or FRD_RESTAPI_ROUTERCATALOG_CLASSPATH,
+        )
+
+    @property
+    def catalog(self) -> type[RouterCatalogInterface]:
+        import importlib
+
+        module = importlib.import_module(self.classpath)
+        catalog_class = getattr(module, self.classname, None)
+        if catalog_class is None:
+            raise ImportError(f"Could not find class '{self.classname}' in module '{self.classpath}'")
+        if not issubclass(catalog_class, RouterCatalogInterface):
+            raise TypeError(f"Class '{self.classname}' is not a subclass of RouterCatalogInterface")
+        return catalog_class

fred_oss-0.56.0/src/main/fred/rest/router/catalog/default/__init__.py

@@ -0,0 +1,10 @@
+from fred.rest.router.catalog.default.catalog import RouterCatalog
+from fred.settings import logger_manager
+
+logger = logger_manager.get_logger(name=__name__)
+
+
+logger.warning(
+    "You are using the default router catalog intended for demonstration purposes only. "
+    "For production use, please implement a custom router catalog."
+)

fred_oss-0.56.0/src/main/fred/rest/router/catalog/default/_base.py

@@ -0,0 +1,29 @@
+from fred.rest.router.interface import RouterInterfaceMixin
+from fred.rest.router.endpoint import RouterEndpointAnnotation
+
+
+class RouterBaseMixin(RouterInterfaceMixin):
+
+    @RouterEndpointAnnotation.set(
+        path="/",
+        methods=["GET"],
+        summary="Base endpoint",
+        description="A simple base endpoint to check service availability."
+    )
+    def base(self, include_telemetry: bool = False, **kwargs) -> dict:
+        if include_telemetry:
+            from fred.utils.runtime import RuntimeProfilingSnapshot
+            kwargs["telemetry"] = RuntimeProfilingSnapshot.auto().to_dict()
+        return {
+            "ok": True,
+            **kwargs
+        }
+
+    @RouterEndpointAnnotation.set(
+        path="/passthrough",
+        methods=["POST"],
+        summary="Passthrough POST endpoint",
+        description="An endpoint that returns all received parameters.",
+    )
+    def passthrough(self, **kwargs) -> dict:
+        return kwargs

fred_oss-0.56.0/src/main/fred/rest/router/catalog/default/_example.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from fred.rest.router.interface import RouterInterfaceMixin
+from fred.rest.router.endpoint import RouterEndpointAnnotation
+
+
+class RouterExampleMixin(RouterInterfaceMixin):
+
+    @RouterEndpointAnnotation.set(
+        path="/ping",
+        methods=["GET"],
+        summary="Ping endpoint",
+        description="A simple ping endpoint to check service availability.",
+    )
+    def ping(self, pong: Optional[str] = None, **kwargs) -> dict:
+        from fred.utils.dateops import datetime_utcnow
+
+        return {
+            "ping_time": datetime_utcnow().isoformat(),
+            "ping_response": pong or "pong",
+        }
+
+    @RouterEndpointAnnotation.set(
+        path="/passthrough",
+        methods=["GET"],
+        summary="Passthrough GET endpoint",
+        description="An endpoint that returns all received parameters.",
+    )
+    def passthrough(self, **kwargs) -> dict:
+        return kwargs

fred_oss-0.56.0/src/main/fred/rest/router/catalog/default/catalog.py

@@ -0,0 +1,20 @@
+import enum
+
+from fred.rest.router.config import RouterConfig
+from fred.rest.router.catalog.default._base import RouterBaseMixin
+from fred.rest.router.catalog.default._example import RouterExampleMixin
+
+from fred.rest.router.catalog.interface import RouterCatalogInterface
+
+
+class RouterCatalog(RouterCatalogInterface, enum.Enum):
+    BASE = RouterConfig.auto(prefix="")(apply=RouterBaseMixin)
+    EXAMPLE = RouterConfig.auto(prefix="/example")(apply=RouterExampleMixin)
+
+    def get_kwargs(self) -> dict:
+        match self:
+            case RouterCatalog.EXAMPLE:
+                # Disable the backend for the example router
+                return {"disregard_backend": True}
+            case _:
+                return {}

fred_oss-0.56.0/src/main/fred/rest/router/catalog/interface.py

@@ -0,0 +1,11 @@
+from fred.rest.router.interface import RouterInterface
+
+
+class RouterCatalogInterface:
+
+    def auto(self, **kwargs) -> RouterInterface:
+        return self.value.reference.auto(**kwargs)
+
+    def get_kwargs(self) -> dict:
+        # This method can be overridden to provide specific kwargs for router initialization
+        return {}

fred_oss-0.56.0/src/main/fred/rest/router/config.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+from fred.rest.router.interface import RouterInterface, RouterInterfaceMixin
+
+
+@dataclass(frozen=True, slots=True)
+class RouterConfig:
+    prefix: str
+    other: dict
+
+    @classmethod
+    def auto(cls, prefix: str = "", **kwargs) -> "RouterConfig":
+        return cls(
+            prefix=prefix,
+            other=kwargs,
+        )
+
+    def get_applied_configs(self, router_class: type[RouterInterface]) -> "AppliedRouterConfig":
+        return AppliedRouterConfig(
+            reference=router_class,
+            config=self,
+        )
+
+    def __call__(self, apply: type[RouterInterfaceMixin]) -> "AppliedRouterConfig":
+        router_classname = f"Router{apply.__name__}"
+        return self.get_applied_configs(
+            router_class=type(router_classname, (RouterInterface, apply, ), {})
+        )
+    
+    def get_configs(self) -> dict:
+        return {
+            "prefix": self.prefix,
+            **self.other,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class AppliedRouterConfig:
+    reference: type[RouterInterface]
+    config: RouterConfig