fred-core 1.3.1__py3-none-any.whl

fred_core/__init__.py

@@ -0,0 +1,166 @@
+# Copyright Thales 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from fred_core.filesystem.local_filesystem import LocalFilesystem
+from fred_core.filesystem.minio_filesystem import MinioFilesystem
+from fred_core.filesystem.structures import (
+    BaseFilesystem,
+    FilesystemResourceInfo,
+    FilesystemResourceInfoResult,
+)
+from fred_core.logs.base_log_store import BaseLogStore
+from fred_core.logs.log_setup import StoreEmitHandler, log_setup
+from fred_core.logs.log_structures import (
+    InMemoryLogStorageConfig,
+    LogEventDTO,
+    LogFilter,
+    LogQuery,
+    LogQueryResult,
+    LogStorageConfig,
+    TailFileResponse,
+)
+from fred_core.logs.memory_log_store import RamLogStore
+from fred_core.logs.opensearch_log_store import OpenSearchLogStore
+from fred_core.model.factory import get_embeddings, get_model, get_structured_chain
+from fred_core.model.models import ModelProvider
+from fred_core.security.authorization import (
+    NO_AUTHZ_CHECK_USER,
+    TODO_PASS_REAL_USER,
+    Action,
+    AuthorizationError,
+    Resource,
+    authorize_or_raise,
+    is_authorized,
+    require_admin,
+)
+from fred_core.security.authorization_decorator import authorize
+from fred_core.security.backend_to_backend_auth import (
+    M2MAuthConfig,
+    M2MBearerAuth,
+    M2MTokenProvider,
+    make_m2m_asgi_client,
+)
+from fred_core.security.keycloak.keycloack_admin_client import (
+    KeycloackDisabled,
+    create_keycloak_admin,
+)
+from fred_core.security.oidc import (
+    decode_jwt,
+    get_current_user,
+    get_keycloak_client_id,
+    get_keycloak_url,
+    initialize_user_security,
+    oauth2_scheme,
+    split_realm_url,
+)
+from fred_core.security.outbound import BearerAuth, ClientCredentialsProvider
+from fred_core.security.rbac import RBACProvider
+from fred_core.security.rebac.openfga_engine import OpenFgaRebacEngine
+from fred_core.security.rebac.rebac_engine import (
+    ORGANIZATION_ID,
+    AgentPermission,
+    DocumentPermission,
+    OrganizationPermission,
+    RebacDisabledResult,
+    RebacEngine,
+    RebacPermission,
+    RebacReference,
+    Relation,
+    RelationType,
+    TagPermission,
+    TeamPermission,
+)
+from fred_core.security.rebac.rebac_factory import rebac_factory
+from fred_core.security.structure import (
+    KeycloakUser,
+    M2MSecurity,
+    OpenFgaRebacConfig,
+    RebacConfiguration,
+    SecurityConfiguration,
+    UserSecurity,
+)
+from fred_core.session import SessionSchema
+from fred_core.session.stores import BaseSessionStore, PostgresSessionStore
+
+__all__ = [
+    "BaseLogStore",
+    "LogEventDTO",
+    "LogFilter",
+    "LogQuery",
+    "LogQueryResult",
+    "OpenSearchLogStore",
+    "RamLogStore",
+    "StoreEmitHandler",
+    "TailFileResponse",
+    "log_setup",
+    "LogStorageConfig",
+    "InMemoryLogStorageConfig",
+    "get_current_user",
+    "decode_jwt",
+    "initialize_user_security",
+    "KeycloakUser",
+    "SecurityConfiguration",
+    "M2MSecurity",
+    "RebacConfiguration",
+    "UserSecurity",
+    "TODO_PASS_REAL_USER",
+    "NO_AUTHZ_CHECK_USER",
+    "BaseFilesystem",
+    "LocalFilesystem",
+    "MinioFilesystem",
+    "FilesystemResourceInfoResult",
+    "FilesystemResourceInfo",
+    "RBACProvider",
+    "require_admin",
+    "Action",
+    "Resource",
+    "AuthorizationError",
+    "is_authorized",
+    "authorize_or_raise",
+    "authorize",
+    "oauth2_scheme",
+    "ClientCredentialsProvider",
+    "BearerAuth",
+    "M2MAuthConfig",
+    "M2MTokenProvider",
+    "M2MBearerAuth",
+    "make_m2m_asgi_client",
+    "split_realm_url",
+    "get_model",
+    "get_structured_chain",
+    "get_embeddings",
+    "ModelProvider",
+    "BaseSessionStore",
+    "PostgresSessionStore",
+    "SessionSchema",
+    "RebacReference",
+    "Relation",
+    "RelationType",
+    "TagPermission",
+    "DocumentPermission",
+    "TeamPermission",
+    "ORGANIZATION_ID",
+    "AgentPermission",
+    "OrganizationPermission",
+    "RebacPermission",
+    "RebacDisabledResult",
+    "RebacEngine",
+    "OpenFgaRebacEngine",
+    "OpenFgaRebacConfig",
+    "rebac_factory",
+    "get_keycloak_url",
+    "get_keycloak_client_id",
+    "KeycloackDisabled",
+    "create_keycloak_admin",
+]

fred_core/common/__init__.py

@@ -0,0 +1,62 @@
+# Copyright Thales 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .config_files import ConfigFiles
+from .config_loader import (
+    load_configuration_with_config_files,
+    parse_yaml_mapping_file,
+)
+from .env import coerce_bool, read_env_bool
+from .fastapi_handlers import register_exception_handlers
+from .lru_cache import ThreadSafeLRUCache
+from .structures import (
+    BaseModelWithId,
+    DuckdbStoreConfig,
+    LogStoreConfig,
+    ModelConfiguration,
+    OpenSearchIndexConfig,
+    OpenSearchStoreConfig,
+    OwnerFilter,
+    PostgresStoreConfig,
+    PostgresTableConfig,
+    SQLStorageConfig,
+    StoreConfig,
+    TemporalSchedulerConfig,
+)
+from .team_id import TeamId
+from .utils import raise_internal_error
+
+__all__ = [
+    "BaseModelWithId",
+    "ConfigFiles",
+    "DuckdbStoreConfig",
+    "LogStoreConfig",
+    "ModelConfiguration",
+    "OpenSearchIndexConfig",
+    "OpenSearchStoreConfig",
+    "OwnerFilter",
+    "PostgresStoreConfig",
+    "PostgresTableConfig",
+    "SQLStorageConfig",
+    "StoreConfig",
+    "TeamId",
+    "TemporalSchedulerConfig",
+    "ThreadSafeLRUCache",
+    "coerce_bool",
+    "load_configuration_with_config_files",
+    "parse_yaml_mapping_file",
+    "raise_internal_error",
+    "read_env_bool",
+    "register_exception_handlers",
+]

fred_core/common/config_files.py

@@ -0,0 +1,135 @@
+# Copyright Thales 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import logging
+import os
+
+from dotenv import load_dotenv
+
+
+class ConfigFiles:
+    """Resolve and track startup config paths for Fred backends.
+
+    Why this exists:
+    - Every backend starts the same way: load environment variables, then load a
+      YAML configuration file.
+    - Developers and operators should see the exact files that were used.
+
+    Example:
+    - `ENV_FILE=./config/.env.prod`
+    - `CONFIG_FILE=./config/configuration_prod.yaml`
+    """
+
+    def __init__(
+        self,
+        *,
+        logger: logging.Logger,
+        default_env_file: str = "./config/.env",
+        default_config_file: str = "./config/configuration.yaml",
+        env_var_name: str = "ENV_FILE",
+        config_var_name: str = "CONFIG_FILE",
+        log_prefix: str = "[CONFIG]",
+    ) -> None:
+        """Create a resolver with Fred defaults.
+
+        Example:
+        - Keep defaults for regular startup.
+        - Override `default_config_file` in tests to point to a fixture.
+        """
+        self._logger = logger
+        self._default_env_file = default_env_file
+        self._default_config_file = default_config_file
+        self._env_var_name = env_var_name
+        self._config_var_name = config_var_name
+        self._log_prefix = log_prefix
+        self._loaded_env_file_path: str | None = None
+        self._loaded_config_file_path: str | None = None
+
+    def get_loaded_env_file_path(self) -> str | None:
+        """Return the effective env file path used at runtime.
+
+        Example:
+        - Returns `./config/.env` when no override is provided.
+        - Returns `/etc/fred/agentic.env` in a production deployment override.
+        """
+        return self._loaded_env_file_path
+
+    def get_loaded_config_file_path(self) -> str | None:
+        """Return the effective YAML config file path used at runtime.
+
+        Example:
+        - `./config/configuration.yaml` in local mode.
+        - `./config/configuration_worker.yaml` for worker startup.
+        """
+        return self._loaded_config_file_path
+
+    def load_environment(self, dotenv_path: str | None = None) -> str:
+        """Load environment variables from the selected env file.
+
+        Selection order:
+        1. Explicit `dotenv_path` argument.
+        2. `ENV_FILE` environment variable.
+        3. Default `./config/.env`.
+
+        Example:
+        - Calling `load_environment()` with `ENV_FILE=./config/.env.prod`
+          loads production secrets and returns that path.
+        """
+        env_path = dotenv_path or os.getenv(self._env_var_name, self._default_env_file)
+        if load_dotenv(env_path):
+            self._logger.info(
+                "%s Loaded environment variables from: %s",
+                self._log_prefix,
+                env_path,
+            )
+        else:
+            self._logger.warning(
+                "%s No .env file found at: %s",
+                self._log_prefix,
+                env_path,
+            )
+        self._loaded_env_file_path = env_path
+        return env_path
+
+    def resolve_config_file_path(self, config_file: str | None = None) -> str:
+        """Resolve and validate the YAML configuration path.
+
+        Selection order:
+        1. Explicit `config_file` argument.
+        2. `CONFIG_FILE` environment variable.
+        3. Default `./config/configuration.yaml`.
+
+        Raises:
+        - `FileNotFoundError` if the resolved file does not exist.
+        """
+        resolved = config_file or os.getenv(
+            self._config_var_name, self._default_config_file
+        )
+        if not os.path.exists(resolved):
+            raise FileNotFoundError(f"Configuration file not found: {resolved}")
+        return resolved
+
+    def mark_config_loaded(self, config_file: str) -> None:
+        """Record and log the configuration file effectively loaded.
+
+        Example:
+        - After parsing `configuration_prod.yaml`, call this method so startup
+          logs and diagnostics expose the exact profile in use.
+        """
+        self._loaded_config_file_path = config_file
+        self._logger.info(
+            "%s Loaded configuration from: %s", self._log_prefix, config_file
+        )

fred_core/common/config_loader.py

@@ -0,0 +1,47 @@
+# Copyright Thales 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Callable, TypeVar
+
+import yaml
+
+from .config_files import ConfigFiles
+
+TConfig = TypeVar("TConfig")
+
+
+def parse_yaml_mapping_file(config_file: str) -> dict:
+    """Load a YAML file and ensure it is a non-empty mapping."""
+    with open(config_file, encoding="utf-8") as file:
+        payload = yaml.safe_load(file)
+    if payload is None:
+        raise ValueError(f"Configuration file is empty: {config_file}")
+    if not isinstance(payload, dict):
+        raise ValueError(f"Configuration file must be a mapping object: {config_file}")
+    return payload
+
+
+def load_configuration_with_config_files(
+    config_files: ConfigFiles,
+    parser: Callable[[str], TConfig],
+    dotenv_path: str | None = None,
+) -> TConfig:
+    """Load env + config path using ConfigFiles and parse via callback."""
+    config_files.load_environment(dotenv_path)
+    config_file = config_files.resolve_config_file_path()
+    configuration = parser(config_file)
+    config_files.mark_config_loaded(config_file)
+    return configuration

fred_core/common/env.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import os
+from typing import Any
+
+"""
+Central boolean parsing helpers for environment flags and loose runtime payloads.
+
+Why this file exists:
+- The same string checks (`"1"`, `"true"`, `"yes"`, `"on"`) were duplicated in
+  multiple services.
+- One shared helper keeps behavior consistent and removes repeated code.
+
+How to use:
+```python
+from fred_core.common import read_env_bool, coerce_bool
+
+docs_enabled = read_env_bool("PRODUCTION_FASTAPI_DOCS_ENABLED", default=True)
+requires_approval = coerce_bool(payload.get("requires_approval"), default=False)
+```
+"""
+
+_TRUTHY_STRINGS: frozenset[str] = frozenset({"1", "true", "yes", "on", "y"})
+_FALSY_STRINGS: frozenset[str] = frozenset({"0", "false", "no", "off", "n"})
+
+
+def coerce_bool(value: Any, default: bool = False) -> bool:
+    """
+    Convert common runtime inputs to bool.
+
+    Supported input types:
+    - `bool`: returned as-is
+    - `int` / `float`: zero => False, non-zero => True
+    - `str`: normalized and matched against known true/false tokens
+    - anything else: fallback to `default`
+    """
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, (int, float)):
+        return value != 0
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        if normalized in _TRUTHY_STRINGS:
+            return True
+        if normalized in _FALSY_STRINGS:
+            return False
+    return default
+
+
+def read_env_bool(name: str, default: bool) -> bool:
+    """
+    Read and parse a boolean environment variable.
+
+    Use this for runtime flags loaded from environment variables.
+    """
+    return coerce_bool(os.getenv(name), default=default)

fred_core/common/fastapi_handlers.py

@@ -0,0 +1,74 @@
+# Copyright Thales 2025
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+
+from fred_core.security.models import AuthorizationError, Resource
+
+logger = logging.getLogger(__name__)
+
+_TEAM_PERMISSION_MESSAGES: dict[str, str] = {
+    "can_update_resources": "You are not allowed to manage resources in this team. Ask a team owner or manager.",
+    "can_update_agents": "You are not allowed to manage agents in this team. Ask a team owner or manager.",
+    "can_administer_members": "You are not allowed to manage members in this team.",
+    "can_administer_managers": "You are not allowed to manage managers in this team.",
+    "can_administer_owners": "You are not allowed to manage owners in this team.",
+    "can_read_members": "You are not allowed to view team members.",
+    "can_update_info": "You are not allowed to update this team.",
+}
+
+
+def _humanize_action(action: str) -> str:
+    return action.replace(":", " ").replace("_", " ")
+
+
+def _authorization_detail_for_client(exc: AuthorizationError) -> str:
+    action = str(exc.action)
+    if exc.resource == Resource.TEAM and action in _TEAM_PERMISSION_MESSAGES:
+        return _TEAM_PERMISSION_MESSAGES[action]
+
+    readable_action = _humanize_action(action)
+    readable_resource = exc.resource.value.replace("_", " ")
+    return f"You are not allowed to {readable_action} {readable_resource}."
+
+
+def register_exception_handlers(app: FastAPI) -> None:
+    """Register authorization and generic exception handlers for FastAPI application."""
+
+    @app.exception_handler(AuthorizationError)
+    async def authorization_error_handler(
+        request: Request, exc: AuthorizationError
+    ) -> JSONResponse:
+        """Handle AuthorizationError by returning a 403 Forbidden response."""
+        logger.warning(f"Authorization denied for user {exc.user_id}: {exc}")
+        return JSONResponse(
+            status_code=403,
+            content={"detail": _authorization_detail_for_client(exc)},
+        )
+
+    @app.exception_handler(Exception)
+    async def generic_exception_handler(
+        request: Request, exc: Exception
+    ) -> JSONResponse:
+        """Handle all unhandled exceptions by logging and returning 500."""
+        logger.error(
+            f"Unhandled exception in {request.method} {request.url}: {exc}",
+            exc_info=True,
+        )
+        return JSONResponse(
+            status_code=500, content={"detail": "Internal server error"}
+        )

fred_core/common/lru_cache.py

@@ -0,0 +1,43 @@
+from collections import OrderedDict
+from threading import Lock
+from typing import Generic, Optional, TypeVar
+
+K = TypeVar("K")
+V = TypeVar("V")
+
+
+class ThreadSafeLRUCache(Generic[K, V]):
+    def __init__(self, max_size: int = 1000):
+        self._max_size = max_size
+        self._lock = Lock()
+        self._cache: OrderedDict[K, V] = OrderedDict()
+
+    def get(self, key: K) -> V | None:
+        with self._lock:
+            value = self._cache.get(key)
+            if value is not None:
+                self._cache.move_to_end(key)  # Mark as recently used
+            return value
+
+    def set(self, key: K, value: V) -> None:
+        with self._lock:
+            self._cache[key] = value
+            self._cache.move_to_end(key)
+            if len(self._cache) > self._max_size:
+                self._cache.popitem(last=False)  # Remove LRU
+
+    def delete(self, key: K) -> Optional[V]:
+        with self._lock:
+            return self._cache.pop(key, None)
+
+    def keys(self) -> list[K]:
+        with self._lock:
+            return list(self._cache.keys())
+
+    def clear(self) -> None:
+        with self._lock:
+            self._cache.clear()
+
+    def __contains__(self, key: K) -> bool:
+        with self._lock:
+            return key in self._cache