clue-api 1.5.0.dev231__tar.gz → 1.5.0.dev238__tar.gz

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: clue-api
-Version: 1.5.0.dev231
+Version: 1.5.0.dev238
 Summary: Clue distributed enrichment service
 License: MIT
 License-File: LICENSE
@@ -39,6 +39,7 @@ Requires-Dist: prometheus-client (>=0.20.0,<0.21.0) ; extra == "server"
 Requires-Dist: pydantic (>=2.7.1,<3.0.0)
 Requires-Dist: pydantic-settings[yaml] (>=2.3.4,<3.0.0)
 Requires-Dist: pyjwt (>=2.8.0,<3.0.0) ; extra == "server"
+Requires-Dist: pymongo (>=4.16.0,<5.0.0) ; extra == "server"
 Requires-Dist: pyroute2 (>=0.7.12,<0.8.0) ; extra == "server"
 Requires-Dist: python-baseconv (>=1.2.2,<2.0.0) ; extra == "server"
 Requires-Dist: pytz (>=2024.1,<2025.0) ; extra == "server"

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/api/__init__.py

@@ -1,9 +1,10 @@
 from sys import exc_info
 from traceback import format_tb
-from typing import Any, Union
+from typing import Any, Union, cast
 
 from flask import Blueprint, Response, make_response, request
 from prometheus_client import Counter
+from pydantic import BaseModel
 
 from clue.common.forge import APP_NAME
 from clue.common.logging import get_logger, log_with_traceback
@@ -33,6 +34,13 @@ def _make_api_response(
         err = "".join(["\n"] + format_tb(trace) + ["%s: %s\n" % (err.__class__.__name__, str(err))]).rstrip("\n")
         log_with_traceback(trace, "Exception", is_exception=True)
 
+    if isinstance(data, BaseModel):
+        data = data.model_dump(mode="json", by_alias=True, exclude_none=True)
+    elif isinstance(data, list) and len(data) > 0 and isinstance(data[0], BaseModel):
+        data = [
+            entry.model_dump(mode="json", by_alias=True, exclude_none=True) for entry in cast(list[BaseModel], data)
+        ]
+
     resp = make_response(
         ClueResponse(response=data, error_message=err, warning=warnings, status_code=status_code).model_dump(
             mode="json", by_alias=True, exclude_none=True

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/api/v1/static.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 from flask import request
@@ -12,10 +13,12 @@ from clue.security.utils import is_path_traversal
 
 SUB_API = "static"
 static_api = make_subapi_blueprint(SUB_API, api_version=1)
-static_api._doc = "Fetch static documentation"
+static_api._doc = "Fetch static documentation"  # type: ignore
 
 CORS(static_api, origins=config.ui.cors_origins, supports_credentials=True)
 
+DOCUMENTATION_FOLDER = (Path(os.environ.get("CLUE_DOCUMENTATION_PATH", Path.cwd() / "docs"))).resolve()
+
 logger = get_logger(__file__)
 
 
@@ -39,23 +42,15 @@ def serve_documentation(**kwargs) -> dict[str, str]:
     """
     docs_filter = request.args.get("filter")
 
-    documentation_folder = Path.cwd() / "docs"
-
     returned_files = {}
 
-    if docs_filter is None:
-        for file in documentation_folder.rglob("*"):
-            if file.is_file():
-                content = file.read_text(encoding="utf-8")
-                returned_files[file.name] = content
-    else:
-        for file in documentation_folder.rglob("*"):
-            if file.is_file() and docs_filter in file.name:
-                try:
-                    content = file.read_text(encoding="utf-8")
-                    returned_files[file.name] = content
-                except FileNotFoundError:
-                    return not_found(err="The file was not found")
+    for file in DOCUMENTATION_FOLDER.rglob("*"):
+        if file.is_file():
+            if docs_filter and docs_filter not in file.name:
+                continue
+
+            content = file.read_text(encoding="utf-8")
+            returned_files[file.name] = content
 
     return ok(returned_files)
 
@@ -78,17 +73,15 @@ def serve_documentation_file(filename: str, **kwargs) -> dict[str, str]:
     {"markdown": "Markdown documentation of howler-docs.md"}
 
     """
-    documentation_folder = (Path.cwd() / "docs").resolve()
-
-    docs_path = (documentation_folder / filename).resolve()
+    docs_path = (DOCUMENTATION_FOLDER / filename).resolve()
 
-    if is_path_traversal(documentation_folder, docs_path):
+    if is_path_traversal(DOCUMENTATION_FOLDER, docs_path):
         return not_found(err="The file does not exist or is typed incorrectly within the relative path.")
 
-    if documentation_folder.exists():
-        content = documentation_folder.read_text(encoding="utf-8")
+    if docs_path.exists():
+        content = docs_path.read_text(encoding="utf-8")
 
         return ok({"markdown": content})
 
-    logger.info("File %s does not exist", documentation_folder)
+    logger.info("File %s does not exist", docs_path)
     return not_found(err="The file does not exist or is typed incorrectly.")

clue_api-1.5.0.dev238/clue/api/v1/sync.py

@@ -0,0 +1,155 @@
+from typing import Any
+
+from flask import request
+from flask_cors import CORS
+from pydantic import TypeAdapter, ValidationError
+
+from clue.api import bad_request, forbidden, internal_error, make_subapi_blueprint, ok
+from clue.common.logging import get_logger
+from clue.common.swagger import generate_swagger_docs
+from clue.config import config
+from clue.models.sync import ChangeRow
+from clue.security import api_login
+from clue.services import mongo_service
+
+SUB_API = "sync"
+sync_api = make_subapi_blueprint(SUB_API, api_version=1)
+sync_api._doc = "Replication functionality for RxDB"  # type: ignore
+
+CORS(sync_api, origins=config.ui.cors_origins, supports_credentials=True)
+
+logger = get_logger(__file__)
+
+
+@generate_swagger_docs()
+@sync_api.route("/<collection>", methods=["GET"])
+@api_login()
+def pull(collection: str, user: dict[str, Any] | None = None, **kwargs) -> dict[str, str]:
+    """Pull replicated changes from a collection since a specified checkpoint.
+
+    Variables:
+    collection => The name of the collection to pull from.
+
+    Optional Arguments:
+    updated_at: int     => Timestamp of the last checkpoint. [Default: 0]
+    id: string          => Document ID of the last checkpoint for pagination.
+    limit: int          => Maximum number of records to return per batch. [Default: 10]
+    omit_deleted        => If present, omit deleted records from the results.
+
+    Result Example:
+    [   # List of SelectorDocument records since the given checkpoint
+        {
+            "id": "<document id>",
+            "updated_at": 1234567890,
+            "_deleted": false,
+            ...
+        },
+        ...
+    ]
+    """
+    if not user:
+        return forbidden(err="You must be logged in as a valid user.")
+
+    if collection not in mongo_service.ALLOWED_COLLECTIONS:
+        return bad_request(err=f"Unknown collection: {collection}")
+
+    updated_at = request.args.get("updated_at", 0, type=int)
+    id: str | None = request.args.get("id", None)
+    limit = request.args.get("limit", 10, type=int)
+    omit_deleted = "omit_deleted" in request.args
+
+    return ok(
+        mongo_service.pull(user["uname"], collection, id, updated_at, batch_size=limit, omit_deleted=omit_deleted)
+    )
+
+
+@generate_swagger_docs()
+@sync_api.route("/<collection>/stream", methods=["GET"])
+@api_login()
+def stream(collection: str, user: dict[str, Any] | None = None, **kwargs):
+    """Stream replicated changes from a collection as server-sent events.
+
+    Variables:
+    collection => The name of the collection to stream from.
+
+    Arguments:
+    None
+
+    Result Example:
+    {
+        # A continuous text/event-stream (SSE) of JSON-encoded change events
+        "id": "<event id>",
+        "documents": [{...}, ...],
+        "checkpoint": {
+            "id": "<id>",
+            "updated_at": 1234567890
+        }
+    }
+    """
+    if not user:
+        return forbidden(err="You must be logged in as a valid user.")
+
+    if collection not in mongo_service.ALLOWED_COLLECTIONS:
+        return bad_request(err=f"Unknown collection: {collection}")
+
+    logger.info("Initializing event source stream")
+
+    return mongo_service.event_stream(user["uname"], collection)
+
+
+@generate_swagger_docs()
+@sync_api.route("/<collection>", methods=["POST"])
+@api_login()
+def push(collection: str, user: dict[str, Any] | None = None, **kwargs) -> dict[str, str]:
+    """Push replicated changes to a collection.
+
+    Variables:
+    collection => The name of the collection to push to.
+
+    Arguments:
+    None
+
+    Data Block:
+    [   # List of change rows for RxDB replication
+        {
+            "newDocumentState": {   # Required. The new state of the document.
+                "id": "<document id>",
+                "updated_at": 1234567890,
+                "_deleted": false,
+                ...
+            },
+            "assumedMasterState": { # Optional. The assumed current server state for conflict detection.
+                "id": "<document id>",
+                "updated_at": 1234567890,
+                ...
+            }
+        },
+        ...
+    ]
+
+    Result Example:
+    [   # List of conflicting SelectorDocuments that were not applied
+        {
+            "id": "<document id>",
+            "updated_at": 1234567890,
+            "_deleted": false,
+            ...
+        },
+        ...
+    ]
+    """
+    if not user:
+        return forbidden(err="You must be logged in as a valid user.")
+
+    if collection not in mongo_service.ALLOWED_COLLECTIONS:
+        return bad_request(err=f"Unknown collection: {collection}")
+
+    try:
+        change_rows = TypeAdapter(list[ChangeRow]).validate_python(request.json, strict=True, by_alias=True)
+
+        return ok(mongo_service.push(user["uname"], collection, change_rows))
+    except ValidationError:
+        logger.exception("Validation exception on push")
+        return bad_request(err="Invalid replication data.")
+    except Exception:
+        return internal_error(err="Failed to process replication data.")

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/app.py

@@ -59,6 +59,7 @@ from clue.api.v1.fetchers import fetchers_api
 from clue.api.v1.lookup import lookup_api
 from clue.api.v1.registration import registration_api
 from clue.api.v1.static import static_api
+from clue.api.v1.sync import sync_api
 from clue.common.logging import get_logger
 from clue.cronjobs import setup_jobs as setup_cron_jobs
 from clue.error import errors
@@ -132,6 +133,7 @@ app.register_blueprint(fetchers_api)
 app.register_blueprint(lookup_api)
 app.register_blueprint(registration_api)
 app.register_blueprint(static_api)
+app.register_blueprint(sync_api)
 
 
 logger.info("Checking extensions for initialization and additional routes")

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/common/forge.py

@@ -1,7 +1,6 @@
 # This file contains the loaders for the different components of the system
 from __future__ import annotations
 
-import logging
 import os
 from pathlib import Path
 from string import Template
@@ -10,7 +9,7 @@ from typing import TYPE_CHECKING
 from flask_caching import Cache
 
 from clue.common.dict_utils import recursive_update
-from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
+from clue.common.logging import get_module_logger
 from clue.common.str_utils import default_string_value
 
 APP_NAME: str = default_string_value(env_name="APP_NAME", default="clue")  # type: ignore[assignment]
@@ -23,12 +22,7 @@ cache = Cache(config={"CACHE_TYPE": "SimpleCache"})
 
 classification_engines: dict[Path, Classification] = {}
 
-logger = logging.getLogger(f"{APP_NAME}.common.forge")
-logger.setLevel(logging.INFO)
-console = logging.StreamHandler()
-console.setLevel(logging.INFO)
-console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
-logger.addHandler(console)
+logger = get_module_logger(f"{APP_NAME}.common.forge")
 
 
 def __get_yml_path(yml_config: str | None = None) -> Path | None:  # noqa: C901

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/common/logging/__init__.py

@@ -29,6 +29,29 @@ LOG_LEVEL_MAP = {
 DEBUG = False
 
 
+def get_module_logger(name: str, level: int = logging.INFO) -> logging.Logger:
+    """Create and return a logger with a pre-configured console handler.
+
+    A StreamHandler is added only when the logger has no handlers yet, so
+    calling this function multiple times for the same name is safe.
+
+    Args:
+        name: The dotted logger name (e.g. ``"clue.models.config"``).
+        level: The logging level to set on both the logger and the handler.
+
+    Returns:
+        The configured :class:`logging.Logger` instance.
+    """
+    _logger = logging.getLogger(name)
+    _logger.setLevel(level)
+    if not _logger.handlers:
+        _console = logging.StreamHandler()
+        _console.setLevel(level)
+        _console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
+        _logger.addHandler(_console)
+    return _logger
+
+
 class JsonFormatter(logging.Formatter):
     """A custom implementation of logging.Formatter that supports json logs as well as traceback for exceptions.
 
@@ -61,7 +84,7 @@ class JsonFormatter(logging.Formatter):
 
         return self._style.format(record)
 
-    def formatException(self, exc_info):  # noqa: N802
+    def formatException(self, exc_info):  # type: ignore # noqa: N802
         """Formats the exception using traceback
 
         Args:

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/config.py

@@ -1,3 +1,7 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
 import os
 
 from clue.common import forge

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/extensions/config.py

@@ -1,4 +1,3 @@
-import logging
 from typing import Any
 
 from pydantic import BaseModel, ImportString, model_validator
@@ -8,14 +7,9 @@ from pydantic_settings import (
     YamlConfigSettingsSource,
 )
 
-from clue.common.logging import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
+from clue.common.logging import get_module_logger
 
-logger = logging.getLogger("clue.extensions.config")
-logger.setLevel(logging.INFO)
-console = logging.StreamHandler()
-console.setLevel(logging.INFO)
-console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
-logger.addHandler(console)
+logger = get_module_logger("clue.extensions.config")
 
 
 class Modules(BaseModel):

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/models/config.py

@@ -1,10 +1,9 @@
 # ruff: noqa: D101
-import logging
 import os
 from email.utils import parseaddr
 from enum import Enum
 from pathlib import Path
-from typing import Self
+from typing import Annotated, Any, Self
 from uuid import uuid4
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
@@ -18,7 +17,7 @@ from pydantic_settings import (
 
 from clue.common import forge
 from clue.common.exceptions import ClueValueError
-from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
+from clue.common.logging import get_module_logger
 from clue.common.str_utils import default_string_value
 
 AUTO_PROPERTY_TYPE = ["access", "classification", "type", "role", "remove_role", "group"]
@@ -28,6 +27,8 @@ DEFAULT_USER_NAME_FIELDS = ["name", "displayName"]
 APP_NAME = default_string_value(env_name="APP_NAME", default="clue").replace("-dev", "")  # type: ignore[union-attr]
 CLASSIFICATION = forge.get_classification()
 
+logger = get_module_logger("clue.models.config")
+
 
 class PasswordRequirement(BaseModel):
     lower: bool = Field(description="Password must contain lowercase letters", default=False)
@@ -213,7 +214,52 @@ class APMServer(BaseModel):
 class Metrics(BaseModel):
     apm_server: APMServer = APMServer()
     export_interval: int = Field(description="How often should we be exporting metrics?", default=5)
-    redis: RedisServer = RedisServer()
+
+
+class MongoDB(BaseModel):
+    host: str = Field(description="Hostname of the MongoDB instance", default="mongodb")
+    port: int = Field(description="Port of the MongoDB instance", default=27017, ge=1, le=65535)
+    user: str | None = Field(description="Username to use to connect to the MongoDB instance", default=None)
+    password: str | None = Field(description="Password to use to connect to the MongoDB instance", default=None)
+    database: str = Field(description="The database to use in the mongodb instance", default="clue")
+    max_retries: int = Field(
+        description="Controls the maximum number of retries to use when an initial connection fails", default=2
+    )
+    connect_timeout: int = Field(
+        description="Controls how long (in milliseconds) to wait when connecting a new socket", default=3000
+    )
+    server_selection_timeout: int = Field(
+        description="Controls how long (in milliseconds) to wait for a suitable server to be found", default=3000
+    )
+
+    def __repr__(self):
+        auth = ""
+        if self.user and self.password:
+            auth = f"{self.user}:***@"
+
+        return f"mongodb://{auth}{self.host}:{self.port}"
+
+    def connection(self) -> dict[str, Any]:
+        """Generate MongoDB connection string and authentication parameters.
+
+        Returns:
+            dict[str, str | int]: A dictionary of connection parameters including host, port,
+                connection timeouts, and optionally username and password if available.
+        """
+        params: dict[str, str | int] = {
+            "host": self.host,
+            "port": self.port,
+            "connectTimeoutMS": self.connect_timeout,
+            "serverSelectionTimeoutMS": self.server_selection_timeout,
+        }
+
+        if self.user and self.password:
+            params["username"] = self.user
+            params["password"] = self.password
+        else:
+            logger.warning("No authentication used for mongodb.")
+
+        return params
 
 
 class Core(BaseModel):
@@ -225,6 +271,9 @@ class Core(BaseModel):
     redis: RedisServer = RedisServer()
     "Configuration for Redis instances"
 
+    mongodb: MongoDB = MongoDB()
+    "Configuration for MongoDB instance"
+
 
 class LogLevel(str, Enum):
     DEBUG = "DEBUG"
@@ -363,6 +412,7 @@ class OBOService(BaseModel):
 
 class UI(BaseModel):
     cors_origins: list[str] = Field(default=[], description="List of valid deployments")
+    replication: bool = Field(default=True, description="Should server-side replication be enabled?")
 
 
 class API(BaseModel):
@@ -386,6 +436,13 @@ class API(BaseModel):
     )
 
 
+class Retention(BaseModel):
+    enabled: Annotated[bool, Field(description="Should records be cached for users?")] = True
+    default_ttl: Annotated[
+        int, Field(description="The number of seconds a record with no set expiry should be cached")
+    ] = 3600
+
+
 root_path = Path("/etc") / APP_NAME
 
 config_locations = [
@@ -396,13 +453,6 @@ config_locations = [
 if os.getenv("AZURE_TEST_CONFIG", None) is not None:
     import re
 
-    logger = logging.getLogger("clue.models.config")
-    logger.setLevel(logging.INFO)
-    console = logging.StreamHandler()
-    console.setLevel(logging.INFO)
-    console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
-    logger.addHandler(console)
-
     logger.info("Azure build environment detected, adding additional config path")
 
     work_dir_parent = Path("/__w")
@@ -436,6 +486,7 @@ class Config(BaseSettings):
     auth: Auth = Auth()
     core: Core = Core()
     logging: Logging = Logging()
+    retention: Retention = Retention()
 
     model_config = SettingsConfigDict(
         yaml_file=config_locations,

{clue_api-1.5.0.dev231 → clue_api-1.5.0.dev238}/clue/models/fetchers.py

@@ -106,7 +106,10 @@ class FetcherResult(BaseModel, Generic[DATA]):
         description="Did the fetcher succeed or fail, or is it pending?"
     )
     data: DATA | None = Field(description="The output of the fetcher.", default=None)
-    error: str | None = Field(description="If the fetcher failed, contains the relevant error message.", default=None)
+    error: str | None = Field(
+        description="If the fetcher failed, contains the relevant error message.",
+        default=None,
+    )
     format: str | None = Field(
         description="What is the format of the output? Used to indicate what component to use when rendering "
         "the output.",