diracx-testing 0.0.1a16__tar.gz → 0.0.1a18__tar.gz

{diracx_testing-0.0.1a16 → diracx_testing-0.0.1a18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: diracx-testing
-Version: 0.0.1a16
+Version: 0.0.1a18
 Summary: TODO
 License: GPL-3.0-only
 Classifier: Intended Audience :: Science/Research
@@ -8,7 +8,7 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Distributed Computing
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: pytest
 Requires-Dist: pytest-asyncio

{diracx_testing-0.0.1a16 → diracx_testing-0.0.1a18}/pyproject.toml

@@ -2,7 +2,7 @@
 name = "diracx-testing"
 description = "TODO"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 keywords = []
 license = {text = "GPL-3.0-only"}
 classifiers = [

{diracx_testing-0.0.1a16 → diracx_testing-0.0.1a18}/src/diracx/testing/__init__.py

@@ -8,6 +8,7 @@ import os
 import re
 import subprocess
 from datetime import datetime, timedelta, timezone
+from functools import partial
 from html.parser import HTMLParser
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -18,6 +19,7 @@ import pytest
 import requests
 
 if TYPE_CHECKING:
+    from diracx.core.settings import DevelopmentSettings
     from diracx.routers.job_manager.sandboxes import SandboxStoreSettings
     from diracx.routers.utils.users import AuthorizedUserInfo, AuthSettings
 
@@ -46,9 +48,7 @@ def pytest_addoption(parser):
 
 
 def pytest_collection_modifyitems(config, items):
-    """
-    Disable the test_regenerate_client if not explicitly asked for
-    """
+    """Disable the test_regenerate_client if not explicitly asked for."""
     if config.getoption("--regenerate-client"):
         # --regenerate-client given in cli: allow client re-generation
         return
@@ -59,11 +59,11 @@ def pytest_collection_modifyitems(config, items):
 
 
 @pytest.fixture(scope="session")
-def rsa_private_key_pem() -> str:
+def private_key_pem() -> str:
     from cryptography.hazmat.primitives import serialization
-    from cryptography.hazmat.primitives.asymmetric import rsa
+    from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
 
-    private_key = rsa.generate_private_key(public_exponent=65537, key_size=4096)
+    private_key = Ed25519PrivateKey.generate()
     return private_key.private_bytes(
         encoding=serialization.Encoding.PEM,
         format=serialization.PrivateFormat.PKCS8,
@@ -79,11 +79,19 @@ def fernet_key() -> str:
 
 
 @pytest.fixture(scope="session")
-def test_auth_settings(rsa_private_key_pem, fernet_key) -> AuthSettings:
+def test_dev_settings() -> DevelopmentSettings:
+    from diracx.core.settings import DevelopmentSettings
+
+    yield DevelopmentSettings()
+
+
+@pytest.fixture(scope="session")
+def test_auth_settings(private_key_pem, fernet_key) -> AuthSettings:
     from diracx.routers.utils.users import AuthSettings
 
     yield AuthSettings(
-        token_key=rsa_private_key_pem,
+        token_algorithm="EdDSA",
+        token_key=private_key_pem,
         state_key=fernet_key,
         allowed_redirects=[
             "http://diracx.test.invalid:8000/api/docs/oauth2-redirect",
@@ -93,7 +101,7 @@ def test_auth_settings(rsa_private_key_pem, fernet_key) -> AuthSettings:
 
 @pytest.fixture(scope="session")
 def aio_moto(worker_id):
-    """Start the moto server in a separate thread and return the base URL
+    """Start the moto server in a separate thread and return the base URL.
 
     The mocking provided by moto doesn't play nicely with aiobotocore so we use
     the server directly. See https://github.com/aio-libs/aiobotocore/issues/755
@@ -142,18 +150,20 @@ class ClientFactory:
         with_config_repo,
         test_auth_settings,
         test_sandbox_settings,
+        test_dev_settings,
     ):
         from diracx.core.config import ConfigSource
         from diracx.core.extensions import select_from_extension
         from diracx.core.settings import ServiceSettingsBase
+        from diracx.db.os.utils import BaseOSDB
         from diracx.db.sql.utils import BaseSQLDB
         from diracx.routers import create_app_inner
         from diracx.routers.access_policies import BaseAccessPolicy
 
+        from .mock_osdb import fake_available_osdb_implementations
+
         class AlwaysAllowAccessPolicy(BaseAccessPolicy):
-            """
-            Dummy access policy
-            """
+            """Dummy access policy."""
 
             async def policy(
                 policy_name: str, user_info: AuthorizedUserInfo, /, **kwargs
@@ -171,9 +181,21 @@ class ClientFactory:
             e.name: "sqlite+aiosqlite:///:memory:"
             for e in select_from_extension(group="diracx.db.sql")
         }
+        # TODO: Monkeypatch this in a less stupid way
+        # TODO: Only use this if opensearch isn't available
+        os_database_conn_kwargs = {
+            e.name: {"sqlalchemy_dsn": "sqlite+aiosqlite:///:memory:"}
+            for e in select_from_extension(group="diracx.db.os")
+        }
+        BaseOSDB.available_implementations = partial(
+            fake_available_osdb_implementations,
+            real_available_implementations=BaseOSDB.available_implementations,
+        )
+
         self._cache_dir = tmp_path_factory.mktemp("empty-dbs")
 
         self.test_auth_settings = test_auth_settings
+        self.test_dev_settings = test_dev_settings
 
         all_access_policies = {
             e.name: [AlwaysAllowAccessPolicy]
@@ -186,11 +208,10 @@ class ClientFactory:
             all_service_settings=[
                 test_auth_settings,
                 test_sandbox_settings,
+                test_dev_settings,
             ],
             database_urls=database_urls,
-            os_database_conn_kwargs={
-                # TODO: JobParametersDB
-            },
+            os_database_conn_kwargs=os_database_conn_kwargs,
             config_source=ConfigSource.create_from_url(
                 backend_url=f"git+file://{with_config_repo}"
             ),
@@ -202,14 +223,20 @@ class ClientFactory:
         for obj in self.all_dependency_overrides:
             assert issubclass(
                 obj.__self__,
-                (ServiceSettingsBase, BaseSQLDB, ConfigSource, BaseAccessPolicy),
+                (
+                    ServiceSettingsBase,
+                    BaseSQLDB,
+                    BaseOSDB,
+                    ConfigSource,
+                    BaseAccessPolicy,
+                ),
             ), obj
 
         self.all_lifetime_functions = self.app.lifetime_functions[:]
         self.app.lifetime_functions = []
         for obj in self.all_lifetime_functions:
             assert isinstance(
-                obj.__self__, (ServiceSettingsBase, BaseSQLDB, ConfigSource)
+                obj.__self__, (ServiceSettingsBase, BaseSQLDB, BaseOSDB, ConfigSource)
             ), obj
 
     @contextlib.contextmanager
@@ -227,6 +254,7 @@ class ClientFactory:
                 self.app.dependency_overrides[k] = UnavailableDependency(class_name)
 
         for obj in self.all_lifetime_functions:
+            # TODO: We should use the name of the entry point instead of the class name
             if obj.__self__.__class__.__name__ in enabled_dependencies:
                 self.app.lifetime_functions.append(obj)
 
@@ -235,14 +263,15 @@ class ClientFactory:
         # already been ran
         self.app.lifetime_functions.append(self.create_db_schemas)
 
-        yield
-
-        self.app.dependency_overrides = {}
-        self.app.lifetime_functions = []
+        try:
+            yield
+        finally:
+            self.app.dependency_overrides = {}
+            self.app.lifetime_functions = []
 
     @contextlib.asynccontextmanager
     async def create_db_schemas(self):
-        """Create DB schema's based on the DBs available in app.dependency_overrides"""
+        """Create DB schema's based on the DBs available in app.dependency_overrides."""
         import aiosqlite
         import sqlalchemy
         from sqlalchemy.util.concurrency import greenlet_spawn
@@ -349,12 +378,18 @@ def session_client_factory(
     test_sandbox_settings,
     with_config_repo,
     tmp_path_factory,
+    test_dev_settings,
 ):
-    """
-    TODO
+    """TODO.
+    ----
+
     """
     yield ClientFactory(
-        tmp_path_factory, with_config_repo, test_auth_settings, test_sandbox_settings
+        tmp_path_factory,
+        with_config_repo,
+        test_auth_settings,
+        test_sandbox_settings,
+        test_dev_settings,
     )
 
 
@@ -378,7 +413,7 @@ def with_config_repo(tmp_path_factory):
 
     repo = Repo.init(tmp_path, initial_branch="master")
     cs_file = tmp_path / "default.yml"
-    example_cs = Config.parse_obj(
+    example_cs = Config.model_validate(
         {
             "DIRAC": {},
             "Registry": {
@@ -418,7 +453,7 @@ def with_config_repo(tmp_path_factory):
             "Operations": {"Defaults": {}},
         }
     )
-    cs_file.write_text(example_cs.json())
+    cs_file.write_text(example_cs.model_dump_json())
     repo.index.add([cs_file])  # add it to the index
     repo.index.commit("Added a new file")
     yield tmp_path
@@ -443,7 +478,7 @@ def demo_urls(demo_dir):
 
 @pytest.fixture(scope="session")
 def demo_kubectl_env(demo_dir):
-    """Get the dictionary of environment variables for kubectl to control the demo"""
+    """Get the dictionary of environment variables for kubectl to control the demo."""
     kube_conf = demo_dir / "kube.conf"
     if not kube_conf.exists():
         raise RuntimeError(f"Could not find {kube_conf}, is the demo running?")
@@ -465,7 +500,7 @@ def demo_kubectl_env(demo_dir):
 
 @pytest.fixture
 def cli_env(monkeypatch, tmp_path, demo_urls, demo_dir):
-    """Set up the environment for the CLI"""
+    """Set up the environment for the CLI."""
     import httpx
 
     from diracx.core.preferences import get_diracx_preferences
@@ -561,7 +596,7 @@ async def test_login(monkeypatch, capfd, cli_env):
 
 
 def do_device_flow_with_dex(url: str, ca_path: str) -> None:
-    """Do the device flow with dex"""
+    """Do the device flow with dex."""
 
     class DexLoginFormParser(HTMLParser):
         def handle_starttag(self, tag, attrs):

diracx_testing-0.0.1a18/src/diracx/testing/dummy_osdb.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import secrets
+
+from diracx.db.os.utils import BaseOSDB
+
+
+class DummyOSDB(BaseOSDB):
+    """Example DiracX OpenSearch database class for testing.
+
+    A new random prefix is created each time the class is defined to ensure
+    test runs are independent of each other.
+    """
+
+    fields = {
+        "DateField": {"type": "date"},
+        "IntField": {"type": "long"},
+        "KeywordField0": {"type": "keyword"},
+        "KeywordField1": {"type": "keyword"},
+        "KeywordField2": {"type": "keyword"},
+        "TextField": {"type": "text"},
+    }
+
+    def __init__(self, *args, **kwargs):
+        # Randomize the index prefix to ensure tests are independent
+        self.index_prefix = f"dummy_{secrets.token_hex(8)}"
+        super().__init__(*args, **kwargs)
+
+    def index_name(self, doc_id: int) -> str:
+        return f"{self.index_prefix}-{doc_id // 1e6:.0f}m"

diracx_testing-0.0.1a18/src/diracx/testing/mock_osdb.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+__all__ = (
+    "MockOSDBMixin",
+    "fake_available_osdb_implementations",
+)
+
+import contextlib
+from datetime import datetime, timezone
+from functools import partial
+from typing import Any, AsyncIterator
+
+from sqlalchemy import select
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+
+from diracx.core.models import SearchSpec, SortSpec
+from diracx.db.sql import utils as sql_utils
+
+
+class MockOSDBMixin:
+    """A subclass of DummyOSDB that hacks it to use sqlite as a backed.
+
+    This is only intended for testing and development purposes to avoid the
+    need to run a full OpenSearch instance. This class is used by defining a
+    new class that inherits from this mixin as well the real DB class, i.e.
+
+    .. code-block:: python
+
+        class JobParametersDB(MockOSDBMixin, JobParametersDB):
+            pass
+
+    or
+
+    .. code-block:: python
+
+        JobParametersDB = type("JobParametersDB", (MockOSDBMixin, JobParametersDB), {})
+    """
+
+    def __init__(self, connection_kwargs: dict[str, Any]) -> None:
+        from sqlalchemy import JSON, Column, Integer, MetaData, String, Table
+
+        from diracx.db.sql.utils import DateNowColumn
+
+        # Dynamically create a subclass of BaseSQLDB so we get clearer errors
+        MockedDB = type(f"Mocked{self.__class__.__name__}", (sql_utils.BaseSQLDB,), {})
+        self._sql_db = MockedDB(connection_kwargs["sqlalchemy_dsn"])
+
+        # Dynamically create the table definition based on the fields
+        columns = [
+            Column("doc_id", Integer, primary_key=True),
+            Column("extra", JSON, default={}, nullable=False),
+        ]
+        for field, field_type in self.fields.items():
+            match field_type["type"]:
+                case "date":
+                    ColumnType = DateNowColumn
+                case "long":
+                    ColumnType = partial(Column, type_=Integer)
+                case "keyword":
+                    ColumnType = partial(Column, type_=String(255))
+                case "text":
+                    ColumnType = partial(Column, type_=String(64 * 1024))
+                case _:
+                    raise NotImplementedError(f"Unknown field type: {field_type=}")
+            columns.append(ColumnType(field, default=None))
+        self._sql_db.metadata = MetaData()
+        self._table = Table("dummy", self._sql_db.metadata, *columns)
+
+    @contextlib.asynccontextmanager
+    async def client_context(self) -> AsyncIterator[None]:
+        async with self._sql_db.engine_context():
+            yield
+
+    async def __aenter__(self):
+        await self._sql_db.__aenter__()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self._sql_db.__aexit__(exc_type, exc_value, traceback)
+
+    async def create_index_template(self) -> None:
+        async with self._sql_db.engine.begin() as conn:
+            await conn.run_sync(self._sql_db.metadata.create_all)
+
+    async def upsert(self, doc_id, document) -> None:
+        async with self:
+            values = {}
+            for key, value in document.items():
+                if key in self.fields:
+                    values[key] = value
+                else:
+                    values.setdefault("extra", {})[key] = value
+
+            stmt = sqlite_insert(self._table).values(doc_id=doc_id, **values)
+            # TODO: Upsert the JSON blob properly
+            stmt = stmt.on_conflict_do_update(index_elements=["doc_id"], set_=values)
+            await self._sql_db.conn.execute(stmt)
+
+    async def search(
+        self,
+        parameters: list[str] | None,
+        search: list[SearchSpec],
+        sorts: list[SortSpec],
+        *,
+        distinct: bool = False,
+        per_page: int = 100,
+        page: int | None = None,
+    ) -> tuple[int, list[dict[Any, Any]]]:
+        async with self:
+            # Apply selection
+            if parameters:
+                columns = []
+                for p in parameters:
+                    if p in self.fields:
+                        columns.append(self._table.columns[p])
+                    else:
+                        columns.append(self._table.columns["extra"][p].label(p))
+            else:
+                columns = self._table.columns
+            stmt = select(*columns)
+            if distinct:
+                stmt = stmt.distinct()
+
+            # Apply filtering
+            stmt = sql_utils.apply_search_filters(
+                self._table.columns.__getitem__, stmt, search
+            )
+
+            # Apply sorting
+            stmt = sql_utils.apply_sort_constraints(
+                self._table.columns.__getitem__, stmt, sorts
+            )
+
+            # Apply pagination
+            if page is not None:
+                stmt = stmt.offset((page - 1) * per_page).limit(per_page)
+
+            results = []
+            async for row in await self._sql_db.conn.stream(stmt):
+                result = dict(row._mapping)
+                result.pop("doc_id", None)
+                if "extra" in result:
+                    result.update(result.pop("extra"))
+                for k, v in list(result.items()):
+                    if isinstance(v, datetime) and v.tzinfo is None:
+                        result[k] = v.replace(tzinfo=timezone.utc)
+                    if v is None:
+                        result.pop(k)
+                results.append(result)
+        return results
+
+    async def ping(self):
+        return await self._sql_db.ping()
+
+
+def fake_available_osdb_implementations(name, *, real_available_implementations):
+    implementations = real_available_implementations(name)
+
+    # Dynamically generate a class that inherits from the first implementation
+    # but that also has the MockOSDBMixin
+    MockParameterDB = type(name, (MockOSDBMixin, implementations[0]), {})
+
+    return [MockParameterDB] + implementations

{diracx_testing-0.0.1a16 → diracx_testing-0.0.1a18}/src/diracx/testing/osdb.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
-import secrets
 import socket
 from subprocess import PIPE, Popen, check_output
 
 import pytest
 
-from diracx.db.os.utils import BaseOSDB
+from .dummy_osdb import DummyOSDB
+from .mock_osdb import MockOSDBMixin
 
 OPENSEARCH_PORT = 28000
 
@@ -18,31 +18,6 @@ def require_port_availability(port: int) -> bool:
             raise RuntimeError(f"This test requires port {port} to be available")
 
 
-class DummyOSDB(BaseOSDB):
-    """Example DiracX OpenSearch database class for testing.
-
-    A new random prefix is created each time the class is defined to ensure
-    test runs are independent of each other.
-    """
-
-    fields = {
-        "DateField": {"type": "date"},
-        "IntField": {"type": "long"},
-        "KeywordField0": {"type": "keyword"},
-        "KeywordField1": {"type": "keyword"},
-        "KeywordField2": {"type": "keyword"},
-        "TextField": {"type": "text"},
-    }
-
-    def __init__(self, *args, **kwargs):
-        # Randomize the index prefix to ensure tests are independent
-        self.index_prefix = f"dummy_{secrets.token_hex(8)}"
-        super().__init__(*args, **kwargs)
-
-    def index_name(self, doc_id: int) -> str:
-        return f"{self.index_prefix}-{doc_id // 1e6:.0f}m"
-
-
 @pytest.fixture(scope="session")
 def opensearch_conn_kwargs(demo_kubectl_env):
     """Fixture to get the OpenSearch connection kwargs.
@@ -108,3 +83,19 @@ async def dummy_opensearch_db(dummy_opensearch_db_without_template):
     await db.create_index_template()
     yield db
     await db.client.indices.delete_index_template(name=db.index_prefix)
+
+
+@pytest.fixture
+async def sql_opensearch_db():
+    """Fixture which returns a SQLOSDB object."""
+
+    class MockDummyOSDB(MockOSDBMixin, DummyOSDB):
+        pass
+
+    db = MockDummyOSDB(
+        connection_kwargs={"sqlalchemy_dsn": "sqlite+aiosqlite:///:memory:"}
+    )
+    async with db.client_context():
+        await db.create_index_template()
+        yield db
+        # No need to cleanup as this uses an in-memory sqlite database

diracx_testing-0.0.1a18/src/diracx/testing/routers.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from functools import partial
+
+from diracx.testing.mock_osdb import fake_available_osdb_implementations
+
+
+@asynccontextmanager
+async def ensure_dbs_exist():
+    from diracx.db.__main__ import init_os, init_sql
+
+    await init_sql()
+    await init_os()
+    yield
+
+
+def create_app():
+    """Create a FastAPI application for testing purposes.
+
+    This is a wrapper around diracx.routers.create_app that:
+     * adds a lifetime function to ensure the DB schemas are initialized
+     * replaces the parameter DBs with sqlite-backed versions
+    """
+    from diracx.db.os.utils import BaseOSDB
+    from diracx.routers import create_app
+
+    BaseOSDB.available_implementations = partial(
+        fake_available_osdb_implementations,
+        real_available_implementations=BaseOSDB.available_implementations,
+    )
+
+    app = create_app()
+    app.lifetime_functions.append(ensure_dbs_exist)
+
+    return app

{diracx_testing-0.0.1a16 → diracx_testing-0.0.1a18}/src/diracx_testing.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: diracx-testing
-Version: 0.0.1a16
+Version: 0.0.1a18
 Summary: TODO
 License: GPL-3.0-only
 Classifier: Intended Audience :: Science/Research
@@ -8,7 +8,7 @@ Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: System :: Distributed Computing
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 Requires-Dist: pytest
 Requires-Dist: pytest-asyncio

{diracx_testing-0.0.1a16 → diracx_testing-0.0.1a18}/src/diracx_testing.egg-info/SOURCES.txt

@@ -1,7 +1,10 @@
 README.md
 pyproject.toml
 src/diracx/testing/__init__.py
+src/diracx/testing/dummy_osdb.py
+src/diracx/testing/mock_osdb.py
 src/diracx/testing/osdb.py
+src/diracx/testing/routers.py
 src/diracx_testing.egg-info/PKG-INFO
 src/diracx_testing.egg-info/SOURCES.txt
 src/diracx_testing.egg-info/dependency_links.txt