diracx-testing 0.0.1a8__py3-none-any.whl → 0.0.1a10__py3-none-any.whl

diracx/testing/__init__.py

@@ -5,11 +5,11 @@ import contextlib
 import os
 import re
 import subprocess
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from html.parser import HTMLParser
 from pathlib import Path
 from typing import TYPE_CHECKING
-from urllib.parse import urljoin
+from urllib.parse import parse_qs, urljoin, urlparse
 from uuid import uuid4
 
 import pytest
@@ -55,22 +55,25 @@ def pytest_collection_modifyitems(config, items):
             item.add_marker(skip_regen)
 
 
-@pytest.fixture
-def test_auth_settings() -> AuthSettings:
+@pytest.fixture(scope="session")
+def rsa_private_key_pem() -> str:
     from cryptography.hazmat.primitives import serialization
     from cryptography.hazmat.primitives.asymmetric import rsa
 
-    from diracx.routers.auth import AuthSettings
-
     private_key = rsa.generate_private_key(public_exponent=65537, key_size=4096)
-    pem = private_key.private_bytes(
+    return private_key.private_bytes(
         encoding=serialization.Encoding.PEM,
         format=serialization.PrivateFormat.PKCS8,
         encryption_algorithm=serialization.NoEncryption(),
     ).decode()
 
+
+@pytest.fixture(scope="session")
+def test_auth_settings(rsa_private_key_pem) -> AuthSettings:
+    from diracx.routers.auth import AuthSettings
+
     yield AuthSettings(
-        token_key=pem,
+        token_key=rsa_private_key_pem,
         allowed_redirects=[
             "http://diracx.test.invalid:8000/api/docs/oauth2-redirect",
         ],
@@ -78,7 +81,7 @@ def test_auth_settings() -> AuthSettings:
 
 
 @pytest.fixture(scope="session")
-def aio_moto():
+def aio_moto(worker_id):
     """Start the moto server in a separate thread and return the base URL
 
     The mocking provided by moto doesn't play nicely with aiobotocore so we use
@@ -87,6 +90,8 @@ def aio_moto():
     from moto.server import ThreadedMotoServer
 
     port = 27132
+    if worker_id != "master":
+        port += int(worker_id.replace("gw", "")) + 1
     server = ThreadedMotoServer(port=port)
     server.start()
     yield {
@@ -97,7 +102,7 @@ def aio_moto():
     server.stop()
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def test_sandbox_settings(aio_moto) -> SandboxStoreSettings:
     from diracx.routers.job_manager.sandboxes import SandboxStoreSettings
 
@@ -108,70 +113,232 @@ def test_sandbox_settings(aio_moto) -> SandboxStoreSettings:
     )
 
 
-@pytest.fixture
-def with_app(test_auth_settings, test_sandbox_settings, with_config_repo):
-    """
-    Create a DiracxApp with hard coded configuration for test
-    """
-    from diracx.core.config import ConfigSource
-    from diracx.routers import create_app_inner
-
-    app = create_app_inner(
-        enabled_systems={".well-known", "auth", "config", "jobs"},
-        all_service_settings=[
-            test_auth_settings,
-            test_sandbox_settings,
-        ],
-        database_urls={
-            "JobDB": "sqlite+aiosqlite:///:memory:",
-            "JobLoggingDB": "sqlite+aiosqlite:///:memory:",
-            "TaskQueueDB": "sqlite+aiosqlite:///:memory:",
-            "AuthDB": "sqlite+aiosqlite:///:memory:",
-            "SandboxMetadataDB": "sqlite+aiosqlite:///:memory:",
-        },
-        os_database_conn_kwargs={
-            # TODO: JobParametersDB
-        },
-        config_source=ConfigSource.create_from_url(
-            backend_url=f"git+file://{with_config_repo}"
-        ),
-    )
+class UnavailableDependency:
+    def __init__(self, key):
+        self.key = key
+
+    def __call__(self):
+        raise NotImplementedError(
+            f"{self.key} has not been made available to this test!"
+        )
+
+
+class ClientFactory:
+    def __init__(
+        self,
+        tmp_path_factory,
+        with_config_repo,
+        test_auth_settings,
+        test_sandbox_settings,
+    ):
+        from diracx.core.config import ConfigSource
+        from diracx.core.extensions import select_from_extension
+        from diracx.core.settings import ServiceSettingsBase
+        from diracx.db.sql.utils import BaseSQLDB
+        from diracx.routers import create_app_inner
+
+        enabled_systems = {
+            e.name for e in select_from_extension(group="diracx.services")
+        }
+        database_urls = {
+            e.name: "sqlite+aiosqlite:///:memory:"
+            for e in select_from_extension(group="diracx.db.sql")
+        }
+        self._cache_dir = tmp_path_factory.mktemp("empty-dbs")
+
+        self.test_auth_settings = test_auth_settings
+
+        self.app = create_app_inner(
+            enabled_systems=enabled_systems,
+            all_service_settings=[
+                test_auth_settings,
+                test_sandbox_settings,
+            ],
+            database_urls=database_urls,
+            os_database_conn_kwargs={
+                # TODO: JobParametersDB
+            },
+            config_source=ConfigSource.create_from_url(
+                backend_url=f"git+file://{with_config_repo}"
+            ),
+        )
+
+        self.all_dependency_overrides = self.app.dependency_overrides.copy()
+        self.app.dependency_overrides = {}
+        for obj in self.all_dependency_overrides:
+            assert issubclass(
+                obj.__self__, (ServiceSettingsBase, BaseSQLDB, ConfigSource)
+            ), obj
+
+        self.all_lifetime_functions = self.app.lifetime_functions[:]
+        self.app.lifetime_functions = []
+        for obj in self.all_lifetime_functions:
+            assert isinstance(
+                obj.__self__, (ServiceSettingsBase, BaseSQLDB, ConfigSource)
+            ), obj
+
+    @contextlib.contextmanager
+    def configure(self, enabled_dependencies):
+        assert (
+            self.app.dependency_overrides == {} and self.app.lifetime_functions == []
+        ), "configure cannot be nested"
+
+        for k, v in self.all_dependency_overrides.items():
+            class_name = k.__self__.__name__
+            if class_name in enabled_dependencies:
+                self.app.dependency_overrides[k] = v
+            else:
+                self.app.dependency_overrides[k] = UnavailableDependency(class_name)
+
+        for obj in self.all_lifetime_functions:
+            if obj.__self__.__class__.__name__ in enabled_dependencies:
+                self.app.lifetime_functions.append(obj)
+
+        # Add create_db_schemas to the end of the lifetime_functions so that the
+        # other lifetime_functions (i.e. those which run db.engine_context) have
+        # already been ran
+        self.app.lifetime_functions.append(self.create_db_schemas)
+
+        yield
+
+        self.app.dependency_overrides = {}
+        self.app.lifetime_functions = []
 
     @contextlib.asynccontextmanager
-    async def create_db_schemas(app=app):
+    async def create_db_schemas(self):
         """Create DB schema's based on the DBs available in app.dependency_overrides"""
+        import aiosqlite
+        import sqlalchemy
+        from sqlalchemy.util.concurrency import greenlet_spawn
+
         from diracx.db.sql.utils import BaseSQLDB
 
-        for k, v in app.dependency_overrides.items():
+        for k, v in self.app.dependency_overrides.items():
             # Ignore dependency overrides which aren't BaseSQLDB.transaction
-            if k.__func__ != BaseSQLDB.transaction.__func__:
+            if (
+                isinstance(v, UnavailableDependency)
+                or k.__func__ != BaseSQLDB.transaction.__func__
+            ):
                 continue
             # The first argument of the overridden BaseSQLDB.transaction is the DB object
             db = v.args[0]
             assert isinstance(db, BaseSQLDB), (k, db)
-            # Fill the DB schema
-            async with db.engine.begin() as conn:
-                # set PRAGMA foreign_keys=ON if sqlite
-                if db._db_url.startswith("sqlite"):
-                    await conn.exec_driver_sql("PRAGMA foreign_keys=ON")
-                await conn.run_sync(db.metadata.create_all)
+
+            # set PRAGMA foreign_keys=ON if sqlite
+            if db.engine.url.drivername.startswith("sqlite"):
+
+                def set_sqlite_pragma(dbapi_connection, connection_record):
+                    cursor = dbapi_connection.cursor()
+                    cursor.execute("PRAGMA foreign_keys=ON")
+                    cursor.close()
+
+                sqlalchemy.event.listen(
+                    db.engine.sync_engine, "connect", set_sqlite_pragma
+                )
+
+            # We maintain a cache of the populated DBs in empty_db_dir so that
+            # we don't have to recreate them for every test. This speeds up the
+            # tests by a considerable amount.
+            ref_db = self._cache_dir / f"{k.__self__.__name__}.db"
+            if ref_db.exists():
+                async with aiosqlite.connect(ref_db) as ref_conn:
+                    conn = await db.engine.raw_connection()
+                    await ref_conn.backup(conn.driver_connection)
+                    await greenlet_spawn(conn.close)
+            else:
+                async with db.engine.begin() as conn:
+                    await conn.run_sync(db.metadata.create_all)
+
+                async with aiosqlite.connect(ref_db) as ref_conn:
+                    conn = await db.engine.raw_connection()
+                    await conn.driver_connection.backup(ref_conn)
+                    await greenlet_spawn(conn.close)
 
         yield
 
-    # Add create_db_schemas to the end of the lifetime_functions so that the
-    # other lifetime_functions (i.e. those which run db.engine_context) have
-    # already been ran
-    app.lifetime_functions.append(create_db_schemas)
+    @contextlib.contextmanager
+    def unauthenticated(self):
+        from fastapi.testclient import TestClient
+
+        with TestClient(self.app) as client:
+            yield client
+
+    @contextlib.contextmanager
+    def normal_user(self):
+        from diracx.core.properties import NORMAL_USER
+        from diracx.routers.auth import create_token
+
+        with self.unauthenticated() as client:
+            payload = {
+                "sub": "testingVO:yellow-sub",
+                "exp": datetime.now(tz=timezone.utc)
+                + timedelta(self.test_auth_settings.access_token_expire_minutes),
+                "aud": AUDIENCE,
+                "iss": ISSUER,
+                "dirac_properties": [NORMAL_USER],
+                "jti": str(uuid4()),
+                "preferred_username": "preferred_username",
+                "dirac_group": "test_group",
+                "vo": "lhcb",
+            }
+            token = create_token(payload, self.test_auth_settings)
+
+            client.headers["Authorization"] = f"Bearer {token}"
+            client.dirac_token_payload = payload
+            yield client
+
+    @contextlib.contextmanager
+    def admin_user(self):
+        from diracx.core.properties import JOB_ADMINISTRATOR
+        from diracx.routers.auth import create_token
+
+        with self.unauthenticated() as client:
+            payload = {
+                "sub": "testingVO:yellow-sub",
+                "aud": AUDIENCE,
+                "iss": ISSUER,
+                "dirac_properties": [JOB_ADMINISTRATOR],
+                "jti": str(uuid4()),
+                "preferred_username": "preferred_username",
+                "dirac_group": "test_group",
+                "vo": "lhcb",
+            }
+            token = create_token(payload, self.test_auth_settings)
+            client.headers["Authorization"] = f"Bearer {token}"
+            client.dirac_token_payload = payload
+            yield client
 
-    yield app
+
+@pytest.fixture(scope="session")
+def session_client_factory(
+    test_auth_settings, test_sandbox_settings, with_config_repo, tmp_path_factory
+):
+    """
+    TODO
+    """
+    yield ClientFactory(
+        tmp_path_factory, with_config_repo, test_auth_settings, test_sandbox_settings
+    )
 
 
 @pytest.fixture
-def with_config_repo(tmp_path):
+def client_factory(session_client_factory, request):
+    marker = request.node.get_closest_marker("enabled_dependencies")
+    if marker is None:
+        raise RuntimeError("This test requires the enabled_dependencies marker")
+    (enabled_dependencies,) = marker.args
+    with session_client_factory.configure(enabled_dependencies=enabled_dependencies):
+        yield session_client_factory
+
+
+@pytest.fixture(scope="session")
+def with_config_repo(tmp_path_factory):
     from git import Repo
 
     from diracx.core.config import Config
 
+    tmp_path = tmp_path_factory.mktemp("cs-repo")
+
     repo = Repo.init(tmp_path, initial_branch="master")
     cs_file = tmp_path / "default.yml"
     example_cs = Config.parse_obj(
@@ -220,58 +387,6 @@ def with_config_repo(tmp_path):
     yield tmp_path
 
 
-@pytest.fixture
-def test_client(with_app):
-    from fastapi.testclient import TestClient
-
-    with TestClient(with_app) as test_client:
-        yield test_client
-
-
-@pytest.fixture
-def normal_user_client(test_client, test_auth_settings):
-    from diracx.core.properties import NORMAL_USER
-    from diracx.routers.auth import create_token
-
-    payload = {
-        "sub": "testingVO:yellow-sub",
-        "exp": datetime.now()
-        + timedelta(test_auth_settings.access_token_expire_minutes),
-        "aud": AUDIENCE,
-        "iss": ISSUER,
-        "dirac_properties": [NORMAL_USER],
-        "jti": str(uuid4()),
-        "preferred_username": "preferred_username",
-        "dirac_group": "test_group",
-        "vo": "lhcb",
-    }
-    token = create_token(payload, test_auth_settings)
-    test_client.headers["Authorization"] = f"Bearer {token}"
-    test_client.dirac_token_payload = payload
-    yield test_client
-
-
-@pytest.fixture
-def admin_user_client(test_client, test_auth_settings):
-    from diracx.core.properties import JOB_ADMINISTRATOR
-    from diracx.routers.auth import create_token
-
-    payload = {
-        "sub": "testingVO:yellow-sub",
-        "aud": AUDIENCE,
-        "iss": ISSUER,
-        "dirac_properties": [JOB_ADMINISTRATOR],
-        "jti": str(uuid4()),
-        "preferred_username": "preferred_username",
-        "dirac_group": "test_group",
-        "vo": "lhcb",
-    }
-    token = create_token(payload, test_auth_settings)
-    test_client.headers["Authorization"] = f"Bearer {token}"
-    test_client.dirac_token_payload = payload
-    yield test_client
-
-
 @pytest.fixture(scope="session")
 def demo_dir(request) -> Path:
     demo_dir = request.config.getoption("--demo-dir")
@@ -436,5 +551,13 @@ def do_device_flow_with_dex(url: str, ca_path: str) -> None:
         verify=ca_path,
     )
     r.raise_for_status()
+    approval_url = r.url  # This is not the same as URL as we redirect to dex
+    # Do the actual approval
+    r = requests.post(
+        approval_url,
+        {"approval": "approve", "req": parse_qs(urlparse(r.url).query)["req"][0]},
+        verify=ca_path,
+    )
+
     # This should have redirected to the DiracX page that shows the login is complete
     assert "Please close the window" in r.text

{diracx_testing-0.0.1a8.dist-info → diracx_testing-0.0.1a10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: diracx-testing
-Version: 0.0.1a8
+Version: 0.0.1a10
 Summary: TODO
 License: GPL-3.0-only
 Classifier: Intended Audience :: Science/Research
@@ -13,6 +13,7 @@ Description-Content-Type: text/markdown
 Requires-Dist: pytest
 Requires-Dist: pytest-asyncio
 Requires-Dist: pytest-cov
+Requires-Dist: pytest-xdist
 Provides-Extra: testing
 Requires-Dist: diracx-testing ; extra == 'testing'
 

diracx_testing-0.0.1a10.dist-info/RECORD

@@ -0,0 +1,6 @@
+diracx/testing/__init__.py,sha256=wXrihINkyIbE2dn4XyKDeQO4cp8bKPkeUIi3ssod0qs,19026
+diracx/testing/osdb.py,sha256=-EFZNyEY07Zq7HdQGZxS3H808Y94aaUhmo0x-Y8xo3Q,3592
+diracx_testing-0.0.1a10.dist-info/METADATA,sha256=BE-nfEHYD7kakG38j-w0mdUUPOFGjycamahyTKEABaM,615
+diracx_testing-0.0.1a10.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+diracx_testing-0.0.1a10.dist-info/top_level.txt,sha256=vJx10tdRlBX3rF2Psgk5jlwVGZNcL3m_7iQWwgPXt-U,7
+diracx_testing-0.0.1a10.dist-info/RECORD,,

diracx_testing-0.0.1a8.dist-info/RECORD

@@ -1,6 +0,0 @@
-diracx/testing/__init__.py,sha256=teeGQ2G3stJ7hNyxV8SyaNJytN3cxL6u8hOWub212VU,14173
-diracx/testing/osdb.py,sha256=-EFZNyEY07Zq7HdQGZxS3H808Y94aaUhmo0x-Y8xo3Q,3592
-diracx_testing-0.0.1a8.dist-info/METADATA,sha256=H6zq2hStmuebu_DZiboFGmxwy9lMILOMPsYBXE-GBi0,586
-diracx_testing-0.0.1a8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-diracx_testing-0.0.1a8.dist-info/top_level.txt,sha256=vJx10tdRlBX3rF2Psgk5jlwVGZNcL3m_7iQWwgPXt-U,7
-diracx_testing-0.0.1a8.dist-info/RECORD,,