diracx-db 0.0.1a49__tar.gz → 0.0.2__tar.gz

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: diracx-db
-Version: 0.0.1a49
+Version: 0.0.2
 Summary: TODO
 License: GPL-3.0-only
 Classifier: Intended Audience :: Science/Research
@@ -12,8 +12,11 @@ Requires-Python: >=3.11
 Requires-Dist: diracx-core
 Requires-Dist: opensearch-py[async]
 Requires-Dist: pydantic>=2.10
+Requires-Dist: python-dateutil
 Requires-Dist: sqlalchemy[aiomysql,aiosqlite]>=2
 Requires-Dist: uuid-utils
 Provides-Extra: testing
 Requires-Dist: diracx-testing; extra == 'testing'
 Requires-Dist: freezegun; extra == 'testing'
+Provides-Extra: types
+Requires-Dist: types-python-dateutil; extra == 'types'

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/pyproject.toml

@@ -18,11 +18,13 @@ dependencies = [
     "pydantic >=2.10",
     "sqlalchemy[aiomysql,aiosqlite] >= 2",
     "uuid-utils",
+    "python-dateutil",
 ]
 dynamic = ["version"]
 
 [project.optional-dependencies]
 testing = ["diracx-testing", "freezegun"]
+types = ["types-python-dateutil"]
 
 [project.entry-points."diracx.dbs.sql"]
 AuthDB = "diracx.db.sql:AuthDB"

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/src/diracx/db/__main__.py

@@ -39,6 +39,7 @@ async def init_sql():
                 if db._db_url.startswith("sqlite"):
                     await conn.exec_driver_sql("PRAGMA foreign_keys=ON")
                 await conn.run_sync(db.metadata.create_all)
+                await db.post_create(conn)
 
 
 async def init_os():

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/src/diracx/db/sql/auth/db.py

@@ -1,16 +1,21 @@
 from __future__ import annotations
 
+import logging
 import secrets
+from datetime import UTC, datetime
+from itertools import pairwise
 
-from sqlalchemy import insert, select, update
+from dateutil.rrule import MONTHLY, rrule
+from sqlalchemy import insert, select, text, update
 from sqlalchemy.exc import IntegrityError, NoResultFound
+from sqlalchemy.ext.asyncio import AsyncConnection
 from uuid_utils import UUID, uuid7
 
 from diracx.core.exceptions import (
     AuthorizationError,
     TokenNotFoundError,
 )
-from diracx.db.sql.utils import BaseSQLDB, hash, substract_date
+from diracx.db.sql.utils import BaseSQLDB, hash, substract_date, uuid7_from_datetime
 
 from .schema import (
     AuthorizationFlows,
@@ -25,10 +30,72 @@ from .schema import Base as AuthDBBase
 USER_CODE_ALPHABET = "BCDFGHJKLMNPQRSTVWXZ"
 MAX_RETRY = 5
 
+logger = logging.getLogger(__name__)
+
 
 class AuthDB(BaseSQLDB):
     metadata = AuthDBBase.metadata
 
+    @classmethod
+    async def post_create(cls, conn: AsyncConnection) -> None:
+        """Create partitions if it is a MySQL DB and it does not have
+        it yet and the table does not have any data yet.
+        We do this as a post_create step as sqlalchemy does not support
+        partition so well.
+        """
+        if conn.dialect.name == "mysql":
+            check_partition_query = text(
+                "SELECT PARTITION_NAME FROM information_schema.partitions "
+                "WHERE TABLE_NAME = 'RefreshTokens' AND PARTITION_NAME is not NULL"
+            )
+            partition_names = (await conn.execute(check_partition_query)).all()
+
+            if not partition_names:
+                # Create a monthly partition from today until 2 years
+                # The partition are named p_<year>_<month>
+                start_date = datetime.now(tz=UTC).replace(
+                    day=1, hour=0, minute=0, second=0, microsecond=0
+                )
+                end_date = start_date.replace(year=start_date.year + 2)
+
+                dates = [
+                    dt for dt in rrule(MONTHLY, dtstart=start_date, until=end_date)
+                ]
+
+                partition_list = []
+                for name, limit in pairwise(dates):
+                    partition_list.append(
+                        f"PARTITION p_{name.year}_{name.month} "
+                        f"VALUES LESS THAN ('{str(uuid7_from_datetime(limit, randomize=False)).replace('-', '')}')"
+                    )
+                partition_list.append("PARTITION p_future VALUES LESS THAN (MAXVALUE)")
+
+                alter_query = text(
+                    f"ALTER TABLE RefreshTokens PARTITION BY RANGE COLUMNS (JTI) ({','.join(partition_list)})"
+                )
+
+                check_table_empty_query = text("SELECT * FROM RefreshTokens LIMIT 1")
+                refresh_table_content = (
+                    await conn.execute(check_table_empty_query)
+                ).all()
+                if refresh_table_content:
+                    logger.warning(
+                        "RefreshTokens table not empty. Run the following query yourself"
+                    )
+                    logger.warning(alter_query)
+                    return
+
+                await conn.execute(alter_query)
+
+                partition_names = (
+                    await conn.execute(
+                        check_partition_query, {"table_name": "RefreshTokens"}
+                    )
+                ).all()
+                assert partition_names, (
+                    f"There should be partitions now {partition_names}"
+                )
+
     async def device_flow_validate_user_code(
         self, user_code: str, max_validity: int
     ) -> str:

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/src/diracx/db/sql/auth/schema.py

@@ -10,7 +10,12 @@ from sqlalchemy import (
 )
 from sqlalchemy.orm import declarative_base
 
-from diracx.db.sql.utils import Column, DateNowColumn, EnumColumn, NullColumn
+from diracx.db.sql.utils import (
+    Column,
+    DateNowColumn,
+    EnumColumn,
+    NullColumn,
+)
 
 USER_CODE_LENGTH = 8
 
@@ -92,7 +97,6 @@ class RefreshTokens(Base):
     status = EnumColumn(
         "Status", RefreshTokenStatus, server_default=RefreshTokenStatus.CREATED.name
     )
-    creation_time = DateNowColumn("CreationTime", index=True)
     scope = Column("Scope", String(1024))
 
     # User attributes bound to the refresh token

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/src/diracx/db/sql/utils/__init__.py

@@ -1,16 +1,6 @@
 from __future__ import annotations
 
-from .base import (
-    BaseSQLDB,
-    SQLDBUnavailableError,
-    _get_columns,
-    apply_search_filters,
-    apply_sort_constraints,
-)
-from .functions import hash, substract_date, utcnow
-from .types import Column, DateNowColumn, EnumBackedBool, EnumColumn, NullColumn
-
-__all__ = (
+__all__ = [
     "_get_columns",
     "utcnow",
     "Column",
@@ -24,4 +14,18 @@ __all__ = (
     "substract_date",
     "hash",
     "SQLDBUnavailableError",
+    "uuid7_from_datetime",
+    "uuid7_to_datetime",
+]
+
+from .base import (
+    BaseSQLDB,
+    SQLDBUnavailableError,
+    _get_columns,
+    apply_search_filters,
+    apply_sort_constraints,
+    uuid7_from_datetime,
+    uuid7_to_datetime,
 )
+from .functions import hash, substract_date, utcnow
+from .types import Column, DateNowColumn, EnumBackedBool, EnumColumn, NullColumn

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/src/diracx/db/sql/utils/base.py

@@ -7,13 +7,15 @@ import re
 from abc import ABCMeta
 from collections.abc import AsyncIterator
 from contextvars import ContextVar
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, Self, cast
+from uuid import UUID as StdUUID  # noqa: N811
 
 from pydantic import TypeAdapter
 from sqlalchemy import DateTime, MetaData, func, select
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
+from uuid_utils import UUID, uuid7
 
 from diracx.core.exceptions import InvalidQueryError
 from diracx.core.extensions import select_from_extension
@@ -153,6 +155,11 @@ class BaseSQLDB(metaclass=ABCMeta):
                     raise
         return db_urls
 
+    @classmethod
+    async def post_create(cls, conn: AsyncConnection) -> None:
+        """Execute actions after the schema has been created."""
+        return
+
     @classmethod
     def transaction(cls) -> Self:
         raise NotImplementedError("This should never be called")
@@ -416,3 +423,33 @@ def apply_sort_constraints(column_mapping, stmt, sorts):
     if sort_columns:
         stmt = stmt.order_by(*sort_columns)
     return stmt
+
+
+def uuid7_to_datetime(uuid: UUID | StdUUID | str) -> datetime:
+    """Convert a UUIDv7 to a datetime."""
+    if isinstance(uuid, StdUUID):
+        # Convert stdlib UUID to uuid_utils.UUID
+        uuid = UUID(str(uuid))
+    elif not isinstance(uuid, UUID):
+        # Convert string or other types to uuid_utils.UUID
+        uuid = UUID(uuid)
+    if uuid.version != 7:
+        raise ValueError(f"UUID {uuid} is not a UUIDv7")
+    return datetime.fromtimestamp(uuid.timestamp / 1000.0, tz=timezone.utc)
+
+
+def uuid7_from_datetime(dt: datetime, *, randomize: bool = True) -> UUID:
+    """Generate a UUIDv7 corresponding to the given datetime.
+
+    If randomize is True, the standard uuid7 function is used resulting in the
+    lowest 62-bits being random. If randomize is False, the UUIDv7 will be the
+    lowest possible UUIDv7 for the given datetime.
+    """
+    timestamp = dt.timestamp()
+    if randomize:
+        uuid = uuid7(int(timestamp), int((timestamp % 1) * 1e9))
+    else:
+        time_high = int(timestamp * 1000) >> 16
+        time_low = int(timestamp * 1000) & 0xFFFF
+        uuid = UUID.from_fields((time_high, time_low, 0x7000, 0x80, 0, 0))
+    return uuid

{diracx_db-0.0.1a49 → diracx_db-0.0.2}/tests/auth/test_refresh_token.py

@@ -56,7 +56,6 @@ async def test_get(auth_db: AuthDB):
             refresh_token_details["sub"],
             refresh_token_details["scope"],
         )
-        creation_time = (await auth_db.get_refresh_token(jti))["CreationTime"]
 
     # Enrich the dict with the generated refresh token attributes
     expected_refresh_token = {
@@ -64,7 +63,6 @@ async def test_get(auth_db: AuthDB):
         "Scope": refresh_token_details["scope"],
         "JTI": jti,
         "Status": RefreshTokenStatus.CREATED,
-        "CreationTime": creation_time,
     }
 
     # Get refresh token details

diracx_db-0.0.2/tests/utils/test_uuid.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from uuid import UUID as StdUUID  # noqa: N811
+
+import freezegun
+import pytest
+from uuid_utils import UUID, uuid7
+
+from diracx.db.sql.utils import uuid7_from_datetime, uuid7_to_datetime
+
+
+def frozen_uuid7() -> UUID:
+    """Create a UUID7 in a way which respects the freezegun context."""
+    timestamp = datetime.now(tz=timezone.utc).timestamp()
+    return uuid7(int(timestamp), int((timestamp % 1) * 1e9))
+
+
+class TestDatetimeToUuid7:
+    """Test cases for the datetime_to_uuid7 function."""
+
+    def test_datetime_to_uuid7_random(self):
+        """Test that the datetime_to_uuid7 function returns a UUID7 with the current timestamp."""
+        dt = datetime.fromisoformat("2024-01-15T12:30:45.123456+00:00")
+        result = uuid7_from_datetime(dt, randomize=True)
+        assert str(result)[:15] == "018d0d1a-5183-7"
+        assert len(set(str(result)[15:])) > 4
+
+    def test_datetime_to_uuid7_deterministic(self):
+        """Test that the datetime_to_uuid7 function returns a UUID7 with the current timestamp."""
+        dt = datetime.fromisoformat("2024-01-15T12:30:45.123456+00:00")
+        result = uuid7_from_datetime(dt, randomize=False)
+        assert str(result) == "018d0d1a-5183-7000-8000-000000000000"
+
+
+class TestUuid7ToDatetime:
+    """Test cases for the uuid7_to_datetime function."""
+
+    def test_uuid7_now(self):
+        """Test that the uuid7 function returns a UUID7 with the current timestamp."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+        assert result.tzinfo == timezone.utc
+        assert result - datetime.now(tz=timezone.utc) < timedelta(milliseconds=2)
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_with_uuid_utils_uuid(self):
+        """Test conversion with uuid_utils.UUID object."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+
+        assert isinstance(result, datetime)
+        assert result.tzinfo == timezone.utc
+        assert result.isoformat() == "2024-01-15T12:30:45.123000+00:00"
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_with_stdlib_uuid(self):
+        """Test conversion with standard library UUID object."""
+        test_uuid = frozen_uuid7()
+        stdlib_uuid = StdUUID(str(test_uuid))
+
+        result = uuid7_to_datetime(stdlib_uuid)
+        assert isinstance(result, datetime)
+        assert result.tzinfo == timezone.utc
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_with_string_uuid(self):
+        """Test conversion with string UUID that gets converted to uuid_utils.UUID."""
+        test_uuid = frozen_uuid7()
+        uuid_string = str(test_uuid)
+
+        result = uuid7_to_datetime(uuid_string)
+        assert isinstance(result, datetime)
+        assert result.tzinfo == timezone.utc
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_precision(self):
+        """Test that the conversion maintains reasonable precision."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+
+        assert isinstance(result, datetime)
+        assert result.tzinfo == timezone.utc
+        assert result.isoformat() == "2024-01-15T12:30:45.123000+00:00"
+
+    @freezegun.freeze_time("1970-01-01 00:00:00.000000")
+    def test_uuid7_to_datetime_epoch_boundary(self):
+        """Test conversion with UUID7 at epoch boundary."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+
+        assert isinstance(result, datetime)
+        assert result.tzinfo == timezone.utc
+        assert result.isoformat() == "1970-01-01T00:00:00+00:00"
+
+    @freezegun.freeze_time("2100-01-01 00:00:00.000000")
+    def test_uuid7_to_datetime_future_timestamp(self):
+        """Test conversion with future timestamp."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+
+        assert isinstance(result, datetime)
+        assert result.tzinfo == timezone.utc
+        assert result.isoformat() == "2100-01-01T00:00:00+00:00"
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_consistency(self):
+        """Test that multiple calls with the same UUID return the same result."""
+        test_uuid = frozen_uuid7()
+
+        result1 = uuid7_to_datetime(test_uuid)
+        result2 = uuid7_to_datetime(test_uuid)
+
+        assert result1.isoformat() == result2.isoformat()
+        assert result1 == result2
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_different_input_types_same_result(self):
+        """Test that different input types representing the same UUID return the same result."""
+        test_uuid = frozen_uuid7()
+        uuid_string = str(test_uuid)
+        stdlib_uuid = StdUUID(uuid_string)
+
+        result_from_uuid_utils = uuid7_to_datetime(test_uuid)
+        result_from_string = uuid7_to_datetime(uuid_string)
+        result_from_stdlib = uuid7_to_datetime(stdlib_uuid)
+
+        # All results should be identical
+        assert result_from_uuid_utils == result_from_string == result_from_stdlib
+
+    def test_uuid7_to_datetime_invalid_input_type(self):
+        """Test that invalid input types raise appropriate errors."""
+        with pytest.raises(TypeError):
+            uuid7_to_datetime(123)  # type: ignore
+
+        with pytest.raises(TypeError):
+            uuid7_to_datetime(123.45)  # type: ignore
+
+        with pytest.raises(TypeError):
+            uuid7_to_datetime([])  # type: ignore
+
+        with pytest.raises(TypeError):
+            uuid7_to_datetime({})  # type: ignore
+
+    def test_uuid7_to_datetime_invalid_uuid_string(self):
+        """Test that invalid UUID strings raise appropriate errors."""
+        with pytest.raises(ValueError):
+            uuid7_to_datetime("not-a-uuid")
+
+        with pytest.raises(ValueError):
+            uuid7_to_datetime("12345")
+
+        with pytest.raises(ValueError):
+            uuid7_to_datetime("")
+
+    def test_uuid7_to_datetime_non_uuid7_uuid(self):
+        """Test behavior with non-UUID7 UUIDs."""
+        uuid4 = StdUUID("550e8400-e29b-41d4-a716-446655440000")
+
+        with pytest.raises(ValueError, match="is not a UUIDv7"):
+            uuid7_to_datetime(uuid4)
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_timezone_awareness(self):
+        """Test that the returned datetime is always timezone-aware and in UTC."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+
+        assert result.tzinfo is not None
+
+        assert result.tzinfo == timezone.utc
+
+        assert result.tzinfo.utcoffset(result) is not None
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_millisecond_precision(self):
+        """Test that the conversion handles millisecond precision correctly."""
+        uuids = [frozen_uuid7() for _ in range(5)]
+        results = [uuid7_to_datetime(uuid) for uuid in uuids]
+
+        for result in results:
+            assert isinstance(result, datetime)
+            assert result.tzinfo == timezone.utc
+
+        timestamps = [r.timestamp() for r in results]
+        assert timestamps == sorted(timestamps)
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_timestamp_property(self):
+        """Test that the function correctly uses the timestamp property."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+
+        expected_timestamp = test_uuid.timestamp / 1000.0
+        actual_timestamp = result.timestamp()
+
+        assert abs(actual_timestamp - expected_timestamp) < 0.001
+
+    @freezegun.freeze_time("2024-01-15 12:30:45.123456")
+    def test_uuid7_to_datetime_deterministic_with_freeze(self):
+        """Test that the function works deterministically with freezegun."""
+        with freezegun.freeze_time("2024-01-15 12:30:45.123000"):
+            uuid1 = frozen_uuid7()
+            result1 = uuid7_to_datetime(uuid1)
+
+        with freezegun.freeze_time("2024-01-15 12:30:45.123000"):
+            uuid2 = frozen_uuid7()
+            result2 = uuid7_to_datetime(uuid2)
+
+        time_diff = abs((result1 - result2).total_seconds())
+        assert time_diff < 1.0
+
+    @freezegun.freeze_time("1970-01-01 00:00:00")
+    def test_uuid7_to_datetime_edge_case_zero_timestamp(self):
+        """Test edge case with very small timestamp values."""
+        test_uuid = frozen_uuid7()
+        result = uuid7_to_datetime(test_uuid)
+
+        assert isinstance(result, datetime)
+        assert result.tzinfo == timezone.utc
+        assert result.isoformat() == "1970-01-01T00:00:00+00:00"