diracx-db 0.0.1a39__tar.gz → 0.0.1a40__tar.gz

{diracx_db-0.0.1a39 → diracx_db-0.0.1a40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: diracx-db
-Version: 0.0.1a39
+Version: 0.0.1a40
 Summary: TODO
 License: GPL-3.0-only
 Classifier: Intended Audience :: Science/Research

{diracx_db-0.0.1a39 → diracx_db-0.0.1a40}/src/diracx/db/sql/job_logging/db.py

@@ -1,28 +1,15 @@
 from __future__ import annotations
 
-import time
-from datetime import timezone
-from typing import TYPE_CHECKING
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import Iterable
 
 from sqlalchemy import delete, func, select
 
-if TYPE_CHECKING:
-    pass
-
-from collections import defaultdict
-
-from diracx.core.models import (
-    JobLoggingRecord,
-    JobStatusReturn,
-)
+from diracx.core.models import JobLoggingRecord, JobStatusReturn
 
 from ..utils import BaseSQLDB
-from .schema import (
-    JobLoggingDBBase,
-    LoggingInfo,
-)
-
-MAGIC_EPOC_NUMBER = 1270000000
+from .schema import JobLoggingDBBase, LoggingInfo
 
 
 class JobLoggingDB(BaseSQLDB):
@@ -35,14 +22,6 @@ class JobLoggingDB(BaseSQLDB):
         records: list[JobLoggingRecord],
     ):
         """Bulk insert entries to the JobLoggingDB table."""
-
-        def get_epoc(date):
-            return (
-                time.mktime(date.timetuple())
-                + date.microsecond / 1000000.0
-                - MAGIC_EPOC_NUMBER
-            )
-
         # First, fetch the maximum SeqNums for the given job_ids
         seqnum_stmt = (
             select(
@@ -70,7 +49,7 @@ class JobLoggingDB(BaseSQLDB):
                     "MinorStatus": record.minor_status,
                     "ApplicationStatus": record.application_status[:255],
                     "StatusTime": record.date,
-                    "StatusTimeOrder": get_epoc(record.date),
+                    "StatusTimeOrder": record.date,
                     "StatusSource": record.source[:32],
                 }
             )
@@ -159,21 +138,16 @@ class JobLoggingDB(BaseSQLDB):
         stmt = delete(LoggingInfo).where(LoggingInfo.job_id.in_(job_ids))
         await self.conn.execute(stmt)
 
-    async def get_wms_time_stamps(self, job_ids):
+    async def get_wms_time_stamps(
+        self, job_ids: Iterable[int]
+    ) -> dict[int, dict[str, datetime]]:
         """Get TimeStamps for job MajorState transitions for multiple jobs at once
         return a {JobID: {State:timestamp}} dictionary.
         """
-        result = defaultdict(dict)
+        result: defaultdict[int, dict[str, datetime]] = defaultdict(dict)
         stmt = select(
-            LoggingInfo.job_id,
-            LoggingInfo.status,
-            LoggingInfo.status_time_order,
+            LoggingInfo.job_id, LoggingInfo.status, LoggingInfo.status_time_order
         ).where(LoggingInfo.job_id.in_(job_ids))
-        rows = await self.conn.execute(stmt)
-        if not rows.rowcount:
-            return {}
-
-        for job_id, event, etime in rows:
-            result[job_id][event] = str(etime + MAGIC_EPOC_NUMBER)
-
-        return result
+        for job_id, event, etime in await self.conn.execute(stmt):
+            result[job_id][event] = etime
+        return dict(result)

diracx_db-0.0.1a40/src/diracx/db/sql/job_logging/schema.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import Integer, Numeric, PrimaryKeyConstraint, String, TypeDecorator
+from sqlalchemy.orm import declarative_base
+
+from ..utils import Column, DateNowColumn
+
+JobLoggingDBBase = declarative_base()
+
+
+class MagicEpochDateTime(TypeDecorator):
+    """A SQLAlchemy type that stores a datetime as a numeric value representing the
+    seconds elapsed since MAGIC_EPOC_NUMBER. The underlying column is defined as
+    Numeric(12,3) which provides a fixed-precision representation.
+    """
+
+    impl = Numeric(12, 3)
+    cache_ok = True
+
+    MAGIC_EPOC_NUMBER = 1270000000
+
+    def process_bind_param(self, value, dialect):
+        """Convert a Python datetime to a numeric value: (timestamp - MAGIC_EPOC_NUMBER).
+        The result is rounded to three decimal places.
+        """
+        if value is None:
+            return None
+        if isinstance(value, datetime):
+            # Convert datetime to seconds since the Unix epoch, subtract our magic epoch,
+            # and round to three decimal places.
+            epoch_seconds = (
+                value.replace(tzinfo=UTC).timestamp() - self.MAGIC_EPOC_NUMBER
+            )
+            return round(epoch_seconds, 3)
+        raise ValueError(
+            "Expected a datetime object for MagicEpochDateTime bind parameter."
+        )
+
+    def process_result_value(self, value, dialect):
+        """Convert the numeric database value back into a Python datetime by reversing the
+        stored difference (adding MAGIC_EPOC_NUMBER).
+        """
+        if value is None:
+            return None
+        # Carefully convert from Decimal to datetime to avoid loosing precision
+        value += self.MAGIC_EPOC_NUMBER
+        value_int = int(value)
+        result = datetime.fromtimestamp(value_int, tz=UTC)
+        return result.replace(microsecond=int((value - value_int) * 1_000_000))
+
+
+class LoggingInfo(JobLoggingDBBase):
+    __tablename__ = "LoggingInfo"
+    job_id = Column("JobID", Integer)
+    seq_num = Column("SeqNum", Integer)
+    status = Column("Status", String(32), default="")
+    minor_status = Column("MinorStatus", String(128), default="")
+    application_status = Column("ApplicationStatus", String(255), default="")
+    status_time = DateNowColumn("StatusTime")
+    status_time_order = Column("StatusTimeOrder", MagicEpochDateTime, default=0)
+    source = Column("StatusSource", String(32), default="Unknown")
+    __table_args__ = (PrimaryKeyConstraint("JobID", "SeqNum"),)

{diracx_db-0.0.1a39 → diracx_db-0.0.1a40}/src/diracx_db.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: diracx-db
-Version: 0.0.1a39
+Version: 0.0.1a40
 Summary: TODO
 License: GPL-3.0-only
 Classifier: Intended Audience :: Science/Research

{diracx_db-0.0.1a39 → diracx_db-0.0.1a40}/tests/jobs/test_job_logging_db.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 
 import pytest
 
@@ -18,11 +18,11 @@ async def job_logging_db():
 
 
 async def test_insert_records(job_logging_db: JobLoggingDB):
-    async with job_logging_db as job_logging_db:
-        # Arrange
-        date = datetime.now(timezone.utc)
+    date_1 = datetime.now(timezone.utc) - timedelta(minutes=1)
+    date_2 = datetime.now(timezone.utc) - timedelta(seconds=1)
+    date_3 = datetime.now(timezone.utc)
 
-        # Act
+    async with job_logging_db as job_logging_db:
         records = []
         for i in range(50):
             records.append(
@@ -31,7 +31,7 @@ async def test_insert_records(job_logging_db: JobLoggingDB):
                     status=JobStatus.RECEIVED,
                     minor_status="received_minor_status",
                     application_status="application_status",
-                    date=date,
+                    date=date_1,
                     source="pytest",
                 )
             )
@@ -41,7 +41,7 @@ async def test_insert_records(job_logging_db: JobLoggingDB):
                     status=JobStatus.SUBMITTING,
                     minor_status="submitted_minor_status",
                     application_status="application_status",
-                    date=date,
+                    date=date_2,
                     source="pytest",
                 )
             )
@@ -51,13 +51,13 @@ async def test_insert_records(job_logging_db: JobLoggingDB):
                     status=JobStatus.RUNNING,
                     minor_status="running_minor_status",
                     application_status="application_status",
-                    date=date,
+                    date=date_3,
                     source="pytest",
                 )
             )
         await job_logging_db.insert_records(records)
 
-        # Assert
+    async with job_logging_db as job_logging_db:
         res = await job_logging_db.get_records([i for i in range(50)])
 
         assert len(res) == 50
@@ -65,26 +65,32 @@ async def test_insert_records(job_logging_db: JobLoggingDB):
         assert res[0][0].Status == JobStatus.RECEIVED.value
         assert res[0][0].MinorStatus == "received_minor_status"
         assert res[0][0].ApplicationStatus == "application_status"
-        assert res[0][0].StatusTime == date
+        assert res[0][0].StatusTime == date_1
         assert res[0][0].Source == "pytest"
 
         # Check the first job - second record
         assert res[0][1].Status == JobStatus.SUBMITTING.value
         assert res[0][1].MinorStatus == "submitted_minor_status"
         assert res[0][1].ApplicationStatus == "application_status"
-        assert res[0][1].StatusTime == date
+        assert res[0][1].StatusTime == date_2
         assert res[0][1].Source == "pytest"
 
         # Check the first job - third record
         assert res[0][2].Status == JobStatus.RUNNING.value
         assert res[0][2].MinorStatus == "running_minor_status"
         assert res[0][2].ApplicationStatus == "application_status"
-        assert res[0][2].StatusTime == date
+        assert res[0][2].StatusTime == date_3
         assert res[0][2].Source == "pytest"
 
         # Check the last job - third record
         assert res[49][2].Status == JobStatus.RUNNING.value
         assert res[49][2].MinorStatus == "running_minor_status"
         assert res[49][2].ApplicationStatus == "application_status"
-        assert res[49][2].StatusTime == date
+        assert res[49][2].StatusTime == date_3
         assert res[49][2].Source == "pytest"
+
+    async with job_logging_db as job_logging_db:
+        res = await job_logging_db.get_wms_time_stamps([1])
+        assert abs(res[1]["Received"] - date_1) < timedelta(microseconds=1000)
+        assert abs(res[1]["Submitting"] - date_2) < timedelta(microseconds=1000)
+        assert abs(res[1]["Running"] - date_3) < timedelta(microseconds=1000)

diracx_db-0.0.1a39/src/diracx/db/sql/job_logging/schema.py

@@ -1,29 +0,0 @@
-from __future__ import annotations
-
-from sqlalchemy import (
-    Integer,
-    Numeric,
-    PrimaryKeyConstraint,
-    String,
-)
-from sqlalchemy.orm import declarative_base
-
-from ..utils import Column, DateNowColumn
-
-JobLoggingDBBase = declarative_base()
-
-
-class LoggingInfo(JobLoggingDBBase):
-    __tablename__ = "LoggingInfo"
-    job_id = Column("JobID", Integer)
-    seq_num = Column("SeqNum", Integer)
-    status = Column("Status", String(32), default="")
-    minor_status = Column("MinorStatus", String(128), default="")
-    application_status = Column("ApplicationStatus", String(255), default="")
-    status_time = DateNowColumn("StatusTime")
-    # TODO: Check that this corresponds to the DOUBLE(12,3) type in MySQL
-    status_time_order = Column(
-        "StatusTimeOrder", Numeric(precision=12, scale=3), default=0
-    )
-    source = Column("StatusSource", String(32), default="Unknown")
-    __table_args__ = (PrimaryKeyConstraint("JobID", "SeqNum"),)