diracx-db 0.0.1a17__py3-none-any.whl → 0.0.1a19__py3-none-any.whl

{diracx_db-0.0.1a17.dist-info → diracx_db-0.0.1a19.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.43.0)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{diracx_db-0.0.1a17.dist-info → diracx_db-0.0.1a19.dist-info}/entry_points.txt

@@ -5,5 +5,6 @@ JobParametersDB = diracx.db.os:JobParametersDB
 AuthDB = diracx.db.sql:AuthDB
 JobDB = diracx.db.sql:JobDB
 JobLoggingDB = diracx.db.sql:JobLoggingDB
+PilotAgentsDB = diracx.db.sql:PilotAgentsDB
 SandboxMetadataDB = diracx.db.sql:SandboxMetadataDB
 TaskQueueDB = diracx.db.sql:TaskQueueDB

diracx/db/sql/jobs/schema.py

@@ -1,290 +0,0 @@
-import sqlalchemy.types as types
-from sqlalchemy import (
-    BigInteger,
-    Boolean,
-    DateTime,
-    Enum,
-    Float,
-    ForeignKey,
-    Index,
-    Integer,
-    Numeric,
-    PrimaryKeyConstraint,
-    String,
-    Text,
-)
-from sqlalchemy.orm import declarative_base
-
-from ..utils import Column, DateNowColumn, NullColumn
-
-JobDBBase = declarative_base()
-JobLoggingDBBase = declarative_base()
-TaskQueueDBBase = declarative_base()
-
-
-class EnumBackedBool(types.TypeDecorator):
-    """Maps a ``EnumBackedBool()`` column to True/False in Python"""
-
-    impl = types.Enum
-    cache_ok: bool = True
-
-    def __init__(self) -> None:
-        super().__init__("True", "False")
-
-    def process_bind_param(self, value, dialect) -> str:
-        if value is True:
-            return "True"
-        elif value is False:
-            return "False"
-        else:
-            raise NotImplementedError(value, dialect)
-
-    def process_result_value(self, value, dialect) -> bool:
-        if value == "True":
-            return True
-        elif value == "False":
-            return False
-        else:
-            raise NotImplementedError(f"Unknown {value=}")
-
-
-class Jobs(JobDBBase):
-    __tablename__ = "Jobs"
-
-    JobID = Column(
-        "JobID",
-        Integer,
-        ForeignKey("JobJDLs.JobID", ondelete="CASCADE"),
-        primary_key=True,
-        default=0,
-    )
-    JobType = Column("JobType", String(32), default="user")
-    JobGroup = Column("JobGroup", String(32), default="00000000")
-    Site = Column("Site", String(100), default="ANY")
-    JobName = Column("JobName", String(128), default="Unknown")
-    Owner = Column("Owner", String(64), default="Unknown")
-    OwnerGroup = Column("OwnerGroup", String(128), default="Unknown")
-    VO = Column("VO", String(32))
-    SubmissionTime = NullColumn("SubmissionTime", DateTime)
-    RescheduleTime = NullColumn("RescheduleTime", DateTime)
-    LastUpdateTime = NullColumn("LastUpdateTime", DateTime)
-    StartExecTime = NullColumn("StartExecTime", DateTime)
-    HeartBeatTime = NullColumn("HeartBeatTime", DateTime)
-    EndExecTime = NullColumn("EndExecTime", DateTime)
-    Status = Column("Status", String(32), default="Received")
-    MinorStatus = Column("MinorStatus", String(128), default="Unknown")
-    ApplicationStatus = Column("ApplicationStatus", String(255), default="Unknown")
-    UserPriority = Column("UserPriority", Integer, default=0)
-    RescheduleCounter = Column("RescheduleCounter", Integer, default=0)
-    VerifiedFlag = Column("VerifiedFlag", EnumBackedBool(), default=False)
-    # TODO: Should this be True/False/"Failed"? Or True/False/Null?
-    AccountedFlag = Column(
-        "AccountedFlag", Enum("True", "False", "Failed"), default="False"
-    )
-
-    __table_args__ = (
-        Index("JobType", "JobType"),
-        Index("JobGroup", "JobGroup"),
-        Index("Site", "Site"),
-        Index("Owner", "Owner"),
-        Index("OwnerGroup", "OwnerGroup"),
-        Index("Status", "Status"),
-        Index("MinorStatus", "MinorStatus"),
-        Index("ApplicationStatus", "ApplicationStatus"),
-        Index("StatusSite", "Status", "Site"),
-        Index("LastUpdateTime", "LastUpdateTime"),
-    )
-
-
-class JobJDLs(JobDBBase):
-    __tablename__ = "JobJDLs"
-    JobID = Column(Integer, autoincrement=True, primary_key=True)
-    JDL = Column(Text)
-    JobRequirements = Column(Text)
-    OriginalJDL = Column(Text)
-
-
-class InputData(JobDBBase):
-    __tablename__ = "InputData"
-    JobID = Column(
-        Integer, ForeignKey("Jobs.JobID", ondelete="CASCADE"), primary_key=True
-    )
-    LFN = Column(String(255), default="", primary_key=True)
-    Status = Column(String(32), default="AprioriGood")
-
-
-class JobParameters(JobDBBase):
-    __tablename__ = "JobParameters"
-    JobID = Column(
-        Integer, ForeignKey("Jobs.JobID", ondelete="CASCADE"), primary_key=True
-    )
-    Name = Column(String(100), primary_key=True)
-    Value = Column(Text)
-
-
-class OptimizerParameters(JobDBBase):
-    __tablename__ = "OptimizerParameters"
-    JobID = Column(
-        Integer, ForeignKey("Jobs.JobID", ondelete="CASCADE"), primary_key=True
-    )
-    Name = Column(String(100), primary_key=True)
-    Value = Column(Text)
-
-
-class AtticJobParameters(JobDBBase):
-    __tablename__ = "AtticJobParameters"
-    JobID = Column(
-        Integer, ForeignKey("Jobs.JobID", ondelete="CASCADE"), primary_key=True
-    )
-    Name = Column(String(100), primary_key=True)
-    Value = Column(Text)
-    RescheduleCycle = Column(Integer)
-
-
-class SiteMask(JobDBBase):
-    __tablename__ = "SiteMask"
-    Site = Column(String(64), primary_key=True)
-    Status = Column(String(64))
-    LastUpdateTime = Column(DateTime)
-    Author = Column(String(255))
-    Comment = Column(Text)
-
-
-class SiteMaskLogging(JobDBBase):
-    __tablename__ = "SiteMaskLogging"
-    Site = Column(String(64), primary_key=True)
-    UpdateTime = Column(DateTime, primary_key=True)
-    Status = Column(String(64))
-    Author = Column(String(255))
-    Comment = Column(Text)
-
-
-class HeartBeatLoggingInfo(JobDBBase):
-    __tablename__ = "HeartBeatLoggingInfo"
-    JobID = Column(
-        Integer, ForeignKey("Jobs.JobID", ondelete="CASCADE"), primary_key=True
-    )
-    Name = Column(String(100), primary_key=True)
-    Value = Column(Text)
-    HeartBeatTime = Column(DateTime, primary_key=True)
-
-
-class JobCommands(JobDBBase):
-    __tablename__ = "JobCommands"
-    JobID = Column(
-        Integer, ForeignKey("Jobs.JobID", ondelete="CASCADE"), primary_key=True
-    )
-    Command = Column(String(100))
-    Arguments = Column(String(100))
-    Status = Column(String(64), default="Received")
-    ReceptionTime = Column(DateTime, primary_key=True)
-    ExecutionTime = NullColumn(DateTime)
-
-
-class LoggingInfo(JobLoggingDBBase):
-    __tablename__ = "LoggingInfo"
-    JobID = Column(Integer)
-    SeqNum = Column(Integer)
-    Status = Column(String(32), default="")
-    MinorStatus = Column(String(128), default="")
-    ApplicationStatus = Column(String(255), default="")
-    StatusTime = DateNowColumn()
-    # TODO: Check that this corresponds to the DOUBLE(12,3) type in MySQL
-    StatusTimeOrder = Column(Numeric(precision=12, scale=3), default=0)
-    Source = Column(String(32), default="Unknown", name="StatusSource")
-    __table_args__ = (PrimaryKeyConstraint("JobID", "SeqNum"),)
-
-
-class TaskQueues(TaskQueueDBBase):
-    __tablename__ = "tq_TaskQueues"
-    TQId = Column(Integer, primary_key=True)
-    Owner = Column(String(255), nullable=False)
-    OwnerGroup = Column(String(32), nullable=False)
-    VO = Column(String(32), nullable=False)
-    CPUTime = Column(BigInteger, nullable=False)
-    Priority = Column(Float, nullable=False)
-    Enabled = Column(Boolean, nullable=False, default=0)
-    __table_args__ = (Index("TQOwner", "Owner", "OwnerGroup", "CPUTime"),)
-
-
-class JobsQueue(TaskQueueDBBase):
-    __tablename__ = "tq_Jobs"
-    TQId = Column(
-        Integer, ForeignKey("tq_TaskQueues.TQId", ondelete="CASCADE"), primary_key=True
-    )
-    JobId = Column(Integer, primary_key=True)
-    Priority = Column(Integer, nullable=False)
-    RealPriority = Column(Float, nullable=False)
-    __table_args__ = (Index("TaskIndex", "TQId"),)
-
-
-class SitesQueue(TaskQueueDBBase):
-    __tablename__ = "tq_TQToSites"
-    TQId = Column(
-        Integer, ForeignKey("tq_TaskQueues.TQId", ondelete="CASCADE"), primary_key=True
-    )
-    Value = Column(String(64), primary_key=True)
-    __table_args__ = (
-        Index("SitesTaskIndex", "TQId"),
-        Index("SitesIndex", "Value"),
-    )
-
-
-class GridCEsQueue(TaskQueueDBBase):
-    __tablename__ = "tq_TQToGridCEs"
-    TQId = Column(
-        Integer, ForeignKey("tq_TaskQueues.TQId", ondelete="CASCADE"), primary_key=True
-    )
-    Value = Column(String(64), primary_key=True)
-    __table_args__ = (
-        Index("GridCEsTaskIndex", "TQId"),
-        Index("GridCEsValueIndex", "Value"),
-    )
-
-
-class BannedSitesQueue(TaskQueueDBBase):
-    __tablename__ = "tq_TQToBannedSites"
-    TQId = Column(
-        Integer, ForeignKey("tq_TaskQueues.TQId", ondelete="CASCADE"), primary_key=True
-    )
-    Value = Column(String(64), primary_key=True)
-    __table_args__ = (
-        Index("BannedSitesTaskIndex", "TQId"),
-        Index("BannedSitesValueIndex", "Value"),
-    )
-
-
-class PlatformsQueue(TaskQueueDBBase):
-    __tablename__ = "tq_TQToPlatforms"
-    TQId = Column(
-        Integer, ForeignKey("tq_TaskQueues.TQId", ondelete="CASCADE"), primary_key=True
-    )
-    Value = Column(String(64), primary_key=True)
-    __table_args__ = (
-        Index("PlatformsTaskIndex", "TQId"),
-        Index("PlatformsValueIndex", "Value"),
-    )
-
-
-class JobTypesQueue(TaskQueueDBBase):
-    __tablename__ = "tq_TQToJobTypes"
-    TQId = Column(
-        Integer, ForeignKey("tq_TaskQueues.TQId", ondelete="CASCADE"), primary_key=True
-    )
-    Value = Column(String(64), primary_key=True)
-    __table_args__ = (
-        Index("JobTypesTaskIndex", "TQId"),
-        Index("JobTypesValueIndex", "Value"),
-    )
-
-
-class TagsQueue(TaskQueueDBBase):
-    __tablename__ = "tq_TQToTags"
-    TQId = Column(
-        Integer, ForeignKey("tq_TaskQueues.TQId", ondelete="CASCADE"), primary_key=True
-    )
-    Value = Column(String(64), primary_key=True)
-    __table_args__ = (
-        Index("TagsTaskIndex", "TQId"),
-        Index("TagsValueIndex", "Value"),
-    )

diracx/db/sql/utils.py

@@ -1,236 +0,0 @@
-from __future__ import annotations
-
-__all__ = ("utcnow", "Column", "NullColumn", "DateNowColumn", "BaseSQLDB")
-
-import contextlib
-import logging
-import os
-from abc import ABCMeta
-from contextvars import ContextVar
-from datetime import datetime, timedelta, timezone
-from functools import partial
-from typing import TYPE_CHECKING, AsyncIterator, Self, cast
-
-from pydantic import TypeAdapter
-from sqlalchemy import Column as RawColumn
-from sqlalchemy import DateTime, Enum, MetaData, select
-from sqlalchemy.exc import OperationalError
-from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
-from sqlalchemy.ext.compiler import compiles
-from sqlalchemy.sql import expression
-
-from diracx.core.exceptions import InvalidQueryError
-from diracx.core.extensions import select_from_extension
-from diracx.core.settings import SqlalchemyDsn
-from diracx.db.exceptions import DBUnavailable
-
-if TYPE_CHECKING:
-    from sqlalchemy.types import TypeEngine
-
-logger = logging.getLogger(__name__)
-
-
-class utcnow(expression.FunctionElement):
-    type: TypeEngine = DateTime()
-    inherit_cache: bool = True
-
-
-@compiles(utcnow, "postgresql")
-def pg_utcnow(element, compiler, **kw) -> str:
-    return "TIMEZONE('utc', CURRENT_TIMESTAMP)"
-
-
-@compiles(utcnow, "mssql")
-def ms_utcnow(element, compiler, **kw) -> str:
-    return "GETUTCDATE()"
-
-
-@compiles(utcnow, "mysql")
-def mysql_utcnow(element, compiler, **kw) -> str:
-    return "(UTC_TIMESTAMP)"
-
-
-@compiles(utcnow, "sqlite")
-def sqlite_utcnow(element, compiler, **kw) -> str:
-    return "DATETIME('now')"
-
-
-def substract_date(**kwargs: float) -> datetime:
-    return datetime.now(tz=timezone.utc) - timedelta(**kwargs)
-
-
-Column: partial[RawColumn] = partial(RawColumn, nullable=False)
-NullColumn: partial[RawColumn] = partial(RawColumn, nullable=True)
-DateNowColumn = partial(Column, DateTime(timezone=True), server_default=utcnow())
-
-
-def EnumColumn(enum_type, **kwargs):
-    return Column(Enum(enum_type, native_enum=False, length=16), **kwargs)
-
-
-class SQLDBError(Exception):
-    pass
-
-
-class SQLDBUnavailable(DBUnavailable, SQLDBError):
-    """Used whenever we encounter a problem with the B connection"""
-
-
-class BaseSQLDB(metaclass=ABCMeta):
-    """This should be the base class of all the DiracX DBs"""
-
-    # engine: AsyncEngine
-    # TODO: Make metadata an abstract property
-    metadata: MetaData
-
-    def __init__(self, db_url: str) -> None:
-        # We use a ContextVar to make sure that self._conn
-        # is specific to each context, and avoid parallel
-        # route executions to overlap
-        self._conn: ContextVar[AsyncConnection | None] = ContextVar(
-            "_conn", default=None
-        )
-        self._db_url = db_url
-        self._engine: AsyncEngine | None = None
-
-    @classmethod
-    def available_implementations(cls, db_name: str) -> list[type[BaseSQLDB]]:
-        """Return the available implementations of the DB in reverse priority order."""
-        db_classes: list[type[BaseSQLDB]] = [
-            entry_point.load()
-            for entry_point in select_from_extension(
-                group="diracx.db.sql", name=db_name
-            )
-        ]
-        if not db_classes:
-            raise NotImplementedError(f"Could not find any matches for {db_name=}")
-        return db_classes
-
-    @classmethod
-    def available_urls(cls) -> dict[str, str]:
-        """Return a dict of available database urls.
-
-        The list of available URLs is determined by environment variables
-        prefixed with ``DIRACX_DB_URL_{DB_NAME}``.
-        """
-        db_urls: dict[str, str] = {}
-        for entry_point in select_from_extension(group="diracx.db.sql"):
-            db_name = entry_point.name
-            var_name = f"DIRACX_DB_URL_{entry_point.name.upper()}"
-            if var_name in os.environ:
-                try:
-                    db_url = os.environ[var_name]
-                    if db_url == "sqlite+aiosqlite:///:memory:":
-                        db_urls[db_name] = db_url
-                    else:
-                        db_urls[db_name] = str(
-                            TypeAdapter(SqlalchemyDsn).validate_python(db_url)
-                        )
-                except Exception:
-                    logger.error("Error loading URL for %s", db_name)
-                    raise
-        return db_urls
-
-    @classmethod
-    def transaction(cls) -> Self:
-        raise NotImplementedError("This should never be called")
-
-    @property
-    def engine(self) -> AsyncEngine:
-        """The engine to use for database operations.
-
-        It is normally not necessary to use the engine directly,
-        unless you are doing something special, like writing a
-        test fixture that gives you a db.
-
-
-        Requires that the engine_context has been entered.
-
-        """
-        assert self._engine is not None, "engine_context must be entered"
-        return self._engine
-
-    @contextlib.asynccontextmanager
-    async def engine_context(self) -> AsyncIterator[None]:
-        """Context manage to manage the engine lifecycle.
-        This is called once at the application startup
-        (see ``lifetime_functions``)
-        """
-        assert self._engine is None, "engine_context cannot be nested"
-
-        # Set the pool_recycle to 30mn
-        # That should prevent the problem of MySQL expiring connection
-        # after 60mn by default
-        engine = create_async_engine(self._db_url, pool_recycle=60 * 30)
-        self._engine = engine
-
-        yield
-
-        self._engine = None
-        await engine.dispose()
-
-    @property
-    def conn(self) -> AsyncConnection:
-        if self._conn.get() is None:
-            raise RuntimeError(f"{self.__class__} was used before entering")
-        return cast(AsyncConnection, self._conn.get())
-
-    async def __aenter__(self) -> Self:
-        """
-        Create a connection.
-        This is called by the Dependency mechanism (see ``db_transaction``),
-        It will create a new connection/transaction for each route call.
-        """
-        assert self._conn.get() is None, "BaseSQLDB context cannot be nested"
-        try:
-            self._conn.set(await self.engine.connect().__aenter__())
-        except Exception as e:
-            raise SQLDBUnavailable("Cannot connect to DB") from e
-
-        return self
-
-    async def __aexit__(self, exc_type, exc, tb):
-        """
-        This is called when exciting a route.
-        If there was no exception, the changes in the DB are committed.
-        Otherwise, they are rollbacked.
-        """
-        if exc_type is None:
-            await self._conn.get().commit()
-        await self._conn.get().__aexit__(exc_type, exc, tb)
-        self._conn.set(None)
-
-    async def ping(self):
-        """
-        Check whether the connection to the DB is still working.
-        We could enable the ``pre_ping`` in the engine, but this would
-        be ran at every query.
-        """
-        try:
-            await self.conn.scalar(select(1))
-        except OperationalError as e:
-            raise SQLDBUnavailable("Cannot ping the DB") from e
-
-
-def apply_search_filters(table, stmt, search):
-    # Apply any filters
-    for query in search:
-        column = table.columns[query["parameter"]]
-        if query["operator"] == "eq":
-            expr = column == query["value"]
-        elif query["operator"] == "neq":
-            expr = column != query["value"]
-        elif query["operator"] == "gt":
-            expr = column > query["value"]
-        elif query["operator"] == "lt":
-            expr = column < query["value"]
-        elif query["operator"] == "in":
-            expr = column.in_(query["values"])
-        elif query["operator"] in "like":
-            expr = column.like(query["value"])
-        elif query["operator"] in "ilike":
-            expr = column.ilike(query["value"])
-        else:
-            raise InvalidQueryError(f"Unknown filter {query=}")
-        stmt = stmt.where(expr)
-    return stmt

diracx_db-0.0.1a17.dist-info/RECORD

@@ -1,27 +0,0 @@
-diracx/db/__init__.py,sha256=2oeUeVwZq53bo_ZOflEYZsBn7tcR5Tzb2AIu0TAWELM,109
-diracx/db/__main__.py,sha256=3yaUP1ig-yaPSQM4wy6CtSXXHivQg-hIz2FeBt7joBc,1714
-diracx/db/exceptions.py,sha256=-LSkEwsvjwU7vXqx-xeLvLKInTRAhjwB7K_AKfQcIH8,41
-diracx/db/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-diracx/db/os/__init__.py,sha256=IZr6z6SefrRvuC8sTC4RmB3_wwOyEt1GzpDuwSMH8O4,112
-diracx/db/os/job_parameters.py,sha256=Knca19uT2G-5FI7MOFlaOAXeHn4ecPVLIH30TiwhaTw,858
-diracx/db/os/utils.py,sha256=mau0_2uRi-I3geefmKQRWFKo4JcIkIUADvnwBiQX700,9129
-diracx/db/sql/__init__.py,sha256=R6tk5lo1EHbt8joGDesesYHcc1swIq9T4AaSixhh7lA,252
-diracx/db/sql/utils.py,sha256=F55qv4u5JQMV5Cl-CNq4nXRCODlCZWarM8Dnkrx55T8,7930
-diracx/db/sql/auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-diracx/db/sql/auth/db.py,sha256=mKjy5B8orw0yu6nOwxyzbBqyeE-J9iYq6fKjuELmr9g,10273
-diracx/db/sql/auth/schema.py,sha256=JCkSa2IRzqMHTpaSc9aB9h33XsFyEM_Ohsenex6xagY,2835
-diracx/db/sql/dummy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-diracx/db/sql/dummy/db.py,sha256=5PIPv6aKY7CGIwmvnGKowjVr9ZQWpbjFSd2PIX7YOUw,1627
-diracx/db/sql/dummy/schema.py,sha256=uEkGDNVZbmJecytkHY1CO-M1MiKxe5w1_h0joJMPC9E,680
-diracx/db/sql/jobs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-diracx/db/sql/jobs/db.py,sha256=CyQIPX2g5ancBIBEMLijAyTi5HZxTgeVH0qQZ3p3KcU,30722
-diracx/db/sql/jobs/schema.py,sha256=YkxIdjTkvLlEZ9IQt86nj80eMvOPbcrfk9aisjmNpqY,9275
-diracx/db/sql/jobs/status_utility.py,sha256=w3caA9ng4y-Bzp0DOasj_DQ_M6JxAzJoHVmyJbEZ2qE,10534
-diracx/db/sql/sandbox_metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-diracx/db/sql/sandbox_metadata/db.py,sha256=0EDFMfOW_O3pEPTShqBCME9z4j-JKpyYM6-BBccr27E,6303
-diracx/db/sql/sandbox_metadata/schema.py,sha256=rngYYkJxBhjETBHGLD1CTipDGe44mRYR0wdaFoAJwp0,1400
-diracx_db-0.0.1a17.dist-info/METADATA,sha256=NarGcFXcBgKbSaHiyBPBVRj7dJ4xQ7OpJ8k0oIA7cHI,691
-diracx_db-0.0.1a17.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-diracx_db-0.0.1a17.dist-info/entry_points.txt,sha256=xEFGu_zgmPgQPlUeFtdahQfQIboJ1ugFOK8eMio9gtw,271
-diracx_db-0.0.1a17.dist-info/top_level.txt,sha256=vJx10tdRlBX3rF2Psgk5jlwVGZNcL3m_7iQWwgPXt-U,7
-diracx_db-0.0.1a17.dist-info/RECORD,,