python3-cyberfusion-queue-support 1.7.0__tar.gz → 2.2__tar.gz

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/PKG-INFO

@@ -1,11 +1,15 @@
 Metadata-Version: 2.4
 Name: python3-cyberfusion-queue-support
-Version: 1.7.0
+Version: 2.2
 Summary: Library to queue actions.
 Author-email: Cyberfusion <support@cyberfusion.io>
 Project-URL: Source, https://github.com/CyberfusionIO/python3-cyberfusion-queue-support
 Description-Content-Type: text/markdown
 Requires-Dist: python3-cyberfusion-systemd-support~=2.1
+Requires-Dist: python3-cyberfusion-database-support~=3.0
+Requires-Dist: alembic==1.8.1
+Requires-Dist: SQLAlchemy==1.4.46
+Requires-Dist: pydantic[dotenv]==1.10.4
 
 # python3-cyberfusion-queue-support
 
@@ -46,6 +50,10 @@ Run the following command to install the package from PyPI:
 
     pip3 install python3-cyberfusion-queue-support
 
+Then, run database migrations:
+
+    bin/queue-support-migrate
+
 ## Debian
 
 Run the following commands to build a Debian package:

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/README.md

@@ -37,6 +37,10 @@ Run the following command to install the package from PyPI:
 
     pip3 install python3-cyberfusion-queue-support
 
+Then, run database migrations:
+
+    bin/queue-support-migrate
+
 ## Debian
 
 Run the following commands to build a Debian package:

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "python3-cyberfusion-queue-support"
-version = "1.7.0"
+version = "2.2"
 description = "Library to queue actions."
 readme = "README.md"
 authors = [
@@ -12,7 +12,14 @@ authors = [
 ]
 dependencies = [
     "python3-cyberfusion-systemd-support~=2.1",
+    "python3-cyberfusion-database-support~=3.0",
+    "alembic==1.8.1",
+    "SQLAlchemy==1.4.46",
+    "pydantic[dotenv]==1.10.4",
 ]
 
 [project.urls]
 "Source" = "https://github.com/CyberfusionIO/python3-cyberfusion-queue-support"
+
+[project.scripts]
+queue-support-migrate = "cyberfusion.QueueSupport.database:run_migrations"

python3_cyberfusion_queue_support-2.2/src/cyberfusion/QueueSupport/__init__.py

@@ -0,0 +1,150 @@
+"""Classes for queue."""
+
+import logging
+from copy import copy
+from dataclasses import dataclass
+from typing import List
+
+from cyberfusion.QueueSupport.database import (
+    Queue as QueueModel,
+    QueueItem,
+    make_database_session,
+    QueueItemOutcome,
+    QueueProcess,
+)
+from cyberfusion.QueueSupport.exceptions import QueueFulfillFailed
+from cyberfusion.QueueSupport.interfaces import OutcomeInterface
+from cyberfusion.QueueSupport.items import _Item
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class QueueItemMapping:
+    """Queue item mapping."""
+
+    item: _Item
+    database_object: QueueItem
+
+
+class Queue:
+    """Represents queue."""
+
+    def __init__(self) -> None:
+        """Set attributes."""
+        self.item_mappings: List[QueueItemMapping] = []
+
+        self._database_session = make_database_session()
+
+        object_ = QueueModel()
+
+        self._database_session.add(object_)
+        self._database_session.commit()
+
+        self.queue_database_object = object_
+
+    def add(self, item: _Item, *, run_duplicate_last: bool = True) -> None:
+        """Add item to queue."""
+        deduplicated = False
+
+        existing_item_index = next(
+            (
+                index
+                for index, item_mapping in enumerate(self.item_mappings)
+                if item_mapping.item == item
+            ),
+            None,
+        )
+
+        if existing_item_index is not None:
+            if run_duplicate_last:
+                self.item_mappings[
+                    existing_item_index
+                ].database_object.deduplicated = True
+
+                self._database_session.commit()
+            else:
+                deduplicated = True
+
+        item_dict = copy(item.__dict__)
+
+        del item_dict["_reference"]
+        del item_dict["_hide_outcomes"]
+
+        object_ = QueueItem(
+            queue=self.queue_database_object,
+            type=item.__class__.__name__,
+            reference=item.reference,
+            hide_outcomes=item.hide_outcomes,
+            deduplicated=deduplicated,
+            attributes=item_dict,
+        )
+
+        self._database_session.add(object_)
+        self._database_session.commit()
+
+        self.item_mappings.append(QueueItemMapping(item, object_))
+
+    def process(self, preview: bool) -> List[OutcomeInterface]:
+        """Process items."""
+        logger.debug("Processing items")
+
+        process_object = QueueProcess(
+            queue_id=self.queue_database_object.id,
+            preview=preview,
+        )
+
+        self._database_session.add(process_object)
+        self._database_session.commit()
+
+        outcomes = []
+
+        for item_mapping in [
+            item_mapping
+            for item_mapping in self.item_mappings
+            if not item_mapping.database_object.deduplicated
+        ]:
+            logger.debug(
+                "Processing item with id '%s'", item_mapping.database_object.id
+            )
+
+            if preview:
+                if not item_mapping.item.hide_outcomes:
+                    outcomes.extend(item_mapping.item.outcomes)
+            else:
+                try:
+                    logger.debug(
+                        "Fulfilling item with id '%s'", item_mapping.database_object.id
+                    )
+
+                    fulfill_outcomes = item_mapping.item.fulfill()
+
+                    if not item_mapping.item.hide_outcomes:
+                        outcomes.extend(fulfill_outcomes)
+
+                    logger.debug(
+                        "Fulfilled item with id '%s'", item_mapping.database_object.id
+                    )
+                except QueueFulfillFailed:
+                    raise
+                except Exception as e:
+                    raise QueueFulfillFailed(
+                        item_mapping.item,
+                    ) from e
+
+            for outcome in outcomes:
+                outcome_object = QueueItemOutcome(
+                    queue_item=item_mapping.database_object,
+                    queue_process=process_object,
+                    type=outcome.__class__.__name__,
+                    attributes=outcome.__dict__,
+                )
+
+                self._database_session.add(outcome_object)
+                self._database_session.commit()
+
+            logger.debug("Processed item with id '%s'", item_mapping.database_object.id)
+
+        logger.debug("Processed items")
+
+        return outcomes

python3_cyberfusion_queue_support-2.2/src/cyberfusion/QueueSupport/database.py

@@ -0,0 +1,112 @@
+from alembic.config import Config
+from alembic import command
+import sqlite3
+from datetime import datetime, timezone
+from sqlalchemy.pool.base import _ConnectionRecord
+from sqlalchemy import ForeignKey, MetaData, Boolean
+from sqlalchemy import create_engine, Column, DateTime, Integer, String
+from sqlalchemy.orm import Session, sessionmaker, relationship
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.engine import Engine
+from sqlalchemy import event
+from sqlalchemy.types import JSON
+
+from cyberfusion.QueueSupport.settings import settings
+
+
+@event.listens_for(Engine, "connect")  # type: ignore[misc]
+def set_sqlite_pragma(
+    dbapi_connection: sqlite3.Connection, connection_record: _ConnectionRecord
+) -> None:
+    """Enable foreign key support.
+
+    This is needed for cascade deletes to work.
+
+    See https://docs.sqlalchemy.org/en/13/dialects/sqlite.html#sqlite-foreign-keys
+    """
+    cursor = dbapi_connection.cursor()
+
+    cursor.execute("PRAGMA foreign_keys=ON")
+
+    cursor.close()
+
+
+def run_migrations() -> None:
+    """Upgrade database schema to latest version."""
+    alembic_config = Config(file_=settings.alembic_config_file_path)
+    alembic_config.set_main_option("sqlalchemy.url", settings.database_path)
+
+    command.upgrade(alembic_config, "head")
+
+
+def make_database_session() -> Session:
+    engine = create_engine(
+        settings.database_path, connect_args={"check_same_thread": False}
+    )
+
+    return sessionmaker(bind=engine)()
+
+
+naming_convention = {
+    "ix": "ix_%(column_0_label)s",
+    "uq": "uq_%(table_name)s_%(column_0_name)s",
+    "ck": "ck_%(table_name)s_%(constraint_name)s",
+    "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+    "pk": "pk_%(table_name)s",
+}
+
+metadata = MetaData(naming_convention=naming_convention)
+
+Base = declarative_base(metadata=metadata)
+
+
+class BaseModel(Base):  # type: ignore[misc, valid-type]
+    """Base model."""
+
+    __abstract__ = True
+
+    id = Column(Integer, primary_key=True)
+    created_at = Column(DateTime, default=datetime.now(timezone.utc), nullable=False)
+
+
+class Queue(BaseModel):
+    """Queue model."""
+
+    __tablename__ = "queues"
+
+
+class QueueProcess(BaseModel):
+    """QueueProcess model."""
+
+    __tablename__ = "queue_processes"
+
+    queue_id = Column(Integer, ForeignKey("queues.id"), nullable=False)
+    queue = relationship("Queue")
+    preview = Column(Boolean, nullable=False)
+
+
+class QueueItem(BaseModel):
+    """QueueItem model."""
+
+    __tablename__ = "queue_items"
+
+    queue_id = Column(Integer, ForeignKey("queues.id"), nullable=False)
+    queue = relationship("Queue")
+    type = Column(String(length=255), nullable=False)
+    reference = Column(String(length=255), nullable=True)
+    hide_outcomes = Column(Boolean, nullable=False)
+    deduplicated = Column(Boolean, nullable=False)
+    attributes = Column(JSON, nullable=False)
+
+
+class QueueItemOutcome(BaseModel):
+    """QueueItemOutcome model."""
+
+    __tablename__ = "queue_item_outcomes"
+
+    queue_item_id = Column(Integer, ForeignKey("queue_items.id"), nullable=False)
+    queue_item = relationship("QueueItem")
+    queue_process_id = Column(Integer, ForeignKey("queue_processes.id"), nullable=False)
+    queue_process = relationship("QueueProcess")
+    type = Column(String(length=255), nullable=False)
+    attributes = Column(JSON, nullable=False)

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/src/cyberfusion/QueueSupport/interfaces.py

@@ -28,7 +28,7 @@ class ItemInterface(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def fulfill(self) -> None:  # pragma: no cover
+    def fulfill(self) -> List[OutcomeInterface]:  # pragma: no cover
         """Fulfill outcomes."""
         pass
 

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/src/cyberfusion/QueueSupport/items/__init__.py

@@ -1,6 +1,7 @@
 """Items."""
 
 import logging
+from abc import ABC, abstractmethod
 from typing import Optional
 
 from cyberfusion.QueueSupport.interfaces import ItemInterface
@@ -8,9 +9,20 @@ from cyberfusion.QueueSupport.interfaces import ItemInterface
 logger = logging.getLogger(__name__)
 
 
-class _Item(ItemInterface):
+class _Item(ItemInterface, ABC):
     """Represents base item."""
 
+    @abstractmethod
+    def __init__(
+        self,
+        *,
+        reference: Optional[str] = None,
+        hide_outcomes: bool = False,
+    ) -> None:  # pragma: no cover
+        """Set attributes."""
+        self._reference = reference
+        self._hide_outcomes = hide_outcomes
+
     @property
     def hide_outcomes(self) -> bool:
         """Get if outcomes should be hidden."""

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/src/cyberfusion/QueueSupport/items/chmod.py

@@ -56,11 +56,15 @@ class ChmodItem(_Item):
 
         return outcomes
 
-    def fulfill(self) -> None:
+    def fulfill(self) -> List[ChmodItemModeChangeOutcome]:
         """Fulfill outcomes."""
-        for outcome in self.outcomes:
+        outcomes = self.outcomes
+
+        for outcome in outcomes:
             os.chmod(outcome.path, outcome.new_mode)
 
+        return outcomes
+
     def __eq__(self, other: object) -> bool:
         """Get equality based on attributes."""
         if not isinstance(other, ChmodItem):

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/src/cyberfusion/QueueSupport/items/chown.py

@@ -106,9 +106,13 @@ class ChownItem(_Item):
 
         return outcomes
 
-    def fulfill(self) -> None:
+    def fulfill(
+        self,
+    ) -> List[ChownItemOwnerChangeOutcome | ChownItemGroupChangeOutcome]:
         """Fulfill outcomes."""
-        for outcome in self.outcomes:
+        outcomes = self.outcomes
+
+        for outcome in outcomes:
             if isinstance(outcome, ChownItemOwnerChangeOutcome):
                 os.chown(
                     outcome.path,
@@ -122,6 +126,8 @@ class ChownItem(_Item):
                     gid=get_gid(outcome.new_group_name),
                 )
 
+        return outcomes
+
     def __eq__(self, other: object) -> bool:
         """Get equality based on attributes."""
         if not isinstance(other, ChownItem):

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/src/cyberfusion/QueueSupport/items/command.py

@@ -35,9 +35,11 @@ class CommandItem(_Item):
 
         return outcomes
 
-    def fulfill(self) -> None:
+    def fulfill(self) -> List[CommandItemRunOutcome]:
         """Fulfill outcomes."""
-        for outcome in self.outcomes:
+        outcomes = self.outcomes
+
+        for outcome in outcomes:
             try:
                 output = subprocess.run(
                     outcome.command,
@@ -53,6 +55,8 @@ class CommandItem(_Item):
                     self, command=outcome.command, stdout=e.stdout, stderr=e.stderr
                 ) from e
 
+        return outcomes
+
     def __eq__(self, other: object) -> bool:
         """Get equality based on attributes."""
         if not isinstance(other, CommandItem):

{python3_cyberfusion_queue_support-1.7.0 → python3_cyberfusion_queue_support-2.2}/src/cyberfusion/QueueSupport/items/copy.py

@@ -94,11 +94,15 @@ class CopyItem(_Item):
 
         return outcomes
 
-    def fulfill(self) -> None:
+    def fulfill(self) -> List[CopyItemCopyOutcome]:
         """Fulfill outcomes."""
-        for outcome in self.outcomes:
+        outcomes = self.outcomes
+
+        for outcome in outcomes:
             shutil.copyfile(outcome.source, outcome.destination)
 
+        return outcomes
+
     def __eq__(self, other: object) -> bool:
         """Get equality based on attributes."""
         if not isinstance(other, CopyItem):

python3_cyberfusion_queue_support-2.2/src/cyberfusion/QueueSupport/items/database_create.py

@@ -0,0 +1,65 @@
+"""Item."""
+
+import logging
+from typing import List, Optional
+
+from cyberfusion.QueueSupport.items import _Item
+from cyberfusion.QueueSupport.outcomes import DatabaseCreateItemCreateOutcome
+
+from cyberfusion.DatabaseSupport.databases import Database
+from cyberfusion.DatabaseSupport import DatabaseSupport
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseCreateItem(_Item):
+    """Represents item."""
+
+    def __init__(
+        self,
+        *,
+        server_software_name: str,
+        name: str,
+        reference: Optional[str] = None,
+        hide_outcomes: bool = False,
+    ) -> None:
+        """Set attributes."""
+        self.server_software_name = server_software_name
+        self.name = name
+        self._reference = reference
+        self._hide_outcomes = hide_outcomes
+
+        self.database = Database(
+            support=DatabaseSupport(server_software_names=[self.server_software_name]),
+            name=self.name,
+            server_software_name=self.server_software_name,
+        )
+
+    @property
+    def outcomes(self) -> List[DatabaseCreateItemCreateOutcome]:
+        """Get outcomes of item."""
+        outcomes = []
+
+        if not self.database.exists:
+            outcomes.append(DatabaseCreateItemCreateOutcome(database=self.database))
+
+        return outcomes
+
+    def fulfill(self) -> List[DatabaseCreateItemCreateOutcome]:
+        """Fulfill outcomes."""
+        outcomes = self.outcomes
+
+        for outcome in outcomes:
+            outcome.database.create()
+
+        return outcomes
+
+    def __eq__(self, other: object) -> bool:
+        """Get equality based on attributes."""
+        if not isinstance(other, DatabaseCreateItem):
+            return False
+
+        return (
+            other.server_software_name == self.server_software_name
+            and other.name == self.name
+        )

python3_cyberfusion_queue_support-2.2/src/cyberfusion/QueueSupport/items/database_drop.py

@@ -0,0 +1,65 @@
+"""Item."""
+
+import logging
+from typing import List, Optional
+
+from cyberfusion.QueueSupport.items import _Item
+from cyberfusion.QueueSupport.outcomes import DatabaseDropItemDropOutcome
+
+from cyberfusion.DatabaseSupport.databases import Database
+from cyberfusion.DatabaseSupport import DatabaseSupport
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseDropItem(_Item):
+    """Represents item."""
+
+    def __init__(
+        self,
+        *,
+        server_software_name: str,
+        name: str,
+        reference: Optional[str] = None,
+        hide_outcomes: bool = False,
+    ) -> None:
+        """Set attributes."""
+        self.server_software_name = server_software_name
+        self.name = name
+        self._reference = reference
+        self._hide_outcomes = hide_outcomes
+
+        self.database = Database(
+            support=DatabaseSupport(server_software_names=[self.server_software_name]),
+            name=self.name,
+            server_software_name=self.server_software_name,
+        )
+
+    @property
+    def outcomes(self) -> List[DatabaseDropItemDropOutcome]:
+        """Get outcomes of item."""
+        outcomes = []
+
+        if self.database.exists:
+            outcomes.append(DatabaseDropItemDropOutcome(database=self.database))
+
+        return outcomes
+
+    def fulfill(self) -> List[DatabaseDropItemDropOutcome]:
+        """Fulfill outcomes."""
+        outcomes = self.outcomes
+
+        for outcome in outcomes:
+            outcome.database.drop()
+
+        return outcomes
+
+    def __eq__(self, other: object) -> bool:
+        """Get equality based on attributes."""
+        if not isinstance(other, DatabaseDropItem):
+            return False
+
+        return (
+            other.server_software_name == self.server_software_name
+            and other.name == self.name
+        )

python3_cyberfusion_queue_support-2.2/src/cyberfusion/QueueSupport/items/database_user_drop.py

@@ -0,0 +1,81 @@
+"""Item."""
+
+import logging
+from typing import List, Optional
+
+from cyberfusion.DatabaseSupport.database_users import DatabaseUser
+from cyberfusion.DatabaseSupport.servers import Server
+
+from cyberfusion.QueueSupport.items import _Item
+from cyberfusion.QueueSupport.outcomes import (
+    DatabaseUserDropItemDropOutcome,
+)
+
+from cyberfusion.DatabaseSupport import DatabaseSupport
+
+logger = logging.getLogger(__name__)
+
+
+class DatabaseUserDropItem(_Item):
+    """Represents item."""
+
+    def __init__(
+        self,
+        *,
+        server_software_name: str,
+        name: str,
+        host: Optional[str] = None,
+        reference: Optional[str] = None,
+        hide_outcomes: bool = False,
+    ) -> None:
+        """Set attributes."""
+        self.server_software_name = server_software_name
+        self.name = name
+        self._host = host
+        self._reference = reference
+        self._hide_outcomes = hide_outcomes
+
+        self.database_user = DatabaseUser(
+            server=Server(
+                support=DatabaseSupport(
+                    server_software_names=[self.server_software_name]
+                )
+            ),
+            name=self.name,
+            server_software_name=self.server_software_name,
+            host=self._host,
+        )
+
+    @property
+    def outcomes(
+        self,
+    ) -> List[DatabaseUserDropItemDropOutcome]:
+        """Get outcomes of item."""
+        outcomes = []
+
+        if self.database_user.exists:
+            outcomes.append(
+                DatabaseUserDropItemDropOutcome(database_user=self.database_user)
+            )
+
+        return outcomes
+
+    def fulfill(self) -> List[DatabaseUserDropItemDropOutcome]:
+        """Fulfill outcomes."""
+        outcomes = self.outcomes
+
+        for outcome in outcomes:
+            outcome.database_user.drop()
+
+        return outcomes
+
+    def __eq__(self, other: object) -> bool:
+        """Get equality based on attributes."""
+        if not isinstance(other, DatabaseUserDropItem):
+            return False
+
+        return (
+            other.server_software_name == self.server_software_name
+            and other.name == self.name
+            and other._host == self._host
+        )