queuerPy 0.19.0__tar.gz → 0.20.0__tar.gz

{queuerpy-0.19.0/queuerPy.egg-info → queuerpy-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.19.0
+Version: 0.20.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>

{queuerpy-0.19.0 → queuerpy-0.20.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.19.0"
+__version__ = "0.20.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

{queuerpy-0.19.0 → queuerpy-0.20.0}/database/db_job.py

@@ -9,7 +9,7 @@ from typing import List, Optional, Tuple
 from uuid import UUID
 from psycopg.rows import dict_row
 
-from ..helper.database import Database
+from ..helper.database import Database, DatabaseConfiguration
 from ..helper.error import QueuerError
 from ..helper.sql import SQLLoader, run_ddl
 from ..model.job import Job
@@ -25,7 +25,7 @@ class JobDBHandler:
     def __init__(
         self,
         db_connection: Database,
-        with_table_drop: bool = False,
+        db_config: DatabaseConfiguration,
         encryption_key: str = "",
     ):
         """Initialize job database handler."""
@@ -35,12 +35,13 @@ class JobDBHandler:
             raise ValueError("Database connection is not established")
 
         self.encryption_key: str = encryption_key
+        self.with_timescale: bool = db_config.with_timescale
 
         sql_loader: SQLLoader = SQLLoader()
-        sql_loader.load_notify_sql(self.db.instance, force=with_table_drop)
-        sql_loader.load_job_sql(self.db.instance, force=with_table_drop)
+        sql_loader.load_notify_sql(self.db.instance, force=db_config.with_table_drop)
+        sql_loader.load_job_sql(self.db.instance, force=db_config.with_table_drop)
 
-        if with_table_drop:
+        if db_config.with_table_drop:
             self.drop_tables()
 
         self.create_table()
@@ -72,7 +73,7 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        run_ddl(self.db.instance, "SELECT init_job();")
+        run_ddl(self.db.instance, f"SELECT init_job({self.with_timescale});")
 
     def drop_tables(self) -> None:
         """Drop job tables with DDL deadlock protection."""
@@ -379,9 +380,15 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor() as cur:
-            cur.execute("SELECT add_retention_archive(%s);", (days,))
-        self.db.instance.commit()
+        try:
+            with self.db.instance.cursor() as cur:
+                cur.execute(
+                    "SELECT add_retention_archive(%s, %s);", (days, self.with_timescale)
+                )
+            self.db.instance.commit()
+        except Exception as e:
+            self.db.instance.rollback()
+            raise e
 
     def remove_retention_archive(self) -> None:
         """
@@ -391,9 +398,15 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor() as cur:
-            cur.execute("SELECT remove_retention_archive();")
-        self.db.instance.commit()
+        try:
+            with self.db.instance.cursor() as cur:
+                cur.execute(
+                    "SELECT remove_retention_archive(%s);", (self.with_timescale,)
+                )
+            self.db.instance.commit()
+        except Exception as e:
+            self.db.instance.rollback()
+            raise e
 
     def select_job_from_archive(self, rid: UUID) -> Optional[Job]:
         """

{queuerpy-0.19.0 → queuerpy-0.20.0}/helper/database.py

@@ -30,6 +30,7 @@ class DatabaseConfiguration:
     schema: str = "public"
     sslmode: str = "require"
     with_table_drop: bool = False
+    with_timescale: bool = True
 
     @classmethod
     def from_env(cls) -> "DatabaseConfiguration":
@@ -45,6 +46,9 @@ class DatabaseConfiguration:
         with_table_drop = (
             os.getenv("QUEUER_DB_WITH_TABLE_DROP", "false").lower() == "true"
         )
+        with_timescale = (
+            os.getenv("QUEUER_DB_WITH_TIMESCALE", "true").lower() != "false"
+        )
 
         # Validate required fields
         if not all(
@@ -72,6 +76,7 @@ class DatabaseConfiguration:
             schema=schema,
             sslmode=sslmode,
             with_table_drop=with_table_drop,
+            with_timescale=with_timescale,
         )
 
     def connection_string(self) -> str:

{queuerpy-0.19.0 → queuerpy-0.20.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queuerPy"
-version = "0.19.0"
+version = "0.20.0"
 description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
 readme = "README.md"
 license = "Apache-2.0"
@@ -92,7 +92,7 @@ pythonVersion = "3.8"
 pythonPlatform = "All"
 
 [tool.mypy]
-python_version = "0.19.0"
+python_version = "0.20.0"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{queuerpy-0.19.0 → queuerpy-0.20.0/queuerPy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.19.0
+Version: 0.20.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>

{queuerpy-0.19.0 → queuerpy-0.20.0}/queuer_global.py

@@ -56,7 +56,7 @@ class QueuerGlobalMixin:
 
         # Database handlers
         self.db_job: JobDBHandler = JobDBHandler(
-            self.database, db_config.with_table_drop, encryption_key
+            self.database, db_config, encryption_key
         )
         self.db_worker: WorkerDBHandler = WorkerDBHandler(
             self.database, db_config.with_table_drop

{queuerpy-0.19.0 → queuerpy-0.20.0}/queuer_master.py

@@ -38,14 +38,27 @@ class QueuerMasterMixin(QueuerGlobalMixin):
 
         # Handle retention archive configuration
         try:
-            if old_master.settings.retention_archive == 0:
-                self.db_job.add_retention_archive(master_settings.retention_archive)
-            elif (
-                old_master.settings.retention_archive
-                != master_settings.retention_archive
-            ):
-                self.db_job.remove_retention_archive()
-                self.db_job.add_retention_archive(master_settings.retention_archive)
+            if master_settings.retention_archive > 0:
+                if old_master.settings.retention_archive == 0:
+                    self.db_job.add_retention_archive(master_settings.retention_archive)
+                elif (
+                    old_master.settings.retention_archive
+                    != master_settings.retention_archive
+                ):
+                    try:
+                        self.db_job.remove_retention_archive()
+                    except Exception as e:
+                        logger.warning(
+                            f"Could not remove old retention policy (may not exist): {e}"
+                        )
+                    self.db_job.add_retention_archive(master_settings.retention_archive)
+            elif old_master.settings.retention_archive > 0:
+                try:
+                    self.db_job.remove_retention_archive()
+                except Exception as e:
+                    logger.warning(
+                        f"Could not remove retention policy (may not exist): {e}"
+                    )
         except Exception as e:
             raise QueuerError("managing retention archive", e)
 
@@ -83,10 +96,9 @@ class QueuerMasterMixin(QueuerGlobalMixin):
                     logger.error(f"Error in master ticker task: {e}")
 
             ticker = Ticker(
-                task=ticker_task,
-                name="_master_ticker",
                 interval=timedelta(seconds=master_settings.master_poll_interval),
-                func=ticker_task,
+                task=ticker_task,
+                use_mp=False,  # Use threading instead of multiprocessing to avoid pickling issues
             )
             logger.info("Starting master ticker...")
             ticker.go()
@@ -150,10 +162,9 @@ class QueuerMasterMixin(QueuerGlobalMixin):
 
             # Create and start the polling ticker
             ticker = Ticker(
-                task=ticker_task,
-                name="_poll_master_ticker",
                 interval=timedelta(seconds=master_settings.master_poll_interval),
-                func=ticker_task,
+                task=ticker_task,
+                use_mp=False,  # Use threading instead of multiprocessing to avoid pickling issues
             )
 
             logger.info("Starting master poll ticker...")

{queuerpy-0.19.0 → queuerpy-0.20.0}/queuer_worker.py

@@ -15,6 +15,19 @@ from .queuer_global import QueuerGlobalMixin
 logger = logging.getLogger(__name__)
 
 
+def get_current_worker_rid(self) -> Optional[UUID]:
+    """
+    Get the current worker's RID safely with mutex locking.
+    Mirrors Go's GetCurrentWorkerRID method.
+
+    :return: Worker RID or None if no worker
+    """
+    with self.worker_mutex:
+        if self.worker is not None:
+            return self.worker.rid
+        return None
+
+
 class QueuerWorkerMixin(QueuerGlobalMixin):
     """
     Mixin class containing worker-related methods for the Queuer.

{queuerpy-0.19.0 → queuerpy-0.20.0}/sql/job.sql

@@ -1,7 +1,9 @@
-CREATE OR REPLACE FUNCTION init_job() RETURNS VOID AS $$
+CREATE OR REPLACE FUNCTION init_job(enable_timescaledb BOOLEAN DEFAULT TRUE) RETURNS VOID AS $$
 BEGIN
     -- Create TimescaleDB extension
-    CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+    IF enable_timescaledb THEN
+        CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+    END IF;
 
     -- Create pgcrypto extension
     CREATE EXTENSION IF NOT EXISTS pgcrypto;
@@ -37,7 +39,9 @@ BEGIN
     );
     
     -- Create hypertable (TimescaleDB feature)
-    PERFORM create_hypertable('job_archive', by_range('updated_at'), if_not_exists => TRUE);
+    IF enable_timescaledb THEN
+        PERFORM create_hypertable('job_archive', by_range('updated_at'), if_not_exists => TRUE);
+    END IF;
     
     -- Create triggers for notifications
     CREATE OR REPLACE TRIGGER job_notify_event
@@ -698,17 +702,64 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION add_retention_archive(input_retention_days INT)
+CREATE OR REPLACE FUNCTION add_retention_archive(input_retention_days INT, enable_timescaledb BOOLEAN DEFAULT TRUE)
 RETURNS VOID AS $$
 BEGIN
-    PERFORM add_retention_policy('job_archive', (input_retention_days * INTERVAL '1 day'));
+    IF enable_timescaledb THEN
+        -- Use TimescaleDB retention policy
+        PERFORM add_retention_policy('job_archive', (input_retention_days * INTERVAL '1 day'));
+    ELSE
+        -- Create sequence for tracking inserts
+        CREATE SEQUENCE IF NOT EXISTS job_archive_insert_counter;
+        
+        -- Create trigger function that deletes old records every 10th insert
+        EXECUTE format('
+            CREATE OR REPLACE FUNCTION cleanup_job_archive_trigger()
+            RETURNS TRIGGER AS $func$
+            DECLARE
+                insert_count BIGINT;
+                deleted_count INT;
+            BEGIN
+                -- Only run cleanup every 10th insert
+                insert_count := nextval(''job_archive_insert_counter'');
+                IF insert_count %% 10 = 0 THEN
+                    DELETE FROM job_archive
+                    WHERE updated_at < (CURRENT_TIMESTAMP - INTERVAL ''%s days'');
+                    
+                    GET DIAGNOSTICS deleted_count = ROW_COUNT;
+                    
+                    -- Log cleanup if any rows were deleted (optional)
+                    IF deleted_count > 0 THEN
+                        RAISE NOTICE ''Cleaned up %% old job_archive records'', deleted_count;
+                    END IF;
+                END IF;
+                RETURN NEW;
+            END;
+            $func$ LANGUAGE plpgsql;
+        ', input_retention_days);
+        
+        -- Create trigger on job_archive
+        DROP TRIGGER IF EXISTS job_archive_cleanup_trigger ON job_archive;
+        CREATE TRIGGER job_archive_cleanup_trigger
+            AFTER INSERT ON job_archive
+            FOR EACH ROW
+            EXECUTE FUNCTION cleanup_job_archive_trigger();
+    END IF;
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION remove_retention_archive()
+CREATE OR REPLACE FUNCTION remove_retention_archive(enable_timescaledb BOOLEAN DEFAULT TRUE)
 RETURNS VOID AS $$
 BEGIN
-    PERFORM remove_retention_policy('job_archive');
+    IF enable_timescaledb THEN
+        -- Remove TimescaleDB retention policy
+        PERFORM remove_retention_policy('job_archive');
+    ELSE
+        -- Drop trigger and related objects
+        DROP TRIGGER IF EXISTS job_archive_cleanup_trigger ON job_archive;
+        DROP FUNCTION IF EXISTS cleanup_job_archive_trigger();
+        DROP SEQUENCE IF EXISTS job_archive_insert_counter;
+    END IF;
 END;
 $$ LANGUAGE plpgsql;