queuerPy 0.18.0__tar.gz → 0.20.0__tar.gz

{queuerpy-0.18.0/queuerPy.egg-info → queuerpy-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.18.0
+Version: 0.20.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -58,9 +58,7 @@ The job table contains only queued, scheduled and running tasks. The ended jobs
 
 ---
 
-# 🛠️ Installation
-
-## PyPI Installation (Recommended)
+## 🛠️ Installation
 
 ```bash
 pip install queuerPy
@@ -134,10 +132,10 @@ You can find a full example in the example folder.
 def new_queuer(name: str, max_concurrency: int, *options: OnError) -> Queuer
 
 def new_queuer_with_db(
-    name: str, 
-    max_concurrency: int, 
-    encryption_key: str, 
-    db_config: DatabaseConfiguration, 
+    name: str,
+    max_concurrency: int,
+    encryption_key: str,
+    db_config: DatabaseConfiguration,
     *options: OnError
 ) -> Queuer
 ```
@@ -149,6 +147,7 @@ def new_queuer_with_db(
 - `options`: Optional `OnError` configurations to apply to the worker.
 
 This function performs the following setup:
+
 - Initializes a logger.
 - Sets up the database connection using the provided `db_config` or environment variables.
 - Creates `JobDBHandler`, `WorkerDBHandler` instances for database interactions.
@@ -167,6 +166,7 @@ def start(self) -> None
 ```
 
 Upon calling `start`:
+
 - It performs a basic check to ensure internal listeners are initialized.
 - Database listeners are created to listen to job events (inserts, updates, deletes) via PostgreSQL NOTIFY/LISTEN.
 - It starts a poller to periodically poll the database for new jobs to process.
@@ -185,7 +185,50 @@ The `stop` method gracefully shuts down the Queuer instance, releasing resources
 def stop(self) -> None
 ```
 
-The `stop` method cancels all jobs, closes database listeners, and cleans up resources.
+The `stop` method cancels all jobs, closes database listeners, and cleans up resources. **Note:** This method can only be used to stop the current worker instance that the code is running in. To stop other workers, use `stop_worker` or `stop_worker_gracefully`.
+
+---
+
+## stop_worker
+
+The `stop_worker` method immediately stops a worker by setting its status to `STOPPED`. This will cancel all running jobs on that worker.
+
+```python
+def stop_worker(self, worker_rid: UUID) -> None
+```
+
+- `worker_rid`: The `UUID` identifying the worker to stop.
+
+When a worker is stopped:
+
+- The worker status is immediately set to `STOPPED` in the database
+- The heartbeat ticker detects the `STOPPED` status and calls `stop()` on that worker
+- All running jobs on that worker are cancelled immediately
+- The worker will no longer accept new jobs
+
+This method is useful for immediately shutting down a worker, for example in emergency situations or when you need to take a worker offline quickly.
+
+---
+
+## stop_worker_gracefully
+
+The `stop_worker_gracefully` method gracefully stops a worker by setting its status to `STOPPING`. This allows currently running jobs to complete before the worker shuts down.
+
+```python
+def stop_worker_gracefully(self, worker_rid: UUID) -> None
+```
+
+- `worker_rid`: The `UUID` identifying the worker to stop gracefully.
+
+When a worker is stopped gracefully:
+
+- The worker status is set to `STOPPING` in the database
+- The heartbeat ticker detects the `STOPPING` status and sets `max_concurrency` to `0`
+- Currently running jobs are allowed to complete normally
+- No new jobs will be accepted by this worker
+- Once all running jobs have finished, the worker status is automatically set to `STOPPED` and the worker shuts down
+
+This method is ideal for maintenance scenarios where you want to ensure all in-progress work completes before shutting down the worker.
 
 ---
 
@@ -212,8 +255,8 @@ The `add_job` method adds a new job to the queue for execution. Jobs are units o
 
 ```python
 def add_job(
-    self, 
-    task: Union[Callable, str], 
+    self,
+    task: Union[Callable, str],
     *parameters: Any,
     **parameters_keyed: Any
 ) -> Job
@@ -322,7 +365,7 @@ class RetryBackoff(str, Enum):
 
 - `RETRY_BACKOFF_NONE`: No backoff. The retry_delay remains constant for all retries.
 - `RETRY_BACKOFF_LINEAR`: The retry delay increases linearly with each attempt (e.g., delay, 2*delay, 3*delay).
-- `RETRY_BACKOFF_EXPONENTIAL`: The retry delay increases exponentially with each attempt (e.g., delay, delay*2, delay*2*2).
+- `RETRY_BACKOFF_EXPONENTIAL`: The retry delay increases exponentially with each attempt (e.g., delay, delay*2, delay*2\*2).
 
 ---
 

{queuerpy-0.18.0 → queuerpy-0.20.0}/README.md

@@ -15,9 +15,7 @@ The job table contains only queued, scheduled and running tasks. The ended jobs
 
 ---
 
-# 🛠️ Installation
-
-## PyPI Installation (Recommended)
+## 🛠️ Installation
 
 ```bash
 pip install queuerPy
@@ -91,10 +89,10 @@ You can find a full example in the example folder.
 def new_queuer(name: str, max_concurrency: int, *options: OnError) -> Queuer
 
 def new_queuer_with_db(
-    name: str, 
-    max_concurrency: int, 
-    encryption_key: str, 
-    db_config: DatabaseConfiguration, 
+    name: str,
+    max_concurrency: int,
+    encryption_key: str,
+    db_config: DatabaseConfiguration,
     *options: OnError
 ) -> Queuer
 ```
@@ -106,6 +104,7 @@ def new_queuer_with_db(
 - `options`: Optional `OnError` configurations to apply to the worker.
 
 This function performs the following setup:
+
 - Initializes a logger.
 - Sets up the database connection using the provided `db_config` or environment variables.
 - Creates `JobDBHandler`, `WorkerDBHandler` instances for database interactions.
@@ -124,6 +123,7 @@ def start(self) -> None
 ```
 
 Upon calling `start`:
+
 - It performs a basic check to ensure internal listeners are initialized.
 - Database listeners are created to listen to job events (inserts, updates, deletes) via PostgreSQL NOTIFY/LISTEN.
 - It starts a poller to periodically poll the database for new jobs to process.
@@ -142,7 +142,50 @@ The `stop` method gracefully shuts down the Queuer instance, releasing resources
 def stop(self) -> None
 ```
 
-The `stop` method cancels all jobs, closes database listeners, and cleans up resources.
+The `stop` method cancels all jobs, closes database listeners, and cleans up resources. **Note:** This method can only be used to stop the current worker instance that the code is running in. To stop other workers, use `stop_worker` or `stop_worker_gracefully`.
+
+---
+
+## stop_worker
+
+The `stop_worker` method immediately stops a worker by setting its status to `STOPPED`. This will cancel all running jobs on that worker.
+
+```python
+def stop_worker(self, worker_rid: UUID) -> None
+```
+
+- `worker_rid`: The `UUID` identifying the worker to stop.
+
+When a worker is stopped:
+
+- The worker status is immediately set to `STOPPED` in the database
+- The heartbeat ticker detects the `STOPPED` status and calls `stop()` on that worker
+- All running jobs on that worker are cancelled immediately
+- The worker will no longer accept new jobs
+
+This method is useful for immediately shutting down a worker, for example in emergency situations or when you need to take a worker offline quickly.
+
+---
+
+## stop_worker_gracefully
+
+The `stop_worker_gracefully` method gracefully stops a worker by setting its status to `STOPPING`. This allows currently running jobs to complete before the worker shuts down.
+
+```python
+def stop_worker_gracefully(self, worker_rid: UUID) -> None
+```
+
+- `worker_rid`: The `UUID` identifying the worker to stop gracefully.
+
+When a worker is stopped gracefully:
+
+- The worker status is set to `STOPPING` in the database
+- The heartbeat ticker detects the `STOPPING` status and sets `max_concurrency` to `0`
+- Currently running jobs are allowed to complete normally
+- No new jobs will be accepted by this worker
+- Once all running jobs have finished, the worker status is automatically set to `STOPPED` and the worker shuts down
+
+This method is ideal for maintenance scenarios where you want to ensure all in-progress work completes before shutting down the worker.
 
 ---
 
@@ -169,8 +212,8 @@ The `add_job` method adds a new job to the queue for execution. Jobs are units o
 
 ```python
 def add_job(
-    self, 
-    task: Union[Callable, str], 
+    self,
+    task: Union[Callable, str],
     *parameters: Any,
     **parameters_keyed: Any
 ) -> Job
@@ -279,7 +322,7 @@ class RetryBackoff(str, Enum):
 
 - `RETRY_BACKOFF_NONE`: No backoff. The retry_delay remains constant for all retries.
 - `RETRY_BACKOFF_LINEAR`: The retry delay increases linearly with each attempt (e.g., delay, 2*delay, 3*delay).
-- `RETRY_BACKOFF_EXPONENTIAL`: The retry delay increases exponentially with each attempt (e.g., delay, delay*2, delay*2*2).
+- `RETRY_BACKOFF_EXPONENTIAL`: The retry delay increases exponentially with each attempt (e.g., delay, delay*2, delay*2\*2).
 
 ---
 

{queuerpy-0.18.0 → queuerpy-0.20.0}/_version.py

@@ -1,6 +1,6 @@
 """Version information for queuerPy package."""
 
-__version__ = "0.18.0"
+__version__ = "0.20.0"
 __version_info__ = tuple(int(part) for part in __version__.split("."))
 
 # For compatibility

{queuerpy-0.18.0 → queuerpy-0.20.0}/database/db_job.py

@@ -9,7 +9,7 @@ from typing import List, Optional, Tuple
 from uuid import UUID
 from psycopg.rows import dict_row
 
-from ..helper.database import Database
+from ..helper.database import Database, DatabaseConfiguration
 from ..helper.error import QueuerError
 from ..helper.sql import SQLLoader, run_ddl
 from ..model.job import Job
@@ -25,7 +25,7 @@ class JobDBHandler:
     def __init__(
         self,
         db_connection: Database,
-        with_table_drop: bool = False,
+        db_config: DatabaseConfiguration,
         encryption_key: str = "",
     ):
         """Initialize job database handler."""
@@ -35,12 +35,13 @@ class JobDBHandler:
             raise ValueError("Database connection is not established")
 
         self.encryption_key: str = encryption_key
+        self.with_timescale: bool = db_config.with_timescale
 
         sql_loader: SQLLoader = SQLLoader()
-        sql_loader.load_notify_sql(self.db.instance, force=with_table_drop)
-        sql_loader.load_job_sql(self.db.instance, force=with_table_drop)
+        sql_loader.load_notify_sql(self.db.instance, force=db_config.with_table_drop)
+        sql_loader.load_job_sql(self.db.instance, force=db_config.with_table_drop)
 
-        if with_table_drop:
+        if db_config.with_table_drop:
             self.drop_tables()
 
         self.create_table()
@@ -72,7 +73,7 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        run_ddl(self.db.instance, "SELECT init_job();")
+        run_ddl(self.db.instance, f"SELECT init_job({self.with_timescale});")
 
     def drop_tables(self) -> None:
         """Drop job tables with DDL deadlock protection."""
@@ -379,9 +380,15 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor() as cur:
-            cur.execute("SELECT add_retention_archive(%s);", (days,))
-        self.db.instance.commit()
+        try:
+            with self.db.instance.cursor() as cur:
+                cur.execute(
+                    "SELECT add_retention_archive(%s, %s);", (days, self.with_timescale)
+                )
+            self.db.instance.commit()
+        except Exception as e:
+            self.db.instance.rollback()
+            raise e
 
     def remove_retention_archive(self) -> None:
         """
@@ -391,9 +398,15 @@ class JobDBHandler:
         if self.db.instance is None:
             raise ValueError("Database connection is not established")
 
-        with self.db.instance.cursor() as cur:
-            cur.execute("SELECT remove_retention_archive();")
-        self.db.instance.commit()
+        try:
+            with self.db.instance.cursor() as cur:
+                cur.execute(
+                    "SELECT remove_retention_archive(%s);", (self.with_timescale,)
+                )
+            self.db.instance.commit()
+        except Exception as e:
+            self.db.instance.rollback()
+            raise e
 
     def select_job_from_archive(self, rid: UUID) -> Optional[Job]:
         """

{queuerpy-0.18.0 → queuerpy-0.20.0}/helper/database.py

@@ -30,6 +30,7 @@ class DatabaseConfiguration:
     schema: str = "public"
     sslmode: str = "require"
     with_table_drop: bool = False
+    with_timescale: bool = True
 
     @classmethod
     def from_env(cls) -> "DatabaseConfiguration":
@@ -45,6 +46,9 @@ class DatabaseConfiguration:
         with_table_drop = (
             os.getenv("QUEUER_DB_WITH_TABLE_DROP", "false").lower() == "true"
         )
+        with_timescale = (
+            os.getenv("QUEUER_DB_WITH_TIMESCALE", "true").lower() != "false"
+        )
 
         # Validate required fields
         if not all(
@@ -72,6 +76,7 @@ class DatabaseConfiguration:
             schema=schema,
             sslmode=sslmode,
             with_table_drop=with_table_drop,
+            with_timescale=with_timescale,
         )
 
     def connection_string(self) -> str:

{queuerpy-0.18.0 → queuerpy-0.20.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "queuerPy"
-version = "0.18.0"
+version = "0.20.0"
 description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
 readme = "README.md"
 license = "Apache-2.0"
@@ -92,7 +92,7 @@ pythonVersion = "3.8"
 pythonPlatform = "All"
 
 [tool.mypy]
-python_version = "0.18.0"
+python_version = "0.20.0"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true

{queuerpy-0.18.0 → queuerpy-0.20.0/queuerPy.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queuerPy
-Version: 0.18.0
+Version: 0.20.0
 Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
 Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
 Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
@@ -58,9 +58,7 @@ The job table contains only queued, scheduled and running tasks. The ended jobs
 
 ---
 
-# 🛠️ Installation
-
-## PyPI Installation (Recommended)
+## 🛠️ Installation
 
 ```bash
 pip install queuerPy
@@ -134,10 +132,10 @@ You can find a full example in the example folder.
 def new_queuer(name: str, max_concurrency: int, *options: OnError) -> Queuer
 
 def new_queuer_with_db(
-    name: str, 
-    max_concurrency: int, 
-    encryption_key: str, 
-    db_config: DatabaseConfiguration, 
+    name: str,
+    max_concurrency: int,
+    encryption_key: str,
+    db_config: DatabaseConfiguration,
     *options: OnError
 ) -> Queuer
 ```
@@ -149,6 +147,7 @@ def new_queuer_with_db(
 - `options`: Optional `OnError` configurations to apply to the worker.
 
 This function performs the following setup:
+
 - Initializes a logger.
 - Sets up the database connection using the provided `db_config` or environment variables.
 - Creates `JobDBHandler`, `WorkerDBHandler` instances for database interactions.
@@ -167,6 +166,7 @@ def start(self) -> None
 ```
 
 Upon calling `start`:
+
 - It performs a basic check to ensure internal listeners are initialized.
 - Database listeners are created to listen to job events (inserts, updates, deletes) via PostgreSQL NOTIFY/LISTEN.
 - It starts a poller to periodically poll the database for new jobs to process.
@@ -185,7 +185,50 @@ The `stop` method gracefully shuts down the Queuer instance, releasing resources
 def stop(self) -> None
 ```
 
-The `stop` method cancels all jobs, closes database listeners, and cleans up resources.
+The `stop` method cancels all jobs, closes database listeners, and cleans up resources. **Note:** This method can only be used to stop the current worker instance that the code is running in. To stop other workers, use `stop_worker` or `stop_worker_gracefully`.
+
+---
+
+## stop_worker
+
+The `stop_worker` method immediately stops a worker by setting its status to `STOPPED`. This will cancel all running jobs on that worker.
+
+```python
+def stop_worker(self, worker_rid: UUID) -> None
+```
+
+- `worker_rid`: The `UUID` identifying the worker to stop.
+
+When a worker is stopped:
+
+- The worker status is immediately set to `STOPPED` in the database
+- The heartbeat ticker detects the `STOPPED` status and calls `stop()` on that worker
+- All running jobs on that worker are cancelled immediately
+- The worker will no longer accept new jobs
+
+This method is useful for immediately shutting down a worker, for example in emergency situations or when you need to take a worker offline quickly.
+
+---
+
+## stop_worker_gracefully
+
+The `stop_worker_gracefully` method gracefully stops a worker by setting its status to `STOPPING`. This allows currently running jobs to complete before the worker shuts down.
+
+```python
+def stop_worker_gracefully(self, worker_rid: UUID) -> None
+```
+
+- `worker_rid`: The `UUID` identifying the worker to stop gracefully.
+
+When a worker is stopped gracefully:
+
+- The worker status is set to `STOPPING` in the database
+- The heartbeat ticker detects the `STOPPING` status and sets `max_concurrency` to `0`
+- Currently running jobs are allowed to complete normally
+- No new jobs will be accepted by this worker
+- Once all running jobs have finished, the worker status is automatically set to `STOPPED` and the worker shuts down
+
+This method is ideal for maintenance scenarios where you want to ensure all in-progress work completes before shutting down the worker.
 
 ---
 
@@ -212,8 +255,8 @@ The `add_job` method adds a new job to the queue for execution. Jobs are units o
 
 ```python
 def add_job(
-    self, 
-    task: Union[Callable, str], 
+    self,
+    task: Union[Callable, str],
     *parameters: Any,
     **parameters_keyed: Any
 ) -> Job
@@ -322,7 +365,7 @@ class RetryBackoff(str, Enum):
 
 - `RETRY_BACKOFF_NONE`: No backoff. The retry_delay remains constant for all retries.
 - `RETRY_BACKOFF_LINEAR`: The retry delay increases linearly with each attempt (e.g., delay, 2*delay, 3*delay).
-- `RETRY_BACKOFF_EXPONENTIAL`: The retry delay increases exponentially with each attempt (e.g., delay, delay*2, delay*2*2).
+- `RETRY_BACKOFF_EXPONENTIAL`: The retry delay increases exponentially with each attempt (e.g., delay, delay*2, delay*2\*2).
 
 ---
 

{queuerpy-0.18.0 → queuerpy-0.20.0}/queuer_global.py

@@ -56,7 +56,7 @@ class QueuerGlobalMixin:
 
         # Database handlers
         self.db_job: JobDBHandler = JobDBHandler(
-            self.database, db_config.with_table_drop, encryption_key
+            self.database, db_config, encryption_key
         )
         self.db_worker: WorkerDBHandler = WorkerDBHandler(
             self.database, db_config.with_table_drop

{queuerpy-0.18.0 → queuerpy-0.20.0}/queuer_master.py

@@ -38,14 +38,27 @@ class QueuerMasterMixin(QueuerGlobalMixin):
 
         # Handle retention archive configuration
         try:
-            if old_master.settings.retention_archive == 0:
-                self.db_job.add_retention_archive(master_settings.retention_archive)
-            elif (
-                old_master.settings.retention_archive
-                != master_settings.retention_archive
-            ):
-                self.db_job.remove_retention_archive()
-                self.db_job.add_retention_archive(master_settings.retention_archive)
+            if master_settings.retention_archive > 0:
+                if old_master.settings.retention_archive == 0:
+                    self.db_job.add_retention_archive(master_settings.retention_archive)
+                elif (
+                    old_master.settings.retention_archive
+                    != master_settings.retention_archive
+                ):
+                    try:
+                        self.db_job.remove_retention_archive()
+                    except Exception as e:
+                        logger.warning(
+                            f"Could not remove old retention policy (may not exist): {e}"
+                        )
+                    self.db_job.add_retention_archive(master_settings.retention_archive)
+            elif old_master.settings.retention_archive > 0:
+                try:
+                    self.db_job.remove_retention_archive()
+                except Exception as e:
+                    logger.warning(
+                        f"Could not remove retention policy (may not exist): {e}"
+                    )
         except Exception as e:
             raise QueuerError("managing retention archive", e)
 
@@ -83,10 +96,9 @@ class QueuerMasterMixin(QueuerGlobalMixin):
                     logger.error(f"Error in master ticker task: {e}")
 
             ticker = Ticker(
-                task=ticker_task,
-                name="_master_ticker",
                 interval=timedelta(seconds=master_settings.master_poll_interval),
-                func=ticker_task,
+                task=ticker_task,
+                use_mp=False,  # Use threading instead of multiprocessing to avoid pickling issues
             )
             logger.info("Starting master ticker...")
             ticker.go()
@@ -150,10 +162,9 @@ class QueuerMasterMixin(QueuerGlobalMixin):
 
             # Create and start the polling ticker
             ticker = Ticker(
-                task=ticker_task,
-                name="_poll_master_ticker",
                 interval=timedelta(seconds=master_settings.master_poll_interval),
-                func=ticker_task,
+                task=ticker_task,
+                use_mp=False,  # Use threading instead of multiprocessing to avoid pickling issues
             )
 
             logger.info("Starting master poll ticker...")

{queuerpy-0.18.0 → queuerpy-0.20.0}/queuer_worker.py

@@ -15,6 +15,19 @@ from .queuer_global import QueuerGlobalMixin
 logger = logging.getLogger(__name__)
 
 
+def get_current_worker_rid(self) -> Optional[UUID]:
+    """
+    Get the current worker's RID safely with mutex locking.
+    Mirrors Go's GetCurrentWorkerRID method.
+
+    :return: Worker RID or None if no worker
+    """
+    with self.worker_mutex:
+        if self.worker is not None:
+            return self.worker.rid
+        return None
+
+
 class QueuerWorkerMixin(QueuerGlobalMixin):
     """
     Mixin class containing worker-related methods for the Queuer.

{queuerpy-0.18.0 → queuerpy-0.20.0}/sql/job.sql

@@ -1,7 +1,9 @@
-CREATE OR REPLACE FUNCTION init_job() RETURNS VOID AS $$
+CREATE OR REPLACE FUNCTION init_job(enable_timescaledb BOOLEAN DEFAULT TRUE) RETURNS VOID AS $$
 BEGIN
     -- Create TimescaleDB extension
-    CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+    IF enable_timescaledb THEN
+        CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+    END IF;
 
     -- Create pgcrypto extension
     CREATE EXTENSION IF NOT EXISTS pgcrypto;
@@ -37,7 +39,9 @@ BEGIN
     );
     
     -- Create hypertable (TimescaleDB feature)
-    PERFORM create_hypertable('job_archive', by_range('updated_at'), if_not_exists => TRUE);
+    IF enable_timescaledb THEN
+        PERFORM create_hypertable('job_archive', by_range('updated_at'), if_not_exists => TRUE);
+    END IF;
     
     -- Create triggers for notifications
     CREATE OR REPLACE TRIGGER job_notify_event
@@ -698,17 +702,64 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION add_retention_archive(input_retention_days INT)
+CREATE OR REPLACE FUNCTION add_retention_archive(input_retention_days INT, enable_timescaledb BOOLEAN DEFAULT TRUE)
 RETURNS VOID AS $$
 BEGIN
-    PERFORM add_retention_policy('job_archive', (input_retention_days * INTERVAL '1 day'));
+    IF enable_timescaledb THEN
+        -- Use TimescaleDB retention policy
+        PERFORM add_retention_policy('job_archive', (input_retention_days * INTERVAL '1 day'));
+    ELSE
+        -- Create sequence for tracking inserts
+        CREATE SEQUENCE IF NOT EXISTS job_archive_insert_counter;
+        
+        -- Create trigger function that deletes old records every 10th insert
+        EXECUTE format('
+            CREATE OR REPLACE FUNCTION cleanup_job_archive_trigger()
+            RETURNS TRIGGER AS $func$
+            DECLARE
+                insert_count BIGINT;
+                deleted_count INT;
+            BEGIN
+                -- Only run cleanup every 10th insert
+                insert_count := nextval(''job_archive_insert_counter'');
+                IF insert_count %% 10 = 0 THEN
+                    DELETE FROM job_archive
+                    WHERE updated_at < (CURRENT_TIMESTAMP - INTERVAL ''%s days'');
+                    
+                    GET DIAGNOSTICS deleted_count = ROW_COUNT;
+                    
+                    -- Log cleanup if any rows were deleted (optional)
+                    IF deleted_count > 0 THEN
+                        RAISE NOTICE ''Cleaned up %% old job_archive records'', deleted_count;
+                    END IF;
+                END IF;
+                RETURN NEW;
+            END;
+            $func$ LANGUAGE plpgsql;
+        ', input_retention_days);
+        
+        -- Create trigger on job_archive
+        DROP TRIGGER IF EXISTS job_archive_cleanup_trigger ON job_archive;
+        CREATE TRIGGER job_archive_cleanup_trigger
+            AFTER INSERT ON job_archive
+            FOR EACH ROW
+            EXECUTE FUNCTION cleanup_job_archive_trigger();
+    END IF;
 END;
 $$ LANGUAGE plpgsql;
 
-CREATE OR REPLACE FUNCTION remove_retention_archive()
+CREATE OR REPLACE FUNCTION remove_retention_archive(enable_timescaledb BOOLEAN DEFAULT TRUE)
 RETURNS VOID AS $$
 BEGIN
-    PERFORM remove_retention_policy('job_archive');
+    IF enable_timescaledb THEN
+        -- Remove TimescaleDB retention policy
+        PERFORM remove_retention_policy('job_archive');
+    ELSE
+        -- Drop trigger and related objects
+        DROP TRIGGER IF EXISTS job_archive_cleanup_trigger ON job_archive;
+        DROP FUNCTION IF EXISTS cleanup_job_archive_trigger();
+        DROP SEQUENCE IF EXISTS job_archive_insert_counter;
+    END IF;
 END;
 $$ LANGUAGE plpgsql;