p1-taskqueue 0.1.1__tar.gz → 0.1.3__tar.gz

{p1_taskqueue-0.1.1 → p1_taskqueue-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: p1-taskqueue
-Version: 0.1.1
+Version: 0.1.3
 Summary: A Task Queue Wrapper for Dekoruma Backend
 Author-email: Chalvin <engineering@dekoruma.com>
 Project-URL: Homepage, https://github.com/Dekoruma/p1-taskqueue

{p1_taskqueue-0.1.1 → p1_taskqueue-0.1.3}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "p1-taskqueue"
 # DO NOT CHANGE THIS VERSION - it gets automatically replaced by CI/CD with the git tag version
-version = "0.1.1"
+version = "0.1.3"
 description = "A Task Queue Wrapper for Dekoruma Backend"
 authors = [
     {name = "Chalvin", email = "engineering@dekoruma.com"}

{p1_taskqueue-0.1.1 → p1_taskqueue-0.1.3}/src/p1_taskqueue.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: p1-taskqueue
-Version: 0.1.1
+Version: 0.1.3
 Summary: A Task Queue Wrapper for Dekoruma Backend
 Author-email: Chalvin <engineering@dekoruma.com>
 Project-URL: Homepage, https://github.com/Dekoruma/p1-taskqueue

{p1_taskqueue-0.1.1 → p1_taskqueue-0.1.3}/src/p1_taskqueue.egg-info/SOURCES.txt

@@ -8,6 +8,8 @@ src/p1_taskqueue.egg-info/top_level.txt
 src/taskqueue/__init__.py
 src/taskqueue/celery_app.py
 src/taskqueue/cmanager.py
+src/taskqueue/libs/__init__.py
+src/taskqueue/libs/helper_test.py
 tests/test_celery_app.py
 tests/test_cmanager.py
 tests/test_test_utils.py

p1_taskqueue-0.1.3/src/taskqueue/libs/helper_test.py

@@ -0,0 +1,88 @@
+import logging
+from typing import List
+
+from celery import current_app
+from kombu.serialization import loads
+
+logger = logging.getLogger(__name__)
+
+
+def clear_all_celery_queues():
+    app = current_app
+    all_queue_names = list(app.amqp.queues.keys())
+    with app.connection_for_read() as conn:
+        with conn.channel() as chan:
+            for queue_name in all_queue_names:
+                queue = app.amqp.queues[queue_name](chan)
+                queue.purge()
+
+
+def celery_worker_burst(include_func_names: List[str], channel: str = "default"):
+    # This doesn't use celery as celery doesn't support filtering out functions
+    # this use kombu to get the message from the queue and then execute the task manually
+    app = current_app
+    included_set = set(include_func_names)
+    processed_count = 0
+    executed_count = 0
+
+    try:
+        with app.connection_for_read() as conn:
+            with conn.channel() as chan:
+                queue = app.amqp.queues[channel](chan)
+
+                while True:
+                    message = queue.get(no_ack=False)
+                    if not message:
+                        break
+
+                    processed_count += 1
+                    task_name = message.headers.get("task")
+
+                    if not task_name or task_name not in app.tasks:
+                        # task is not registered in celery
+                        logger.warning(
+                            f"Invalid task '{task_name}'. Skipping.")
+                        message.ack()
+                        continue
+
+                    try:
+                        task_obj = app.tasks[task_name]
+                        accept = {"application/json",
+                                  "application/x-python-serialize"}
+                        decoded_body = loads(
+                            message.body, message.content_type, message.content_encoding, accept=accept
+                        )
+
+                        task_args = decoded_body[0] if decoded_body else []
+                        task_kwargs = decoded_body[1] if len(
+                            decoded_body) > 1 else {}
+
+                        full_func_name = ""
+                        if task_name.endswith("dynamic_function_executor") and len(task_args) >= 2:
+                            full_func_name = f"{task_args[0]}.{task_args[1]}"
+                        elif task_name.endswith("dynamic_class_method_executor") and len(task_args) >= 3:
+                            full_func_name = f"{task_args[0]}.{task_args[1]}.{task_args[2]}"
+
+                        should_execute = full_func_name in included_set if full_func_name else False
+
+                        if should_execute:
+                            logger.info(f"Executing task: {full_func_name}")
+                            message.ack()
+                            task_obj.apply(args=task_args, kwargs=task_kwargs)
+                            executed_count += 1
+                            logger.info(
+                                f"Successfully executed task: {full_func_name}")
+                        else:
+                            logger.debug(
+                                f"Skipping: {full_func_name or task_name}")
+                            message.ack()
+
+                    except Exception as e:
+                        logger.error(
+                            f"Failed to process task {task_name}: {type(e).__name__}: {e}")
+                        if message and not message.acknowledged:
+                            message.ack()
+
+    except Exception as e:
+        logger.error(
+            f"Failed to connect to queue {channel}: {type(e).__name__}: {e}")

{p1_taskqueue-0.1.1 → p1_taskqueue-0.1.3}/tests/test_test_utils.py

@@ -1,13 +1,13 @@
 from unittest.mock import MagicMock
 from unittest.mock import patch
 
-from libs.test_utils import celery_worker_burst
-from libs.test_utils import clear_all_celery_queues
+from taskqueue.libs.helper_test import celery_worker_burst
+from taskqueue.libs.helper_test import clear_all_celery_queues
 
 
 class TestClearAllCeleryQueues:
 
-    @patch('libs.test_utils.current_app')
+    @patch('taskqueue.libs.helper_test.current_app')
     def test_clear_all_celery_queues_given_multiple_queues_expect_all_purged(self, mock_current_app):
         mock_queue1 = MagicMock()
         mock_queue2 = MagicMock()
@@ -32,7 +32,7 @@ class TestClearAllCeleryQueues:
         mock_queue1.purge.assert_called_once()
         mock_queue2.purge.assert_called_once()
 
-    @patch('libs.test_utils.current_app')
+    @patch('taskqueue.libs.helper_test.current_app')
     def test_clear_all_celery_queues_given_empty_queues_expect_no_purge_calls(self, mock_current_app):
         mock_current_app.amqp.queues.keys.return_value = []
         mock_current_app.amqp.queues = {}
@@ -49,8 +49,8 @@ class TestClearAllCeleryQueues:
 
 class TestCeleryWorkerBurst:
 
-    @patch('libs.test_utils.current_app')
-    @patch('libs.test_utils.loads')
+    @patch('taskqueue.libs.helper_test.current_app')
+    @patch('taskqueue.libs.helper_test.loads')
     def test_celery_worker_burst_given_matching_function_executor_expect_execution(self, mock_loads, mock_current_app):
         mock_task = MagicMock()
         mock_current_app.tasks = {
@@ -77,7 +77,7 @@ class TestCeleryWorkerBurst:
         mock_current_app.connection_for_read.return_value.__enter__.return_value = mock_conn
         mock_conn.channel.return_value.__enter__.return_value = mock_chan
 
-        with patch('libs.test_utils.logger') as mock_logger:
+        with patch('taskqueue.libs.helper_test.logger') as mock_logger:
             celery_worker_burst(['module.submodule.test_function'])
 
             mock_logger.info.assert_any_call(
@@ -91,8 +91,8 @@ class TestCeleryWorkerBurst:
             kwargs={}
         )
 
-    @patch('libs.test_utils.current_app')
-    @patch('libs.test_utils.loads')
+    @patch('taskqueue.libs.helper_test.current_app')
+    @patch('taskqueue.libs.helper_test.loads')
     def test_celery_worker_burst_given_matching_class_method_executor_expect_execution(self, mock_loads, mock_current_app):
         mock_task = MagicMock()
         mock_current_app.tasks = {
@@ -119,7 +119,7 @@ class TestCeleryWorkerBurst:
         mock_current_app.connection_for_read.return_value.__enter__.return_value = mock_conn
         mock_conn.channel.return_value.__enter__.return_value = mock_chan
 
-        with patch('libs.test_utils.logger') as mock_logger:
+        with patch('taskqueue.libs.helper_test.logger') as mock_logger:
             celery_worker_burst(['module.submodule.TestClass.test_method'])
 
             mock_logger.info.assert_any_call(
@@ -133,8 +133,8 @@ class TestCeleryWorkerBurst:
             kwargs={}
         )
 
-    @patch('libs.test_utils.current_app')
-    @patch('libs.test_utils.loads')
+    @patch('taskqueue.libs.helper_test.current_app')
+    @patch('taskqueue.libs.helper_test.loads')
     def test_celery_worker_burst_given_non_matching_function_expect_skip(self, mock_loads, mock_current_app):
         mock_task = MagicMock()
         mock_current_app.tasks = {
@@ -166,7 +166,7 @@ class TestCeleryWorkerBurst:
         mock_message.ack.assert_called_once()
         mock_task.apply.assert_not_called()
 
-    @patch('libs.test_utils.current_app')
+    @patch('taskqueue.libs.helper_test.current_app')
     def test_celery_worker_burst_given_invalid_task_name_expect_warning_and_skip(self, mock_current_app):
         mock_current_app.tasks = {}
 
@@ -187,14 +187,14 @@ class TestCeleryWorkerBurst:
         mock_current_app.connection_for_read.return_value.__enter__.return_value = mock_conn
         mock_conn.channel.return_value.__enter__.return_value = mock_chan
 
-        with patch('libs.test_utils.logger') as mock_logger:
+        with patch('taskqueue.libs.helper_test.logger') as mock_logger:
             celery_worker_burst(['some.function'])
 
             mock_logger.warning.assert_called_once_with(
                 "Invalid task 'invalid.task.name'. Skipping.")
             mock_message.ack.assert_called_once()
 
-    @patch('libs.test_utils.current_app')
+    @patch('taskqueue.libs.helper_test.current_app')
     def test_celery_worker_burst_given_no_messages_expect_no_processing(self, mock_current_app):
         mock_queue = MagicMock()
         mock_queue.get.return_value = None
@@ -210,8 +210,8 @@ class TestCeleryWorkerBurst:
 
         mock_queue.get.assert_called_once_with(no_ack=False)
 
-    @patch('libs.test_utils.current_app')
-    @patch('libs.test_utils.loads')
+    @patch('taskqueue.libs.helper_test.current_app')
+    @patch('taskqueue.libs.helper_test.loads')
     def test_celery_worker_burst_given_custom_channel_expect_correct_queue_used(self, mock_loads, mock_current_app):
         mock_task = MagicMock()
         mock_current_app.tasks = {
@@ -238,7 +238,7 @@ class TestCeleryWorkerBurst:
         mock_current_app.connection_for_read.return_value.__enter__.return_value = mock_conn
         mock_conn.channel.return_value.__enter__.return_value = mock_chan
 
-        with patch('libs.test_utils.logger') as mock_logger:
+        with patch('taskqueue.libs.helper_test.logger') as mock_logger:
             celery_worker_burst(
                 ['module.submodule.test_function'], channel='custom_queue')
 
@@ -249,8 +249,8 @@ class TestCeleryWorkerBurst:
 
         mock_queue_factory.assert_called_once_with(mock_chan)
 
-    @patch('libs.test_utils.current_app')
-    @patch('libs.test_utils.loads')
+    @patch('taskqueue.libs.helper_test.current_app')
+    @patch('taskqueue.libs.helper_test.loads')
     def test_celery_worker_burst_given_task_processing_error_expect_error_logged_and_ack(self, mock_loads, mock_current_app):
         mock_task = MagicMock()
         mock_task.apply.side_effect = Exception("Task execution failed")
@@ -283,7 +283,7 @@ class TestCeleryWorkerBurst:
         mock_current_app.connection_for_read.return_value.__enter__.return_value = mock_conn
         mock_conn.channel.return_value.__enter__.return_value = mock_chan
 
-        with patch('libs.test_utils.logger') as mock_logger:
+        with patch('taskqueue.libs.helper_test.logger') as mock_logger:
             celery_worker_burst(['module.submodule.test_function'])
 
             mock_logger.info.assert_any_call(