kew 0.1.0__tar.gz

kew-0.1.0/.gitignore

@@ -0,0 +1,31 @@
+# Optional: create a .gitignore (save as kew/.gitignore)
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.idea/
+.vscode/

kew-0.1.0/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.3
+Name: kew
+Version: 0.1.0
+Summary: A flexible async task queue manager for Python applications
+Project-URL: Homepage, https://github.com/justrach/kew
+Project-URL: Bug Tracker, https://github.com/justrach/kew/issues
+Author-email: Rach Pradhan <rach@rachit.ai>
+Keywords: async,concurrent,manager,queue,task
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: AsyncIO
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.7
+Requires-Dist: typing; python_version < '3.5'
+Description-Content-Type: text/markdown
+
+# Kew Task Queue Manager
+
+A flexible and robust asynchronous task queue manager for Python applications.
+
+## Features
+
+- Asynchronous task execution
+- Configurable worker pool
+- Task status tracking and monitoring
+- Automatic cleanup of completed tasks
+- Thread-safe operations
+- Comprehensive logging
+
+## Installation
+
+```bash
+pip install kew
+```
+
+## Quick Start
+
+```python
+import asyncio
+from kew import TaskQueueManager
+
+async def example_task(x: int):
+    await asyncio.sleep(1)
+    return x * 2
+
+async def main():
+    # Initialize the task queue manager
+    manager = TaskQueueManager(max_workers=2)
+    
+    # Submit a task
+    task_info = await manager.submit_task(
+        task_id="task1",
+        task_type="multiplication",
+        task_func=example_task,
+        x=5
+    )
+    
+    # Wait for result
+    await asyncio.sleep(2)
+    status = manager.get_task_status("task1")
+    print(f"Result: {status.result}")
+    
+    # Cleanup
+    await manager.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Advanced Usage
+
+### Concurrent Task Execution
+
+```python
+async def main():
+    manager = TaskQueueManager(max_workers=2)
+    
+    # Submit multiple tasks
+    tasks = []
+    for i in range(5):
+        task_info = await manager.submit_task(
+            task_id=f"task{i}",
+            task_type="example",
+            task_func=example_task,
+            x=i
+        )
+        tasks.append(task_info)
+    
+    # Wait for all tasks to complete
+    await manager.wait_for_all_tasks()
+```
+
+### Task Status Monitoring
+
+```python
+status = manager.get_task_status("task1")
+print(f"Status: {status.status}")
+print(f"Result: {status.result}")
+print(f"Error: {status.error}")
+```
+
+## API Reference
+
+### TaskQueueManager
+
+- `__init__(max_workers=2, queue_size=1000, task_timeout=3600)`
+- `async submit_task(task_id, task_type, task_func, *args, **kwargs)`
+- `get_task_status(task_id)`
+- `async wait_for_task(task_id, timeout=None)`
+- `async wait_for_all_tasks(timeout=None)`
+- `cleanup_old_tasks(max_age_hours=24)`
+- `async shutdown()`
+
+### TaskStatus
+
+Enum with states:
+- `QUEUED`
+- `PROCESSING`
+- `COMPLETED`
+- `FAILED`
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

kew-0.1.0/README.md

@@ -0,0 +1,112 @@
+# Kew Task Queue Manager
+
+A flexible and robust asynchronous task queue manager for Python applications.
+
+## Features
+
+- Asynchronous task execution
+- Configurable worker pool
+- Task status tracking and monitoring
+- Automatic cleanup of completed tasks
+- Thread-safe operations
+- Comprehensive logging
+
+## Installation
+
+```bash
+pip install kew
+```
+
+## Quick Start
+
+```python
+import asyncio
+from kew import TaskQueueManager
+
+async def example_task(x: int):
+    await asyncio.sleep(1)
+    return x * 2
+
+async def main():
+    # Initialize the task queue manager
+    manager = TaskQueueManager(max_workers=2)
+    
+    # Submit a task
+    task_info = await manager.submit_task(
+        task_id="task1",
+        task_type="multiplication",
+        task_func=example_task,
+        x=5
+    )
+    
+    # Wait for result
+    await asyncio.sleep(2)
+    status = manager.get_task_status("task1")
+    print(f"Result: {status.result}")
+    
+    # Cleanup
+    await manager.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Advanced Usage
+
+### Concurrent Task Execution
+
+```python
+async def main():
+    manager = TaskQueueManager(max_workers=2)
+    
+    # Submit multiple tasks
+    tasks = []
+    for i in range(5):
+        task_info = await manager.submit_task(
+            task_id=f"task{i}",
+            task_type="example",
+            task_func=example_task,
+            x=i
+        )
+        tasks.append(task_info)
+    
+    # Wait for all tasks to complete
+    await manager.wait_for_all_tasks()
+```
+
+### Task Status Monitoring
+
+```python
+status = manager.get_task_status("task1")
+print(f"Status: {status.status}")
+print(f"Result: {status.result}")
+print(f"Error: {status.error}")
+```
+
+## API Reference
+
+### TaskQueueManager
+
+- `__init__(max_workers=2, queue_size=1000, task_timeout=3600)`
+- `async submit_task(task_id, task_type, task_func, *args, **kwargs)`
+- `get_task_status(task_id)`
+- `async wait_for_task(task_id, timeout=None)`
+- `async wait_for_all_tasks(timeout=None)`
+- `cleanup_old_tasks(max_age_hours=24)`
+- `async shutdown()`
+
+### TaskStatus
+
+Enum with states:
+- `QUEUED`
+- `PROCESSING`
+- `COMPLETED`
+- `FAILED`
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

kew-0.1.0/example.py

@@ -0,0 +1,36 @@
+# example.py
+import asyncio
+from kew import TaskQueueManager
+
+async def example_task(x: int):
+    await asyncio.sleep(1)
+    return x * 2
+
+async def main():
+    # Initialize the task queue manager
+    manager = TaskQueueManager(max_workers=2)
+    
+    # Submit a task
+    task_info = await manager.submit_task(
+        task_id="task1",
+        task_type="multiplication",
+        task_func=example_task,
+        x=5
+    )
+    
+    # Get task status
+    status = manager.get_task_status("task1")
+    print(f"Task status: {status.status}")
+    
+    # Wait a bit to see the result
+    await asyncio.sleep(2)
+    status = manager.get_task_status("task1")
+    print(f"Final status: {status.status}")
+    print(f"Result: {status.result}")
+    
+    # Cleanup and shutdown - now properly awaited
+    manager.cleanup_old_tasks()
+    await manager.shutdown()
+
+if __name__ == "__main__":
+    asyncio.run(main())

kew-0.1.0/kew/__init__.py

@@ -0,0 +1,5 @@
+from .manager import TaskQueueManager
+from .models import TaskStatus, TaskInfo
+
+__version__ = "0.1.0"
+__all__ = ["TaskQueueManager", "TaskStatus", "TaskInfo"]

kew-0.1.0/kew/exceptions.py

@@ -0,0 +1,11 @@
+class TaskQueueError(Exception):
+    """Base exception for task queue errors"""
+    pass
+
+class TaskAlreadyExistsError(TaskQueueError):
+    """Raised when attempting to add a task with an ID that already exists"""
+    pass
+
+class TaskNotFoundError(TaskQueueError):
+    """Raised when attempting to access a non-existent task"""
+    pass

kew-0.1.0/kew/manager.py

@@ -0,0 +1,150 @@
+from concurrent.futures import ThreadPoolExecutor
+from queue import Queue
+import threading
+from typing import Optional, Dict, Any, Callable
+from datetime import datetime
+import logging
+import asyncio
+from .models import TaskStatus, TaskInfo
+from .exceptions import TaskAlreadyExistsError, TaskNotFoundError
+
+logger = logging.getLogger(__name__)
+
+class TaskQueueManager:
+    def __init__(
+        self, 
+        max_workers: int = 2,
+        queue_size: int = 1000,
+        task_timeout: int = 3600
+    ):
+        self.executor = ThreadPoolExecutor(max_workers=max_workers)
+        self.queue = Queue(maxsize=queue_size)
+        self.task_timeout = task_timeout
+        self.tasks: Dict[str, TaskInfo] = {}
+        self._lock = threading.Lock()
+        self._shutdown = False
+        self._setup_logging()
+
+    def _setup_logging(self):
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        logger.setLevel(logging.INFO)
+
+    async def submit_task(
+        self,
+        task_id: str,
+        task_type: str,
+        task_func: Callable,
+        *args,
+        **kwargs
+    ) -> TaskInfo:
+        with self._lock:
+            if task_id in self.tasks:
+                raise TaskAlreadyExistsError(
+                    f"Task with ID {task_id} already exists"
+                )
+            
+            task_info = TaskInfo(task_id, task_type)
+            self.tasks[task_id] = task_info
+            logger.info(f"Task {task_id} ({task_type}) submitted")
+
+        # Create task coroutine
+        task = asyncio.create_task(self._execute_task(
+            task_id,
+            task_func,
+            *args,
+            **kwargs
+        ))
+        
+        # Store the task for later cleanup
+        task_info._task = task
+        
+        return task_info
+
+    async def _execute_task(
+        self,
+        task_id: str,
+        task_func: Callable,
+        *args,
+        **kwargs
+    ):
+        try:
+            with self._lock:
+                task_info = self.tasks[task_id]
+                task_info.status = TaskStatus.PROCESSING
+                task_info.started_time = datetime.now()
+                logger.info(f"Starting execution of task {task_id}")
+
+            # Execute the task function
+            result = await task_func(*args, **kwargs)
+
+            with self._lock:
+                task_info.result = result
+                task_info.status = TaskStatus.COMPLETED
+                task_info.completed_time = datetime.now()
+                logger.info(f"Task {task_id} completed successfully with result: {result}")
+
+            return result
+
+        except Exception as e:
+            logger.error(f"Error executing task {task_id}: {str(e)}")
+            with self._lock:
+                task_info = self.tasks[task_id]
+                task_info.status = TaskStatus.FAILED
+                task_info.error = str(e)
+                task_info.completed_time = datetime.now()
+            raise
+
+    async def wait_for_task(self, task_id: str, timeout: Optional[float] = None) -> TaskInfo:
+        """Wait for a specific task to complete"""
+        task_info = self.get_task_status(task_id)
+        if hasattr(task_info, '_task'):
+            try:
+                await asyncio.wait_for(task_info._task, timeout=timeout)
+            except asyncio.TimeoutError:
+                logger.warning(f"Task {task_id} timed out after {timeout} seconds")
+                raise
+        return task_info
+
+    async def wait_for_all_tasks(self, timeout: Optional[float] = None):
+        """Wait for all tasks to complete"""
+        tasks = [
+            task_info._task for task_info in self.tasks.values()
+            if hasattr(task_info, '_task') and not task_info._task.done()
+        ]
+        if tasks:
+            await asyncio.wait(tasks, timeout=timeout)
+
+    def get_task_status(self, task_id: str) -> TaskInfo:
+        with self._lock:
+            task_info = self.tasks.get(task_id)
+            if not task_info:
+                raise TaskNotFoundError(f"Task {task_id} not found")
+            return task_info
+
+    def get_queue_length(self) -> int:
+        return self.queue.qsize()
+
+    def cleanup_old_tasks(self, max_age_hours: int = 24):
+        current_time = datetime.now()
+        cleaned_count = 0
+        with self._lock:
+            for task_id, task_info in list(self.tasks.items()):
+                if task_info.completed_time:
+                    age = current_time - task_info.completed_time
+                    if age.total_seconds() > max_age_hours * 3600:
+                        del self.tasks[task_id]
+                        cleaned_count += 1
+        
+        logger.info(f"Cleaned up {cleaned_count} old tasks")
+
+    async def shutdown(self):
+        logger.info("Shutting down TaskQueueManager")
+        self._shutdown = True
+        # Wait for pending tasks
+        await self.wait_for_all_tasks(timeout=5.0)
+        self.executor.shutdown(wait=True)

kew-0.1.0/kew/models.py

@@ -0,0 +1,35 @@
+from datetime import datetime
+from enum import Enum
+from typing import Optional, TypeVar, Generic
+
+T = TypeVar('T')  # Generic type for task result
+
+class TaskStatus(Enum):
+    QUEUED = "queued"
+    PROCESSING = "processing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+class TaskInfo(Generic[T]):
+    def __init__(self, task_id: str, task_type: str):
+        self.task_id = task_id
+        self.task_type = task_type
+        self.status = TaskStatus.QUEUED
+        self.queued_time = datetime.now()
+        self.started_time: Optional[datetime] = None
+        self.completed_time: Optional[datetime] = None
+        self.result: Optional[T] = None
+        self.error: Optional[str] = None
+    
+    def to_dict(self):
+        """Convert TaskInfo to dictionary representation"""
+        return {
+            "task_id": self.task_id,
+            "task_type": self.task_type,
+            "status": self.status.value,
+            "queued_time": self.queued_time.isoformat(),
+            "started_time": self.started_time.isoformat() if self.started_time else None,
+            "completed_time": self.completed_time.isoformat() if self.completed_time else None,
+            "result": self.result,
+            "error": self.error
+        }

kew-0.1.0/pyproject.toml

@@ -0,0 +1,35 @@
+# First, let's create pyproject.toml (save as kew/pyproject.toml)
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "kew"
+version = "0.1.0"
+authors = [
+  { name="Rach Pradhan", email="rach@rachit.ai" },
+]
+description = "A flexible async task queue manager for Python applications"
+readme = "README.md"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Framework :: AsyncIO",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+keywords = ["async", "queue", "task", "manager", "concurrent"]
+dependencies = [
+    "typing;python_version<'3.5'",
+]
+
+[project.urls]
+"Homepage" = "https://github.com/justrach/kew"
+"Bug Tracker" = "https://github.com/justrach/kew/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["kew"]

kew-0.1.0/setup.py

@@ -0,0 +1,21 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="kew",
+    version="0.1.0",
+    packages=find_packages(),
+    install_requires=[
+        "typing;python_version<'3.5'",
+    ],
+    author="Your Name",
+    author_email="your.email@example.com",
+    description="A flexible async task queue manager for Python applications",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.7",
+)

kew-0.1.0/test_multiple.py

@@ -0,0 +1,60 @@
+# test_multiple.py
+import asyncio
+from kew import TaskQueueManager, TaskStatus
+import random
+
+async def long_task(task_num: int, sleep_time: int) -> dict:
+    """Simulate a long-running task"""
+    print(f"Starting task {task_num} (will take {sleep_time} seconds)")
+    await asyncio.sleep(sleep_time)
+    result = sleep_time * 2
+    print(f"Task {task_num} completed with result: {result}")
+    return {"task_num": task_num, "result": result}
+
+async def main():
+    # Initialize manager with 2 workers (so only 2 tasks can run simultaneously)
+    manager = TaskQueueManager(max_workers=2)
+    
+    # Submit 5 tasks with different durations
+    tasks = []
+    for i in range(5):
+        sleep_time = random.randint(3, 7)  # Random duration between 3-7 seconds
+        task_info = await manager.submit_task(
+            task_id=f"task{i+1}",
+            task_type="long_calculation",
+            task_func=long_task,
+            task_num=i+1,
+            sleep_time=sleep_time
+        )
+        tasks.append(task_info)
+        print(f"Submitted task {i+1}")
+
+    # Monitor task progress
+    while True:
+        all_completed = True
+        print("\nCurrent status:")
+        for task in tasks:
+            status = manager.get_task_status(task.task_id)
+            print(f"{task.task_id}: {status.status.value} - Result: {status.result}")
+            if status.status not in (TaskStatus.COMPLETED, TaskStatus.FAILED):
+                all_completed = False
+        
+        if all_completed:
+            break
+            
+        await asyncio.sleep(1)  # Wait a second before checking again
+
+    # Final results
+    print("\nFinal results:")
+    for task in tasks:
+        status = manager.get_task_status(task.task_id)
+        print(f"{task.task_id}: {status.result}")
+
+    # Properly await shutdown
+    await manager.shutdown()
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down gracefully...")