task-lattice 0.0.1a0__tar.gz

task_lattice-0.0.1a0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nicholas Williams
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

task_lattice-0.0.1a0/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.4
+Name: task-lattice
+Version: 0.0.1a0
+Summary: Distributed Task Framework
+Keywords: tasks,workflow,dag,distributed-tasks
+Author: Nicholas Williams
+Author-email: Nicholas Williams <noreply@users.noreply.github.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: OS Independent
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+
+<p>
+  <h3 style="font-size: 3.0em; margin: 0;">Task Lattice</h3>
+  <em>Distributed Task Framework for distributing work across workers</em>
+</p>
+
+<p align="left">
+  <img src="https://github.com/nicholasfelixwilliams/task-lattice/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI">
+  <img src="https://img.shields.io/pypi/v/task-lattice?color=%2334D058&label=pypi%20package" alt="Package version">
+  <img src="https://img.shields.io/pypi/pyversions/task-lattice.svg?color=%2334D058" alt="Supported Python versions">
+  <img src="https://img.shields.io/static/v1?label=code%20style&message=ruff&color=black">
+</p>
+
+---
+
+### 🚀 Key Features
+Task Lattice's key features include:
+
+- **Broker Support** - Following brokers are supported:
+    - Solace
+    - Kafka (Day 2)
+- **Queues** - Multiple queue/priority queue support
+- **Async** - Async tasks supported incl. execution in event loop
+- **Customisation** - Extensive customisation of queue, worker and tasks including:
+    - Automated task retry
+    - Dead letter queues
+    - Worker concurrency
+    - Queue capacity
+    - ...
+- **Minimal code** - Minimal code is required to use task lattice
+- **DAG support** - Supports DAG (directed acyclical graph) workflow execution
+- **Monitoring** - Supports live monitoring of the queues, workers, tasks
+
+---
+
+### ℹ️ Installation
+
+```sh
+# Using pip
+pip install task-lattice
+
+# Using poetry
+poetry add task-lattice
+
+# Using uv
+uv add task-lattice
+```
+
+---
+
+### 📦 Dependencies
+
+TBD
+
+---
+
+### 📘 How to use
+
+**Step 1 -** Define your Task Lattice application
+
+```python
+from task_lattice import TaskLattice, SolaceConnectionDetails, QueueDetails
+
+app = TaskLattice(
+    SolaceConnectionDetails(host="localhost", port=55555, vpn="default", username="admin", password="admin"),
+    TaskLatticeConfig(
+      queues=[
+        QueueConfig(name="default", topic="tasks.default"),
+      ], 
+      default_queue="default"
+    ),
+)
+```
+
+**Step 2 -** Define your tasks:
+
+```python
+@app.task
+def sync_function():
+  ...
+
+@app.task
+async def async_function():
+  ...
+```
+
+**Step 3 -** Enqueue a task:
+```python
+task = sync_function.create()
+
+app.enqueue(task)
+```
+
+**Step 4 -** Run a worker to process tasks:
+```python
+app.start_worker()
+```
+
+---
+
+### 📘 Logging
+
+TBD
+
+---
+
+### 📘 Extensions
+
+TBD
+
+---
+
+### ℹ️ License
+
+This project is licensed under the terms of the MIT license.

task_lattice-0.0.1a0/README.md

@@ -0,0 +1,114 @@
+<p>
+  <h3 style="font-size: 3.0em; margin: 0;">Task Lattice</h3>
+  <em>Distributed Task Framework for distributing work across workers</em>
+</p>
+
+<p align="left">
+  <img src="https://github.com/nicholasfelixwilliams/task-lattice/actions/workflows/ci.yml/badge.svg?branch=main" alt="CI">
+  <img src="https://img.shields.io/pypi/v/task-lattice?color=%2334D058&label=pypi%20package" alt="Package version">
+  <img src="https://img.shields.io/pypi/pyversions/task-lattice.svg?color=%2334D058" alt="Supported Python versions">
+  <img src="https://img.shields.io/static/v1?label=code%20style&message=ruff&color=black">
+</p>
+
+---
+
+### 🚀 Key Features
+Task Lattice's key features include:
+
+- **Broker Support** - Following brokers are supported:
+    - Solace
+    - Kafka (Day 2)
+- **Queues** - Multiple queue/priority queue support
+- **Async** - Async tasks supported incl. execution in event loop
+- **Customisation** - Extensive customisation of queue, worker and tasks including:
+    - Automated task retry
+    - Dead letter queues
+    - Worker concurrency
+    - Queue capacity
+    - ...
+- **Minimal code** - Minimal code is required to use task lattice
+- **DAG support** - Supports DAG (directed acyclical graph) workflow execution
+- **Monitoring** - Supports live monitoring of the queues, workers, tasks
+
+---
+
+### ℹ️ Installation
+
+```sh
+# Using pip
+pip install task-lattice
+
+# Using poetry
+poetry add task-lattice
+
+# Using uv
+uv add task-lattice
+```
+
+---
+
+### 📦 Dependencies
+
+TBD
+
+---
+
+### 📘 How to use
+
+**Step 1 -** Define your Task Lattice application
+
+```python
+from task_lattice import TaskLattice, SolaceConnectionDetails, QueueDetails
+
+app = TaskLattice(
+    SolaceConnectionDetails(host="localhost", port=55555, vpn="default", username="admin", password="admin"),
+    TaskLatticeConfig(
+      queues=[
+        QueueConfig(name="default", topic="tasks.default"),
+      ], 
+      default_queue="default"
+    ),
+)
+```
+
+**Step 2 -** Define your tasks:
+
+```python
+@app.task
+def sync_function():
+  ...
+
+@app.task
+async def async_function():
+  ...
+```
+
+**Step 3 -** Enqueue a task:
+```python
+task = sync_function.create()
+
+app.enqueue(task)
+```
+
+**Step 4 -** Run a worker to process tasks:
+```python
+app.start_worker()
+```
+
+---
+
+### 📘 Logging
+
+TBD
+
+---
+
+### 📘 Extensions
+
+TBD
+
+---
+
+### ℹ️ License
+
+This project is licensed under the terms of the MIT license.

task_lattice-0.0.1a0/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "task-lattice"
+version = "0.0.1a"
+description = "Distributed Task Framework"
+readme = "README.md"
+authors = [
+    { name = "Nicholas Williams", email = "noreply@users.noreply.github.com" }
+]
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["tasks", "workflow", "dag", "distributed-tasks"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Operating System :: OS Independent",
+    "Typing :: Typed",
+]
+requires-python = ">=3.12"
+dependencies = []
+
+[build-system]
+requires = ["uv_build>=0.9.11,<0.10.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "mypy>=1.20.2",
+    "pytest>=9.0.3",
+    "pytest-asyncio>=1.3.0",
+    "pytest-cov>=7.1.0",
+    "ruff>=0.15.12",
+    "solace-pubsubplus>=1.11.0",
+]

task_lattice-0.0.1a0/src/task_lattice/app.py

@@ -0,0 +1,95 @@
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from inspect import iscoroutinefunction
+from typing import Callable
+
+from .broker import SolaceBroker
+from .config import SolaceConnectionDetails, TaskLatticeConfig
+from .task import Task, TaskInstance
+
+
+class TaskLattice:
+    _task_registry: dict[str, Task]
+
+    def __init__(
+        self, connection_details: SolaceConnectionDetails, config: TaskLatticeConfig
+    ):
+        self.connection_details = connection_details
+        self.config = config
+        self.broker = SolaceBroker(self.connection_details)
+
+        self._task_registry = {}
+
+    def close(self):
+        self.broker.disconnect()
+
+    def task(self, f: Callable | None = None, *, name: str | None = None):
+        """Decorator to register a python function as a TaskLattice task.
+
+        This must be applied to every task (sync or async) in the following way:
+
+            app = TaskLattice(...)
+
+            @app.task
+            def function(): ...
+
+            @app.task()
+            def function(): ...
+        """
+
+        def decorator(func: Callable):
+            task_name = name or func.__name__
+
+            if task_name in self._task_registry:
+                raise ValueError(f"Task {task_name} is already registered")
+
+            task = Task(name=task_name, func=func, is_async=iscoroutinefunction(func))
+
+            self._task_registry[task.name] = task
+
+            # Attach TaskLattice methods
+            def create_task_instance(
+                args: list | None = None, kwargs: dict | None = None
+            ):
+                return TaskInstance(task.name, self.config, args or [], kwargs or {})
+
+            func.create = create_task_instance
+
+            return func
+
+        if f is not None:
+            return decorator(f)
+
+        return decorator
+
+    def enqueue(self, task: TaskInstance):
+        self.broker.publish(task)
+
+    def start_worker(self):
+        loop = asyncio.get_event_loop()
+        asyncio.set_event_loop(loop)
+
+        executor = ThreadPoolExecutor()
+
+        def handle_message(message: dict):
+            task = self._task_registry.get(message["task_name"])
+
+            if task is None:
+                print(f"Unknown task: {message['task_name']}")
+                return
+
+            if task.is_async:
+                asyncio.run_coroutine_threadsafe(
+                    task.func(*message["args"], **message["kwargs"]), loop
+                )
+            else:
+                loop.run_in_executor(
+                    executor,
+                    lambda: task.func(*message["args"], **message["kwargs"]),
+                )
+
+        # start broker consumer
+        self.broker.start_consumer(handle_message)
+
+        print("Worker started")
+        loop.run_forever()

task_lattice-0.0.1a0/src/task_lattice/broker.py

@@ -0,0 +1,80 @@
+import json
+from solace.messaging.messaging_service import MessagingService
+from solace.messaging.receiver.message_receiver import MessageHandler
+from solace.messaging.resources.topic import Topic
+from solace.messaging.resources.queue import Queue
+from solace.messaging.publisher.persistent_message_publisher import (
+    PersistentMessagePublisher,
+)
+
+from .config import SolaceConnectionDetails
+from .task import TaskInstance
+
+
+class SolaceBroker:
+    publisher: PersistentMessagePublisher
+
+    def __init__(self, connection_details: SolaceConnectionDetails):
+        self.connection_details = connection_details
+
+        config = {
+            "solace.messaging.transport.host": f"tcp://{self.connection_details.host}:{self.connection_details.port}",
+            "solace.messaging.service.vpn-name": self.connection_details.vpn,
+            "solace.messaging.authentication.scheme.basic.username": self.connection_details.username,
+            "solace.messaging.authentication.scheme.basic.password": self.connection_details.password,
+        }
+
+        self.service = MessagingService.builder().from_properties(config).build()
+        self.publisher = (
+            self.service.create_persistent_message_publisher_builder().build()
+        )
+
+    def connect(self):
+        self.service.connect()
+        self.publisher.start()
+
+    def ensure_connected(self):
+        if not self.service.is_connected:
+            self.service.connect()
+        if not self.publisher.is_ready():
+            self.publisher.start()
+
+    def disconnect(self):
+        self.publisher.terminate()
+        self.service.disconnect()
+
+    def publish(self, task: TaskInstance):
+        self.ensure_connected()
+
+        # Build the message
+        msg = (
+            self.service.message_builder()
+            .with_priority(task.priority)
+            .build(json.dumps(task.message))
+        )
+
+        # Publish the message
+        self.publisher.publish(msg, Topic.of("tasks.default"))
+
+    def start_consumer(self, handler):
+        self.ensure_connected()
+        receiver = self.service.create_persistent_message_receiver_builder().build(
+            Queue.durable_exclusive_queue("task_queue")
+        )
+
+        class MyHandler(MessageHandler):
+            def on_message(_, message):
+                payload = message.get_payload_as_string()
+                print("HI")
+
+                # deserialize
+                data = json.loads(payload)
+
+                print(data)
+
+                handler(data)
+
+                receiver.ack(message)
+
+        receiver.start()
+        receiver.receive_async(MyHandler())

task_lattice-0.0.1a0/src/task_lattice/brokers/solace.py

@@ -0,0 +1,83 @@
+import asyncio
+from dataclasses import dataclass
+import json
+from solace.messaging.messaging_service import MessagingService
+from solace.messaging.receiver.message_receiver import MessageHandler
+from solace.messaging.resources.topic import Topic
+from solace.messaging.resources.queue import Queue
+
+
+@dataclass(frozen=True)
+class SolaceConnectionDetails:
+    host: str
+    port: int
+    vpn: str
+    username: str
+    password: str
+
+
+class SolaceBroker:
+    def __init__(self, connection_details: SolaceConnectionDetails):
+        self.connection_details = connection_details
+
+        config = {
+            "solace.messaging.transport.host": f"tcp://{self.connection_details.host}:{self.connection_details.port}",
+            "solace.messaging.service.vpn-name": self.connection_details.vpn,
+            "solace.messaging.authentication.scheme.basic.username": self.connection_details.username,
+            "solace.messaging.authentication.scheme.basic.password": self.connection_details.password,
+        }
+
+        self.service = MessagingService.builder().from_properties(config).build()
+        self.publisher = (
+            self.service.create_persistent_message_publisher_builder().build()
+        )
+
+    def connect(self):
+        self.service.connect()
+        self.publisher.start()
+
+    def disconnect(self):
+        self.publisher.terminate()
+        self.service.disconnect()
+
+    def publish(self, message: dict, topic: str, priority: int):
+        sol_topic = Topic.of(topic)
+
+        msg_builder = self.service.message_builder()
+        msg_builder.with_priority(priority)
+        msg = msg_builder.build(json.dumps(message))
+
+        self.publisher.publish(msg, sol_topic)
+
+        # print(f"Published {msg} to {sol_topic}")
+
+    def listen_to_queue(self, queue: str):
+        q = Queue.durable_exclusive_queue(queue)
+        receiver = self.service.create_persistent_message_receiver_builder().build(q)
+        loop = asyncio.get_event_loop()
+
+        semaphore = asyncio.Semaphore(5)
+
+        async def process_message(payload: str):
+            print("⚙️ Processing async:", payload)
+            await asyncio.sleep(0.01)
+
+        # Proper handler class
+        class MyHandler(MessageHandler):
+            def on_message(self, message):
+                payload = message.get_payload_as_string()
+                print("📩 Received:", payload)
+
+                async def wrapper():
+                    async with semaphore:
+                        await process_message(payload)
+                        receiver.ack(message)
+
+                asyncio.run_coroutine_threadsafe(wrapper(), loop)
+
+        handler = MyHandler()
+
+        receiver.start()
+        receiver.receive_async(handler)
+
+        loop.run_forever()

task_lattice-0.0.1a0/src/task_lattice/config.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class SolaceConnectionDetails:
+    host: str
+    port: int
+    vpn: str
+
+    # TODO: Allow multiple auth schemes
+    username: str
+    password: str
+
+
+@dataclass(frozen=True)
+class QueueConfig:
+    name: str
+    topic: str
+
+
+@dataclass(frozen=True)
+class TaskLatticeConfig:
+    queues: list[QueueConfig]
+
+    default_queue: str

task_lattice-0.0.1a0/src/task_lattice/task.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+from typing import Callable
+
+from .config import TaskLatticeConfig
+
+
+@dataclass
+class Task:
+    name: str
+    func: Callable
+    is_async: bool
+
+
+class TaskInstance:
+    def __init__(
+        self, task_name: str, config: TaskLatticeConfig, args: list, kwargs: dict
+    ) -> None:
+        self.task_name = task_name
+        self.args = args
+        self.kwargs = kwargs
+
+        self.priority = 2
+        self.topic = "tasks/default"
+
+    @property
+    def message(self) -> dict:
+        return {"task_name": self.task_name, "args": self.args, "kwargs": self.kwargs}