ibm-watsonx-orchestrate 1.2.0__py3-none-any.whl → 1.4.2__py3-none-any.whl

ibm_watsonx_orchestrate/client/tools/tool_client.py

@@ -21,6 +21,10 @@ class ToolClient(BaseAPIClient):
     def upload_tools_artifact(self, tool_id: str, file_path: str) -> dict:
         return self._post(f"/tools/{tool_id}/upload", files={"file": (f"{tool_id}.zip", open(file_path, "rb"), "application/zip", {"Expires": "0"})})
     
+    def download_tools_artifact(self, tool_id: str) -> bytes:
+        response = self._get(f"/tools/{tool_id}/download", return_raw=True)
+        return response.content
+
     def get_draft_by_name(self, tool_name: str) -> List[dict]:
         return self.get_drafts_by_names([tool_name])
 
@@ -39,3 +43,7 @@ class ToolClient(BaseAPIClient):
                 if e.response.status_code == 404 and "not found with the given name" in e.response.text:
                     return ""
                 raise(e)
+    
+    def get_drafts_by_ids(self, tool_ids: List[str]) -> List[dict]:
+        formatted_tool_ids = [f"ids={x}" for x in tool_ids]
+        return self._get(f"/tools?{'&'.join(formatted_tool_ids)}")

ibm_watsonx_orchestrate/docker/compose-lite.yml

@@ -49,6 +49,17 @@ services:
     ports:
       - 3001:3001
 
+  ai-gateway:
+    image: ${AI_GATEWAY_REGISTRY:-us.icr.io/watson-orchestrate-private}/ai-gateway:${AI_GATEWAY_TAG:-latest}
+    platform: linux/amd64
+    restart: unless-stopped
+    ports:
+      - "8787:8787"
+    environment:  
+      WATSONX_API_KEY: ${WATSONX_APIKEY}
+      WATSONX_URL: ${WATSONX_URL}
+      WATSONX_SPACE_ID: ${WATSONX_SPACE_ID}
+
   ui:
     image: ${UI_REGISTRY:-us.icr.io/watson-orchestrate-private}/wxo-chat:${UITAG:-latest}
     platform: linux/amd64
@@ -95,6 +106,7 @@ services:
     environment:
       WXO_DEPLOYMENT_TYPE: 'laptop'
       AGENT_RUNTIME_ENDPOINT: http://wxo-server:4321
+      TEMPUS_RUNTIME_ENDPOINT: http://wxo-tempus-runtime:9044
     command: 'npm start'
     ports:
       - '4025:4025'
@@ -210,14 +222,21 @@ services:
     volumes:
       - ./sdk:/packages
       - tools:/tools
+    
     command: >
-        bash -c "mkdir -p /tmp/certs && 
-                  openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /tmp/certs/key.pem -out /tmp/certs/cert.pem -subj '/CN=localhost' && 
-                  uvicorn --host 0.0.0.0 --port 4322 --ssl-keyfile /tmp/certs/key.pem --ssl-certfile /tmp/certs/cert.pem --log-config /app/config/logs/log_conf.yaml --workers 2 --log-level debug wo_archer.api.main:app & 
-                  (for i in {1..20}; do curl --silent --fail -k https://127.0.0.1:4322/health/alive && echo '[INFO] HTTPS Service ready' && break || echo '[INFO] Waiting for HTTPS service...' && sleep 30; done; curl --silent --fail -k https://127.0.0.1:4322/health/alive || (echo '[ERROR] HTTPS service failed to start' && exit 1) ) &&
-                  uvicorn --host 0.0.0.0 --port 4321 --log-config /app/config/logs/log_conf.yaml --log-level debug --workers 5 wo_archer.api.main:app & 
-                  (for i in {1..20}; do curl --silent --fail -k http://127.0.0.1:4321/health/alive && echo '[INFO] HTTP Service ready' && break || echo '[INFO] Waiting for HTTP service...' && sleep 30; done; curl --silent --fail -k http://127.0.0.1:4321/health/alive || (echo '[ERROR] HTTP service failed to start' && exit 1) ) && 
-                  wait"
+       bash -c '
+          export PYTHONPATH="/tmp/monkey:$PYTHONPATH"
+          mkdir -p /tmp/monkey &&
+
+          echo -e "from uvicorn.supervisors.multiprocess import Process\n_original_is_alive = Process.is_alive\ndef patched_is_alive(self, timeout: float = 30):\n    return _original_is_alive(self, timeout=30)\nProcess.is_alive = patched_is_alive" > /tmp/monkey/sitecustomize.py &&
+
+          uvicorn --host 0.0.0.0 --port 4321 --log-config /app/config/logs/log_conf.yaml --log-level debug --workers 5 wo_archer.api.main:app &
+
+          for i in {1..40}; do curl --silent --fail http://127.0.0.1:4321/health/alive && echo "[INFO] HTTP Service ready" && break || echo "[INFO] Waiting for HTTP service..." && sleep 10; done;
+          curl --silent --fail http://127.0.0.1:4321/health/alive || (echo "[ERROR] HTTP service failed to start" && exit 1);
+
+          wait
+        '
     depends_on:
       wxo-server-redis:
           condition: service_started
@@ -294,6 +313,15 @@ services:
       MILVUS_DB_NAME: default
       MILVUS_USERNAME: root
       MILVUS_PASSWORD: Milvus
+      WO_API_KEY: ${WO_API_KEY}
+      WO_USERNAME: ${WO_USERNAME}
+      WO_PASSWORD: ${WO_PASSWORD}
+      WO_INSTANCE: ${WO_INSTANCE}
+      USE_SAAS_ML_TOOLS_RUNTIME: ${USE_SAAS_ML_TOOLS_RUNTIME}
+      AUTHORIZATION_URL: ${AUTHORIZATION_URL}
+      WO_AUTH_TYPE: ${WO_AUTH_TYPE}
+      AI_GATEWAY_BASE_URL: ${AI_GATEWAY_BASE_URL}
+      AI_GATEWAY_ENABLED : ${AI_GATEWAY_ENABLED}
 
   wxo-server-worker:
     image: ${WORKER_REGISTRY:-us.icr.io/watson-orchestrate-private}/wxo-server-conversation_controller:${WORKER_TAG:-latest}
@@ -311,6 +339,8 @@ services:
       - ./sdk:/packages
       - tools:/tools
     environment:
+      AI_GATEWAY_ENABLED : ${AI_GATEWAY_ENABLED}
+      AI_GATEWAY_BASE_URL: ${AI_GATEWAY_BASE_URL}
       IS_WXO_LITE: 'TRUE'
       TRM_BASE_URL: http://tools-runtime-manager:8080
       AGENT_STEP_DETAILS: redis://wxo-server-redis:6379/0
@@ -372,6 +402,13 @@ services:
       MILVUS_DB_NAME: default
       MILVUS_USERNAME: root
       MILVUS_PASSWORD: Milvus
+      WO_API_KEY: ${WO_API_KEY}
+      WO_USERNAME: ${WO_USERNAME}
+      WO_PASSWORD: ${WO_PASSWORD}
+      WO_INSTANCE: ${WO_INSTANCE}
+      USE_SAAS_ML_TOOLS_RUNTIME: ${USE_SAAS_ML_TOOLS_RUNTIME}
+      AUTHORIZATION_URL: ${AUTHORIZATION_URL}
+      WO_AUTH_TYPE: ${WO_AUTH_TYPE}
   
   tools-runtime-manager:
     image: ${TRM_REGISTRY:-us.icr.io/watson-orchestrate-private}/tools-runtime-manager:${TRM_TAG:-latest}
@@ -407,6 +444,12 @@ services:
       VOLUME_MOUNT_PATH: "/shared-data"
       TOOLS_RUNTIME_MANAGER_BASE_URL: ${TOOLS_RUNTIME_MANAGER_BASE_URL}
       CONNECTION_SERVICE_BASE_URL: http://wxo-server-connection-manager:3001
+      STORAGE_S3_BUCKET: ${STORAGE_S3_BUCKET:-wxo-server-storage-bucket}  
+      STORAGE_S3_ENDPOINT: http://wxo-server-minio:9000
+      STORAGE_S3_FORCE_PATH_STYLE: 'true'
+      STORAGE_S3_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-minioadmin}
+      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-watsonxorchestrate}
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
@@ -432,6 +475,12 @@ services:
       DB_PASSWORD: ${DB_PASSWORD:-postgres}
       DB_NAME: ${DB_NAME:-postgres}
       DB_SSLMODE: ${DB_SSLMODE:-disable}  # Disable SSL if not configured
+      STORAGE_S3_BUCKET: ${STORAGE_S3_BUCKET:-wxo-server-storage-bucket}  
+      STORAGE_S3_ENDPOINT: http://wxo-server-minio:9000
+      STORAGE_S3_FORCE_PATH_STYLE: 'true'
+      STORAGE_S3_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-minioadmin}
+      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-watsonxorchestrate}
 
 ########################
 # LANGFUSE dependencies
@@ -531,11 +580,8 @@ services:
   wxo-tempus-runtime:
     image: ${FLOW_RUMTIME_REGISTRY:-us.icr.io/watson-orchestrate-private}/wxo-tempus-runtime:${FLOW_RUNTIME_TAG:-latest}
     restart: unless-stopped
-    platform: linux/amd64
-    profiles: [with-tempus-runtime]
     environment:
       NODE_TLS_REJECT_UNAUTHORIZED: "0"
-      COMMON_CONFIG_FILE: "/shared/common-config.yaml"
       SERVER_MODE: production
       SERVER_ENVIRONMENT: SDK
       SERVER_HTTP_PORT: "9044"
@@ -557,6 +603,17 @@ services:
       PG_DATABASE: ${POSTGRES_USER:-postgres}
       USER_MGMT_BASE_URL: "" # TODO
       WXO_SERVER_BASE_URL: http://wxo-server:4321
+      TRM_SERVER_BASE_URL: http://tools-runtime-manager:8080
+      CONNECTION_MANAGER_SERVER_BASE_URL: http://wxo-server-connection-manager:3001
+      REDIS_ENDPOINT: wxo-server-redis:6379
+      REDIS_USER:
+      REDIS_PASSWORD:
+      REDIS_TLS: false
+      LANGFUSE_ENABLED: ${LANGFUSE_ENABLED:-false}
+      LANGFUSE_HOST: ${LANGFUSE_HOST:-http://langfuse-web:3000}
+      LANGFUSE_PUBLIC_KEY:  ${LANGFUSE_PUBLIC_KEY:-pk-lf-7417757e-d6df-421b-957e-683b76acb5df}
+      LANGFUSE_SECRET_KEY:  ${LANGFUSE_PRIVATE_KEY:-sk-lf-7bc4da63-7b2b-40c0-b5eb-1e0cf64f9af2}
+      LOG_LEVEL: info
     healthcheck:
       test: curl -k http://localhost:9044/readiness --fail
       interval: 5s
@@ -564,8 +621,6 @@ services:
       retries: 5
     ports:
       - 9044:9044
-    volumes:
-      - ./tempus/common-config.yaml:/shared/common-config.yaml
     depends_on:
       - wxo-server-db
     
@@ -595,4 +650,4 @@ volumes:
 networks:
   default:
     name: wxo-server
-    
+    

ibm_watsonx_orchestrate/docker/default.env

@@ -46,38 +46,46 @@ CELERY_RESULTS_TTL="3600"
 EVENT_BROKER_TTL="-1"
 
 # START -- IMAGE REGISTRIES AND TAGS
-# THE VALUES FOR REGISTRY_URL AND *_REGISTRY ARE NOT SET HERE; THEY ARE EITHER PROVIDED BY THE USER OR DETERMINED AT RUNTIME BASED ON WO_DEVELOPER_EDITION_SOURCE.
+# The registry URL to pull the private images from, including the name of the repository in the registry.
+# e.g. cp.icr.io/cp/wxo-lite
+# If the registry URL is not set here or by the user, then it will be set automatically based on the value of WO_DEVELOPER_EDITION_SOURCE
+# The *_REGISTRY variables are used to set the registry URL for each component. If not set, the URL here will be used.
+# See get_default_registry_env_vars_by_dev_edition_source() in src/ibm_watsonx_orchestrate/cli/commands/server/server_command.py for more details.
 REGISTRY_URL=
 
-SERVER_TAG=03-05-2025
+SERVER_TAG=22-05-2025
 SERVER_REGISTRY=
 
-WORKER_TAG=03-05-2025
+WORKER_TAG=22-05-2025
 WORKER_REGISTRY=
 
+AI_GATEWAY_TAG=20-05-2025
+AI_GATEWAY_REGISTRY=
+
 DB_REGISTRY=
 # If you build multiarch set all three of these to the same, we have a pr against main
 # to not have this separation, but we can merge it later
-DBTAG=02-05-2025
-AMDDBTAG=02-05-2025
-ARM64DBTAG=02-05-2025
+DBTAG=21-05-2025
+AMDDBTAG=21-05-2025
+ARM64DBTAG=21-05-2025
 
 UI_REGISTRY=
-UITAG=01-05-2025
+UITAG=23-05-2025
 
 CM_REGISTRY=
-CM_TAG=30-04-2025
+CM_TAG=13-05-2025
 
-TRM_TAG=02-05-2025
+TRM_TAG=23-05-2025
 TRM_REGISTRY=
 
-TR_TAG=02-05-2025
+TR_TAG=23-05-2025
 TR_REGISTRY=
 
+BUILDER_TAG=21-05-2025
 BUILDER_REGISTRY=
-BUILDER_TAG=02-05-2025
 
-FLOW_RUNTIME_TAG=31-03-2025
+
+FLOW_RUNTIME_TAG=13-05-2025
 FLOW_RUMTIME_REGISTRY=
 
 # END -- IMAGE REGISTRIES AND TAGS
@@ -110,6 +118,8 @@ WXO_BASE_URL=http://wxo-server:4321
 RUNTIME_MANAGER_API_KEY="testapikey"
 TOOLS_RUNTIME_MANAGER_BASE_URL="http://tools-runtime-manager:8080"
 CONNECTION_SERVICE_BASE_URL="http://wxo-server-connection-manager:3001"
+AI_GATEWAY_BASE_URL="http://ai-gateway:8787/v1"
+AI_GATEWAY_ENABLED=True
 
 #To Prevent warnings
 VECTOR_STORE_PROVIDER=

ibm_watsonx_orchestrate/docker/tempus/common-config.yaml

@@ -1 +1 @@
-NODE_LOG_LEVEL: trace
+NODE_LOG_LEVEL: audit

ibm_watsonx_orchestrate/experimental/flow_builder/flows/__init__.py

@@ -0,0 +1,41 @@
+from .constants import START, END, RESERVED
+from ..types import FlowContext, TaskData, TaskEventType
+from ..node import UserNode, AgentNode, StartNode, EndNode
+from .flow import Flow, CompiledFlow, FlowRun, FlowEvent, FlowEventType, FlowFactory, MatchPolicy, WaitPolicy, ForeachPolicy, Branch, Foreach, Loop
+from .decorators import user, flow_spec, flow
+from .data_map import Assignment, DataMapSpec
+
+
+__all__ = [
+    "START",
+    "END",
+    "RESERVED",
+
+    "FlowContext",
+    "TaskData",
+    "TaskEventType",
+
+    "UserNode",
+    "AgentNode",
+    "StartNode",
+    "EndNode",
+    "Assignment",
+    "DataMapSpec",
+
+    "Flow",    
+    "CompiledFlow",
+    "FlowRun",
+    "FlowEvent",
+    "FlowEventType",
+    "FlowFactory",
+    "MatchPolicy",
+    "WaitPolicy",
+    "ForeachPolicy",
+    "Branch",
+    "Foreach",
+    "Loop",
+
+    "user",
+    "flow_spec",
+    "flow"
+]

ibm_watsonx_orchestrate/experimental/flow_builder/flows/constants.py

@@ -0,0 +1,17 @@
+"""
+Predefined constants for Flow.
+"""
+import sys
+
+START = sys.intern("__start__")
+END = sys.intern("__end__")
+
+ANY_USER = sys.intern("__any_user__")
+FLOW_CONTEXT = sys.intern("FlowContext")
+
+RESERVED = {
+    START,
+    END,
+    FLOW_CONTEXT,
+    ANY_USER
+}

ibm_watsonx_orchestrate/experimental/flow_builder/flows/data_map.py

@@ -0,0 +1,91 @@
+from typing import Any, List, cast
+import uuid
+import json
+import yaml
+from pydantic import BaseModel, Field, SerializeAsAny
+
+from ibm_watsonx_orchestrate.agent_builder.tools.types import (
+    ToolRequestBody, ToolResponseBody, JsonSchemaObject
+)
+from ..types import (
+    _to_json_from_input_schema, _to_json_from_output_schema, SchemaRef, Assignment
+)
+
+class DataMapSpec(BaseModel):
+ 
+    name: str
+
+    def __init__(self, **data: Any) -> None:
+        if 'id' not in data:
+            data['id'] = str(uuid.uuid4())
+        super().__init__(**data)
+
+    def to_json(self) -> dict[Any, dict]:
+        '''Create a JSON object representing the data'''
+        model_spec = {}
+        model_spec["name"] = self.name
+        return model_spec
+
+
+
+class AssignmentDataMapSpec(DataMapSpec):
+ 
+    maps: List[Assignment] 
+
+    def to_json(self) -> dict[str, Any]:
+        '''Create a JSON object representing the data'''
+        model_spec = super().to_json()
+
+        if self.maps:
+            model_spec["maps"] = [assignment.model_dump() for assignment in self.maps]
+
+        return model_spec
+
+
+class DataMap(BaseModel):
+ 
+    spec: SerializeAsAny[DataMapSpec]
+
+    def __call__(self, **kwargs):
+        pass
+
+    def dump_spec(self, file: str) -> None:
+     
+        dumped = self.spec.model_dump(mode='json',
+                                      exclude_unset=True, exclude_none=True, by_alias=True)
+        with open(file, 'w', encoding="utf-8") as f:
+            if file.endswith('.yaml') or file.endswith('.yml'):
+                yaml.dump(dumped, f)
+            elif file.endswith('.json'):
+                json.dump(dumped, f, indent=2)
+            else:
+                raise ValueError('file must end in .json, .yaml, or .yml')
+
+    def dumps_spec(self) -> str:
+     
+        dumped = self.spec.model_dump(mode='json',
+                                      exclude_unset=True, exclude_none=True, by_alias=True)
+        return json.dumps(dumped, indent=2)
+
+    def __repr__(self):
+        return f"DataMap(name='{self.spec.name}', description='{self.spec.description}')"
+
+    def to_json(self) -> dict[str, Any]:
+     
+        obj = self.get_spec().to_json()
+
+        return { "spec": obj }
+    
+    def get_spec(self) -> DataMapSpec:
+     
+        return self.spec
+    
+class AssignmentDataMap(DataMap):
+    def get_spec(self) -> AssignmentDataMapSpec:
+        return cast(AssignmentDataMapSpec, self.spec)
+    
+    def to_json(self) -> dict[str, Any]:
+     
+        obj = super().to_json()
+        return obj
+    

ibm_watsonx_orchestrate/experimental/flow_builder/flows/decorators.py

@@ -0,0 +1,143 @@
+"""
+A set of decorators to help define different Flow constructs.
+"""
+
+import asyncio
+from functools import wraps
+import logging
+import inspect
+from typing import Callable, Optional, Sequence
+from pydantic import BaseModel
+from ..types import extract_node_spec, UserNodeSpec, FlowSpec
+
+from .flow import FlowFactory, Flow
+
+logger = logging.getLogger(__name__)
+
+
+class FlowWrapper:
+    def __init__(self, func, a_model):
+        self.func = func
+        self.a_model = a_model
+        wraps(func)(self)  # Preserve metadata
+
+    def __call__(self, *args, **kwargs):
+        result = self.func(self.a_model)
+        if not isinstance(result, Flow):
+            raise ValueError("Return value must be of type Flow")
+        return result
+    
+def user(*args, name: str|None=None, description: str|None=None, owners: Sequence[str] = ()):
+    """Decorator to mark a function as a task specification."""
+
+    def decorator(func: Callable):
+        node_spec = extract_node_spec(func, name, description)
+        func.__user_spec__ = UserNodeSpec(type = "user",
+                                          name = node_spec.name,
+                                          display_name = node_spec.display_name,
+                                          description = node_spec.description,
+                                          input_schema = node_spec.input_schema,
+                                          output_schema = node_spec.output_schema,
+                                          output_schema_object = node_spec.output_schema_object,
+                                          owners=owners)
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            logger.error(f"User node {name} is not supported yet.")            
+            r = func(*args, **kwargs)
+            return r
+
+        return wrapper
+
+    if len(args) == 1 and callable(args[0]):
+        return decorator(args[0])
+    else:
+        return decorator
+
+
+def flow_spec(*args,
+              name: Optional[str]=None,
+              description: str|None=None,
+              initiators: Sequence[str] = ()):
+    """Decorator to mark a function as a flow specification."""
+
+    def decorator(func: Callable):
+        node_spec = extract_node_spec(func, name, description)
+        a_spec = FlowSpec(type = "flow",
+                             name = node_spec.name,
+                             display_name = node_spec.display_name,
+                             description = node_spec.description,
+                             input_schema = node_spec.input_schema,
+                             output_schema = node_spec.output_schema,
+                             output_schema_object = node_spec.output_schema_object,
+                             initiators = initiators)
+
+        # we should also check a flow is async
+        if not asyncio.iscoroutinefunction(func):
+            raise ValueError("Flow must be asynchronous.")
+
+        logger.info("Generated flow spec: %s", a_spec)
+        func.__flow_spec__ = a_spec
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            logger.info("Creating flow spec: %s", name)
+            r = func(*args, **kwargs)
+            logger.info("Flow spec %s returned: %s", name, r)
+            return r
+
+        return wrapper
+
+    if len(args) == 1 and callable(args[0]):
+        return decorator(args[0])
+    else:
+        return decorator
+    
+def flow(*args, 
+         name: Optional[str]=None, 
+         display_name: Optional[str]=None,
+         description: str|None=None,
+         input_schema: type[BaseModel] | None = None,
+         output_schema: type[BaseModel] | None = None,
+         initiators: Sequence[str] = ()):
+    """Decorator to mark a function as a flow model builder."""
+
+    def decorator(func: Callable):
+        """
+        Decorator that takes a function as an argument and returns a wrapper function.
+        The wrapper function takes a single argument of type Flow and calls the original function with the created flow as an argument.
+        """
+
+        sig = inspect.signature(func)
+        if len(sig.parameters) != 1:
+            raise ValueError("Only one argument is allowed")
+        param = list(sig.parameters.values())[0]
+        if param.annotation != Flow:
+            raise ValueError("Argument must be of type Flow")
+        if sig.return_annotation != Flow:
+            raise ValueError("Return value must be of type Flow")
+        
+        node_spec = extract_node_spec(func, name, description)
+        a_model = FlowFactory.create_flow(
+                             name = node_spec.name,
+                             display_name = display_name,
+                             description = node_spec.description,
+                             input_schema = input_schema,
+                             output_schema = output_schema,
+                             initiators = initiators)
+
+        # logger.info("Creating flow model: %s", a_model.spec.name)
+
+        # @wraps(func)
+        # def wrapper(*args, **kwargs):
+        #     result = func(a_model)
+        #     if not isinstance(result, Flow):
+        #         raise ValueError("Return value must be of type Flow")
+        #     return result
+
+        return FlowWrapper(func, a_model)
+
+    if len(args) == 1 and callable(args[0]):
+        return decorator(args[0])
+    else:
+        return decorator

ibm_watsonx_orchestrate/experimental/flow_builder/flows/events.py

@@ -0,0 +1,72 @@
+import redis
+import time
+import json
+from dotenv import load_dotenv
+import os
+
+from typing import (
+    AsyncIterator, Union
+)
+
+from ..types import (
+    FlowEventType, TaskEventType, FlowEvent, FlowContext
+)
+
+class StreamConsumer:
+    def __init__(self, instance_id: str):
+        
+        load_dotenv()
+        self.redis_host = os.getenv("REDIS_HOST", "localhost")
+        self.redis_port = os.getenv("REDIS_PORT", 6379)
+        self.redis_db = os.getenv("REDIS_DB", 0)
+        self.redis = redis.Redis(host=self.redis_host, port=self.redis_port, db=self.redis_db)
+        self.instance_id = instance_id
+        self.stream_name = f"tempus:{self.instance_id}"
+        self.last_processed_id = 0
+        
+
+    async def consume(self) -> AsyncIterator[FlowEvent]:
+        
+        while True:
+            try:
+                # XREAD command: Read new messages from the stream
+                messages = self.redis.xread({self.stream_name: self.last_processed_id}, block=5000, count=10)
+
+                for stream, events in messages:
+                    for event_id, event_data in events:
+                        self.last_processed_id = event_id  # Update the last read event ID
+                        flow_event = deserialize_flow_event(event_data)
+                        yield flow_event
+
+            except Exception as e:
+                print(f"Error occurred: {e}")
+                time.sleep(5)  # Wait for 5 seconds before retrying
+
+            time.sleep(1)  # Sleep for 1 second before checking for new messages
+
+def deserialize_flow_event(byte_data: bytes) -> FlowEvent:
+    """Deserialize byte data into a FlowEvent object."""
+    # Decode the byte data
+    decoded_data = byte_data[b'message'].decode('utf-8')
+    
+    # Parse the JSON string into a dictionary
+    parsed_data = json.loads(decoded_data)
+    
+    # Deserialize into FlowEvent using Pydantic's parsing
+    flow_event = FlowEvent(
+        kind=get_event_type(parsed_data["kind"]),
+        context=FlowContext(**parsed_data["context"]) if "context" in parsed_data and parsed_data["context"] != {} else None,
+        error=json.loads(parsed_data["error"]) if "error" in parsed_data and parsed_data["error"] != {} and len(parsed_data["error"]) > 0 else None,
+    )
+    
+    return flow_event
+
+def get_event_type(selected_event_type: str) -> Union[FlowEventType, TaskEventType]:
+    """Selects the right event type from the corresponding enumerator"""
+    eventKind = selected_event_type.upper()
+    if eventKind in FlowEventType.__members__:
+        return FlowEventType(selected_event_type)
+    elif eventKind in TaskEventType.__members__:
+        return TaskEventType(selected_event_type)
+    else:
+        raise ValueError(f"Invalid event type: {eventKind}")