flowcept 0.8.9__py3-none-any.whl → 0.8.11__py3-none-any.whl

flowcept/cli.py

@@ -0,0 +1,460 @@
+"""
+Flowcept CLI.
+
+How to add a new command:
+--------------------------
+1. Write a function with type-annotated arguments and a NumPy-style docstring.
+2. Add it to one of the groups in `COMMAND_GROUPS`.
+3. It will automatically become available as `flowcept --<function-name>` (underscores become hyphens).
+
+Supports:
+- `flowcept --command`
+- `flowcept --command --arg=value`
+- `flowcept -h` or `flowcept` for full help
+- `flowcept --help --command` for command-specific help
+"""
+
+import subprocess
+from time import sleep
+from typing import Dict, Optional
+import argparse
+import os
+import sys
+import json
+import textwrap
+import inspect
+from functools import wraps
+from importlib import resources
+from pathlib import Path
+from typing import List
+
+from flowcept import Flowcept, configs
+
+
+def no_docstring(func):
+    """Decorator to silence linter for missing docstrings."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def show_config():
+    """
+    Show Flowcept configuration.
+    """
+    config_data = {
+        "session_settings_path": configs.SETTINGS_PATH,
+        "env_FLOWCEPT_SETTINGS_PATH": os.environ.get("FLOWCEPT_SETTINGS_PATH", None),
+    }
+    print(f"This is the settings path in this session: {configs.SETTINGS_PATH}")
+    print(
+        f"This is your FLOWCEPT_SETTINGS_PATH environment variable value: {config_data['env_FLOWCEPT_SETTINGS_PATH']}"
+    )
+
+
+def init_settings():
+    """
+    Create a new settings.yaml file in your home directory under ~/.flowcept.
+    """
+    dest_path = Path(os.path.join(configs._SETTINGS_DIR, "settings.yaml"))
+
+    if dest_path.exists():
+        overwrite = input(f"{dest_path} already exists. Overwrite? (y/N): ").strip().lower()
+        if overwrite != "y":
+            print("Operation aborted.")
+            return
+
+    os.makedirs(configs._SETTINGS_DIR, exist_ok=True)
+
+    SAMPLE_SETTINGS_PATH = str(resources.files("resources").joinpath("sample_settings.yaml"))
+
+    with open(SAMPLE_SETTINGS_PATH, "rb") as src_file, open(dest_path, "wb") as dst_file:
+        dst_file.write(src_file.read())
+    print(f"Copied {configs.SETTINGS_PATH} to {dest_path}")
+
+
+def start_consumption_services(bundle_exec_id: str = None, check_safe_stops: bool = False, consumers: List[str] = None):
+    """
+    Start services that consume data from a queue or other source.
+
+    Parameters
+    ----------
+    bundle_exec_id : str, optional
+        The ID of the bundle execution to associate with the consumers.
+    check_safe_stops : bool, optional
+        Whether to check for safe stopping conditions before starting.
+    consumers : list of str, optional
+        List of consumer IDs to start. If not provided, all consumers will be started.
+    """
+    print("Starting consumption services...")
+    print(f"  bundle_exec_id: {bundle_exec_id}")
+    print(f"  check_safe_stops: {check_safe_stops}")
+    print(f"  consumers: {consumers or []}")
+
+    Flowcept.start_consumption_services(
+        bundle_exec_id=bundle_exec_id,
+        check_safe_stops=check_safe_stops,
+        consumers=consumers,
+    )
+
+
+def stop_consumption_services():
+    """
+    Stop the document inserter.
+    """
+    print("Not implemented yet.")
+
+
+def start_services(with_mongo: bool = False):
+    """
+    Start Flowcept services (optionally including MongoDB).
+
+    Parameters
+    ----------
+    with_mongo : bool, optional
+        Whether to also start MongoDB.
+    """
+    print(f"Starting services{' with Mongo' if with_mongo else ''}")
+    print("Not implemented yet.")
+
+
+def stop_services():
+    """
+    Stop Flowcept services.
+    """
+    print("Not implemented yet.")
+
+
+def workflow_count(workflow_id: str):
+    """
+    Count number of documents in the DB.
+
+    Parameters
+    ----------
+    workflow_id : str
+        The ID of the workflow to count tasks for.
+    """
+    result = {
+        "workflow_id": workflow_id,
+        "tasks": len(Flowcept.db.query({"workflow_id": workflow_id})),
+        "workflows": len(Flowcept.db.query({"workflow_id": workflow_id}, collection="workflows")),
+        "objects": len(Flowcept.db.query({"workflow_id": workflow_id}, collection="objects")),
+    }
+    print(json.dumps(result, indent=2))
+
+
+def query(filter: str, project: str = None, sort: str = None, limit: int = 0):
+    """
+    Query the MongoDB task collection with an optional projection, sort, and limit.
+
+    Parameters
+    ----------
+    filter : str
+        A JSON string representing the MongoDB filter query.
+    project : str, optional
+        A JSON string specifying fields to include or exclude in the result (MongoDB projection).
+    sort : str, optional
+        A JSON string specifying sorting criteria (e.g., '[["started_at", -1]]').
+    limit : int, optional
+        Maximum number of documents to return. Default is 0 (no limit).
+
+    Returns
+    -------
+    List[dict]
+        A list of task documents matching the query.
+    """
+    _filter = json.loads(filter)
+    _project = json.loads(project) or None
+    _sort = list(sort) or None
+    print(
+        json.dumps(Flowcept.db.query(filter=_filter, project=_project, sort=_sort, limit=limit), indent=2, default=str)
+    )
+
+
+def get_task(task_id: str):
+    """
+    Query the Document DB to retrieve a task.
+
+    Parameters
+    ----------
+    task_id : str
+        The identifier of the task.
+    """
+    _query = {"task_id": task_id}
+    print(json.dumps(Flowcept.db.query(_query), indent=2, default=str))
+
+
+def start_agent():
+    """Start Flowcept agent."""
+    from flowcept.flowceptor.adapters.agents.flowcept_agent import main
+
+    main()
+
+
+def agent_client(tool_name: str, kwargs: str = None):
+    """Agent Client.
+
+    Parameters.
+    ----------
+    tool_name : str
+        Name of the tool
+    kwargs : str, optional
+        A stringfied JSON containing the kwargs for the tool, if needed.
+    """
+    print(kwargs)
+    if kwargs is not None:
+        kwargs = json.loads(kwargs)
+
+    print(f"Going to run agent tool '{tool_name}'.")
+    if kwargs:
+        print(f"Using kwargs: {kwargs}")
+    print("-----------------")
+    from flowcept.flowceptor.consumers.agent.client_agent import run_tool
+
+    result = run_tool(tool_name, kwargs)[0]
+
+    print(result.text)
+
+
+def check_services():
+    """
+    Run a full diagnostic test on the Flowcept system and its dependencies.
+
+    This function:
+    - Prints the current configuration path.
+    - Checks if required services (e.g., MongoDB, agent) are alive.
+    - Runs a test function wrapped with Flowcept instrumentation.
+    - Verifies MongoDB insertion (if enabled).
+    - Verifies agent communication and LLM connectivity (if enabled).
+
+    Returns
+    -------
+    None
+        Prints diagnostics to stdout; returns nothing.
+    """
+    print(f"Testing with settings at: {configs.SETTINGS_PATH}")
+    from flowcept.configs import MONGO_ENABLED, AGENT, KVDB_ENABLED, INSERTION_BUFFER_TIME
+
+    if not Flowcept.services_alive():
+        print("Some of the enabled services are not alive!")
+        return
+
+    check_safe_stops = KVDB_ENABLED
+
+    from uuid import uuid4
+    from flowcept.instrumentation.flowcept_task import flowcept_task
+
+    workflow_id = str(uuid4())
+
+    @flowcept_task
+    def test_function(n: int) -> Dict[str, int]:
+        return {"output": n + 1}
+
+    with Flowcept(workflow_id=workflow_id, check_safe_stops=check_safe_stops):
+        test_function(2)
+
+    if MONGO_ENABLED:
+        print("MongoDB is enabled, so we are testing it too.")
+        tasks = Flowcept.db.query({"workflow_id": workflow_id})
+        if len(tasks) != 1:
+            print(f"The query result, {len(tasks)}, is not what we expected.")
+            return
+
+    if AGENT.get("enabled", False):
+        print("Agent is enabled, so we are testing it too.")
+        from flowcept.flowceptor.consumers.agent.client_agent import run_tool
+
+        try:
+            print(run_tool("check_liveness"))
+        except Exception as e:
+            print(e)
+            return
+
+        print("Testing LLM connectivity")
+        check_llm_result = run_tool("check_llm")[0]
+        print(check_llm_result.text)
+
+        if "error" in check_llm_result.text.lower():
+            print("There is an error with the LLM communication.")
+            return
+        elif MONGO_ENABLED:
+            print("Testing if llm chat was stored in MongoDB.")
+            response_metadata = json.loads(check_llm_result.text.split("\n")[0])
+            print(response_metadata)
+            sleep(INSERTION_BUFFER_TIME * 1.05)
+            chats = Flowcept.db.query({"workflow_id": response_metadata["agent_id"]})
+            if chats:
+                print(chats)
+            else:
+                print("Could not find chat history. Make sure that the DB Inserter service is on.")
+    print("\n\nAll expected services seem to be working properly!")
+    return
+
+
+COMMAND_GROUPS = [
+    ("Basic Commands", [check_services, show_config, init_settings, start_services, stop_services]),
+    ("Consumption Commands", [start_consumption_services, stop_consumption_services]),
+    ("Database Commands", [workflow_count, query, get_task]),
+    ("Agent Commands", [start_agent, agent_client]),
+]
+
+COMMANDS = set(f for _, fs in COMMAND_GROUPS for f in fs)
+
+
+def _run_command(cmd_str: str, check_output: bool = True, popen_kwargs: Optional[Dict] = None) -> Optional[str]:
+    """
+    Run a shell command with optional output capture.
+
+    Parameters
+    ----------
+    cmd_str : str
+        The command to execute.
+    check_output : bool, optional
+        If True, capture and return the command's standard output.
+        If False, run interactively (stdout/stderr goes to terminal).
+    popen_kwargs : dict, optional
+        Extra keyword arguments to pass to subprocess.run.
+
+    Returns
+    -------
+    output : str or None
+        The standard output of the command if check_output is True, else None.
+
+    Raises
+    ------
+    subprocess.CalledProcessError
+        If the command exits with a non-zero status.
+    """
+    if popen_kwargs is None:
+        popen_kwargs = {}
+
+    kwargs = {"shell": True, "check": True, **popen_kwargs}
+
+    if check_output:
+        kwargs.update({"capture_output": True, "text": True})
+        result = subprocess.run(cmd_str, **kwargs)
+        return result.stdout.strip()
+    else:
+        subprocess.run(cmd_str, **kwargs)
+        return None
+
+
+def _parse_numpy_doc(docstring: str):
+    parsed = {}
+    lines = docstring.splitlines() if docstring else []
+    in_params = False
+    for line in lines:
+        line = line.strip()
+        if line.lower().startswith("parameters"):
+            in_params = True
+            continue
+        if in_params:
+            if " : " in line:
+                name, typeinfo = line.split(" : ", 1)
+                parsed[name.strip()] = {"type": typeinfo.strip(), "desc": ""}
+            elif parsed:
+                last = list(parsed)[-1]
+                parsed[last]["desc"] += " " + line
+    return parsed
+
+
+@no_docstring
+def main():  # noqa: D103
+    parser = argparse.ArgumentParser(
+        description="Flowcept CLI", formatter_class=argparse.RawTextHelpFormatter, add_help=False
+    )
+
+    for func in COMMANDS:
+        doc = func.__doc__ or ""
+        func_name = func.__name__
+        flag = f"--{func_name.replace('_', '-')}"
+        short_help = doc.strip().splitlines()[0] if doc else ""
+        parser.add_argument(flag, action="store_true", help=short_help)
+
+        for pname, param in inspect.signature(func).parameters.items():
+            arg_name = f"--{pname.replace('_', '-')}"
+            params_doc = _parse_numpy_doc(doc).get(pname, {})
+            help_text = f"{params_doc.get('type', '')} - {params_doc.get('desc', '').strip()}"
+            if isinstance(param.annotation, bool):
+                parser.add_argument(arg_name, action="store_true", help=help_text)
+            elif param.annotation == List[str]:
+                parser.add_argument(arg_name, type=lambda s: s.split(","), help=help_text)
+            else:
+                parser.add_argument(arg_name, type=str, help=help_text)
+
+    # Handle --help --command
+    help_flag = "--help" in sys.argv
+    command_flags = {f"--{f.__name__.replace('_', '-')}" for f in COMMANDS}
+    matched_command_flag = next((arg for arg in sys.argv if arg in command_flags), None)
+
+    if help_flag and matched_command_flag:
+        command_func = next(f for f in COMMANDS if f"--{f.__name__.replace('_', '-')}" == matched_command_flag)
+        doc = command_func.__doc__ or ""
+        sig = inspect.signature(command_func)
+        print(f"\nHelp for `flowcept {matched_command_flag}`:\n")
+        print(textwrap.indent(doc.strip(), "  "))
+        print("\n  Arguments:")
+        params = _parse_numpy_doc(doc)
+        for pname, p in sig.parameters.items():
+            meta = params.get(pname, {})
+            opt = p.default != inspect.Parameter.empty
+            print(
+                f"    --{pname:<18} {meta.get('type', 'str')}, "
+                f"{'optional' if opt else 'required'} - {meta.get('desc', '').strip()}"
+            )
+        print()
+        sys.exit(0)
+
+    if len(sys.argv) == 1 or help_flag:
+        print("\nFlowcept CLI\n")
+        for group, funcs in COMMAND_GROUPS:
+            print(f"{group}:\n")
+            for func in funcs:
+                name = func.__name__
+                flag = f"--{name.replace('_', '-')}"
+                doc = func.__doc__ or ""
+                summary = doc.strip().splitlines()[0] if doc else ""
+                sig = inspect.signature(func)
+                print(f"  flowcept {flag}", end="")
+                for pname, p in sig.parameters.items():
+                    is_opt = p.default != inspect.Parameter.empty
+                    print(f" [--{pname.replace('_', '-')}] " if is_opt else f" --{pname.replace('_', '-')}", end="")
+                print(f"\n      {summary}")
+                params = _parse_numpy_doc(doc)
+                if params:
+                    print("      Arguments:")
+                    for argname, meta in params.items():
+                        opt = sig.parameters[argname].default != inspect.Parameter.empty
+                        print(
+                            f"          --"
+                            f"{argname:<18} {meta['type']}, "
+                            f"{'optional' if opt else 'required'} - {meta['desc'].strip()}"
+                        )
+                print()
+        print("Run `flowcept --<command>` to invoke a command.\n")
+        sys.exit(0)
+
+    args = vars(parser.parse_args())
+
+    for func in COMMANDS:
+        flag = f"--{func.__name__.replace('_', '-')}"
+        if args.get(func.__name__.replace("-", "_")):
+            sig = inspect.signature(func)
+            kwargs = {}
+            for pname in sig.parameters:
+                val = args.get(pname.replace("-", "_"))
+                if val is not None:
+                    kwargs[pname] = val
+            func(**kwargs)
+            break
+    else:
+        print("Unknown command. Use `flowcept -h` to see available commands.")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
+    # check_services()

flowcept/commons/daos/keyvalue_dao.py

@@ -1,6 +1,6 @@
 """Key value module."""
 
-from redis import Redis, ConnectionPool
+from flowcept.commons.daos.redis_conn import RedisConn
 
 from flowcept.commons.flowcept_logger import FlowceptLogger
 from flowcept.configs import (
@@ -24,32 +24,13 @@ class KeyValueDAO:
             cls._instance = super(KeyValueDAO, cls).__new__(cls)
         return cls._instance
 
-    @staticmethod
-    def build_redis_conn_pool():
-        """Utility function to build Redis connection."""
-        pool = ConnectionPool(
-            host=KVDB_HOST,
-            port=KVDB_PORT,
-            db=0,
-            password=KVDB_PASSWORD,
-            decode_responses=False,
-            max_connections=10000,  # TODO: Config file
-            socket_keepalive=True,
-            retry_on_timeout=True,
-        )
-        return Redis(connection_pool=pool)
-        # return Redis()
-
     def __init__(self):
         if not hasattr(self, "_initialized"):
             self._initialized = True
             self.logger = FlowceptLogger()
-            if KVDB_URI is not None:
-                # If a URI is provided, use it for connection
-                self.redis_conn = Redis.from_url(KVDB_URI)
-            else:
-                # Otherwise, use the host, port, and password settings
-                self.redis_conn = KeyValueDAO.build_redis_conn_pool()
+            self.redis_conn = RedisConn.build_redis_conn_pool(
+                host=KVDB_HOST, port=KVDB_PORT, password=KVDB_PASSWORD, uri=KVDB_URI
+            )
 
     def delete_set(self, set_name: str):
         """Delete it."""
@@ -133,3 +114,18 @@ class KeyValueDAO:
         None
         """
         self.redis_conn.delete(key)
+
+    def liveness_test(self):
+        """Get the livelyness of it."""
+        try:
+            response = self.redis_conn.ping()
+            if response:
+                return True
+            else:
+                return False
+        except ConnectionError as e:
+            self.logger.exception(e)
+            return False
+        except Exception as e:
+            self.logger.exception(e)
+            return False

flowcept/commons/daos/mq_dao/mq_dao_base.py

@@ -20,6 +20,7 @@ from flowcept.configs import (
     MQ_CHUNK_SIZE,
     MQ_TYPE,
     MQ_TIMING,
+    KVDB_ENABLED,
 )
 
 from flowcept.commons.utils import GenericJSONEncoder
@@ -67,7 +68,13 @@ class MQDao(ABC):
         self.logger = FlowceptLogger()
         self.started = False
         self._adapter_settings = adapter_settings
-        self._keyvalue_dao = KeyValueDAO()
+        if KVDB_ENABLED:
+            self._keyvalue_dao = KeyValueDAO()
+        else:
+            self._keyvalue_dao = None
+            self.logger.warning(
+                "We are going to run without KVDB. If you are running a workflow, this may lead to errors."
+            )
         self._time_based_flushing_started = False
         self.buffer: Union[AutoflushBuffer, List] = None
         if MQ_TIMING:
@@ -138,7 +145,7 @@ class MQDao(ABC):
         """
         self._keyvalue_dao.delete_key("current_campaign_id")
 
-    def init_buffer(self, interceptor_instance_id: str, exec_bundle_id=None):
+    def init_buffer(self, interceptor_instance_id: str, exec_bundle_id=None, check_safe_stops=True):
         """Create the buffer."""
         if not self.started:
             if flowcept.configs.DB_FLUSH_MODE == "online":
@@ -147,7 +154,8 @@ class MQDao(ABC):
                     max_size=MQ_BUFFER_SIZE,
                     flush_interval=MQ_INSERTION_BUFFER_TIME,
                 )
-                self.register_time_based_thread_init(interceptor_instance_id, exec_bundle_id)
+                if check_safe_stops:
+                    self.register_time_based_thread_init(interceptor_instance_id, exec_bundle_id)
                 self._time_based_flushing_started = True
             else:
                 self.buffer = list()
@@ -164,9 +172,9 @@ class MQDao(ABC):
             self.bulk_publish(self.buffer)
             self.buffer = list()
 
-    def _stop_timed(self, interceptor_instance_id: str, bundle_exec_id: int = None):
+    def _stop_timed(self, interceptor_instance_id: str, check_safe_stops: bool = True, bundle_exec_id: int = None):
         t1 = time()
-        self._stop(interceptor_instance_id, bundle_exec_id)
+        self._stop(interceptor_instance_id, check_safe_stops, bundle_exec_id)
         t2 = time()
         self._flush_events.append(["final", t1, t2, t2 - t1, "n/a"])
 
@@ -175,14 +183,14 @@ class MQDao(ABC):
             writer.writerow(["type", "start", "end", "duration", "size"])
             writer.writerows(self._flush_events)
 
-    def _stop(self, interceptor_instance_id: str, bundle_exec_id: int = None):
-        """Stop it."""
-        msg0 = "MQ publisher received stop signal! bundle: "
-        self.logger.debug(msg0 + f"{bundle_exec_id}; interceptor id: {interceptor_instance_id}")
+    def _stop(self, interceptor_instance_id: str, check_safe_stops: bool = True, bundle_exec_id: int = None):
+        """Stop MQ publisher."""
+        self.logger.debug(f"MQ pub received stop sign: bundle={bundle_exec_id}, interceptor={interceptor_instance_id}")
         self._close_buffer()
-        msg = "Flushed MQ for last time! Send stop msg. bundle: "
-        self.logger.debug(msg + f"{bundle_exec_id}; interceptor id: {interceptor_instance_id}")
-        self._send_mq_dao_time_thread_stop(interceptor_instance_id, bundle_exec_id)
+        self.logger.debug("Flushed MQ for the last time!")
+        if check_safe_stops:
+            self.logger.debug(f"Sending stop msg. Bundle: {bundle_exec_id}; interceptor id: {interceptor_instance_id}")
+            self._send_mq_dao_time_thread_stop(interceptor_instance_id, bundle_exec_id)
         self.started = False
 
     def _send_mq_dao_time_thread_stop(self, interceptor_instance_id, exec_bundle_id=None):
@@ -197,10 +205,10 @@ class MQDao(ABC):
         # self.logger.info("Control msg sent: " + str(msg))
         self.send_message(msg)
 
-    def send_document_inserter_stop(self):
+    def send_document_inserter_stop(self, exec_bundle_id=None):
         """Send the document."""
         # These control_messages are handled by the document inserter
-        msg = {"type": "flowcept_control", "info": "stop_document_inserter"}
+        msg = {"type": "flowcept_control", "info": "stop_document_inserter", "exec_bundle_id": exec_bundle_id}
         self.send_message(msg)
 
     @abstractmethod
@@ -223,20 +231,12 @@ class MQDao(ABC):
         """Subscribe to the interception channel."""
         raise NotImplementedError()
 
+    @abstractmethod
+    def unsubscribe(self):
+        """Subscribe to the interception channel."""
+        raise NotImplementedError()
+
     @abstractmethod
     def liveness_test(self) -> bool:
-        """Check whether the base KV store's connection is ready. This is enough for Redis MQ too. Other MQs need
-        further liveness implementation.
-        """
-        try:
-            response = self._keyvalue_dao.redis_conn.ping()
-            if response:
-                return True
-            else:
-                return False
-        except ConnectionError as e:
-            self.logger.exception(e)
-            return False
-        except Exception as e:
-            self.logger.exception(e)
-            return False
+        """Checks if the MQ system is alive."""
+        raise NotImplementedError()

flowcept/commons/daos/mq_dao/mq_dao_kafka.py

@@ -108,12 +108,13 @@ class MQDaoKafka(MQDao):
     def liveness_test(self):
         """Get the livelyness of it."""
         try:
-            if not super().liveness_test():
-                self.logger.error("KV Store not alive!")
-                return False
             admin_client = AdminClient(self._kafka_conf)
             kafka_metadata = admin_client.list_topics(timeout=5)
             return MQ_CHANNEL in kafka_metadata.topics
         except Exception as e:
             self.logger.exception(e)
             return False
+
+    def unsubscribe(self):
+        """Unsubscribes from Kafka topic."""
+        raise NotImplementedError()

flowcept/commons/daos/mq_dao/mq_dao_mofka.py

@@ -104,3 +104,7 @@ class MQDaoMofka(MQDao):
     def liveness_test(self):
         """Test Mofka Liveness."""
         return True
+
+    def unsubscribe(self):
+        """Unsubscribes from Mofka topic."""
+        raise NotImplementedError()