flowcept 0.9.1__py3-none-any.whl → 0.9.3__py3-none-any.whl

flowcept/__init__.py

@@ -16,6 +16,11 @@ def __getattr__(name):
 
         return WorkflowObject
 
+    elif name == "TaskObject":
+        from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
+
+        return TaskObject
+
     elif name == "flowcept_task":
         from flowcept.instrumentation.flowcept_task import flowcept_task
 

flowcept/agents/flowcept_ctx_manager.py

@@ -53,7 +53,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         Current application context holding task state and QA components.
     msgs_counter : int
         Counter tracking how many task messages have been processed.
-    context_size : int
+    context_chunk_size : int
         Number of task messages to collect before triggering QA index building and LLM analysis.
     qa_manager : FlowceptQAManager
         Utility for constructing QA chains from task summaries.
@@ -64,7 +64,7 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
         self.tracker_config = dict(max_examples=3, max_str_len=50)
         self.schema_tracker = DynamicSchemaTracker(**self.tracker_config)
         self.msgs_counter = 0
-        self.context_size = 1
+        self.context_chunk_size = 1  # Should be in the settings
         super().__init__()
 
     def message_handler(self, msg_obj: Dict):
@@ -98,18 +98,22 @@ class FlowceptAgentContextManager(BaseAgentContextManager):
             if len(task_summary.get("tags", [])):
                 self.context.critical_tasks.append(task_summary)
 
-            if self.msgs_counter > 0 and self.msgs_counter % self.context_size == 0:
+            if self.msgs_counter > 0 and self.msgs_counter % self.context_chunk_size == 0:
                 self.logger.debug(
-                    f"Going to add to index! {(self.msgs_counter - self.context_size, self.msgs_counter)}"
+                    f"Going to add to index! {(self.msgs_counter - self.context_chunk_size, self.msgs_counter)}"
                 )
                 try:
                     self.update_schema_and_add_to_df(
-                        tasks=self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]
+                        tasks=self.context.task_summaries[
+                            self.msgs_counter - self.context_chunk_size : self.msgs_counter
+                        ]
                     )
                 except Exception as e:
                     self.logger.error(
                         f"Could not add these tasks to buffer!\n"
-                        f"{self.context.task_summaries[self.msgs_counter - self.context_size : self.msgs_counter]}"
+                        f"{
+                            self.context.task_summaries[self.msgs_counter - self.context_chunk_size : self.msgs_counter]
+                        }"
                     )
                     self.logger.exception(e)
 

flowcept/agents/gui/gui_utils.py

@@ -1,3 +1,5 @@
+import base64
+import ast
 import io
 import json
 
@@ -122,6 +124,53 @@ def display_ai_msg_from_tool(tool_result: ToolResult):
     return agent_reply
 
 
+def _sniff_mime(b: bytes) -> str:
+    if b.startswith(b"\x89PNG\r\n\x1a\n"):
+        return "image/png"
+    if b.startswith(b"\xff\xd8\xff"):
+        return "image/jpeg"
+    if b.startswith(b"GIF87a") or b.startswith(b"GIF89a"):
+        return "image/gif"
+    if b.startswith(b"BM"):
+        return "image/bmp"
+    if b.startswith(b"RIFF") and b[8:12] == b"WEBP":
+        return "image/webp"
+    return "application/octet-stream"
+
+
+def ensure_data_uri(val):
+    r"""Accepts bytes/bytearray/memoryview or a repr like \"b'\\x89PNG...'\" and returns a data URL."""
+    if isinstance(val, str) and val.startswith("data:"):
+        return val
+    if isinstance(val, str) and val.startswith("b'"):
+        try:
+            val = ast.literal_eval(val)  # turn repr into bytes
+        except Exception:
+            return None
+    if isinstance(val, memoryview):
+        val = val.tobytes()
+    if isinstance(val, bytearray):
+        val = bytes(val)
+    if isinstance(val, bytes):
+        mime = _sniff_mime(val)
+        return f"data:{mime};base64,{base64.b64encode(val).decode('ascii')}"
+    return val  # path/URL, etc.
+
+
+def _render_df(df: pd.DataFrame, image_width: int = 90, row_height: int = 90):
+    if "image" in df.columns:
+        df = df.copy()
+        df["image"] = df["image"].apply(ensure_data_uri)
+        st.dataframe(
+            df,
+            column_config={"image": st.column_config.ImageColumn("Preview", width=image_width)},
+            hide_index=True,
+            row_height=row_height,  # make thumbnails visible
+        )
+    else:
+        st.dataframe(df, hide_index=True)
+
+
 def display_df_tool_response(tool_result: ToolResult):
     r"""
     Display the DataFrame contained in a ToolResult.
@@ -170,7 +219,8 @@ def display_df_tool_response(tool_result: ToolResult):
             df = pd.read_csv(io.StringIO(result_df_str))
             print("The result is a df")
             if not df.empty:
-                st.dataframe(df, hide_index=False)
+                _render_df(df)
+
                 print("Columns", str(df.columns))
                 print("Number of columns", len(df.columns))
             else:
@@ -190,6 +240,7 @@ def display_df_tool_response(tool_result: ToolResult):
 
         if summary:
             st.markdown("📝 Summary:")
+            print(f"THIS IS THE SUMMARY\n{summary}")
             st.markdown(summary)
         elif summary_error:
             st.markdown(f"⚠️ Encountered this error when summarizing the result dataframe:\n```text\n{summary_error}")

flowcept/agents/prompts/general_prompts.py

@@ -24,8 +24,8 @@ ROUTING_PROMPT = (
     "Given the following user message, classify it into one of the following routes:\n"
     "- small_talk: if it's casual conversation or some random word (e.g., 'hausdn', 'a', hello, how are you, what can you do, what's your name)\n"
     "- plot: if user is requesting plots (e.g., plot, chart, visualize)\n"
+    "- in_context_query: if the user asks questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
     "- historical_prov_query: if the user wants to query historical provenance data\n"
-    "- in_context_query: if the user appears to ask questions about tasks or data in running workflow (or a workflow that ran recently) or if the user mentions the in-memory 'df' or a dataframe.\n"
     "- in_chat_query: if the user appears to be asking about something that has said recently in this chat.\n"
     "- unknown: if you don't know.\n"
     "Respond with only the route label."

flowcept/agents/prompts/in_memory_query_prompts.py

@@ -15,6 +15,7 @@ COMMON_TASK_FIELDS = """
     | `ended_at`                    | datetime64[ns, UTC] | End time of a task. | 
     | `subtype`                     | string | Subtype of a task. |
     | `tags`                        | List[str] | List of descriptive tags. |
+    | `image`                        | blob | Raw binary data related to an image. |
     | `telemetry_summary.duration_sec` | float | Task duration (seconds). |
     | `telemetry_summary.cpu.percent_all_diff` | float | Difference in overall CPU utilization percentage across all cores between task end and start.|
     | `telemetry_summary.cpu.user_time_diff`   | float |  Difference average per core CPU user time ( seconds ) between task start and end times.|
@@ -27,6 +28,17 @@ COMMON_TASK_FIELDS = """
 DF_FORM = "The user has a pandas DataFrame called `df`, created from flattened task objects using `pd.json_normalize`."
 
 
+def get_example_values_prompt(example_values):
+    values_prompt = f"""    
+           Now, this other dictionary below provides type (t), up to 3 example values (v), and, for lists, shape (s) and element type (et) for each field.
+           Field names do not include `used.` or `generated.` They represent the unprefixed form shared across roles. String values may be truncated if they exceed the length limit.
+           ```python
+           {example_values}
+           ```
+       """
+    return values_prompt
+
+
 def get_df_schema_prompt(dynamic_schema, example_values):
     schema_prompt = f"""
      ## DATAFRAME STRUCTURE
@@ -52,14 +64,7 @@ def get_df_schema_prompt(dynamic_schema, example_values):
         ---
     """
 
-    values_prompt = f"""    
-        Now, this other dictionary below provides type (t), up to 3 example values (v), and, for lists, shape (s) and element type (et) for each field.
-        Field names do not include `used.` or `generated.` They represent the unprefixed form shared across roles. String values may be truncated if they exceed the length limit.
-        ```python
-        {example_values}
-        ```
-    """
-
+    values_prompt = get_example_values_prompt(example_values)
     # values_prompt = ""
     prompt = schema_prompt + values_prompt
     return prompt
@@ -221,7 +226,7 @@ def generate_pandas_code_prompt(query: str, dynamic_schema, example_values):
         f"{JOB}"
         f"{DF_FORM}"
         f"{get_df_schema_prompt(dynamic_schema, example_values)}"  # main tester
-        # f"{QUERY_GUIDELINES}" # main tester
+        f"{QUERY_GUIDELINES}"  # main tester
         f"{FEW_SHOTS}"  # main tester
         f"{OUTPUT_FORMATTING}"
         "User Query:"
@@ -230,9 +235,16 @@ def generate_pandas_code_prompt(query: str, dynamic_schema, example_values):
     return prompt
 
 
-def dataframe_summarizer_context(code, reduced_df, query) -> str:
+def dataframe_summarizer_context(code, reduced_df, dynamic_schema, example_values, query) -> str:
+    job = "You are a Workflow Provenance Specialist analyzing a DataFrame that was obtained to answer a query."
+
+    if "image" in reduced_df.columns:
+        reduced_df = reduced_df.drop(columns=["image"])
+
     prompt = f"""
-    You are a Workflow Provenance Specialist analyzing a DataFrame that was obtained to answer a query. Given:
+    {job}
+    
+     Given:
     
     **User Query**:  
     {query}
@@ -240,19 +252,26 @@ def dataframe_summarizer_context(code, reduced_df, query) -> str:
     **Query_Code**:  
     {code}
     
-    **Reduced DataFrame** (rows sampled from full result):  
+    **Reduced DataFrame `df` contents** (rows sampled from full result):  
     {reduced_df}
     
-    Your task is to:
-    1. Analyze the DataFrame values and columns for any meaningful or notable information.
-    2. Compare the query_code with the data content to understand what the result represents. THIS IS A REDUCED DATAFRAME, the original dataframe, used to answer the query, may be much bigger. IT IS ALREADY KNOWN! Do not need to restate this.
-    3. Provide a concise and direct answer to the user query. Your final response to the query should be  within ```text .
+    **Original df (before reduction) had this schema:
+    {get_df_schema_prompt(dynamic_schema, example_values)}
+    
+    Your task is to find a concise and direct answer as an English sentence to the user query.
+        
+    Only if the answer to the query is complex, provide more explanation by: 
+        1. Analyzing the DataFrame values and columns for any meaningful or notable information. 
+        2. Comparing the query_code with the data content to understand what the result represents. THIS IS A REDUCED DATAFRAME, the original dataframe, used to answer the query, may be much bigger. IT IS ALREADY KNOWN! Do not need to restate this.
+        3. If it makes sense, provide information beyond the recorded provenance, but state it clearly that you are inferring it.
+    
+    In the end, conclude by giving your concise answer as follows: **Response**: <YOUR ANSWER>
 
     Note that the user should not know that this is a reduced dataframe. 
-    
     Keep your response short and focused.
 
     """
+
     return prompt
 
 

flowcept/agents/tools/general_tools.py

@@ -74,7 +74,7 @@ def prompt_handler(message: str) -> ToolResult:
     TextContent
         The AI response or routing feedback.
     """
-    df_key_words = {"save", "result = df", "reset context"}
+    df_key_words = ["df", "save", "result = df", "reset context"]
     for key in df_key_words:
         if key in message:
             return run_df_query(llm=None, query=message, plot=False)

flowcept/agents/tools/in_memory_queries/in_memory_queries_tools.py

@@ -294,6 +294,8 @@ def generate_result_df(llm, query: str, dynamic_schema, example_values, df, atte
     >>> generate_result_df(llm, "bad query", schema, examples, df, attempt_fix=False)
     ToolResult(code=405, result="Failed to parse this as Python code: ...")
     """
+    if llm is None:
+        llm = build_llm_model()
     try:
         prompt = generate_pandas_code_prompt(query, dynamic_schema, example_values)
         response = llm(prompt)
@@ -351,7 +353,14 @@ def generate_result_df(llm, query: str, dynamic_schema, example_values, df, atte
     summary, summary_error = None, None
     if summarize:
         try:
-            tool_result = summarize_result(llm, result_code, result_df, query)
+            tool_result = summarize_result(
+                llm,
+                result_code,
+                result_df,
+                query,
+                dynamic_schema,
+                example_values,
+            )
             if tool_result.is_success():
                 return_code = 301
                 summary = tool_result.result
@@ -570,7 +579,14 @@ def extract_or_fix_json_code(llm, raw_text) -> ToolResult:
 
 
 @mcp_flowcept.tool()
-def summarize_result(llm, code, result, query: str) -> ToolResult:
+def summarize_result(
+    llm,
+    code,
+    result,
+    query: str,
+    dynamic_schema,
+    example_values,
+) -> ToolResult:
     """
     Summarize the pandas result with local reduction for large DataFrames.
     - For wide DataFrames, selects top columns based on variance and uniqueness.
@@ -578,7 +594,7 @@ def summarize_result(llm, code, result, query: str) -> ToolResult:
     - Constructs a detailed prompt for the LLM with original column context.
     """
     summarized_df = summarize_df(result, code)
-    prompt = dataframe_summarizer_context(code, summarized_df, query)
+    prompt = dataframe_summarizer_context(code, summarized_df, dynamic_schema, example_values, query)
     try:
         response = llm(prompt)
         return ToolResult(code=201, result=response)

flowcept/cli.py

@@ -101,17 +101,17 @@ def version():
     print(f"Flowcept {__version__}")
 
 
-def stream_messages(print_messages: bool = False, messages_file_path: Optional[str] = None):
+def stream_messages(messages_file_path: Optional[str] = None, keys_to_show: List[str] = None):
     """
     Listen to Flowcept's message stream and optionally echo/save messages.
 
     Parameters.
     ----------
-    print_messages : bool, optional
-        If True, print each decoded message to stdout.
     messages_file_path : str, optional
         If provided, append each message as JSON (one per line) to this file.
         If the file already exists, a new timestamped file is created instead.
+    keys_to_show : List[str], optional
+        List of object keys to show in the prints. Use comma-separated list: --keys-to-show 'activity_id','workflow_id'
     """
     # Local imports to avoid changing module-level deps
     from flowcept.configs import MQ_TYPE
@@ -123,10 +123,7 @@ def stream_messages(print_messages: bool = False, messages_file_path: Optional[s
     import os
     import json
     from datetime import datetime
-    import redis
-    import msgpack
-    from flowcept.configs import MQ_HOST, MQ_PORT, MQ_CHANNEL, KVDB_URI
-    from flowcept.commons.daos.mq_dao.mq_dao_redis import MQDaoRedis
+    from flowcept.flowceptor.consumers.base_consumer import BaseConsumer
 
     def _timestamped_path_if_exists(path: Optional[str]) -> Optional[str]:
         if not path:
@@ -150,53 +147,53 @@ def stream_messages(print_messages: bool = False, messages_file_path: Optional[s
 
         return json.dumps(obj, ensure_ascii=False, separators=(",", ":"), default=_default)
 
-    # Prepare output file (JSONL)
     out_fh = None
     if messages_file_path:
         out_path = _timestamped_path_if_exists(messages_file_path)
         out_fh = open(out_path, "w", encoding="utf-8", buffering=1)  # line-buffered
 
-    # Connect & subscribe
-    redis_client = redis.from_url(KVDB_URI) if KVDB_URI else redis.Redis(host=MQ_HOST, port=MQ_PORT, db=0)
-    pubsub = redis_client.pubsub()
-    pubsub.subscribe(MQ_CHANNEL)
-
-    print(f"Listening for messages on channel '{MQ_CHANNEL}'... (Ctrl+C to exit)")
-
-    try:
-        for message in pubsub.listen():
-            if not message or message.get("type") in MQDaoRedis.MESSAGE_TYPES_IGNORE:
-                continue
-
-            data = message.get("data")
-            if not isinstance(data, (bytes, bytearray)):
-                print(f"Skipping message with unexpected data type: {type(data)} - {data}")
-                continue
+    class MyConsumer(BaseConsumer):
+        def __init__(self):
+            super().__init__()
 
+        def message_handler(self, msg_obj: Dict) -> bool:
             try:
-                msg_obj = msgpack.loads(data, strict_map_key=False)
-                msg_type = msg_obj.get("type", None)
-                print(f"\nReceived a message! type={msg_type}")
-
-                if print_messages:
-                    print(_json_dumps(msg_obj))
+                if keys_to_show is not None:
+                    obj_to_print = {}
+                    for k in keys_to_show:
+                        v = msg_obj.get(k, None)
+                        if v is not None:
+                            obj_to_print[k] = v
+                    if not obj_to_print:
+                        obj_to_print = msg_obj
+                else:
+                    obj_to_print = msg_obj
+
+                print(_json_dumps(obj_to_print))
 
                 if out_fh is not None:
-                    out_fh.write(_json_dumps(msg_obj))
+                    out_fh.write(_json_dumps(obj_to_print))
                     out_fh.write("\n")
-
+            except KeyboardInterrupt:
+                print("\nGracefully interrupted, shutting down...")
+                return False
             except Exception as e:
-                print(f"Error decoding message: {e}")
+                print(e)
+                return False
+            finally:
+                try:
+                    if out_fh:
+                        out_fh.close()
+                except Exception as e:
+                    print(e)
+                    return False
 
-    except KeyboardInterrupt:
-        print("\nInterrupted, shutting down...")
-    finally:
-        try:
-            if out_fh:
-                out_fh.close()
-            pubsub.close()
-        except Exception:
-            pass
+            return True
+
+    m = f"Printing only the keys {keys_to_show}" if keys_to_show is not None else ""
+    print(f"Listening for messages.{m} Ctrl+C to exit")
+    consumer = MyConsumer()
+    consumer.start(daemon=False)
 
 
 def start_consumption_services(bundle_exec_id: str = None, check_safe_stops: bool = False, consumers: List[str] = None):
@@ -700,3 +697,5 @@ def main():  # noqa: D103
 if __name__ == "__main__":
     main()
     # check_services()
+
+__doc__ = None

flowcept/commons/autoflush_buffer.py

@@ -47,6 +47,11 @@ class AutoflushBuffer:
         if len(buffer) >= self._max_size:
             self._swap_event.set()
 
+    @property
+    def current_buffer(self):
+        """Return the currently active buffer (read-only)."""
+        return self._buffers[self._current_buffer_index]
+
     def time_based_flush(self):
         """Time flush."""
         while not self._stop_event.is_set():

flowcept/commons/daos/docdb_dao/lmdb_dao.py

@@ -56,7 +56,10 @@ class LMDBDAO(DocumentDBDAO):
             t0 = 0
             if PERF_LOG:
                 t0 = time()
-            indexed_buffer = curate_dict_task_messages(docs, indexing_key, t0, convert_times=False)
+            indexed_buffer = curate_dict_task_messages(
+                docs, indexing_key, t0, convert_times=False, keys_to_drop=["data"]
+            )
+
             with self._env.begin(write=True, db=self._tasks_db) as txn:
                 for key, value in indexed_buffer.items():
                     k, v = key.encode(), json.dumps(value).encode()

flowcept/commons/daos/mq_dao/mq_dao_base.py

@@ -102,6 +102,7 @@ class MQDao(object):
 
                 with open(DUMP_BUFFER_PATH, "wb", buffering=1_048_576) as f:
                     for obj in buffer:
+                        obj.pop("data", None)  # We are not going to store data in the buffer file.
                         f.write(orjson.dumps(obj))
                         f.write(b"\n")
                 self.logger.info(f"Saved Flowcept messages into {DUMP_BUFFER_PATH}.")

flowcept/commons/flowcept_dataclasses/task_object.py

@@ -16,45 +16,125 @@ from flowcept.configs import (
 
 
 class TaskObject:
-    """Task class."""
+    """Task object class.
+
+    Represents a single provenance task in Flowcept, including inputs, outputs,
+    execution metadata, telemetry, and environment details.
+    """
 
     type = "task"
+    """Constant type label for this object ("task")."""
+
     subtype: AnyStr = None
-    task_id: AnyStr = None  # Any way to identify a task
+    """Optional subtype of the task (e.g., iteration, ML step, custom)."""
+
+    task_id: AnyStr = None
+    """Unique identifier of the task."""
+
     utc_timestamp: float = None
+    """UTC timestamp when the task object was created."""
+
     adapter_id: AnyStr = None
+    """Identifier of the adapter that produced this task (if any)."""
+
     user: AnyStr = None
+    """User who executed or triggered the task."""
+
     data: Any = None
-    used: Dict[AnyStr, Any] = None  # Used parameter and files
+    """Arbitrary raw data payload associated with the task. It is good practice to add custom_metadata associated with
+    `data`, especially if it contains file contents. 
+    In that case, `custom_metadata` should contain the keys "file_type", "file_content", "file_name", "extension".
+    """
+
+    used: Dict[AnyStr, Any] = None
+    """Inputs consumed by the task (parameters, files, resources)."""
+
     campaign_id: AnyStr = None
-    generated: Dict[AnyStr, Any] = None  # Generated results and files
+    """Campaign identifier grouping related tasks together."""
+
+    generated: Dict[AnyStr, Any] = None
+    """Outputs produced by the task (results, artifacts, files)."""
+
     submitted_at: float = None
+    """Timestamp when the task was submitted."""
+
     started_at: float = None
+    """Timestamp when the task execution started."""
+
     ended_at: float = None
-    registered_at: float = None  # Leave this for dates generated at the DocInserter
+    """Timestamp when the task execution ended."""
+
+    registered_at: float = None
+    """Timestamp when the task was registered by the DocInserter."""
+
     telemetry_at_start: Telemetry = None
+    """Telemetry snapshot captured at the start of the task."""
+
     telemetry_at_end: Telemetry = None
+    """Telemetry snapshot captured at the end of the task."""
+
     workflow_name: AnyStr = None
+    """Name of the workflow this task belongs to."""
+
     workflow_id: AnyStr = None
+    """Identifier of the workflow this task belongs to."""
+
     parent_task_id: AnyStr = None
+    """Identifier of the parent task, if this task is nested or dependent."""
+
     activity_id: AnyStr = None
-    group_id: AnyStr = None  # Utilized especially for loop iteration tasks, to group them.
+    """Activity name (usually the function name) associated with the task."""
+
+    group_id: AnyStr = None
+    """Grouping identifier, often used to link loop iterations together."""
+
     status: Status = None
+    """Execution status of the task (e.g., FINISHED, ERROR)."""
+
     stdout: Union[AnyStr, Dict] = None
+    """Captured standard output from the task, if available."""
+
     stderr: Union[AnyStr, Dict] = None
+    """Captured standard error from the task, if available."""
+
     custom_metadata: Dict[AnyStr, Any] = None
+    """Custom metadata dictionary provided by the developer/user."""
+
     mq_host: str = None
+    """Message queue host associated with the task."""
+
     environment_id: AnyStr = None
+    """Identifier of the environment where the task executed."""
+
     node_name: AnyStr = None
+    """Node name in a distributed system or HPC cluster."""
+
     login_name: AnyStr = None
+    """Login name of the user in the execution environment."""
+
     public_ip: AnyStr = None
+    """Public IP address of the machine executing the task."""
+
     private_ip: AnyStr = None
+    """Private IP address of the machine executing the task."""
+
     hostname: AnyStr = None
+    """Hostname of the machine executing the task."""
+
     address: AnyStr = None
+    """Optional network address associated with the task."""
+
     dependencies: List = None
+    """List of task IDs this task depends on."""
+
     dependents: List = None
+    """List of task IDs that depend on this task."""
+
     tags: List = None
+    """User-defined tags attached to the task."""
+
     agent_id: str = None
+    """Identifier of the agent responsible for executing this task (if any)."""
 
     _DEFAULT_ENRICH_VALUES = {
         "node_name": NODE_NAME,