osbot-utils 2.19.0__tar.gz → 2.21.0__tar.gz

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: osbot_utils
-Version: 2.19.0
+Version: 2.21.0
 Summary: OWASP Security Bot - Utils
 License: MIT
 Author: Dinis Cruz
@@ -23,7 +23,7 @@ Description-Content-Type: text/markdown
 
 Powerful Python util methods and classes that simplify common apis and tasks.
 
-![Current Release](https://img.shields.io/badge/release-v2.19.0-blue)
+![Current Release](https://img.shields.io/badge/release-v2.21.0-blue)
 [![codecov](https://codecov.io/gh/owasp-sbot/OSBot-Utils/graph/badge.svg?token=GNVW0COX1N)](https://codecov.io/gh/owasp-sbot/OSBot-Utils)
 
 

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/README.md

@@ -2,7 +2,7 @@
 
 Powerful Python util methods and classes that simplify common apis and tasks.
 
-![Current Release](https://img.shields.io/badge/release-v2.19.0-blue)
+![Current Release](https://img.shields.io/badge/release-v2.21.0-blue)
 [![codecov](https://codecov.io/gh/owasp-sbot/OSBot-Utils/graph/badge.svg?token=GNVW0COX1N)](https://codecov.io/gh/owasp-sbot/OSBot-Utils)
 
 

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/osbot_utils/helpers/flows/Flow.py

@@ -3,10 +3,12 @@ import logging
 import typing
 
 from osbot_utils.helpers.Dependency_Manager                 import Dependency_Manager
+from osbot_utils.helpers.flows.actions.Flow__Data           import Flow__Data
+from osbot_utils.helpers.flows.models.Flow_Run__Event       import Flow_Run__Event
 from osbot_utils.type_safe.Type_Safe                        import Type_Safe
 from osbot_utils.helpers.CFormat                            import CFormat, f_dark_grey, f_magenta, f_bold
 from osbot_utils.helpers.flows.models.Flow_Run__Config      import Flow_Run__Config
-from osbot_utils.helpers.flows.Flow__Events                 import flow_events
+from osbot_utils.helpers.flows.actions.Flow__Events         import flow_events
 from osbot_utils.helpers.flows.models.Flow_Run__Event_Data  import Flow_Run__Event_Data
 from osbot_utils.testing.Stdout                             import Stdout
 from osbot_utils.utils.Misc                                 import random_id, lower, time_now
@@ -22,6 +24,7 @@ FLOW__LOGGING__DATE_FORMAT = '%H:%M:%S'
 
 # todo: add flow duration
 class Flow(Type_Safe):
+    flow_data          : Flow__Data
     captured_exec_logs : list
     data               : dict                   # dict available to the tasks to add and collect data
     flow_id            : str                    # rename to flow_run_id (or also capture the flow_run_id)
@@ -47,6 +50,10 @@ class Flow(Type_Safe):
                                                    type        = artifact_type or "link"))                                                   # type clashed with built-in type
         event_data.flow_run_id = self.flow_id
         flow_events.on__new_artifact(event_data)
+        self.flow_data.add_artifact(key           = key or 'default'           ,
+                                    description   = description or ''          ,
+                                    data          = data                       ,
+                                    artifact_type = artifact_type or 'default' )
 
 
     def add_flow_result(self, key, description):
@@ -55,6 +62,7 @@ class Flow(Type_Safe):
         event_data.data        = dict(result_data = dict(key         = key         ,
                                                          description = description ))
         flow_events.on__new_result(event_data)
+        self.flow_data.add_result(key, description)
 
     def config_logger(self):
         with self.logger as _:
@@ -79,9 +87,12 @@ class Flow(Type_Safe):
         try:
             with Stdout() as stdout:
                 self.invoke_flow_target()
+            self.flow_data.set_return_value(self.flow_return_value)
+            self.flow_data.set_completed()
         except Exception as error:
             self.flow_error = error
             self.log_error(self.cformat.red(f"Error executing flow: {error}"))
+            self.flow_data.set_error(error)
 
         self.log_captured_stdout        (stdout)
         self.print_flow_return_value    ()
@@ -93,6 +104,9 @@ class Flow(Type_Safe):
         if self.flow_return_value:
             self.add_flow_result(key = 'flow-return-value', description=f'{self.flow_return_value}')
         flow_events.on__flow__stop(self.flow_event_data())
+
+        if self.flow_config.raise_flow_error and self.flow_error:                                       # raise an event if there was an error and raise_flow_error is set
+            raise self.flow_error
         return self
 
     def captured_logs(self):
@@ -153,6 +167,7 @@ class Flow(Type_Safe):
                 self.logger.error(message)
             else:
                 self.logger.info(message)
+            self.flow_data.add_log(log_level, message, task_run_id)
 
     def log_messages(self):
         return ansis_to_texts(self.log_messages_with_colors())
@@ -161,6 +176,8 @@ class Flow(Type_Safe):
         return self.logger.memory_handler_messages()
 
     def print_log_messages(self, use_colors=True):
+        print()
+        print()
         if use_colors:
             if self.captured_exec_logs:
                 for message in self.captured_exec_logs:
@@ -230,9 +247,23 @@ class Flow(Type_Safe):
             _.config_logger   ()
             _.setup_flow_run  ()
             _.set_flow_name   ()
+            _.flow_data.set_flow_id  (self.flow_id)
+            _.flow_data.set_flow_name(self.flow_name)
+            _.add_event_listener()
         return self
 
     def setup_flow_run(self):
         with self as _:
             if not _.flow_id:
                 _.flow_id = self.random_flow_id()
+
+    def add_event_listener(self):
+        flow_events.event_listeners.append(self.event_listener)
+
+    def event_listener(self, flow_event: Flow_Run__Event):
+        if self.flow_config.flow_data__capture_events:
+            self.flow_data.add_event(flow_event)
+
+    def json(self) -> typing.Dict[str, typing.Any]:         # Convert flow data to JSON
+        return self.flow_data.json()
+

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/osbot_utils/helpers/flows/Task.py

@@ -4,7 +4,7 @@ import typing
 from osbot_utils.helpers.flows.models.Flow_Run__Event_Data  import Flow_Run__Event_Data
 from osbot_utils.utils.Misc                                 import random_id, lower
 from osbot_utils.helpers.Dependency_Manager                 import Dependency_Manager
-from osbot_utils.helpers.flows.Flow__Events                 import flow_events
+from osbot_utils.helpers.flows.actions.Flow__Events         import flow_events
 from osbot_utils.testing.Stdout                             import Stdout
 from osbot_utils.helpers.CFormat                            import CFormat, f_dark_grey, f_red, f_blue, f_bold
 from osbot_utils.type_safe.Type_Safe                        import Type_Safe
@@ -49,6 +49,7 @@ class Task(Type_Safe):
         return self.execute__after()
 
     def execute__before(self):
+        self.on_task_start()
         self.task_flow = self.find_flow()
         if self.task_flow is None:
             raise Exception("No Flow found for Task")
@@ -107,17 +108,17 @@ class Task(Type_Safe):
         self.print_task_finished_message()
 
         flow_events.on__task__stop(self.task_event_data())
+        self.on_task_end()
         return self.task_return_value
 
-
-    def find_flow(self):
+    def find_flow(self):                                                            # Find the closest Flow instance in the call stack by examining both self parameters and local variables in each frame
         stack = inspect.stack()
         for frame_info in stack:
             frame = frame_info.frame
-            if 'self' in frame.f_locals:
-                instance = frame.f_locals['self']
-                if type(instance) is Flow:
-                    return instance
+            for var_name, var_value in list(frame.f_locals.items()):                      # Check all local variables in the frame
+                if type(var_value) is Flow:
+                    return var_value
+        return None
 
     def print_task_finished_message(self):
         if self.task_flow.flow_config.print_finished_message:
@@ -131,4 +132,15 @@ class Task(Type_Safe):
 
 
     def random_task_id(self):
-        return lower(random_id(prefix=TASK__RANDOM_ID__PREFIX))
+        return lower(random_id(prefix=TASK__RANDOM_ID__PREFIX))
+
+
+    def on_task_start(self):                                             # Handle task start event
+        self.task_flow.flow_data.add_task(self.task_id, self.task_name)
+
+    def on_task_end(self):                                               # Handle task end event
+        status = 'failed' if self.task_error else 'completed'
+        self.task_flow.flow_data.update_task(self.task_id     ,
+                                             status           ,
+                                             self.task_error  ,
+                                             self.task_return_value)

osbot_utils-2.21.0/osbot_utils/helpers/flows/actions/Flow__Data.py

@@ -0,0 +1,87 @@
+from typing                                                     import Optional, Dict, Any
+from osbot_utils.helpers.Timestamp_Now                          import Timestamp_Now
+from osbot_utils.helpers.flows.models.Flow_Run__Event           import Flow_Run__Event
+from osbot_utils.helpers.flows.models.Schema__Flow__Artifact    import Schema__Flow__Artifact
+from osbot_utils.helpers.flows.models.Schema__Flow__Result      import Schema__Flow__Result
+from osbot_utils.helpers.flows.schemas.Schema__Flow             import Schema__Flow
+from osbot_utils.helpers.flows.schemas.Schema__Flow__Data       import Schema__Flow__Data
+#from osbot_utils.helpers.flows.schemas.Schema__Flow__Event     import Schema__Flow__Event
+from osbot_utils.helpers.flows.schemas.Schema__Flow__Log        import Schema__Flow__Log
+from osbot_utils.helpers.flows.schemas.Schema__Flow__Task__Data import Schema__Flow__Task__Data
+from osbot_utils.type_safe.Type_Safe                            import Type_Safe
+
+
+class Flow__Data(Type_Safe):
+    flow_data : Schema__Flow__Data                                               # Main data container for flow execution
+
+    def set_completed(self):                                                     # Mark flow as completed
+        self.flow_data.end_time = Timestamp_Now()
+        self.flow_data.status   = "completed"
+
+    def set_error(self, error: Exception):                                      # Record flow execution error
+        self.flow_data.end_time = Timestamp_Now()
+        self.flow_data.status   = "failed"
+        self.flow_data.error    = str(error)
+
+    def set_flow_id(self, flow_id: str):                                        # Set flow identifier
+        self.flow_data.flow_id = flow_id
+
+    def set_flow_name(self, flow_name: str):                                    # Set flow name
+        self.flow_data.flow_name = flow_name
+
+    def set_return_value(self, value: Any):                                     # Set flow return value
+        self.flow_data.return_value = value
+
+    def add_task(self, task_id: str, task_name: str):                          # Record start of task execution
+        self.flow_data.tasks[task_id] = Schema__Flow__Task__Data(task_id      = task_id         ,
+                                                                 task_name    = task_name       ,
+                                                                 start_time   = Timestamp_Now() ,
+                                                                 end_time     = None            ,
+                                                                 status       = ""              ,
+                                                                 error_message= None            ,
+                                                                 return_value = None            ,
+                                                                 input_args   = ()              ,
+                                                                 input_kwargs = {}              )
+
+    def update_task(self, task_id: str, status: str,                            # Update task execution status
+                    error: Optional[Exception] = None,
+                    return_value: Any = None):
+        if task_id in self.flow_data.tasks:
+            task_data = self.flow_data.tasks[task_id]
+            task_data.end_time      = Timestamp_Now()
+            task_data.status        = status
+            task_data.error_message = str(error) if error else None
+            task_data.return_value  = return_value
+
+    # def add_event(self, event: Schema__Flow__Event):                          # todo: this needs refactoring with the Flow_Run__Event events used below
+    def add_event(self, event: Flow_Run__Event):                                # Record flow execution event
+         self.flow_data.events.append(event)
+
+    def add_log(self, level: int, message: str,                                 # Record log message
+                task_id: Optional[str] = None):
+        self.flow_data.logs.append(Schema__Flow__Log(
+            timestamp = Timestamp_Now(),
+            level     = level,
+            message   = message,
+            task_id   = task_id))
+
+    def add_result(self, key: str, description: str):                           # Record flow execution result
+        self.flow_data.results.append(Schema__Flow__Result(
+            key         = key,
+            description = description,
+            timestamp   = Timestamp_Now()))
+
+    def add_artifact(self, key          : str,
+                           description  : str,
+                           data         : Any,
+                           artifact_type: str):                             # Record flow execution artifact
+
+        self.flow_data.artifacts.append(Schema__Flow__Artifact(
+            key         = key,
+            description = description,
+            data        = data,
+            type        = artifact_type,
+            timestamp   = Timestamp_Now()))
+
+    def json(self) -> Dict[str, Any]:                                          # Convert flow data to JSON
+        return Schema__Flow(flow_data=self.flow_data).json()

osbot_utils-2.21.0/osbot_utils/helpers/flows/models/Flow_Run__Config.py

@@ -0,0 +1,14 @@
+from osbot_utils.type_safe.Type_Safe import Type_Safe
+
+# todo: this needs a good clean up and refactoring
+class Flow_Run__Config(Type_Safe):
+    add_task_to_self          : bool = True
+    log_to_console            : bool = False
+    log_to_memory             : bool = True
+    logging_enabled           : bool = True
+    print_logs                : bool = False
+    print_none_return_value   : bool = False
+    print_finished_message    : bool = False
+    raise_flow_error          : bool = True
+    flow_data__capture_events : bool = False
+

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/osbot_utils/helpers/flows/models/Flow_Run__Event.py

@@ -1,12 +1,9 @@
-from osbot_utils.helpers.Random_Guid import Random_Guid
-
-from osbot_utils.type_safe.Type_Safe                     import Type_Safe
+from osbot_utils.helpers.Random_Guid                        import Random_Guid
+from osbot_utils.type_safe.Type_Safe                        import Type_Safe
 from osbot_utils.helpers.Timestamp_Now                      import Timestamp_Now
 from osbot_utils.helpers.flows.models.Flow_Run__Event_Data  import Flow_Run__Event_Data
 from osbot_utils.helpers.flows.models.Flow_Run__Event_Type  import Flow_Run__Event_Type
 
-
-
 class Flow_Run__Event(Type_Safe):
     event_id    : Random_Guid
     event_type  : Flow_Run__Event_Type

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/osbot_utils/helpers/flows/models/Flow_Run__Event_Data.py

@@ -1,8 +1,6 @@
 import logging
-
 from osbot_utils.type_safe.Type_Safe import Type_Safe
 
-
 class Flow_Run__Event_Data(Type_Safe):
     data        : dict
     event_source: str

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/osbot_utils/helpers/flows/models/Flow_Run__Event_Type.py

@@ -1,3 +1,4 @@
+
 from enum import Enum
 
 from osbot_utils.type_safe.Type_Safe import Type_Safe

osbot_utils-2.21.0/osbot_utils/helpers/flows/models/Schema__Flow__Artifact.py

@@ -0,0 +1,11 @@
+from typing                             import Any
+from osbot_utils.helpers.Timestamp_Now  import Timestamp_Now
+from osbot_utils.type_safe.Type_Safe    import Type_Safe
+
+
+class Schema__Flow__Artifact(Type_Safe):       # Represents an artifact produced during flow execution
+    key         : str
+    description : str
+    data        : Any
+    type        : str
+    timestamp   : Timestamp_Now

osbot_utils-2.21.0/osbot_utils/helpers/flows/models/Schema__Flow__Result.py

@@ -0,0 +1,7 @@
+from osbot_utils.type_safe.Type_Safe                        import Type_Safe
+from osbot_utils.helpers.Timestamp_Now                      import Timestamp_Now
+
+class Schema__Flow__Result(Type_Safe):                      # Represents a result produced by the flow
+    key         : str
+    description : str
+    timestamp   : Timestamp_Now

osbot_utils-2.21.0/osbot_utils/helpers/flows/schemas/Schema__Flow.py

@@ -0,0 +1,9 @@
+from typing import List
+
+from osbot_utils.helpers.flows.models.Flow_Run__Event     import Flow_Run__Event
+from osbot_utils.helpers.flows.schemas.Schema__Flow__Data import Schema__Flow__Data
+from osbot_utils.type_safe.Type_Safe                      import Type_Safe
+
+class Schema__Flow(Type_Safe):                              # Root schema for flow execution data
+    flow_data  : Schema__Flow__Data
+    flow_events: List[Flow_Run__Event]                      # todo: refactor this Flow_Run__Event class with the respective Schema_* classes

osbot_utils-2.21.0/osbot_utils/helpers/flows/schemas/Schema__Flow__Data.py

@@ -0,0 +1,25 @@
+from typing                                                     import Optional, Dict, List, Any
+from osbot_utils.helpers.Timestamp_Now                          import Timestamp_Now
+from osbot_utils.helpers.flows.models.Flow_Run__Event           import Flow_Run__Event
+from osbot_utils.helpers.flows.models.Schema__Flow__Artifact    import Schema__Flow__Artifact
+from osbot_utils.helpers.flows.models.Schema__Flow__Result      import Schema__Flow__Result
+from osbot_utils.helpers.flows.schemas.Schema__Flow__Log        import Schema__Flow__Log
+from osbot_utils.helpers.flows.schemas.Schema__Flow__Task__Data import Schema__Flow__Task__Data
+from osbot_utils.type_safe.Type_Safe                            import Type_Safe
+#from osbot_utils.helpers.flows.schemas.Schema__Flow__Event      import Schema__Flow__Event
+
+
+class Schema__Flow__Data(Type_Safe):                        # Main container for flow execution data
+    flow_id      : str
+    flow_name    : str
+    start_time   : Timestamp_Now
+    end_time     : Optional[Timestamp_Now]
+    status       : str                                      # 'completed', 'failed', 'running'
+    error        : Optional[str]
+    tasks        : Dict[str, Schema__Flow__Task__Data]      # task_id -> task_data
+    #events       : List[Schema__Flow__Event]               # todo: this needs refactoring with the Flow_Run__Event events used below
+    events       : List[Flow_Run__Event]
+    results      : List[Schema__Flow__Result]
+    artifacts    : List[Schema__Flow__Artifact]
+    logs         : List[Schema__Flow__Log]
+    return_value : Any

osbot_utils-2.21.0/osbot_utils/helpers/flows/schemas/Schema__Flow__Event.py

@@ -0,0 +1,13 @@
+# todo: this needs refactoring with the Flow_Run__Event events stored in the Schema__Flow__Data. object
+
+
+# from osbot_utils.helpers.Timestamp_Now                           import Timestamp_Now
+# from osbot_utils.helpers.flows.schemas.Schema__Flow__Event__Data import Schema__Flow__Event__Data
+# from osbot_utils.type_safe.Type_Safe                             import Type_Safe
+#
+#
+# class Schema__Flow__Event(Type_Safe):               # Represents a single event in the flow execution"""
+#     event_id    : str
+#     event_type  : str
+#     event_data  : Schema__Flow__Event__Data
+#     timestamp   : Timestamp_Now

osbot_utils-2.21.0/osbot_utils/helpers/flows/schemas/Schema__Flow__Event__Data.py

@@ -0,0 +1,14 @@
+# todo: this needs refactoring with the Flow_Run__Event events stored in the Schema__Flow__Data. object
+
+# from typing                             import Dict, Optional, Any
+# from osbot_utils.type_safe.Type_Safe    import Type_Safe
+#
+#
+# class Schema__Flow__Event__Data(Type_Safe):         # Represents the data associated with a flow event
+#     data         : Dict[str, Any]
+#     event_source : str
+#     flow_name    : Optional[str]
+#     flow_run_id  : Optional[str]
+#     log_level    : int
+#     task_name    : Optional[str]
+#     task_run_id  : Optional[str]

osbot_utils-2.21.0/osbot_utils/helpers/flows/schemas/Schema__Flow__Log.py

@@ -0,0 +1,10 @@
+from typing                             import Optional
+from osbot_utils.helpers.Timestamp_Now  import Timestamp_Now
+from osbot_utils.type_safe.Type_Safe    import Type_Safe
+
+
+class Schema__Flow__Log(Type_Safe):                         # Represents a log entry from the flow execution
+    timestamp   : Timestamp_Now
+    level       : int
+    message     : str
+    task_id     : Optional[str]

osbot_utils-2.21.0/osbot_utils/helpers/flows/schemas/Schema__Flow__Task__Data.py

@@ -0,0 +1,15 @@
+from typing                             import Optional, Any
+from osbot_utils.helpers.Timestamp_Now  import Timestamp_Now
+from osbot_utils.type_safe.Type_Safe    import Type_Safe
+
+
+class Schema__Flow__Task__Data(Type_Safe):                      # Represents the data associated with a task execution
+    task_id          : str
+    task_name        : str
+    start_time       : Timestamp_Now
+    end_time         : Optional[Timestamp_Now]
+    status           : str                                      # 'completed', 'failed', 'running'
+    error_message    : Optional[str]
+    return_value     : Any
+    input_args       : tuple
+    input_kwargs     : dict

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/osbot_utils/utils/Http.py

@@ -56,7 +56,6 @@ def http_request(url, data=None, headers=None, method='GET', encoding ='utf-8',
     ssl_request = url.startswith('https://')
     headers = headers or {}
     if data:
-        print()
         if type(data) is not str:                                   # if the data object is not a string
             if headers.get('Content-Type') == "application/json":   # and a json payload is expected
                 data = json.dumps(data)                             # convert it to json
@@ -164,6 +163,17 @@ def POST(url, data='', headers=None):
 def POST_json(*args, **kwargs):
     return json.loads(POST(*args, **kwargs))
 
+def POST_json_get_bytes(url=None, data=None):
+    headers          = {'Content-Type': "application/json"}     # todo add support for providing custom headers
+    kwargs = dict(url     = url   ,
+                  data    = data,
+                  headers = headers      ,
+                  method  = 'POST'       ,
+                  encoding = None
+                  )
+    response = http_request(**kwargs)
+    return response
+
 def PUT(url, data='', headers=None):
     return http_request(url, data, headers, 'PUT')
 

osbot_utils-2.21.0/osbot_utils/version

@@ -0,0 +1 @@
+v2.21.0

{osbot_utils-2.19.0 → osbot_utils-2.21.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name        = "osbot_utils"
-version     = "v2.19.0"
+version     = "v2.21.0"
 description = "OWASP Security Bot - Utils"
 authors     = ["Dinis Cruz <dinis.cruz@owasp.org>"]
 license     = "MIT"

osbot_utils-2.19.0/osbot_utils/helpers/flows/models/Flow_Run__Config.py

@@ -1,12 +0,0 @@
-from osbot_utils.type_safe.Type_Safe import Type_Safe
-
-
-class Flow_Run__Config(Type_Safe):
-    add_task_to_self       : bool = True
-    log_to_console         : bool = False
-    log_to_memory          : bool = True
-    logging_enabled        : bool = True
-    print_logs             : bool = False
-    print_none_return_value: bool = False
-    print_finished_message : bool = False
-

osbot_utils-2.19.0/osbot_utils/version

@@ -1 +0,0 @@
-v2.19.0