flowcept 0.8.10__py3-none-any.whl → 0.8.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. flowcept/cli.py +210 -10
  2. flowcept/commons/daos/keyvalue_dao.py +19 -23
  3. flowcept/commons/daos/mq_dao/mq_dao_base.py +29 -29
  4. flowcept/commons/daos/mq_dao/mq_dao_kafka.py +4 -3
  5. flowcept/commons/daos/mq_dao/mq_dao_mofka.py +4 -0
  6. flowcept/commons/daos/mq_dao/mq_dao_redis.py +38 -5
  7. flowcept/commons/daos/redis_conn.py +47 -0
  8. flowcept/commons/flowcept_dataclasses/task_object.py +36 -8
  9. flowcept/commons/settings_factory.py +2 -4
  10. flowcept/commons/task_data_preprocess.py +200 -0
  11. flowcept/commons/utils.py +1 -1
  12. flowcept/configs.py +8 -4
  13. flowcept/flowcept_api/flowcept_controller.py +30 -13
  14. flowcept/flowceptor/adapters/agents/__init__.py +1 -0
  15. flowcept/flowceptor/adapters/agents/agents_utils.py +89 -0
  16. flowcept/flowceptor/adapters/agents/flowcept_agent.py +292 -0
  17. flowcept/flowceptor/adapters/agents/flowcept_llm_prov_capture.py +186 -0
  18. flowcept/flowceptor/adapters/agents/prompts.py +51 -0
  19. flowcept/flowceptor/adapters/base_interceptor.py +13 -6
  20. flowcept/flowceptor/adapters/brokers/__init__.py +1 -0
  21. flowcept/flowceptor/adapters/brokers/mqtt_interceptor.py +132 -0
  22. flowcept/flowceptor/adapters/mlflow/mlflow_interceptor.py +3 -3
  23. flowcept/flowceptor/adapters/tensorboard/tensorboard_interceptor.py +3 -3
  24. flowcept/flowceptor/consumers/agent/__init__.py +1 -0
  25. flowcept/flowceptor/consumers/agent/base_agent_context_manager.py +101 -0
  26. flowcept/flowceptor/consumers/agent/client_agent.py +48 -0
  27. flowcept/flowceptor/consumers/agent/flowcept_agent_context_manager.py +145 -0
  28. flowcept/flowceptor/consumers/agent/flowcept_qa_manager.py +112 -0
  29. flowcept/flowceptor/consumers/base_consumer.py +90 -0
  30. flowcept/flowceptor/consumers/document_inserter.py +135 -36
  31. flowcept/flowceptor/telemetry_capture.py +1 -1
  32. flowcept/instrumentation/task_capture.py +8 -2
  33. flowcept/version.py +1 -1
  34. {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/METADATA +10 -1
  35. {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/RECORD +39 -27
  36. resources/sample_settings.yaml +37 -13
  37. flowcept/flowceptor/adapters/zambeze/__init__.py +0 -1
  38. flowcept/flowceptor/adapters/zambeze/zambeze_dataclasses.py +0 -41
  39. flowcept/flowceptor/adapters/zambeze/zambeze_interceptor.py +0 -102
  40. {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/WHEEL +0 -0
  41. {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/entry_points.txt +0 -0
  42. {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/licenses/LICENSE +0 -0
@@ -1,102 +0,0 @@
1
- """Zambeze interceptor module."""
2
-
3
- from threading import Thread
4
- from time import sleep
5
- import pika
6
- import json
7
- from typing import Dict
8
-
9
- from flowcept.commons.utils import get_utc_now, get_status_from_str
10
- from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
11
- from flowcept.flowceptor.adapters.base_interceptor import (
12
- BaseInterceptor,
13
- )
14
-
15
-
16
- class ZambezeInterceptor(BaseInterceptor):
17
- """Zambeze interceptor."""
18
-
19
- def __init__(self, plugin_key="zambeze"):
20
- super().__init__(plugin_key)
21
- self._consumer_tag = None
22
- self._channel = None
23
- self._observer_thread: Thread = None
24
-
25
- def prepare_task_msg(self, zambeze_msg: Dict) -> TaskObject:
26
- """Prepare a task."""
27
- task_msg = TaskObject()
28
- task_msg.utc_timestamp = get_utc_now()
29
- task_msg.campaign_id = zambeze_msg.get("campaign_id", None)
30
- task_msg.task_id = zambeze_msg.get("activity_id", None)
31
- task_msg.activity_id = zambeze_msg.get("name", None)
32
- task_msg.dependencies = zambeze_msg.get("depends_on", None)
33
- task_msg.custom_metadata = {"command": zambeze_msg.get("command", None)}
34
- task_msg.status = get_status_from_str(zambeze_msg.get("activity_status", None))
35
- task_msg.used = {
36
- "args": zambeze_msg.get("arguments", None),
37
- "kwargs": zambeze_msg.get("kwargs", None),
38
- "files": zambeze_msg.get("files", None),
39
- }
40
- return task_msg
41
-
42
- def start(self, bundle_exec_id) -> "ZambezeInterceptor":
43
- """Start it."""
44
- super().start(bundle_exec_id)
45
- self._observer_thread = Thread(target=self.observe)
46
- self._observer_thread.start()
47
- return self
48
-
49
- def stop(self) -> bool:
50
- """Stop it."""
51
- self.logger.debug("Interceptor stopping...")
52
- super().stop()
53
- try:
54
- self._channel.stop_consuming()
55
- except Exception as e:
56
- self.logger.warning(f"This exception is expected to occur after channel.basic_cancel: {e}")
57
- sleep(2)
58
- self._observer_thread.join()
59
- self.logger.debug("Interceptor stopped.")
60
- return True
61
-
62
- def observe(self):
63
- """Observe it."""
64
- connection = pika.BlockingConnection(
65
- pika.ConnectionParameters(host=self.settings.host, port=self.settings.port)
66
- )
67
- self._channel = connection.channel()
68
- for queue in self.settings.queue_names:
69
- self._channel.queue_declare(queue=queue)
70
-
71
- # self._consumer_tag =\
72
- for queue in self.settings.queue_names:
73
- self._channel.basic_consume(
74
- queue=queue,
75
- on_message_callback=self.callback,
76
- auto_ack=True,
77
- )
78
- self.logger.debug(f"Waiting for Zambeze messages on queue {queue}")
79
-
80
- try:
81
- self._channel.start_consuming()
82
- except Exception as e:
83
- self.logger.warning(
84
- f"If this exception happens after channel.start_consuming finishes, it is expected:\n {e}"
85
- )
86
-
87
- def _intercept(self, body_obj):
88
- self.logger.debug(f"Zambeze interceptor needs to intercept this:\n\t{json.dumps(body_obj)}")
89
- task_msg = self.prepare_task_msg(body_obj)
90
- self.intercept(task_msg.to_dict())
91
-
92
- def callback(self, ch, method, properties, body):
93
- """Implement the callback."""
94
- body_obj = json.loads(body)
95
- if self.settings.key_values_to_filter is not None:
96
- for key_value in self.settings.key_values_to_filter:
97
- if key_value.key in body_obj:
98
- if body_obj[key_value.key] == key_value.value:
99
- self._intercept(body_obj)
100
- break
101
- else:
102
- self._intercept(body_obj)