flowcept 0.8.10__py3-none-any.whl → 0.8.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flowcept/cli.py +210 -10
- flowcept/commons/daos/keyvalue_dao.py +19 -23
- flowcept/commons/daos/mq_dao/mq_dao_base.py +29 -29
- flowcept/commons/daos/mq_dao/mq_dao_kafka.py +4 -3
- flowcept/commons/daos/mq_dao/mq_dao_mofka.py +4 -0
- flowcept/commons/daos/mq_dao/mq_dao_redis.py +38 -5
- flowcept/commons/daos/redis_conn.py +47 -0
- flowcept/commons/flowcept_dataclasses/task_object.py +36 -8
- flowcept/commons/settings_factory.py +2 -4
- flowcept/commons/task_data_preprocess.py +200 -0
- flowcept/commons/utils.py +1 -1
- flowcept/configs.py +8 -4
- flowcept/flowcept_api/flowcept_controller.py +30 -13
- flowcept/flowceptor/adapters/agents/__init__.py +1 -0
- flowcept/flowceptor/adapters/agents/agents_utils.py +89 -0
- flowcept/flowceptor/adapters/agents/flowcept_agent.py +292 -0
- flowcept/flowceptor/adapters/agents/flowcept_llm_prov_capture.py +186 -0
- flowcept/flowceptor/adapters/agents/prompts.py +51 -0
- flowcept/flowceptor/adapters/base_interceptor.py +13 -6
- flowcept/flowceptor/adapters/brokers/__init__.py +1 -0
- flowcept/flowceptor/adapters/brokers/mqtt_interceptor.py +132 -0
- flowcept/flowceptor/adapters/mlflow/mlflow_interceptor.py +3 -3
- flowcept/flowceptor/adapters/tensorboard/tensorboard_interceptor.py +3 -3
- flowcept/flowceptor/consumers/agent/__init__.py +1 -0
- flowcept/flowceptor/consumers/agent/base_agent_context_manager.py +101 -0
- flowcept/flowceptor/consumers/agent/client_agent.py +48 -0
- flowcept/flowceptor/consumers/agent/flowcept_agent_context_manager.py +145 -0
- flowcept/flowceptor/consumers/agent/flowcept_qa_manager.py +112 -0
- flowcept/flowceptor/consumers/base_consumer.py +90 -0
- flowcept/flowceptor/consumers/document_inserter.py +135 -36
- flowcept/flowceptor/telemetry_capture.py +1 -1
- flowcept/instrumentation/task_capture.py +8 -2
- flowcept/version.py +1 -1
- {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/METADATA +10 -1
- {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/RECORD +39 -27
- resources/sample_settings.yaml +37 -13
- flowcept/flowceptor/adapters/zambeze/__init__.py +0 -1
- flowcept/flowceptor/adapters/zambeze/zambeze_dataclasses.py +0 -41
- flowcept/flowceptor/adapters/zambeze/zambeze_interceptor.py +0 -102
- {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/WHEEL +0 -0
- {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/entry_points.txt +0 -0
- {flowcept-0.8.10.dist-info → flowcept-0.8.11.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
"""Zambeze interceptor module."""
|
|
2
|
-
|
|
3
|
-
from threading import Thread
|
|
4
|
-
from time import sleep
|
|
5
|
-
import pika
|
|
6
|
-
import json
|
|
7
|
-
from typing import Dict
|
|
8
|
-
|
|
9
|
-
from flowcept.commons.utils import get_utc_now, get_status_from_str
|
|
10
|
-
from flowcept.commons.flowcept_dataclasses.task_object import TaskObject
|
|
11
|
-
from flowcept.flowceptor.adapters.base_interceptor import (
|
|
12
|
-
BaseInterceptor,
|
|
13
|
-
)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class ZambezeInterceptor(BaseInterceptor):
|
|
17
|
-
"""Zambeze interceptor."""
|
|
18
|
-
|
|
19
|
-
def __init__(self, plugin_key="zambeze"):
|
|
20
|
-
super().__init__(plugin_key)
|
|
21
|
-
self._consumer_tag = None
|
|
22
|
-
self._channel = None
|
|
23
|
-
self._observer_thread: Thread = None
|
|
24
|
-
|
|
25
|
-
def prepare_task_msg(self, zambeze_msg: Dict) -> TaskObject:
|
|
26
|
-
"""Prepare a task."""
|
|
27
|
-
task_msg = TaskObject()
|
|
28
|
-
task_msg.utc_timestamp = get_utc_now()
|
|
29
|
-
task_msg.campaign_id = zambeze_msg.get("campaign_id", None)
|
|
30
|
-
task_msg.task_id = zambeze_msg.get("activity_id", None)
|
|
31
|
-
task_msg.activity_id = zambeze_msg.get("name", None)
|
|
32
|
-
task_msg.dependencies = zambeze_msg.get("depends_on", None)
|
|
33
|
-
task_msg.custom_metadata = {"command": zambeze_msg.get("command", None)}
|
|
34
|
-
task_msg.status = get_status_from_str(zambeze_msg.get("activity_status", None))
|
|
35
|
-
task_msg.used = {
|
|
36
|
-
"args": zambeze_msg.get("arguments", None),
|
|
37
|
-
"kwargs": zambeze_msg.get("kwargs", None),
|
|
38
|
-
"files": zambeze_msg.get("files", None),
|
|
39
|
-
}
|
|
40
|
-
return task_msg
|
|
41
|
-
|
|
42
|
-
def start(self, bundle_exec_id) -> "ZambezeInterceptor":
|
|
43
|
-
"""Start it."""
|
|
44
|
-
super().start(bundle_exec_id)
|
|
45
|
-
self._observer_thread = Thread(target=self.observe)
|
|
46
|
-
self._observer_thread.start()
|
|
47
|
-
return self
|
|
48
|
-
|
|
49
|
-
def stop(self) -> bool:
|
|
50
|
-
"""Stop it."""
|
|
51
|
-
self.logger.debug("Interceptor stopping...")
|
|
52
|
-
super().stop()
|
|
53
|
-
try:
|
|
54
|
-
self._channel.stop_consuming()
|
|
55
|
-
except Exception as e:
|
|
56
|
-
self.logger.warning(f"This exception is expected to occur after channel.basic_cancel: {e}")
|
|
57
|
-
sleep(2)
|
|
58
|
-
self._observer_thread.join()
|
|
59
|
-
self.logger.debug("Interceptor stopped.")
|
|
60
|
-
return True
|
|
61
|
-
|
|
62
|
-
def observe(self):
|
|
63
|
-
"""Observe it."""
|
|
64
|
-
connection = pika.BlockingConnection(
|
|
65
|
-
pika.ConnectionParameters(host=self.settings.host, port=self.settings.port)
|
|
66
|
-
)
|
|
67
|
-
self._channel = connection.channel()
|
|
68
|
-
for queue in self.settings.queue_names:
|
|
69
|
-
self._channel.queue_declare(queue=queue)
|
|
70
|
-
|
|
71
|
-
# self._consumer_tag =\
|
|
72
|
-
for queue in self.settings.queue_names:
|
|
73
|
-
self._channel.basic_consume(
|
|
74
|
-
queue=queue,
|
|
75
|
-
on_message_callback=self.callback,
|
|
76
|
-
auto_ack=True,
|
|
77
|
-
)
|
|
78
|
-
self.logger.debug(f"Waiting for Zambeze messages on queue {queue}")
|
|
79
|
-
|
|
80
|
-
try:
|
|
81
|
-
self._channel.start_consuming()
|
|
82
|
-
except Exception as e:
|
|
83
|
-
self.logger.warning(
|
|
84
|
-
f"If this exception happens after channel.start_consuming finishes, it is expected:\n {e}"
|
|
85
|
-
)
|
|
86
|
-
|
|
87
|
-
def _intercept(self, body_obj):
|
|
88
|
-
self.logger.debug(f"Zambeze interceptor needs to intercept this:\n\t{json.dumps(body_obj)}")
|
|
89
|
-
task_msg = self.prepare_task_msg(body_obj)
|
|
90
|
-
self.intercept(task_msg.to_dict())
|
|
91
|
-
|
|
92
|
-
def callback(self, ch, method, properties, body):
|
|
93
|
-
"""Implement the callback."""
|
|
94
|
-
body_obj = json.loads(body)
|
|
95
|
-
if self.settings.key_values_to_filter is not None:
|
|
96
|
-
for key_value in self.settings.key_values_to_filter:
|
|
97
|
-
if key_value.key in body_obj:
|
|
98
|
-
if body_obj[key_value.key] == key_value.value:
|
|
99
|
-
self._intercept(body_obj)
|
|
100
|
-
break
|
|
101
|
-
else:
|
|
102
|
-
self._intercept(body_obj)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|