pyxecm 2.0.4__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyxecm might be problematic. Click here for more details.
- pyxecm/coreshare.py +5 -3
- pyxecm/helper/data.py +4 -4
- pyxecm/helper/otel_config.py +26 -0
- pyxecm/helper/web.py +1 -2
- pyxecm/otca.py +1356 -16
- pyxecm/otcs.py +2354 -593
- pyxecm/otds.py +1 -1
- pyxecm/otmm.py +4 -5
- pyxecm/py.typed +0 -0
- pyxecm-3.0.0.dist-info/METADATA +48 -0
- pyxecm-3.0.0.dist-info/RECORD +96 -0
- {pyxecm-2.0.4.dist-info → pyxecm-3.0.0.dist-info}/WHEEL +1 -2
- pyxecm-3.0.0.dist-info/entry_points.txt +4 -0
- {pyxecm/customizer/api → pyxecm_api}/__main__.py +1 -1
- pyxecm_api/agents/__init__.py +7 -0
- pyxecm_api/agents/app.py +13 -0
- pyxecm_api/agents/functions.py +119 -0
- pyxecm_api/agents/models.py +10 -0
- pyxecm_api/agents/otcm_knowledgegraph/functions.py +85 -0
- pyxecm_api/agents/otcm_knowledgegraph/models.py +61 -0
- pyxecm_api/agents/otcm_knowledgegraph/router.py +74 -0
- pyxecm_api/agents/otcm_user_agent/models.py +20 -0
- pyxecm_api/agents/otcm_user_agent/router.py +65 -0
- pyxecm_api/agents/otcm_workspace_agent/models.py +40 -0
- pyxecm_api/agents/otcm_workspace_agent/router.py +200 -0
- pyxecm_api/app.py +221 -0
- {pyxecm/customizer/api → pyxecm_api}/auth/functions.py +10 -2
- {pyxecm/customizer/api → pyxecm_api}/auth/router.py +4 -3
- {pyxecm/customizer/api → pyxecm_api}/common/functions.py +39 -9
- {pyxecm/customizer/api → pyxecm_api}/common/metrics.py +1 -2
- {pyxecm/customizer/api → pyxecm_api}/common/router.py +7 -8
- {pyxecm/customizer/api → pyxecm_api}/settings.py +21 -6
- {pyxecm/customizer/api → pyxecm_api}/terminal/router.py +1 -1
- {pyxecm/customizer/api → pyxecm_api}/v1_csai/router.py +39 -10
- pyxecm_api/v1_csai/statics/bindings/utils.js +189 -0
- pyxecm_api/v1_csai/statics/tom-select/tom-select.complete.min.js +356 -0
- pyxecm_api/v1_csai/statics/tom-select/tom-select.css +334 -0
- pyxecm_api/v1_csai/statics/vis-9.1.2/vis-network.css +1 -0
- pyxecm_api/v1_csai/statics/vis-9.1.2/vis-network.min.js +27 -0
- pyxecm_api/v1_maintenance/__init__.py +1 -0
- {pyxecm/customizer/api → pyxecm_api}/v1_maintenance/functions.py +3 -3
- {pyxecm/customizer/api → pyxecm_api}/v1_maintenance/router.py +8 -8
- pyxecm_api/v1_otcs/__init__.py +1 -0
- {pyxecm/customizer/api → pyxecm_api}/v1_otcs/functions.py +7 -5
- {pyxecm/customizer/api → pyxecm_api}/v1_otcs/router.py +8 -7
- pyxecm_api/v1_payload/__init__.py +1 -0
- {pyxecm/customizer/api → pyxecm_api}/v1_payload/functions.py +10 -7
- {pyxecm/customizer/api → pyxecm_api}/v1_payload/router.py +11 -10
- {pyxecm/customizer → pyxecm_customizer}/__init__.py +8 -0
- {pyxecm/customizer → pyxecm_customizer}/__main__.py +15 -21
- {pyxecm/customizer → pyxecm_customizer}/browser_automation.py +414 -103
- {pyxecm/customizer → pyxecm_customizer}/customizer.py +178 -116
- {pyxecm/customizer → pyxecm_customizer}/guidewire.py +60 -20
- {pyxecm/customizer → pyxecm_customizer}/k8s.py +4 -4
- pyxecm_customizer/knowledge_graph.py +719 -0
- pyxecm_customizer/log.py +35 -0
- {pyxecm/customizer → pyxecm_customizer}/m365.py +41 -33
- {pyxecm/customizer → pyxecm_customizer}/payload.py +2265 -1933
- {pyxecm/customizer/api/common → pyxecm_customizer}/payload_list.py +18 -55
- {pyxecm/customizer → pyxecm_customizer}/salesforce.py +1 -1
- {pyxecm/customizer → pyxecm_customizer}/sap.py +6 -2
- {pyxecm/customizer → pyxecm_customizer}/servicenow.py +2 -4
- {pyxecm/customizer → pyxecm_customizer}/settings.py +7 -6
- {pyxecm/customizer → pyxecm_customizer}/successfactors.py +40 -28
- {pyxecm/customizer → pyxecm_customizer}/translate.py +1 -1
- {pyxecm/maintenance_page → pyxecm_maintenance_page}/__main__.py +1 -1
- {pyxecm/maintenance_page → pyxecm_maintenance_page}/app.py +14 -8
- pyxecm/customizer/api/app.py +0 -157
- pyxecm/customizer/log.py +0 -107
- pyxecm/customizer/nhc.py +0 -1169
- pyxecm/customizer/openapi.py +0 -258
- pyxecm/customizer/pht.py +0 -1357
- pyxecm-2.0.4.dist-info/METADATA +0 -119
- pyxecm-2.0.4.dist-info/RECORD +0 -78
- pyxecm-2.0.4.dist-info/licenses/LICENSE +0 -202
- pyxecm-2.0.4.dist-info/top_level.txt +0 -1
- {pyxecm/customizer/api → pyxecm_api}/__init__.py +0 -0
- {pyxecm/customizer/api/auth → pyxecm_api/agents/otcm_knowledgegraph}/__init__.py +0 -0
- {pyxecm/customizer/api/common → pyxecm_api/agents/otcm_user_agent}/__init__.py +0 -0
- {pyxecm/customizer/api/v1_csai → pyxecm_api/agents/otcm_workspace_agent}/__init__.py +0 -0
- {pyxecm/customizer/api/v1_maintenance → pyxecm_api/auth}/__init__.py +0 -0
- {pyxecm/customizer/api → pyxecm_api}/auth/models.py +0 -0
- {pyxecm/customizer/api/v1_otcs → pyxecm_api/common}/__init__.py +0 -0
- {pyxecm/customizer/api → pyxecm_api}/common/models.py +0 -0
- {pyxecm/customizer/api → pyxecm_api}/terminal/__init__.py +0 -0
- {pyxecm/customizer/api/v1_payload → pyxecm_api/v1_csai}/__init__.py +0 -0
- {pyxecm/customizer/api → pyxecm_api}/v1_csai/models.py +0 -0
- {pyxecm/customizer/api → pyxecm_api}/v1_maintenance/models.py +0 -0
- {pyxecm/customizer/api → pyxecm_api}/v1_payload/models.py +0 -0
- {pyxecm/customizer → pyxecm_customizer}/exceptions.py +0 -0
- {pyxecm/maintenance_page → pyxecm_maintenance_page}/__init__.py +0 -0
- {pyxecm/maintenance_page → pyxecm_maintenance_page}/settings.py +0 -0
- {pyxecm/maintenance_page → pyxecm_maintenance_page}/static/favicon.avif +0 -0
- {pyxecm/maintenance_page → pyxecm_maintenance_page}/templates/maintenance.html +0 -0
|
@@ -0,0 +1,719 @@
|
|
|
1
|
+
"""Knowledge Graph Module to implement a class to build and maintain and knowledge graph.
|
|
2
|
+
|
|
3
|
+
The knowledge graph consists of the OTCS workspaces and their relationships.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
__author__ = "Dr. Marc Diefenbruch"
|
|
7
|
+
__copyright__ = "Copyright (C) 2024-2025, OpenText"
|
|
8
|
+
__credits__ = ["Kai-Philip Gatzweiler"]
|
|
9
|
+
__maintainer__ = "Dr. Marc Diefenbruch"
|
|
10
|
+
__email__ = "mdiefenb@opentext.com"
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
from collections import deque
|
|
14
|
+
|
|
15
|
+
from pyxecm import OTCS
|
|
16
|
+
from pyxecm.helper import Data
|
|
17
|
+
from pyxecm.helper.otel_config import tracer
|
|
18
|
+
|
|
19
|
+
APP_NAME = "pyxecm"
|
|
20
|
+
MODULE_NAME = APP_NAME + ".customizer.knowledge_graph"
|
|
21
|
+
OTEL_TRACING_ATTRIBUTES = {"class": "knowledge_graph"}
|
|
22
|
+
|
|
23
|
+
default_logger = logging.getLogger(MODULE_NAME)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class KnowledgeGraph:
|
|
27
|
+
"""Used to build and maintain a Knowledge Graph for the OTCS workspaces."""
|
|
28
|
+
|
|
29
|
+
logger: logging.Logger = default_logger
|
|
30
|
+
|
|
31
|
+
WORKSPACE_ID_FIELD = "id"
|
|
32
|
+
WORKSPACE_NAME_FIELD = "name"
|
|
33
|
+
WORKSPACE_TYPE_FIELD = "wnf_wksp_type_id"
|
|
34
|
+
|
|
35
|
+
def __init__(self, otcs_object: OTCS, ontology: dict[tuple[str, str, str], list[str]] | None = None) -> None:
|
|
36
|
+
"""Initialize the Knowledge Graph.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
otcs_object (OTCS):
|
|
40
|
+
An instance of the OTCS class providing access to workspace data.
|
|
41
|
+
ontology (dict[tuple[str, str, str], list[str]]):
|
|
42
|
+
A dictionary mapping (source_type, target_type, rel_type) tuples
|
|
43
|
+
to a list of semantic relationship names. source_type and target_type
|
|
44
|
+
are workspace type names from OTCS. rel_type is either "parent" or "child".
|
|
45
|
+
It abstracts the graph structure at the type level.
|
|
46
|
+
|
|
47
|
+
Example:
|
|
48
|
+
ontology = {
|
|
49
|
+
("Vendor", "Material", "child"): ["offers", "supplies", "provides"],
|
|
50
|
+
("Vendor", "Purchase Order", "child"): ["receives", "fulfills"],
|
|
51
|
+
("Vendor", "Purchase Contract", "child"): ["signs", "owns"],
|
|
52
|
+
("Material", "Vendor", "parent"): ["is supplied by", "is offered by"],
|
|
53
|
+
("Purchase Order", "Material", "child"): ["includes", "consists of"],
|
|
54
|
+
("Customer", "Sales Order", "child"): ["orders", "issues", "sends"],
|
|
55
|
+
("Customer", "Sales Contract", "child"): ["signs", "owns"],
|
|
56
|
+
("Sales Order", "Customer", "parent"): ["belongs to", "is initiated by"],
|
|
57
|
+
("Sales Order", "Material", "child"): ["includes", "consists of"],
|
|
58
|
+
("Sales Order", "Delivery", "child"): ["triggers", "is followed by"],
|
|
59
|
+
("Sales Order", "Production Order", "child"): ["triggers", "is followed by"],
|
|
60
|
+
("Sales Contract", "Material", "child"): ["includes", "consists of"],
|
|
61
|
+
("Production Order", "Material", "child"): ["includes", "consists of"],
|
|
62
|
+
("Production Order", "Delivery", "child"): ["triggers", "is followed by"],
|
|
63
|
+
("Production Order", "Goods Movement", "child"): ["triggers", "is followed by"],
|
|
64
|
+
("Delivery", "Goods Movement", "child"): ["triggers", "is followed by"],
|
|
65
|
+
("Delivery", "Material", "child"): ["triggers", "is followed by"],
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
"""
|
|
69
|
+
|
|
70
|
+
# The OTCS object to traverse the workspaces and workspace relationships:
|
|
71
|
+
self._otcs = otcs_object
|
|
72
|
+
|
|
73
|
+
# The ontology abstracts the graph structure at the type level:
|
|
74
|
+
self._ontology: dict[tuple[str, str, str], list[str]] = ontology if ontology else {}
|
|
75
|
+
self._type_graph = self.build_type_graph()
|
|
76
|
+
self._type_graph_inverted = self.invert_type_graph(self._type_graph)
|
|
77
|
+
|
|
78
|
+
# This include the pandas data frames for the node and edge INSTANCES:
|
|
79
|
+
self._nodes = Data() # columns=["id", "name", "type"])
|
|
80
|
+
self._edges = Data() # (columns=["source", "target", "relationship"])
|
|
81
|
+
|
|
82
|
+
# Create a simple dictionary with all workspace types to easily
|
|
83
|
+
# lookup the workspace type name (value) by the workspace type ID (key):
|
|
84
|
+
workspace_types = self._otcs.get_workspace_types()
|
|
85
|
+
self.workspace_types = {
|
|
86
|
+
wt["data"]["properties"]["wksp_type_id"]: wt["data"]["properties"]["wksp_type_name"]
|
|
87
|
+
for wt in workspace_types["results"]
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
# end method definition
|
|
91
|
+
|
|
92
|
+
def get_nodes(self) -> Data:
|
|
93
|
+
"""Return the graph nodes as a Data object.
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
Data:
|
|
97
|
+
Data object with embedded Pandas data frame for the nodes of the Knowledge Graph.
|
|
98
|
+
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
return self._nodes
|
|
102
|
+
|
|
103
|
+
# end method definition
|
|
104
|
+
|
|
105
|
+
def get_edges(self) -> Data:
|
|
106
|
+
"""Return the graph edges as a Data object.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
Data:
|
|
110
|
+
Data object with embedded Pandas data frame for the edges of the Knowledge Graph.
|
|
111
|
+
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
return self._edges
|
|
115
|
+
|
|
116
|
+
# end method definition
|
|
117
|
+
|
|
118
|
+
def get_otcs_object(self) -> OTCS:
|
|
119
|
+
"""Return the OTCS object.
|
|
120
|
+
|
|
121
|
+
Returns:
|
|
122
|
+
OTCS:
|
|
123
|
+
OTCS object.
|
|
124
|
+
|
|
125
|
+
"""
|
|
126
|
+
|
|
127
|
+
return self._otcs
|
|
128
|
+
|
|
129
|
+
# end method definition
|
|
130
|
+
|
|
131
|
+
def get_ontology(self) -> dict[tuple[str, str, str], list[str]] | None:
|
|
132
|
+
"""Return the graph ontology.
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
dict[tuple[str, str, str], list[str]] | None:
|
|
136
|
+
Defined ontology for the knowledge graph.
|
|
137
|
+
|
|
138
|
+
"""
|
|
139
|
+
|
|
140
|
+
return self._ontology
|
|
141
|
+
|
|
142
|
+
# end method definition
|
|
143
|
+
|
|
144
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="build_type_graph")
|
|
145
|
+
def build_type_graph(self, directions: list | None = None) -> dict[str, set[str]]:
|
|
146
|
+
"""Construct a directed type-level graph from the ontology.
|
|
147
|
+
|
|
148
|
+
This uses the ontology's (source_type, target_type, direction) keys to build
|
|
149
|
+
a forward graph of type relationships, avoiding duplicates.
|
|
150
|
+
|
|
151
|
+
Args:
|
|
152
|
+
directions (list | None, optional):
|
|
153
|
+
Which directions the edges should be traversed: "child", "parent", or both.
|
|
154
|
+
Default is ["child"].
|
|
155
|
+
|
|
156
|
+
Returns:
|
|
157
|
+
dict[str, set[str]]:
|
|
158
|
+
A dictionary mapping each entity type to a list of connected types
|
|
159
|
+
based on child/parent ontology relationships.
|
|
160
|
+
|
|
161
|
+
Example:
|
|
162
|
+
Input:
|
|
163
|
+
{
|
|
164
|
+
("Vendor", "Material", "child"): ["supplies"],
|
|
165
|
+
("Material", "Vendor", "parent"): ["is supplied by"]
|
|
166
|
+
}
|
|
167
|
+
Output:
|
|
168
|
+
{
|
|
169
|
+
"Vendor": {"Material"},
|
|
170
|
+
"Material": {"Vendor"}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
"""
|
|
174
|
+
|
|
175
|
+
type_graph = {}
|
|
176
|
+
|
|
177
|
+
if directions is None:
|
|
178
|
+
directions = ["child"]
|
|
179
|
+
|
|
180
|
+
# To avoid duplicate entries, we use a set during graph construction to
|
|
181
|
+
# ensure each target type appears only once per source type.
|
|
182
|
+
# Then convert it back to a list at the end.
|
|
183
|
+
for source_type, target_type, direction in self._ontology:
|
|
184
|
+
# Determine the forward (parent-chld) direction
|
|
185
|
+
if direction == "child" and "child" in directions:
|
|
186
|
+
from_type, to_type = source_type, target_type
|
|
187
|
+
# Add forward edge
|
|
188
|
+
type_graph.setdefault(from_type, set()).add(to_type)
|
|
189
|
+
elif direction == "parent" and "parent" in directions:
|
|
190
|
+
from_type, to_type = target_type, source_type
|
|
191
|
+
# Add reverse edge to allow backward traversal
|
|
192
|
+
type_graph.setdefault(to_type, set()).add(from_type)
|
|
193
|
+
else:
|
|
194
|
+
continue # ignore unknown directions
|
|
195
|
+
|
|
196
|
+
return type_graph
|
|
197
|
+
|
|
198
|
+
# end method definition
|
|
199
|
+
|
|
200
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="invert_type_graph")
|
|
201
|
+
def invert_type_graph(self, type_graph: dict) -> dict[str, set[str]]:
|
|
202
|
+
"""Invert the provided type graph, meaning invert the relationships (edges) in the graph.
|
|
203
|
+
|
|
204
|
+
Args:
|
|
205
|
+
type_graph (dict):
|
|
206
|
+
Existing type graph that should be inverted.
|
|
207
|
+
|
|
208
|
+
Returns:
|
|
209
|
+
dict[str, set[str]]:
|
|
210
|
+
A new inverted type graph.
|
|
211
|
+
|
|
212
|
+
"""
|
|
213
|
+
|
|
214
|
+
type_graph_inverted = {}
|
|
215
|
+
|
|
216
|
+
for start_type, target_types in type_graph.items():
|
|
217
|
+
for target_type in target_types:
|
|
218
|
+
if target_type not in type_graph_inverted:
|
|
219
|
+
type_graph_inverted[target_type] = set()
|
|
220
|
+
type_graph_inverted[target_type].add(start_type)
|
|
221
|
+
|
|
222
|
+
return type_graph_inverted
|
|
223
|
+
|
|
224
|
+
# end method definition
|
|
225
|
+
|
|
226
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="build_type_pathes")
|
|
227
|
+
def build_type_pathes(
|
|
228
|
+
self,
|
|
229
|
+
source_type: str,
|
|
230
|
+
target_type: str,
|
|
231
|
+
direction: str = "child",
|
|
232
|
+
) -> list[list[str]]:
|
|
233
|
+
"""Find all possible paths (acyclic) from start to target type.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
source_type:
|
|
237
|
+
The source type (e.g., 'Material').
|
|
238
|
+
target_type:
|
|
239
|
+
The target type (e.g., 'Customer').
|
|
240
|
+
direction (str, optional):
|
|
241
|
+
Either "child" (source_id to target_id) or "parent" (target_id to source_id).
|
|
242
|
+
"child" is the default.
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
list[list[str]]:
|
|
246
|
+
List of paths (each a list of types), all acyclic.
|
|
247
|
+
|
|
248
|
+
"""
|
|
249
|
+
|
|
250
|
+
if source_type == target_type:
|
|
251
|
+
return [[source_type]]
|
|
252
|
+
|
|
253
|
+
# Do we want parent -> child or child -> parent?
|
|
254
|
+
type_graph = self._type_graph if direction == "child" else self._type_graph_inverted
|
|
255
|
+
|
|
256
|
+
all_pathes = []
|
|
257
|
+
stack = [(source_type, [source_type])] # (current_type, path_so_far)
|
|
258
|
+
|
|
259
|
+
while stack:
|
|
260
|
+
current, path = stack.pop()
|
|
261
|
+
|
|
262
|
+
for neighbor in type_graph.get(current, []):
|
|
263
|
+
if neighbor in path:
|
|
264
|
+
continue # avoid cycles
|
|
265
|
+
new_path = path + [neighbor]
|
|
266
|
+
if neighbor == target_type:
|
|
267
|
+
all_pathes.append(new_path)
|
|
268
|
+
else:
|
|
269
|
+
stack.append((neighbor, new_path))
|
|
270
|
+
|
|
271
|
+
return all_pathes
|
|
272
|
+
|
|
273
|
+
# end method definition
|
|
274
|
+
|
|
275
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="build_graph")
|
|
276
|
+
def build_graph(
|
|
277
|
+
self,
|
|
278
|
+
workspace_type_exclusions: str | list | None = None,
|
|
279
|
+
workspace_type_inclusions: str | list | None = None,
|
|
280
|
+
filter_at_traversal: bool = False,
|
|
281
|
+
relationship_types: list | None = None,
|
|
282
|
+
metadata: bool = False,
|
|
283
|
+
workers: int = 3,
|
|
284
|
+
strategy: str = "BFS",
|
|
285
|
+
max_depth: int | None = None,
|
|
286
|
+
timeout: float = 60.0,
|
|
287
|
+
) -> dict:
|
|
288
|
+
"""Build the knowledge graph by traversing all workspaces and their relationships.
|
|
289
|
+
|
|
290
|
+
Args:
|
|
291
|
+
workspace_type_exclusions (str | list | None, optional):
|
|
292
|
+
List of workspace types to exclude. Can be a single workspace type
|
|
293
|
+
or a list of workspace types. None = inactive.
|
|
294
|
+
workspace_type_inclusions (str | list | None, optional):
|
|
295
|
+
List of workspace types to include. Can be a single workspace type
|
|
296
|
+
or a list of workspace types. None = inactive.
|
|
297
|
+
filter_at_traversal (bool, optional):
|
|
298
|
+
If False (default) the inclusion and exclusion filters are only tested for the
|
|
299
|
+
queue initialization not the traversal via workspace relationships.
|
|
300
|
+
If True the inclusion and exclusion filters are also tested
|
|
301
|
+
during the traversal of workspace relationships.
|
|
302
|
+
relationship_types (list | None, optional):
|
|
303
|
+
The default that will be established if None is provided is ["child", "parent"].
|
|
304
|
+
metadata (bool, optional):
|
|
305
|
+
If True, metadata for the workspace nodes will be included.
|
|
306
|
+
workers (int, optional):
|
|
307
|
+
The number of parallel working threads. Defaults to 3.
|
|
308
|
+
strategy (str, optional):
|
|
309
|
+
Either "DFS" for Depth First Search, or "BFS" for Breadth First Search.
|
|
310
|
+
"BFS" is the default.
|
|
311
|
+
max_depth (int | None, optional):
|
|
312
|
+
The maximum traversal depth. Defaults to None = unlimited.
|
|
313
|
+
timeout (float, optional):
|
|
314
|
+
Wait time for the queue to have items. This is also the time it
|
|
315
|
+
takes at the end to detect the workers are done. So expect delay
|
|
316
|
+
if you raise it high!
|
|
317
|
+
|
|
318
|
+
Returns:
|
|
319
|
+
dict:
|
|
320
|
+
The number of traversed and processed workspaces.
|
|
321
|
+
|
|
322
|
+
"""
|
|
323
|
+
|
|
324
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="build_node")
|
|
325
|
+
def build_node(workspace_node: dict, metadata: bool = False, **kwargs: dict) -> tuple[bool, bool]:
|
|
326
|
+
"""Add a node to the Knowledge Graph by inserting into the nodes data frame.
|
|
327
|
+
|
|
328
|
+
This method will be called back from the OTCS traverse_workspaces_parallel()
|
|
329
|
+
for each traversed workspace node.
|
|
330
|
+
|
|
331
|
+
Args:
|
|
332
|
+
workspace_node (dict):
|
|
333
|
+
The workspace node.
|
|
334
|
+
kwargs (dict):
|
|
335
|
+
Optional additional parameters.
|
|
336
|
+
metadata (bool, optional):
|
|
337
|
+
If True, metadata for the workspace nodes will be included.
|
|
338
|
+
|
|
339
|
+
Returns:
|
|
340
|
+
bool:
|
|
341
|
+
Whether or not the operation was successful.
|
|
342
|
+
bool:
|
|
343
|
+
Whether or not we require further traversal.
|
|
344
|
+
|
|
345
|
+
"""
|
|
346
|
+
|
|
347
|
+
workspace_id = self._otcs.get_result_value(response=workspace_node, key=self.WORKSPACE_ID_FIELD)
|
|
348
|
+
workspace_name = self._otcs.get_result_value(response=workspace_node, key=self.WORKSPACE_NAME_FIELD)
|
|
349
|
+
workspace_type = self.workspace_types[
|
|
350
|
+
self._otcs.get_result_value(response=workspace_node, key=self.WORKSPACE_TYPE_FIELD)
|
|
351
|
+
]
|
|
352
|
+
data = {
|
|
353
|
+
"id": workspace_id,
|
|
354
|
+
"name": workspace_name,
|
|
355
|
+
"type": workspace_type,
|
|
356
|
+
**kwargs, # ← allows adding more attributes from caller
|
|
357
|
+
}
|
|
358
|
+
if metadata:
|
|
359
|
+
response = self._otcs.get_workspace(node_id=workspace_id, fields="categories", metadata=True)
|
|
360
|
+
if response:
|
|
361
|
+
data["attributes"] = self._otcs.extract_category_data(node=response)
|
|
362
|
+
with self._nodes.lock():
|
|
363
|
+
self._nodes.append(data)
|
|
364
|
+
return (True, True)
|
|
365
|
+
|
|
366
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="build_edge")
|
|
367
|
+
def build_edge(
|
|
368
|
+
workspace_node_from: dict, workspace_node_to: dict, rel_type: str = "child", **kwargs: dict | None
|
|
369
|
+
) -> tuple[bool, bool]:
|
|
370
|
+
"""Add an edge to the Knowledge Graph by inserting into the edges data frame.
|
|
371
|
+
|
|
372
|
+
This method will be called back from the OTCS traverse_workspaces_parallel()
|
|
373
|
+
for each traversed workspace relationship (edge).
|
|
374
|
+
|
|
375
|
+
Args:
|
|
376
|
+
workspace_node_from (dict):
|
|
377
|
+
The source workspace node.
|
|
378
|
+
workspace_node_to (dict):
|
|
379
|
+
The target workspace node.
|
|
380
|
+
rel_type (str, optional):
|
|
381
|
+
The relationship type ("child" or "parent").
|
|
382
|
+
kwargs (dict, optional):
|
|
383
|
+
Optional additional parameters.
|
|
384
|
+
|
|
385
|
+
Returns:
|
|
386
|
+
bool:
|
|
387
|
+
Whether or not the operation was successful.
|
|
388
|
+
bool:
|
|
389
|
+
Whether or not we require further traversal.
|
|
390
|
+
|
|
391
|
+
"""
|
|
392
|
+
|
|
393
|
+
workspace_source_id = self._otcs.get_result_value(response=workspace_node_from, key=self.WORKSPACE_ID_FIELD)
|
|
394
|
+
workspace_target_id = self._otcs.get_result_value(response=workspace_node_to, key=self.WORKSPACE_ID_FIELD)
|
|
395
|
+
workspace_source_type = self._otcs.get_result_value(
|
|
396
|
+
response=workspace_node_from, key=self.WORKSPACE_TYPE_FIELD
|
|
397
|
+
)
|
|
398
|
+
workspace_target_type = self._otcs.get_result_value(
|
|
399
|
+
response=workspace_node_to, key=self.WORKSPACE_TYPE_FIELD
|
|
400
|
+
)
|
|
401
|
+
with self._edges.lock():
|
|
402
|
+
self._edges.append(
|
|
403
|
+
{
|
|
404
|
+
"source_type": workspace_source_type,
|
|
405
|
+
"source_id": workspace_source_id,
|
|
406
|
+
"target_type": workspace_target_type,
|
|
407
|
+
"target_id": workspace_target_id,
|
|
408
|
+
"relationship_type": rel_type,
|
|
409
|
+
"relationship_semantics": self.get_semantic_labels(
|
|
410
|
+
source_type=self.workspace_types.get(workspace_source_type, workspace_source_type),
|
|
411
|
+
target_type=self.workspace_types.get(workspace_target_type, workspace_target_type),
|
|
412
|
+
rel_type=rel_type,
|
|
413
|
+
),
|
|
414
|
+
**kwargs, # ← allows adding more attributes from caller
|
|
415
|
+
}
|
|
416
|
+
)
|
|
417
|
+
return (True, True)
|
|
418
|
+
|
|
419
|
+
#
|
|
420
|
+
# Start the actual traversal algorithm in OTCS:
|
|
421
|
+
#
|
|
422
|
+
result = self._otcs.traverse_workspaces_parallel(
|
|
423
|
+
workspace_type_exclusions=workspace_type_exclusions,
|
|
424
|
+
workspace_type_inclusions=workspace_type_inclusions,
|
|
425
|
+
filter_at_traversal=filter_at_traversal,
|
|
426
|
+
node_executables=[build_node],
|
|
427
|
+
relationship_executables=[build_edge],
|
|
428
|
+
relationship_types=relationship_types,
|
|
429
|
+
workers=workers,
|
|
430
|
+
strategy=strategy,
|
|
431
|
+
max_depth=max_depth,
|
|
432
|
+
timeout=timeout,
|
|
433
|
+
metadata=metadata,
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
return result
|
|
437
|
+
|
|
438
|
+
# end method definition
|
|
439
|
+
|
|
440
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="get_semantic_labels")
|
|
441
|
+
def get_semantic_labels(self, source_type: str, target_type: str, rel_type: str) -> list:
|
|
442
|
+
"""Resolve semantic labels from the ontology.
|
|
443
|
+
|
|
444
|
+
Args:
|
|
445
|
+
source_type:
|
|
446
|
+
Type of the source workspace.
|
|
447
|
+
target_type:
|
|
448
|
+
Type of the target workspace.
|
|
449
|
+
rel_type:
|
|
450
|
+
Raw relationship type. Either "parent" or "child".
|
|
451
|
+
|
|
452
|
+
Returns:
|
|
453
|
+
list:
|
|
454
|
+
A list of semantic labels for the relationship.
|
|
455
|
+
|
|
456
|
+
"""
|
|
457
|
+
|
|
458
|
+
key = (source_type, target_type, rel_type)
|
|
459
|
+
|
|
460
|
+
# As fallback option we just return a 1-item list with the technical relationship name.
|
|
461
|
+
return self._ontology.get(key, [rel_type])
|
|
462
|
+
|
|
463
|
+
# end method definition
|
|
464
|
+
|
|
465
|
+
@tracer.start_as_current_span(attributes=OTEL_TRACING_ATTRIBUTES, name="graph_query")
|
|
466
|
+
def graph_query(
|
|
467
|
+
self,
|
|
468
|
+
source_type: str,
|
|
469
|
+
source_value: str | int,
|
|
470
|
+
target_type: str,
|
|
471
|
+
target_value: str | int | None = None,
|
|
472
|
+
max_hops: int | None = None,
|
|
473
|
+
direction: str = "child",
|
|
474
|
+
intermediate_types: list[str] | None = None,
|
|
475
|
+
strict_intermediate_types: bool = False,
|
|
476
|
+
ordered_intermediate_types: bool = False,
|
|
477
|
+
) -> set[tuple[str, int]]:
|
|
478
|
+
"""Find target entities by using the Knowledge Graph.
|
|
479
|
+
|
|
480
|
+
Given a source entity (like Material:M-789), find target entities (like Customer)
|
|
481
|
+
by traversing the graph in BFS manner.
|
|
482
|
+
|
|
483
|
+
Args:
|
|
484
|
+
source_type (str):
|
|
485
|
+
Type of the starting node (e.g. "Material").
|
|
486
|
+
source_value (str | int):
|
|
487
|
+
The ID or name of the source node (e.g. "M-789").
|
|
488
|
+
target_type (str):
|
|
489
|
+
Desired result type (e.g. "Customer").
|
|
490
|
+
target_value (str | int | None, optional):
|
|
491
|
+
The value or name of the source node (e.g. "M-789").
|
|
492
|
+
max_hops (int | None, optional):
|
|
493
|
+
Limit on graph traversal depth. If None (default) there's no limit.
|
|
494
|
+
direction (str, optional):
|
|
495
|
+
Either "child" (source_id to target_id) or "parent" (target_id to source_id).
|
|
496
|
+
"child" is the default.
|
|
497
|
+
intermediate_types (list[str] | None, optional):
|
|
498
|
+
Types that must be traversed.
|
|
499
|
+
strict_intermediate_types (bool, optional):
|
|
500
|
+
Only allow source + intermediate + target types in path.
|
|
501
|
+
ordered_intermediate_types (bool, optional):
|
|
502
|
+
Enforce order of intermediate types.
|
|
503
|
+
|
|
504
|
+
Returns:
|
|
505
|
+
set:
|
|
506
|
+
Set of (name, id) tuples for matching nodes, e.g. {("Customer A", 123)}
|
|
507
|
+
|
|
508
|
+
"""
|
|
509
|
+
|
|
510
|
+
# Keep the linter happy:
|
|
511
|
+
_ = source_type, source_value
|
|
512
|
+
|
|
513
|
+
# Get the nodes and edges of the Knowledge Graph:
|
|
514
|
+
nodes_df = self.get_nodes().get_data_frame()
|
|
515
|
+
edges_df = self.get_edges().get_data_frame()
|
|
516
|
+
|
|
517
|
+
#
|
|
518
|
+
# 1. Find the starting node
|
|
519
|
+
#
|
|
520
|
+
source_node = nodes_df[
|
|
521
|
+
(nodes_df["type"] == source_type)
|
|
522
|
+
& (nodes_df["name"].astype(str).str.contains(str(source_value), case=False, na=False, regex=False))
|
|
523
|
+
]
|
|
524
|
+
if source_node.empty:
|
|
525
|
+
return set()
|
|
526
|
+
start_id = source_node.iloc[0]["id"]
|
|
527
|
+
|
|
528
|
+
#
|
|
529
|
+
# 2. Prepare the data structures for the BFS traversal:
|
|
530
|
+
#
|
|
531
|
+
visited = set()
|
|
532
|
+
queue = deque([(start_id, 0, [])]) # (node_id, depth, path_types)
|
|
533
|
+
results = set()
|
|
534
|
+
# Cache of build_type_pathes results
|
|
535
|
+
type_path_cache = {}
|
|
536
|
+
|
|
537
|
+
#
|
|
538
|
+
# 3. BFS from start_id to target_type:
|
|
539
|
+
#
|
|
540
|
+
while queue:
|
|
541
|
+
# Get the next node from the traversal queue
|
|
542
|
+
# (also updates the current depth and list of traversed types on the current path):
|
|
543
|
+
current_id, current_depth, path_types = queue.popleft()
|
|
544
|
+
|
|
545
|
+
# Check if the maximum depth is exceeded:
|
|
546
|
+
if max_hops is not None and current_depth > max_hops:
|
|
547
|
+
self.logger.info("Traversal has exceeded the given %d max hops!", max_hops)
|
|
548
|
+
continue # the while loop
|
|
549
|
+
|
|
550
|
+
# Get the Knowledge Graph node row for the current ID:
|
|
551
|
+
node_row = nodes_df[nodes_df["id"] == current_id]
|
|
552
|
+
if node_row.empty:
|
|
553
|
+
self.logger.error("Cannot find graph node with ID -> %d. This should never happen!", current_id)
|
|
554
|
+
continue # the while loop
|
|
555
|
+
|
|
556
|
+
current_type = node_row.iloc[0]["type"]
|
|
557
|
+
current_name = node_row.iloc[0]["name"]
|
|
558
|
+
|
|
559
|
+
# Check if this node has been traversed before. If yes, skip.
|
|
560
|
+
if current_id in visited:
|
|
561
|
+
self.logger.debug(
|
|
562
|
+
"Node -> '%s' (%d) of type -> '%s' has been visited before!%s",
|
|
563
|
+
current_name,
|
|
564
|
+
current_id,
|
|
565
|
+
current_type,
|
|
566
|
+
" But we have requirements for intermediate types that need to be checked."
|
|
567
|
+
if intermediate_types
|
|
568
|
+
else " We don't need to further traverse from here.",
|
|
569
|
+
)
|
|
570
|
+
# If we don't have intermediate types then one path to a node is
|
|
571
|
+
# sufficient and we can stop traversal here (otherwise we still
|
|
572
|
+
# stop traversal but want to check if current node matches all requirements):
|
|
573
|
+
if not intermediate_types:
|
|
574
|
+
continue # the while loop
|
|
575
|
+
|
|
576
|
+
# Simplified restriction: skip if current type is not allowed
|
|
577
|
+
if (
|
|
578
|
+
strict_intermediate_types
|
|
579
|
+
and intermediate_types
|
|
580
|
+
and current_type not in [source_type] + intermediate_types + [target_type]
|
|
581
|
+
):
|
|
582
|
+
self.logger.debug(
|
|
583
|
+
"The current node type -> '%s' is not allowed for this query. Stop further traversals for this node.",
|
|
584
|
+
current_type,
|
|
585
|
+
)
|
|
586
|
+
continue # the while loop
|
|
587
|
+
|
|
588
|
+
# Update path types for tracking
|
|
589
|
+
path_types = path_types + [current_type]
|
|
590
|
+
|
|
591
|
+
# Check if current node matches target_type:
|
|
592
|
+
if current_type == target_type:
|
|
593
|
+
# Check intermediate type constraints:
|
|
594
|
+
if intermediate_types:
|
|
595
|
+
if ordered_intermediate_types:
|
|
596
|
+
# Ordered check
|
|
597
|
+
index = 0
|
|
598
|
+
for t in path_types:
|
|
599
|
+
if t == intermediate_types[index]:
|
|
600
|
+
index += 1
|
|
601
|
+
if index == len(intermediate_types):
|
|
602
|
+
break
|
|
603
|
+
if index != len(intermediate_types):
|
|
604
|
+
self.logger.debug(
|
|
605
|
+
"Target node -> '%s' (%d) has the right type -> '%s' but path -> %s has not traversed intermediate types -> %s in the right ordering",
|
|
606
|
+
current_name,
|
|
607
|
+
current_id,
|
|
608
|
+
target_type,
|
|
609
|
+
path_types,
|
|
610
|
+
intermediate_types,
|
|
611
|
+
)
|
|
612
|
+
continue # Ordered constraint not fulfilled
|
|
613
|
+
elif not all(t in path_types for t in intermediate_types):
|
|
614
|
+
self.logger.debug(
|
|
615
|
+
"Target node -> '%s' (%d) has the right type -> '%s' but path -> %s has not traversed intermediate types -> %s",
|
|
616
|
+
current_name,
|
|
617
|
+
current_id,
|
|
618
|
+
target_type,
|
|
619
|
+
path_types,
|
|
620
|
+
intermediate_types,
|
|
621
|
+
)
|
|
622
|
+
continue # the while loop because unordered constraint not fulfilled
|
|
623
|
+
# end if intermediate_types:
|
|
624
|
+
|
|
625
|
+
# This is the actual check for the target instance value:
|
|
626
|
+
if (
|
|
627
|
+
target_value is not None
|
|
628
|
+
and target_value != current_name # exact match
|
|
629
|
+
and target_value.lower() not in current_name.lower() # partial match (substring)
|
|
630
|
+
):
|
|
631
|
+
self.logger.debug(
|
|
632
|
+
"Target node -> '%s' (%d) has the right type -> '%s' but not matching name or attributes (%s)",
|
|
633
|
+
current_name,
|
|
634
|
+
current_id,
|
|
635
|
+
target_type,
|
|
636
|
+
target_value,
|
|
637
|
+
)
|
|
638
|
+
continue
|
|
639
|
+
|
|
640
|
+
results.add((current_name, current_id))
|
|
641
|
+
self.logger.debug(
|
|
642
|
+
"Found node -> '%s' (%d) of desired target type -> '%s'%s%s",
|
|
643
|
+
current_name,
|
|
644
|
+
current_id,
|
|
645
|
+
target_type,
|
|
646
|
+
" and name or attributes -> {}".format(target_value) if target_value else "",
|
|
647
|
+
" via path -> {}.".format(path_types) if path_types else "",
|
|
648
|
+
)
|
|
649
|
+
# end if current_type == target_type:
|
|
650
|
+
|
|
651
|
+
# We need to check this once more because if intermediate_types are specified
|
|
652
|
+
# we need to continue until here...
|
|
653
|
+
if current_id in visited:
|
|
654
|
+
continue
|
|
655
|
+
visited.add(current_id)
|
|
656
|
+
|
|
657
|
+
# Get allowed neighbor types using cached paths
|
|
658
|
+
if (current_type, target_type) not in type_path_cache:
|
|
659
|
+
pathes = self.build_type_pathes(source_type=current_type, target_type=target_type, direction=direction)
|
|
660
|
+
type_path_cache[(current_type, target_type)] = pathes
|
|
661
|
+
else:
|
|
662
|
+
pathes = type_path_cache[(current_type, target_type)]
|
|
663
|
+
|
|
664
|
+
# Get allowed neighbor types from the type graph (pruning)
|
|
665
|
+
neighbor_types = (
|
|
666
|
+
set(self._type_graph.get(current_type, []))
|
|
667
|
+
if direction == "child"
|
|
668
|
+
else set(self._type_graph_inverted.get(current_type, []))
|
|
669
|
+
)
|
|
670
|
+
# Reduce that list to neighor types that are on a path to the target type:
|
|
671
|
+
allowed_types = {
|
|
672
|
+
allowed_type for allowed_type in neighbor_types if any(allowed_type in path for path in pathes)
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
removed_types = neighbor_types - allowed_types # this is a set difference!
|
|
676
|
+
if removed_types:
|
|
677
|
+
self.logger.debug(
|
|
678
|
+
"Remove traverse options -> %s for type node -> '%s' as they are not leading towards target type -> '%s'. Remaining types to travers -> %s. Initial neighbor types -> %s",
|
|
679
|
+
str(removed_types),
|
|
680
|
+
current_type,
|
|
681
|
+
target_type,
|
|
682
|
+
str(allowed_types),
|
|
683
|
+
str(neighbor_types),
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
# Determine all neighbors (= list of target IDs derived from the edges).
|
|
687
|
+
# The expression 'edges_df["source_id"] == current_id' creates a Boolean mask —
|
|
688
|
+
# a Series of True or False values — indicating which rows of the edges_df DataFrame
|
|
689
|
+
# have a source_id equal to the current_id.
|
|
690
|
+
# The double brackets ([['target_id']]) are important to return a DataFrame (a 2D table)
|
|
691
|
+
# and not just a series (column).
|
|
692
|
+
if direction == "child":
|
|
693
|
+
neighbors = edges_df[edges_df["source_id"] == current_id][["target_id"]]
|
|
694
|
+
neighbor_ids = neighbors["target_id"].tolist()
|
|
695
|
+
else: # direction = "parent" - here we switch source and target:
|
|
696
|
+
neighbors = edges_df[edges_df["target_id"] == current_id][["source_id"]]
|
|
697
|
+
neighbor_ids = neighbors["source_id"].tolist()
|
|
698
|
+
|
|
699
|
+
if neighbors.empty:
|
|
700
|
+
continue
|
|
701
|
+
|
|
702
|
+
# Get neighbor nodes with their types
|
|
703
|
+
neighbor_rows = nodes_df[nodes_df["id"].isin(neighbor_ids)][["id", "type"]]
|
|
704
|
+
|
|
705
|
+
# Filter neighbors by allowed types
|
|
706
|
+
filtered_neighbors = [
|
|
707
|
+
nid
|
|
708
|
+
for nid in neighbor_rows.itertuples(index=False)
|
|
709
|
+
if nid.type in allowed_types and nid.id not in visited
|
|
710
|
+
]
|
|
711
|
+
|
|
712
|
+
# Travers edges from current node to neighbors:
|
|
713
|
+
for neighbor in filtered_neighbors:
|
|
714
|
+
queue.append((neighbor.id, current_depth + 1, path_types))
|
|
715
|
+
# end while queue
|
|
716
|
+
|
|
717
|
+
return results
|
|
718
|
+
|
|
719
|
+
# end method definition
|