pyxecm 1.6__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of pyxecm might be problematic. Click here for more details.
- pyxecm/__init__.py +6 -4
- pyxecm/avts.py +673 -246
- pyxecm/coreshare.py +686 -467
- pyxecm/customizer/__init__.py +16 -4
- pyxecm/customizer/__main__.py +58 -0
- pyxecm/customizer/api/__init__.py +5 -0
- pyxecm/customizer/api/__main__.py +6 -0
- pyxecm/customizer/api/app.py +914 -0
- pyxecm/customizer/api/auth.py +154 -0
- pyxecm/customizer/api/metrics.py +92 -0
- pyxecm/customizer/api/models.py +13 -0
- pyxecm/customizer/api/payload_list.py +865 -0
- pyxecm/customizer/api/settings.py +103 -0
- pyxecm/customizer/browser_automation.py +332 -139
- pyxecm/customizer/customizer.py +1007 -1130
- pyxecm/customizer/exceptions.py +35 -0
- pyxecm/customizer/guidewire.py +322 -0
- pyxecm/customizer/k8s.py +713 -378
- pyxecm/customizer/log.py +107 -0
- pyxecm/customizer/m365.py +2867 -909
- pyxecm/customizer/nhc.py +1169 -0
- pyxecm/customizer/openapi.py +258 -0
- pyxecm/customizer/payload.py +16817 -7467
- pyxecm/customizer/pht.py +699 -285
- pyxecm/customizer/salesforce.py +516 -342
- pyxecm/customizer/sap.py +58 -41
- pyxecm/customizer/servicenow.py +593 -371
- pyxecm/customizer/settings.py +442 -0
- pyxecm/customizer/successfactors.py +408 -346
- pyxecm/customizer/translate.py +83 -48
- pyxecm/helper/__init__.py +5 -2
- pyxecm/helper/assoc.py +83 -43
- pyxecm/helper/data.py +2406 -870
- pyxecm/helper/logadapter.py +27 -0
- pyxecm/helper/web.py +229 -101
- pyxecm/helper/xml.py +527 -171
- pyxecm/maintenance_page/__init__.py +5 -0
- pyxecm/maintenance_page/__main__.py +6 -0
- pyxecm/maintenance_page/app.py +51 -0
- pyxecm/maintenance_page/settings.py +28 -0
- pyxecm/maintenance_page/static/favicon.avif +0 -0
- pyxecm/maintenance_page/templates/maintenance.html +165 -0
- pyxecm/otac.py +234 -140
- pyxecm/otawp.py +1436 -557
- pyxecm/otcs.py +7716 -3161
- pyxecm/otds.py +2150 -919
- pyxecm/otiv.py +36 -21
- pyxecm/otmm.py +1272 -325
- pyxecm/otpd.py +231 -127
- pyxecm-2.0.0.dist-info/METADATA +145 -0
- pyxecm-2.0.0.dist-info/RECORD +54 -0
- {pyxecm-1.6.dist-info → pyxecm-2.0.0.dist-info}/WHEEL +1 -1
- pyxecm-1.6.dist-info/METADATA +0 -53
- pyxecm-1.6.dist-info/RECORD +0 -32
- {pyxecm-1.6.dist-info → pyxecm-2.0.0.dist-info/licenses}/LICENSE +0 -0
- {pyxecm-1.6.dist-info → pyxecm-2.0.0.dist-info}/top_level.txt +0 -0
pyxecm/avts.py
CHANGED
|
@@ -1,35 +1,20 @@
|
|
|
1
|
-
"""
|
|
2
|
-
AVTS stands for Aviator Search and is an OpenText offering for LLMM-based search across multiple repositories
|
|
3
|
-
|
|
4
|
-
Class: AVTS
|
|
5
|
-
Methods:
|
|
6
|
-
__init__: class initializer
|
|
7
|
-
request_header: Returns the request header used for Application calls.
|
|
8
|
-
do_request: Call an Aviator Search REST API in a safe way
|
|
9
|
-
parse_request_response: Converts the request response (JSon) to a Python list in a safe way
|
|
10
|
-
authenticate: Authenticate at Search Aviator via oAuth authentication
|
|
11
|
-
repo_create_extended_ecm: Create a new repository to crawl in Aviator Search
|
|
12
|
-
start_crawling: Start crawling of a repository
|
|
13
|
-
stop_crawling: Stop the crawling of a repository
|
|
14
|
-
get_repo_list: Get a list of all repositories
|
|
15
|
-
get_repo_by_name: Get a repository by name
|
|
16
|
-
"""
|
|
1
|
+
"""AVTS stands for Aviator Search and is an OpenText offering for LLMM-based search across multiple repositories."""
|
|
17
2
|
|
|
18
3
|
__author__ = "Dr. Marc Diefenbruch"
|
|
19
|
-
__copyright__ = "Copyright 2024, OpenText"
|
|
4
|
+
__copyright__ = "Copyright (C) 2024-2025, OpenText"
|
|
20
5
|
__credits__ = ["Kai-Philip Gatzweiler"]
|
|
21
6
|
__maintainer__ = "Dr. Marc Diefenbruch"
|
|
22
7
|
__email__ = "mdiefenb@opentext.com"
|
|
23
8
|
|
|
9
|
+
import base64
|
|
24
10
|
import json
|
|
25
11
|
import logging
|
|
26
|
-
import time
|
|
27
12
|
import os
|
|
28
|
-
import
|
|
13
|
+
import time
|
|
29
14
|
|
|
30
15
|
import requests
|
|
31
16
|
|
|
32
|
-
|
|
17
|
+
default_logger = logging.getLogger("pyxecm.customizer.avts")
|
|
33
18
|
|
|
34
19
|
REQUEST_HEADERS = {"Accept": "application/json", "Content-Type": "application/json"}
|
|
35
20
|
|
|
@@ -38,8 +23,10 @@ REQUEST_RETRY_DELAY = 20
|
|
|
38
23
|
REQUEST_MAX_RETRIES = 2
|
|
39
24
|
|
|
40
25
|
|
|
41
|
-
class AVTS
|
|
42
|
-
"""
|
|
26
|
+
class AVTS:
|
|
27
|
+
"""Configure and interact with Aviator Search REST API."""
|
|
28
|
+
|
|
29
|
+
logger: logging.Logger = default_logger
|
|
43
30
|
|
|
44
31
|
_config: dict
|
|
45
32
|
_session = None
|
|
@@ -52,18 +39,33 @@ class AVTS(object):
|
|
|
52
39
|
base_url: str,
|
|
53
40
|
username: str,
|
|
54
41
|
password: str,
|
|
55
|
-
|
|
56
|
-
|
|
42
|
+
logger: logging.Logger = default_logger,
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Initialize the AVTS object.
|
|
57
45
|
|
|
58
46
|
Args:
|
|
59
|
-
otds_url (str):
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
47
|
+
otds_url (str):
|
|
48
|
+
The URL of the OTDS Server used by Aviator Search.
|
|
49
|
+
client_id (str):
|
|
50
|
+
The client ID for the Aviator Search oAuth client.
|
|
51
|
+
client_secret (str):
|
|
52
|
+
The client secret for the Aviator Search oAuth client.
|
|
53
|
+
base_url (str):
|
|
54
|
+
The Aviator Search base URL.
|
|
55
|
+
username (str):
|
|
56
|
+
User with administrative permissions in Aviator Search.
|
|
57
|
+
password (str):
|
|
58
|
+
Password of the user with administrative permissions in Aviator Search.
|
|
59
|
+
logger (logging.Logger, optional):
|
|
60
|
+
The logging object to use for all log messages. Defaults to default_logger.
|
|
61
|
+
|
|
65
62
|
"""
|
|
66
63
|
|
|
64
|
+
if logger != default_logger:
|
|
65
|
+
self.logger = logger.getChild("avts")
|
|
66
|
+
for logfilter in logger.filters:
|
|
67
|
+
self.logger.addFilter(logfilter)
|
|
68
|
+
|
|
67
69
|
avts_config = {}
|
|
68
70
|
|
|
69
71
|
# Store the credentials and parameters in a config dictionary:
|
|
@@ -75,9 +77,7 @@ class AVTS(object):
|
|
|
75
77
|
avts_config["password"] = password
|
|
76
78
|
|
|
77
79
|
avts_config["tokenUrl"] = avts_config["otdsUrl"] + "/otdsws/oauth2/token"
|
|
78
|
-
avts_config["repoUrl"] =
|
|
79
|
-
avts_config["baseUrl"] + "/aviator-gateway/avts-api/admin/v1/repo"
|
|
80
|
-
)
|
|
80
|
+
avts_config["repoUrl"] = avts_config["baseUrl"] + "/aviator-gateway/avts-api/admin/v1/repo"
|
|
81
81
|
|
|
82
82
|
self._config = avts_config
|
|
83
83
|
self._accesstoken = None
|
|
@@ -87,23 +87,34 @@ class AVTS(object):
|
|
|
87
87
|
# end method definition
|
|
88
88
|
|
|
89
89
|
def config(self) -> dict:
|
|
90
|
-
"""
|
|
90
|
+
"""Return the configuration dictionary.
|
|
91
91
|
|
|
92
92
|
Returns:
|
|
93
93
|
dict: Configuration dictionary
|
|
94
|
+
|
|
94
95
|
"""
|
|
96
|
+
|
|
95
97
|
return self._config
|
|
96
98
|
|
|
97
99
|
# end method definition
|
|
98
100
|
|
|
99
101
|
def request_header(self, content_type: str = "") -> dict:
|
|
100
|
-
"""
|
|
101
|
-
|
|
102
|
+
"""Return the request header used for Application calls.
|
|
103
|
+
|
|
104
|
+
Consists of Bearer access token and Content Type
|
|
102
105
|
|
|
103
106
|
Args:
|
|
104
|
-
content_type (str, optional):
|
|
105
|
-
|
|
106
|
-
|
|
107
|
+
content_type (str, optional):
|
|
108
|
+
Custom content type for the request.
|
|
109
|
+
Typical values:
|
|
110
|
+
* application/json - Used for sending JSON-encoded data
|
|
111
|
+
* application/x-www-form-urlencoded - The default for HTML forms.
|
|
112
|
+
Data is sent as key-value pairs in the body of the request, similar to query parameters.
|
|
113
|
+
* multipart/form-data - Used for file uploads or when a form includes non-ASCII characters
|
|
114
|
+
|
|
115
|
+
Returns:
|
|
116
|
+
dict: The request header values.
|
|
117
|
+
|
|
107
118
|
"""
|
|
108
119
|
|
|
109
120
|
request_header = {}
|
|
@@ -135,24 +146,40 @@ class AVTS(object):
|
|
|
135
146
|
max_retries: int = REQUEST_MAX_RETRIES,
|
|
136
147
|
retry_forever: bool = False,
|
|
137
148
|
) -> dict | None:
|
|
138
|
-
"""Call an Aviator Search REST API in a safe way
|
|
149
|
+
"""Call an Aviator Search REST API in a safe way.
|
|
139
150
|
|
|
140
151
|
Args:
|
|
141
|
-
url (str):
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
152
|
+
url (str):
|
|
153
|
+
URL to send the request to.
|
|
154
|
+
method (str, optional):
|
|
155
|
+
HTTP method (GET, POST, etc.). Defaults to "GET".
|
|
156
|
+
headers (dict | None, optional):
|
|
157
|
+
Request headers. Defaults to None.
|
|
158
|
+
data (dict | None, optional):
|
|
159
|
+
Request payload. Defaults to None.
|
|
160
|
+
json_data (dict | None, optional):
|
|
161
|
+
Request payload for the JSON parameter. Defaults to None.
|
|
162
|
+
files (dict | None, optional):
|
|
163
|
+
Dictionary of {"name": file-tuple} for multipart encoding upload.
|
|
164
|
+
The file-tuple can be a 2-tuple ("filename", fileobj) or a 3-tuple
|
|
165
|
+
("filename", fileobj, "content_type").
|
|
166
|
+
timeout (int | None, optional):
|
|
167
|
+
Timeout for the request in seconds. Defaults to REQUEST_TIMEOUT.
|
|
168
|
+
show_error (bool, optional):
|
|
169
|
+
Whether or not an error should be logged in case of a failed REST call.
|
|
170
|
+
If False, then only a warning is logged. Defaults to True.
|
|
171
|
+
failure_message (str, optional):
|
|
172
|
+
Specific error message. Defaults to "".
|
|
173
|
+
success_message (str, optional):
|
|
174
|
+
Specific success message. Defaults to "".
|
|
175
|
+
max_retries (int, optional):
|
|
176
|
+
Number of retries on connection errors. Defaults to REQUEST_MAX_RETRIES.
|
|
177
|
+
retry_forever (bool, optional):
|
|
178
|
+
Whether to wait forever without timeout. Defaults to False.
|
|
153
179
|
|
|
154
180
|
Returns:
|
|
155
181
|
dict | None: Response of Aviator Search REST API or None in case of an error.
|
|
182
|
+
|
|
156
183
|
"""
|
|
157
184
|
|
|
158
185
|
retries = 0
|
|
@@ -170,30 +197,27 @@ class AVTS(object):
|
|
|
170
197
|
|
|
171
198
|
if response.ok:
|
|
172
199
|
if success_message:
|
|
173
|
-
logger.debug(success_message)
|
|
200
|
+
self.logger.debug(success_message)
|
|
174
201
|
return self.parse_request_response(response)
|
|
175
202
|
# Check if Session has expired - then re-authenticate and try once more
|
|
176
203
|
elif response.status_code == 401 and retries == 0:
|
|
177
|
-
logger.debug("Session has expired - try to re-authenticate...")
|
|
204
|
+
self.logger.debug("Session has expired - try to re-authenticate...")
|
|
178
205
|
self.authenticate()
|
|
179
206
|
retries += 1
|
|
180
207
|
else:
|
|
181
208
|
# Handle plain HTML responses to not pollute the logs
|
|
182
209
|
content_type = response.headers.get("content-type", None)
|
|
183
|
-
if content_type == "text/html"
|
|
184
|
-
response_text = "HTML content (see debug log)"
|
|
185
|
-
else:
|
|
186
|
-
response_text = response.text
|
|
210
|
+
response_text = "HTML content (see debug log)" if content_type == "text/html" else response.text
|
|
187
211
|
|
|
188
212
|
if show_error:
|
|
189
|
-
logger.error(
|
|
213
|
+
self.logger.error(
|
|
190
214
|
"%s; status -> %s; error -> %s",
|
|
191
215
|
failure_message,
|
|
192
216
|
response.status_code,
|
|
193
217
|
response_text,
|
|
194
218
|
)
|
|
195
219
|
else:
|
|
196
|
-
logger.warning(
|
|
220
|
+
self.logger.warning(
|
|
197
221
|
"%s; status -> %s; warning -> %s",
|
|
198
222
|
failure_message,
|
|
199
223
|
response.status_code,
|
|
@@ -201,7 +225,7 @@ class AVTS(object):
|
|
|
201
225
|
)
|
|
202
226
|
|
|
203
227
|
if content_type == "text/html":
|
|
204
|
-
logger.debug(
|
|
228
|
+
self.logger.debug(
|
|
205
229
|
"%s; status -> %s; warning -> %s",
|
|
206
230
|
failure_message,
|
|
207
231
|
response.status_code,
|
|
@@ -211,39 +235,39 @@ class AVTS(object):
|
|
|
211
235
|
return None
|
|
212
236
|
except requests.exceptions.Timeout:
|
|
213
237
|
if retries <= max_retries:
|
|
214
|
-
logger.warning(
|
|
238
|
+
self.logger.warning(
|
|
215
239
|
"Request timed out. Retrying in %s seconds...",
|
|
216
240
|
str(REQUEST_RETRY_DELAY),
|
|
217
241
|
)
|
|
218
242
|
retries += 1
|
|
219
243
|
time.sleep(REQUEST_RETRY_DELAY) # Add a delay before retrying
|
|
220
244
|
else:
|
|
221
|
-
logger.error(
|
|
222
|
-
"%s; timeout error",
|
|
245
|
+
self.logger.error(
|
|
246
|
+
"%s; timeout error.",
|
|
223
247
|
failure_message,
|
|
224
248
|
)
|
|
225
249
|
if retry_forever:
|
|
226
250
|
# If it fails after REQUEST_MAX_RETRIES retries we let it wait forever
|
|
227
|
-
logger.warning("Turn timeouts off and wait forever...")
|
|
251
|
+
self.logger.warning("Turn timeouts off and wait forever...")
|
|
228
252
|
timeout = None
|
|
229
253
|
else:
|
|
230
254
|
return None
|
|
231
255
|
except requests.exceptions.ConnectionError:
|
|
232
256
|
if retries <= max_retries:
|
|
233
|
-
logger.warning(
|
|
257
|
+
self.logger.warning(
|
|
234
258
|
"Connection error. Retrying in %s seconds...",
|
|
235
259
|
str(REQUEST_RETRY_DELAY),
|
|
236
260
|
)
|
|
237
261
|
retries += 1
|
|
238
262
|
time.sleep(REQUEST_RETRY_DELAY) # Add a delay before retrying
|
|
239
263
|
else:
|
|
240
|
-
logger.error(
|
|
241
|
-
"%s; connection error",
|
|
264
|
+
self.logger.error(
|
|
265
|
+
"%s; connection error.",
|
|
242
266
|
failure_message,
|
|
243
267
|
)
|
|
244
268
|
if retry_forever:
|
|
245
269
|
# If it fails after REQUEST_MAX_RETRIES retries we let it wait forever
|
|
246
|
-
logger.warning("Turn timeouts off and wait forever...")
|
|
270
|
+
self.logger.warning("Turn timeouts off and wait forever...")
|
|
247
271
|
timeout = None
|
|
248
272
|
time.sleep(REQUEST_RETRY_DELAY) # Add a delay before retrying
|
|
249
273
|
else:
|
|
@@ -257,43 +281,47 @@ class AVTS(object):
|
|
|
257
281
|
additional_error_message: str = "",
|
|
258
282
|
show_error: bool = True,
|
|
259
283
|
) -> list | None:
|
|
260
|
-
"""
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
284
|
+
"""Convert the request response (JSon) to a Python list in a safe way that also handles exceptions.
|
|
285
|
+
|
|
286
|
+
It first tries to load the response.text
|
|
287
|
+
via json.loads() that produces a dict output. Only if response.text is
|
|
288
|
+
not set or is empty it just converts the response_object to a dict using
|
|
289
|
+
the vars() built-in method.
|
|
265
290
|
|
|
266
291
|
Args:
|
|
267
|
-
response_object (object):
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
292
|
+
response_object (object):
|
|
293
|
+
This is reponse object delivered by the request call.
|
|
294
|
+
additional_error_message (str, optional):
|
|
295
|
+
Use a more specific error message in case of an error.
|
|
296
|
+
show_error (bool, optional):
|
|
297
|
+
If True, write an error to the log file.
|
|
298
|
+
If False, write a warning to the log file.
|
|
299
|
+
|
|
272
300
|
Returns:
|
|
273
|
-
list
|
|
301
|
+
list | None:
|
|
302
|
+
The response information or None in case of an error.
|
|
303
|
+
|
|
274
304
|
"""
|
|
275
305
|
|
|
276
306
|
if not response_object:
|
|
277
307
|
return None
|
|
278
308
|
|
|
279
309
|
try:
|
|
280
|
-
if response_object.text
|
|
281
|
-
list_object = json.loads(response_object.text)
|
|
282
|
-
else:
|
|
283
|
-
list_object = vars(response_object)
|
|
310
|
+
list_object = json.loads(response_object.text) if response_object.text else vars(response_object)
|
|
284
311
|
except json.JSONDecodeError as exception:
|
|
285
312
|
if additional_error_message:
|
|
286
313
|
message = "Cannot decode response as JSON. {}; error -> {}".format(
|
|
287
|
-
additional_error_message,
|
|
314
|
+
additional_error_message,
|
|
315
|
+
exception,
|
|
288
316
|
)
|
|
289
317
|
else:
|
|
290
318
|
message = "Cannot decode response as JSON; error -> {}".format(
|
|
291
|
-
exception
|
|
319
|
+
exception,
|
|
292
320
|
)
|
|
293
321
|
if show_error:
|
|
294
|
-
logger.error(message)
|
|
322
|
+
self.logger.error(message)
|
|
295
323
|
else:
|
|
296
|
-
logger.warning(message)
|
|
324
|
+
self.logger.warning(message)
|
|
297
325
|
return None
|
|
298
326
|
else:
|
|
299
327
|
return list_object
|
|
@@ -301,7 +329,13 @@ class AVTS(object):
|
|
|
301
329
|
# end method definition
|
|
302
330
|
|
|
303
331
|
def authenticate(self) -> str | None:
|
|
304
|
-
"""Authenticate at Search
|
|
332
|
+
"""Authenticate at Aviator Search via OAuth.
|
|
333
|
+
|
|
334
|
+
Returns:
|
|
335
|
+
str | None:
|
|
336
|
+
The access token or None in case of an error.
|
|
337
|
+
|
|
338
|
+
"""
|
|
305
339
|
|
|
306
340
|
if not self._session:
|
|
307
341
|
self._session = requests.Session()
|
|
@@ -319,6 +353,7 @@ class AVTS(object):
|
|
|
319
353
|
"client_secret": self.config()["clientSecret"],
|
|
320
354
|
"username": self.config()["username"],
|
|
321
355
|
"password": self.config()["password"],
|
|
356
|
+
"scope": "otds:roles",
|
|
322
357
|
}
|
|
323
358
|
|
|
324
359
|
response = self.do_request(
|
|
@@ -327,7 +362,10 @@ class AVTS(object):
|
|
|
327
362
|
headers=request_header,
|
|
328
363
|
data=request_payload,
|
|
329
364
|
timeout=None,
|
|
330
|
-
failure_message=
|
|
365
|
+
failure_message="Failed to authenticate to OTDS with username -> {} and client_id -> {}".format(
|
|
366
|
+
self.config()["username"],
|
|
367
|
+
self.config()["clientId"],
|
|
368
|
+
),
|
|
331
369
|
)
|
|
332
370
|
|
|
333
371
|
if response is not None:
|
|
@@ -337,7 +375,7 @@ class AVTS(object):
|
|
|
337
375
|
|
|
338
376
|
# end method definition
|
|
339
377
|
|
|
340
|
-
def
|
|
378
|
+
def create_extended_ecm_repo(
|
|
341
379
|
self,
|
|
342
380
|
name: str,
|
|
343
381
|
username: str,
|
|
@@ -345,128 +383,223 @@ class AVTS(object):
|
|
|
345
383
|
otcs_url: str,
|
|
346
384
|
otcs_api_url: str,
|
|
347
385
|
node_id: int,
|
|
348
|
-
version: str = "24.3.0",
|
|
349
386
|
) -> dict | None:
|
|
350
|
-
"""Create a new repository to crawl
|
|
387
|
+
"""Create a new Extended ECM repository to crawl with Aviator Search.
|
|
351
388
|
|
|
352
389
|
Args:
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
username (str):
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
390
|
+
name (str):
|
|
391
|
+
The name of the repository.
|
|
392
|
+
username (str):
|
|
393
|
+
Username to use for crawling.
|
|
394
|
+
password (str):
|
|
395
|
+
Password of the user used for crawling.
|
|
396
|
+
otcs_url (str):
|
|
397
|
+
Base URL of Content Server e.g. https://otcs.base-url.tld/cs/cs
|
|
398
|
+
otcs_api_url (str):
|
|
399
|
+
The REST API URL of Content Server.
|
|
400
|
+
node_id (int):
|
|
401
|
+
Root Node ID for crawling
|
|
402
|
+
version (str, optional):
|
|
403
|
+
TODO: The version number of ???
|
|
359
404
|
|
|
360
405
|
Returns:
|
|
361
|
-
dict | None:
|
|
406
|
+
dict | None:
|
|
407
|
+
Parsed response object from the API or None in case of an error
|
|
408
|
+
|
|
362
409
|
"""
|
|
363
410
|
|
|
364
411
|
payload = {
|
|
365
|
-
"
|
|
366
|
-
"name": name,
|
|
367
|
-
"metadataFields": ["NODE"],
|
|
368
|
-
"socName": "xECM",
|
|
412
|
+
"authType": "Basic",
|
|
369
413
|
"params": [
|
|
370
414
|
{
|
|
371
415
|
"id": "OpenTextApiUrl",
|
|
372
|
-
"label": "
|
|
416
|
+
"label": "Service URL",
|
|
373
417
|
"ctlType": "text",
|
|
418
|
+
"description": "OpenText Content Management API URL",
|
|
374
419
|
"required": True,
|
|
420
|
+
"defaultValue": "localhost",
|
|
421
|
+
"visible": True,
|
|
422
|
+
"editable": False,
|
|
375
423
|
"value": otcs_api_url,
|
|
376
424
|
},
|
|
377
425
|
{
|
|
378
426
|
"id": "Username",
|
|
379
|
-
"label": "
|
|
427
|
+
"label": "Username",
|
|
380
428
|
"ctlType": "text",
|
|
429
|
+
"description": "OpenText Content Management Username",
|
|
381
430
|
"required": True,
|
|
431
|
+
"defaultValue": "",
|
|
432
|
+
"visible": True,
|
|
433
|
+
"editable": True,
|
|
382
434
|
"value": username,
|
|
383
435
|
},
|
|
384
436
|
{
|
|
385
437
|
"id": "Password",
|
|
386
|
-
"label": "
|
|
438
|
+
"label": "Password",
|
|
387
439
|
"ctlType": "password",
|
|
440
|
+
"description": "OpenText Content Management password",
|
|
388
441
|
"required": True,
|
|
442
|
+
"defaultValue": "",
|
|
443
|
+
"visible": True,
|
|
444
|
+
"editable": True,
|
|
389
445
|
"value": password,
|
|
390
446
|
},
|
|
391
447
|
{
|
|
392
|
-
"id": "
|
|
393
|
-
"label": "
|
|
448
|
+
"id": "sourceLink",
|
|
449
|
+
"label": "Source Link",
|
|
394
450
|
"ctlType": "text",
|
|
451
|
+
"description": "Example: <OpenText Content Management API URL>/app/nodes/${NODE}/metadata",
|
|
452
|
+
"required": False,
|
|
453
|
+
"defaultValue": otcs_url + "/app/nodes/${NODE}/metadata",
|
|
454
|
+
"visible": True,
|
|
455
|
+
"editable": True,
|
|
456
|
+
},
|
|
457
|
+
{
|
|
458
|
+
"id": "RootNodeIds",
|
|
459
|
+
"label": "Root Node ID's",
|
|
460
|
+
"ctlType": "text",
|
|
461
|
+
"description": "List of nodes to be crawled(comma seperated)",
|
|
395
462
|
"required": True,
|
|
396
|
-
"
|
|
463
|
+
"defaultValue": "",
|
|
464
|
+
"visible": True,
|
|
465
|
+
"editable": False,
|
|
466
|
+
"value": "2000",
|
|
397
467
|
},
|
|
398
468
|
{
|
|
399
|
-
"id": "
|
|
400
|
-
"label": "
|
|
469
|
+
"id": "proxy",
|
|
470
|
+
"label": "Proxy Service",
|
|
471
|
+
"ctlType": "boolean",
|
|
472
|
+
"description": "",
|
|
473
|
+
"required": False,
|
|
474
|
+
"defaultValue": "false",
|
|
475
|
+
"value": "false",
|
|
476
|
+
"visible": True,
|
|
477
|
+
"editable": True,
|
|
478
|
+
},
|
|
479
|
+
{
|
|
480
|
+
"id": "proxyScheme",
|
|
481
|
+
"label": "Proxy Scheme",
|
|
482
|
+
"ctlType": "select",
|
|
483
|
+
"description": "",
|
|
484
|
+
"required": False,
|
|
485
|
+
"defaultValue": "HTTP",
|
|
486
|
+
"value": "HTTP",
|
|
487
|
+
"visible": True,
|
|
488
|
+
"acceptedValues": ["HTTP", "HTTPS", "SOCKS5"],
|
|
489
|
+
"editable": True,
|
|
490
|
+
},
|
|
491
|
+
{
|
|
492
|
+
"id": "proxyHost",
|
|
493
|
+
"label": "Proxy Host",
|
|
401
494
|
"ctlType": "text",
|
|
495
|
+
"description": "",
|
|
402
496
|
"required": False,
|
|
403
|
-
"defaultValue":
|
|
497
|
+
"defaultValue": "",
|
|
498
|
+
"value": "",
|
|
404
499
|
"visible": True,
|
|
500
|
+
"editable": True,
|
|
501
|
+
},
|
|
502
|
+
{
|
|
503
|
+
"id": "proxyPort",
|
|
504
|
+
"label": "Proxy Port",
|
|
505
|
+
"ctlType": "text",
|
|
506
|
+
"description": "",
|
|
507
|
+
"required": False,
|
|
508
|
+
"defaultValue": "",
|
|
509
|
+
"value": "",
|
|
510
|
+
"visible": True,
|
|
511
|
+
"editable": True,
|
|
512
|
+
},
|
|
513
|
+
{
|
|
514
|
+
"id": "ProxyConfigService",
|
|
515
|
+
"label": "Proxy Config Service",
|
|
516
|
+
"ctlType": "text",
|
|
517
|
+
"description": "",
|
|
518
|
+
"required": False,
|
|
519
|
+
"defaultValue": "",
|
|
520
|
+
"value": "",
|
|
521
|
+
"visible": False,
|
|
522
|
+
"editable": True,
|
|
405
523
|
},
|
|
406
524
|
],
|
|
407
|
-
"
|
|
408
|
-
"
|
|
409
|
-
|
|
410
|
-
|
|
525
|
+
"config": {
|
|
526
|
+
"type": "nifi",
|
|
527
|
+
"id": "xECM",
|
|
528
|
+
"crawlConfig": {
|
|
529
|
+
"name": "GetOpenText",
|
|
530
|
+
"type": "idol.nifi.connector.GetOpenText",
|
|
411
531
|
"group": "idol.nifi.connector",
|
|
412
532
|
"artifact": "idol-nifi-connector-opentext",
|
|
413
|
-
"version":
|
|
533
|
+
"version": "25.1.0-nifi1",
|
|
414
534
|
},
|
|
415
|
-
"
|
|
416
|
-
"name": "
|
|
417
|
-
"type": "idol.nifi.connector.
|
|
535
|
+
"viewConfig": {
|
|
536
|
+
"name": "ViewOpenText",
|
|
537
|
+
"type": "idol.nifi.connector.ViewOpenText",
|
|
418
538
|
"group": "idol.nifi.connector",
|
|
419
539
|
"artifact": "idol-nifi-connector-opentext",
|
|
420
|
-
"version":
|
|
540
|
+
"version": "25.1.0-nifi1",
|
|
421
541
|
},
|
|
422
|
-
"
|
|
542
|
+
"omniConfig": {
|
|
423
543
|
"name": "GetOpenTextGroups",
|
|
424
544
|
"type": "idol.nifi.connector.GetOpenTextGroups",
|
|
425
545
|
"group": "idol.nifi.connector",
|
|
426
546
|
"artifact": "idol-nifi-connector-opentext",
|
|
427
|
-
"version":
|
|
547
|
+
"version": "25.1.0-nifi1",
|
|
548
|
+
"repoName": "ECM",
|
|
428
549
|
},
|
|
429
|
-
|
|
430
|
-
"idolProperties": {
|
|
431
|
-
"view": {
|
|
550
|
+
"crawlProps": {
|
|
432
551
|
"Password": "${Password}",
|
|
433
552
|
"Username": "${UserName}",
|
|
553
|
+
"META:SOURCE": "OPENTEXT",
|
|
554
|
+
"RootNodeIds": "${RootNodeIds}",
|
|
555
|
+
"MappedSecurity": "true",
|
|
434
556
|
"OpenTextApiUrl": "${OpenTextApiUrl}",
|
|
557
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
435
558
|
},
|
|
436
|
-
"
|
|
559
|
+
"viewProps": {
|
|
437
560
|
"Password": "${Password}",
|
|
438
561
|
"Username": "${UserName}",
|
|
439
|
-
"RootNodeId": "${RootNodeId}",
|
|
440
|
-
"META:SOURCE": "OPENTEXT",
|
|
441
|
-
"MappedSecurity": "true",
|
|
442
562
|
"OpenTextApiUrl": "${OpenTextApiUrl}",
|
|
563
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
443
564
|
},
|
|
444
|
-
"
|
|
565
|
+
"omniProps": {
|
|
445
566
|
"Password": "${Password}",
|
|
446
567
|
"Username": "${UserName}",
|
|
447
568
|
"OpenTextApiUrl": "${OpenTextApiUrl}",
|
|
569
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
448
570
|
"OpenTextApiPageSize": "10",
|
|
449
571
|
},
|
|
572
|
+
"metadataFields": ["NODE"],
|
|
450
573
|
},
|
|
574
|
+
"name": name,
|
|
575
|
+
"id": "xECM",
|
|
576
|
+
"sourceId": "xECM",
|
|
451
577
|
}
|
|
452
578
|
|
|
453
579
|
request_header = self.request_header()
|
|
454
580
|
request_url = self.config()["repoUrl"]
|
|
455
581
|
|
|
456
|
-
|
|
582
|
+
response = self.do_request(
|
|
457
583
|
url=request_url,
|
|
458
584
|
method="POST",
|
|
459
585
|
json_data=payload,
|
|
460
586
|
headers=request_header,
|
|
461
587
|
timeout=None,
|
|
462
588
|
failure_message="Failed to create repository -> '{}' ({})".format(
|
|
463
|
-
name,
|
|
589
|
+
name,
|
|
590
|
+
node_id,
|
|
464
591
|
),
|
|
592
|
+
show_error=False,
|
|
465
593
|
)
|
|
466
594
|
|
|
595
|
+
if response is None:
|
|
596
|
+
self.logger.error("Failed to create repository -> %s (%s)", name, node_id)
|
|
597
|
+
|
|
598
|
+
return response
|
|
599
|
+
|
|
467
600
|
# end method definition
|
|
468
601
|
|
|
469
|
-
def
|
|
602
|
+
def create_msteams_repo(
|
|
470
603
|
self,
|
|
471
604
|
name: str,
|
|
472
605
|
client_id: str,
|
|
@@ -477,36 +610,44 @@ class AVTS(object):
|
|
|
477
610
|
index_call_recordings: bool = True,
|
|
478
611
|
index_message_replies: bool = True,
|
|
479
612
|
index_user_chats: bool = True,
|
|
480
|
-
oauth2_site_name: str = "AVTS",
|
|
481
|
-
oauth2_sites_file: str = "",
|
|
482
|
-
version: str = "24.3.0",
|
|
483
613
|
) -> dict | None:
|
|
484
|
-
"""Create a new repository to crawl
|
|
614
|
+
"""Create a new Microsoft Teams repository to crawl with Aviator Search.
|
|
485
615
|
|
|
486
616
|
Args:
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
617
|
+
name (str):
|
|
618
|
+
The name of the repository.
|
|
619
|
+
client_id (str):
|
|
620
|
+
The M365 client ID.
|
|
621
|
+
tenant_id (str):
|
|
622
|
+
The M365 tenant ID.
|
|
623
|
+
certificate_file (str):
|
|
624
|
+
The path to the certificate file.
|
|
625
|
+
certificate_password (str):
|
|
626
|
+
The password for the certificate.
|
|
627
|
+
index_attachments (bool, optional):
|
|
628
|
+
Whether or not to index / crawl attachments.
|
|
629
|
+
index_call_recordings (bool, optional):
|
|
630
|
+
Whether or not to index / crawl meeting recordings.
|
|
631
|
+
index_message_replies (bool, optional):
|
|
632
|
+
Whether or not to index / crawl message replies.
|
|
633
|
+
index_user_chats (bool, optional):
|
|
634
|
+
Whether or not to index / crawl user chats.
|
|
635
|
+
version(str, optional): default 24.3.0
|
|
636
|
+
|
|
637
|
+
# TODO: add more params
|
|
490
638
|
|
|
491
639
|
Returns:
|
|
492
|
-
dict | None:
|
|
640
|
+
dict | None:
|
|
641
|
+
Parsed response object from the API or None in case of an error
|
|
642
|
+
|
|
493
643
|
"""
|
|
494
644
|
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
# Read the content of the file
|
|
499
|
-
certificate_file_content = file.read()
|
|
500
|
-
# Convert the bytes to a base64 string
|
|
501
|
-
certificate_file_content_base64 = base64.b64encode(
|
|
502
|
-
certificate_file_content
|
|
503
|
-
).decode("utf-8")
|
|
645
|
+
certificate_file_content_base64 = self.get_certificate_file_content_base64(
|
|
646
|
+
certificate_file,
|
|
647
|
+
)
|
|
504
648
|
|
|
505
649
|
payload = {
|
|
506
|
-
"id": "MSTeams",
|
|
507
|
-
"socName": "Microsoft Teams",
|
|
508
650
|
"authType": "OAUTH",
|
|
509
|
-
"name": name,
|
|
510
651
|
"params": [
|
|
511
652
|
{
|
|
512
653
|
"id": "OAuth2SiteName",
|
|
@@ -516,6 +657,7 @@ class AVTS(object):
|
|
|
516
657
|
"defaultValue": "AVTS",
|
|
517
658
|
"value": "AVTS",
|
|
518
659
|
"visible": False,
|
|
660
|
+
"editable": True,
|
|
519
661
|
},
|
|
520
662
|
{
|
|
521
663
|
"id": "OAuth2SitesFile",
|
|
@@ -525,6 +667,7 @@ class AVTS(object):
|
|
|
525
667
|
"defaultValue": "",
|
|
526
668
|
"value": "",
|
|
527
669
|
"visible": False,
|
|
670
|
+
"editable": True,
|
|
528
671
|
},
|
|
529
672
|
{
|
|
530
673
|
"id": "sourceLink",
|
|
@@ -533,6 +676,7 @@ class AVTS(object):
|
|
|
533
676
|
"required": False,
|
|
534
677
|
"defaultValue": "",
|
|
535
678
|
"visible": True,
|
|
679
|
+
"editable": True,
|
|
536
680
|
},
|
|
537
681
|
{
|
|
538
682
|
"id": "clientID",
|
|
@@ -543,6 +687,7 @@ class AVTS(object):
|
|
|
543
687
|
"defaultValue": "",
|
|
544
688
|
"value": client_id,
|
|
545
689
|
"visible": True,
|
|
690
|
+
"editable": True,
|
|
546
691
|
},
|
|
547
692
|
{
|
|
548
693
|
"id": "tenant",
|
|
@@ -553,6 +698,7 @@ class AVTS(object):
|
|
|
553
698
|
"defaultValue": "",
|
|
554
699
|
"value": tenant_id,
|
|
555
700
|
"visible": True,
|
|
701
|
+
"editable": False,
|
|
556
702
|
},
|
|
557
703
|
{
|
|
558
704
|
"id": "IndexAttachments",
|
|
@@ -561,8 +707,9 @@ class AVTS(object):
|
|
|
561
707
|
"description": "Specifies whether to index attachments",
|
|
562
708
|
"required": False,
|
|
563
709
|
"defaultValue": "true",
|
|
564
|
-
"value":
|
|
565
|
-
"visible":
|
|
710
|
+
"value": "true",
|
|
711
|
+
"visible": str(index_attachments).lower(),
|
|
712
|
+
"editable": True,
|
|
566
713
|
},
|
|
567
714
|
{
|
|
568
715
|
"id": "IndexCallRecordings",
|
|
@@ -573,6 +720,7 @@ class AVTS(object):
|
|
|
573
720
|
"defaultValue": "true",
|
|
574
721
|
"value": str(index_call_recordings).lower(),
|
|
575
722
|
"visible": True,
|
|
723
|
+
"editable": True,
|
|
576
724
|
},
|
|
577
725
|
{
|
|
578
726
|
"id": "IndexMessageReplies",
|
|
@@ -583,6 +731,7 @@ class AVTS(object):
|
|
|
583
731
|
"defaultValue": "true",
|
|
584
732
|
"value": str(index_message_replies).lower(),
|
|
585
733
|
"visible": True,
|
|
734
|
+
"editable": True,
|
|
586
735
|
},
|
|
587
736
|
{
|
|
588
737
|
"id": "IndexUserChats",
|
|
@@ -593,6 +742,7 @@ class AVTS(object):
|
|
|
593
742
|
"defaultValue": "true",
|
|
594
743
|
"value": str(index_user_chats).lower(),
|
|
595
744
|
"visible": True,
|
|
745
|
+
"editable": True,
|
|
596
746
|
},
|
|
597
747
|
{
|
|
598
748
|
"id": "certificateFile",
|
|
@@ -601,8 +751,9 @@ class AVTS(object):
|
|
|
601
751
|
"description": 'Please upload a valid "*.pfx" certificate file',
|
|
602
752
|
"required": True,
|
|
603
753
|
"defaultValue": "",
|
|
604
|
-
"value": "C:\\fakepath\\certificate.pfx",
|
|
754
|
+
"value": "C:\\fakepath\\certificate 1 3 (1).pfx",
|
|
605
755
|
"visible": True,
|
|
756
|
+
"editable": True,
|
|
606
757
|
"fileDatabase64": f"data:application/x-pkcs12;base64,{certificate_file_content_base64}",
|
|
607
758
|
},
|
|
608
759
|
{
|
|
@@ -613,42 +764,126 @@ class AVTS(object):
|
|
|
613
764
|
"defaultValue": "",
|
|
614
765
|
"value": certificate_password,
|
|
615
766
|
"visible": True,
|
|
767
|
+
"editable": True,
|
|
768
|
+
},
|
|
769
|
+
{
|
|
770
|
+
"id": "proxy",
|
|
771
|
+
"label": "Proxy Service",
|
|
772
|
+
"ctlType": "boolean",
|
|
773
|
+
"description": "",
|
|
774
|
+
"required": False,
|
|
775
|
+
"defaultValue": "false",
|
|
776
|
+
"value": "true",
|
|
777
|
+
"visible": True,
|
|
778
|
+
"editable": True,
|
|
779
|
+
},
|
|
780
|
+
{
|
|
781
|
+
"id": "proxyScheme",
|
|
782
|
+
"label": "Proxy Scheme",
|
|
783
|
+
"ctlType": "select",
|
|
784
|
+
"description": "",
|
|
785
|
+
"required": False,
|
|
786
|
+
"defaultValue": "HTTP",
|
|
787
|
+
"value": "HTTP",
|
|
788
|
+
"visible": True,
|
|
789
|
+
"acceptedValues": [
|
|
790
|
+
"HTTP",
|
|
791
|
+
"HTTPS",
|
|
792
|
+
"SOCKS5",
|
|
793
|
+
],
|
|
794
|
+
"editable": True,
|
|
795
|
+
},
|
|
796
|
+
{
|
|
797
|
+
"id": "proxyHost",
|
|
798
|
+
"label": "Proxy Host",
|
|
799
|
+
"ctlType": "text",
|
|
800
|
+
"description": "",
|
|
801
|
+
"required": False,
|
|
802
|
+
"defaultValue": "",
|
|
803
|
+
"value": "10.194.10.21",
|
|
804
|
+
"visible": True,
|
|
805
|
+
"editable": True,
|
|
806
|
+
},
|
|
807
|
+
{
|
|
808
|
+
"id": "proxyPort",
|
|
809
|
+
"label": "Proxy Port",
|
|
810
|
+
"ctlType": "text",
|
|
811
|
+
"description": "",
|
|
812
|
+
"required": False,
|
|
813
|
+
"defaultValue": "",
|
|
814
|
+
"value": "3128",
|
|
815
|
+
"visible": True,
|
|
816
|
+
"editable": True,
|
|
817
|
+
},
|
|
818
|
+
{
|
|
819
|
+
"id": "ProxyConfigService",
|
|
820
|
+
"label": "Proxy Config Service",
|
|
821
|
+
"ctlType": "text",
|
|
822
|
+
"description": "",
|
|
823
|
+
"required": False,
|
|
824
|
+
"defaultValue": "",
|
|
825
|
+
"value": "",
|
|
826
|
+
"visible": False,
|
|
827
|
+
"editable": True,
|
|
616
828
|
},
|
|
617
829
|
],
|
|
618
|
-
"
|
|
619
|
-
"
|
|
830
|
+
"config": {
|
|
831
|
+
"type": "nifi",
|
|
832
|
+
"id": "MSTeams",
|
|
833
|
+
"crawlConfig": {
|
|
834
|
+
"name": "GetMicrosoftTeams",
|
|
835
|
+
"type": "idol.nifi.connector.GetMicrosoftTeams",
|
|
836
|
+
"group": "idol.nifi.connector",
|
|
837
|
+
"artifact": "idol-nifi-connector-officeteams",
|
|
838
|
+
"version": "25.1.0-nifi1",
|
|
839
|
+
},
|
|
840
|
+
"viewConfig": {
|
|
620
841
|
"name": "ViewMicrosoftTeams",
|
|
621
842
|
"type": "idol.nifi.connector.ViewMicrosoftTeams",
|
|
622
843
|
"group": "idol.nifi.connector",
|
|
623
844
|
"artifact": "idol-nifi-connector-officeteams",
|
|
624
|
-
"version":
|
|
845
|
+
"version": "25.1.0-nifi1",
|
|
625
846
|
},
|
|
626
|
-
"
|
|
627
|
-
"name": "
|
|
628
|
-
"type": "idol.nifi.connector.
|
|
847
|
+
"omniConfig": {
|
|
848
|
+
"name": "GetMicrosoftTeamsGroups",
|
|
849
|
+
"type": "idol.nifi.connector.GetMicrosoftTeamsGroups",
|
|
629
850
|
"group": "idol.nifi.connector",
|
|
630
851
|
"artifact": "idol-nifi-connector-officeteams",
|
|
631
|
-
"version":
|
|
852
|
+
"version": "25.1.0-nifi1",
|
|
853
|
+
"repoName": "OneDrive",
|
|
632
854
|
},
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
"view": {
|
|
636
|
-
"Oauth2SiteName": "${OAuth2SiteName}",
|
|
637
|
-
"Oauth2SitesFile": "${OAuth2SitesFile}",
|
|
638
|
-
"IndexCallRecordings": "true",
|
|
639
|
-
},
|
|
640
|
-
"crawler": {
|
|
641
|
-
"META:SOURCE": "MSTeams",
|
|
855
|
+
"crawlProps": {
|
|
856
|
+
"META:SOURCE": "Microsoft Teams",
|
|
642
857
|
"IndexUserChats": "${IndexUserChats}",
|
|
858
|
+
"MappedSecurity": "true",
|
|
643
859
|
"Oauth2SiteName": "${OAuth2SiteName}",
|
|
644
860
|
"Oauth2SitesFile": "${OAuth2SitesFile}",
|
|
645
861
|
"IndexAttachments": "${IndexAttachments}",
|
|
862
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
646
863
|
"IndexCallRecordings": "${IndexCallRecordings}",
|
|
647
864
|
"IndexMessageReplies": "${IndexMessageReplies}",
|
|
865
|
+
"ChatMessageGroupingSection": "chat",
|
|
866
|
+
"ChannelMessageGroupingSection": "channel",
|
|
867
|
+
"[chat]MessageGroupingInterval": "24 hour",
|
|
868
|
+
"[chat]MessageGroupingStrategy": "Interval",
|
|
869
|
+
"[channel]MessageGroupingInterval": "24 hour",
|
|
870
|
+
"[channel]MessageGroupingStrategy": "Interval",
|
|
871
|
+
},
|
|
872
|
+
"viewProps": {
|
|
873
|
+
"Oauth2SiteName": "${OAuth2SiteName}",
|
|
874
|
+
"Oauth2SitesFile": "${OAuth2SitesFile}",
|
|
875
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
876
|
+
},
|
|
877
|
+
"omniProps": {
|
|
878
|
+
"Oauth2SiteName": "${OAuth2SiteName}",
|
|
879
|
+
"Oauth2SitesFile": "${OAuth2SitesFile}",
|
|
880
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
648
881
|
},
|
|
882
|
+
"metadataFields": [],
|
|
649
883
|
},
|
|
650
|
-
"
|
|
651
|
-
"
|
|
884
|
+
"name": name,
|
|
885
|
+
"id": "MSTeams",
|
|
886
|
+
"sourceId": "MSTeams",
|
|
652
887
|
}
|
|
653
888
|
|
|
654
889
|
request_header = self.request_header()
|
|
@@ -661,9 +896,11 @@ class AVTS(object):
|
|
|
661
896
|
headers=request_header,
|
|
662
897
|
timeout=None,
|
|
663
898
|
failure_message="Failed to create repository -> '{}'".format(name),
|
|
899
|
+
show_error=False,
|
|
664
900
|
)
|
|
665
901
|
|
|
666
902
|
if response is None:
|
|
903
|
+
self.logger.error("Failed to create repository -> %s", name)
|
|
667
904
|
return None
|
|
668
905
|
|
|
669
906
|
self.repo_admin_consent(response["id"])
|
|
@@ -672,7 +909,7 @@ class AVTS(object):
|
|
|
672
909
|
|
|
673
910
|
# end method definition
|
|
674
911
|
|
|
675
|
-
def
|
|
912
|
+
def create_sharepoint_repo(
|
|
676
913
|
self,
|
|
677
914
|
name: str,
|
|
678
915
|
client_id: str,
|
|
@@ -686,34 +923,49 @@ class AVTS(object):
|
|
|
686
923
|
index_user_profiles: bool = True,
|
|
687
924
|
oauth2_site_name: str = "AVTS",
|
|
688
925
|
oauth2_sites_file: str = "",
|
|
689
|
-
version: str = "24.3.0",
|
|
690
926
|
) -> dict | None:
|
|
691
|
-
"""Create a new repository to crawl
|
|
927
|
+
"""Create a new Microsoft SharePoint repository to crawl with Aviator Search.
|
|
692
928
|
|
|
693
929
|
Args:
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
930
|
+
name (str):
|
|
931
|
+
The name of the repository.
|
|
932
|
+
client_id (str):
|
|
933
|
+
The M365 client ID.
|
|
934
|
+
tenant_id (str):
|
|
935
|
+
The M365 tenant ID.
|
|
936
|
+
certificate_file (str):
|
|
937
|
+
TODO: _description_
|
|
938
|
+
certificate_password (int):
|
|
939
|
+
TODO: _description_
|
|
940
|
+
sharepoint_url (str):
|
|
941
|
+
The SharePoint URL.
|
|
942
|
+
sharepoint_url_type (str):
|
|
943
|
+
The SharePoint URL type.
|
|
944
|
+
sharepoint_mysite_url (str):
|
|
945
|
+
The SharePoint MySite URL.
|
|
946
|
+
sharepoint_admin_url (str):
|
|
947
|
+
The SharePoint administration URL.
|
|
948
|
+
index_user_profiles (bool, optional):
|
|
949
|
+
TODO: _description_. Defaults to True.
|
|
950
|
+
oauth2_site_name (str, optional):
|
|
951
|
+
TODO: _description_. Defaults to "AVTS".
|
|
952
|
+
oauth2_sites_file (str, optional):
|
|
953
|
+
TODO: _description_. Defaults to "".
|
|
954
|
+
version (str, optional):
|
|
955
|
+
TODO: _description_. Defaults to "24.3.0".
|
|
697
956
|
|
|
698
957
|
Returns:
|
|
699
|
-
dict | None:
|
|
958
|
+
dict | None:
|
|
959
|
+
Parsed response object from the API or None in case of an error
|
|
960
|
+
|
|
700
961
|
"""
|
|
701
962
|
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
# Read the content of the file
|
|
706
|
-
certificate_file_content = file.read()
|
|
707
|
-
# Convert the bytes to a base64 string
|
|
708
|
-
certificate_file_content_base64 = base64.b64encode(
|
|
709
|
-
certificate_file_content
|
|
710
|
-
).decode("utf-8")
|
|
963
|
+
certificate_file_content_base64 = self.get_certificate_file_content_base64(
|
|
964
|
+
certificate_file,
|
|
965
|
+
)
|
|
711
966
|
|
|
712
967
|
payload = {
|
|
713
|
-
"id": "SharePoint",
|
|
714
|
-
"socName": "SharePoint Online",
|
|
715
968
|
"authType": "OAUTH",
|
|
716
|
-
"name": name,
|
|
717
969
|
"params": [
|
|
718
970
|
{
|
|
719
971
|
"id": "OAuth2SiteName",
|
|
@@ -723,6 +975,7 @@ class AVTS(object):
|
|
|
723
975
|
"defaultValue": "AVTS",
|
|
724
976
|
"value": oauth2_site_name,
|
|
725
977
|
"visible": False,
|
|
978
|
+
"editable": True,
|
|
726
979
|
},
|
|
727
980
|
{
|
|
728
981
|
"id": "OAuth2SitesFile",
|
|
@@ -732,6 +985,7 @@ class AVTS(object):
|
|
|
732
985
|
"defaultValue": "",
|
|
733
986
|
"value": oauth2_sites_file,
|
|
734
987
|
"visible": False,
|
|
988
|
+
"editable": True,
|
|
735
989
|
},
|
|
736
990
|
{
|
|
737
991
|
"id": "sourceLink",
|
|
@@ -741,8 +995,8 @@ class AVTS(object):
|
|
|
741
995
|
"required": False,
|
|
742
996
|
"defaultValue": "",
|
|
743
997
|
"visible": True,
|
|
744
|
-
"
|
|
745
|
-
+ "${FILEDIRREF}/Forms/AllItems.aspx?id=${FILEREF}&parent=${FILEDIRREF}",
|
|
998
|
+
"editable": True,
|
|
999
|
+
"value": sharepoint_url + "${FILEDIRREF}/Forms/AllItems.aspx?id=${FILEREF}&parent=${FILEDIRREF}",
|
|
746
1000
|
},
|
|
747
1001
|
{
|
|
748
1002
|
"id": "clientID",
|
|
@@ -753,6 +1007,7 @@ class AVTS(object):
|
|
|
753
1007
|
"defaultValue": "",
|
|
754
1008
|
"value": client_id,
|
|
755
1009
|
"visible": True,
|
|
1010
|
+
"editable": True,
|
|
756
1011
|
},
|
|
757
1012
|
{
|
|
758
1013
|
"id": "tenant",
|
|
@@ -763,6 +1018,7 @@ class AVTS(object):
|
|
|
763
1018
|
"defaultValue": "",
|
|
764
1019
|
"value": tenant_id,
|
|
765
1020
|
"visible": True,
|
|
1021
|
+
"editable": True,
|
|
766
1022
|
},
|
|
767
1023
|
{
|
|
768
1024
|
"id": "sharePointUrl",
|
|
@@ -771,8 +1027,19 @@ class AVTS(object):
|
|
|
771
1027
|
"description": 'The URL to start synchronizing from. Specify a URL that matches "SharePoint URL type"',
|
|
772
1028
|
"required": True,
|
|
773
1029
|
"defaultValue": "",
|
|
774
|
-
"value":
|
|
1030
|
+
"value": sharepoint_mysite_url,
|
|
1031
|
+
"visible": True,
|
|
1032
|
+
"editable": False,
|
|
1033
|
+
},
|
|
1034
|
+
{
|
|
1035
|
+
"id": "MappedWebApplicationPolicies",
|
|
1036
|
+
"label": "Mapped Web Application Policies",
|
|
1037
|
+
"ctlType": "boolean",
|
|
1038
|
+
"required": False,
|
|
1039
|
+
"defaultValue": "false",
|
|
1040
|
+
"value": "false",
|
|
775
1041
|
"visible": True,
|
|
1042
|
+
"editable": False,
|
|
776
1043
|
},
|
|
777
1044
|
{
|
|
778
1045
|
"id": "sharePointAdminUrl",
|
|
@@ -783,6 +1050,7 @@ class AVTS(object):
|
|
|
783
1050
|
"defaultValue": "",
|
|
784
1051
|
"value": sharepoint_admin_url,
|
|
785
1052
|
"visible": True,
|
|
1053
|
+
"editable": False,
|
|
786
1054
|
},
|
|
787
1055
|
{
|
|
788
1056
|
"id": "sharePointMySiteUrl",
|
|
@@ -793,6 +1061,7 @@ class AVTS(object):
|
|
|
793
1061
|
"defaultValue": "",
|
|
794
1062
|
"value": sharepoint_mysite_url,
|
|
795
1063
|
"visible": True,
|
|
1064
|
+
"editable": False,
|
|
796
1065
|
},
|
|
797
1066
|
{
|
|
798
1067
|
"id": "sharePointOnline",
|
|
@@ -803,15 +1072,7 @@ class AVTS(object):
|
|
|
803
1072
|
"defaultValue": "true",
|
|
804
1073
|
"value": "true",
|
|
805
1074
|
"visible": False,
|
|
806
|
-
|
|
807
|
-
{
|
|
808
|
-
"id": "MappedWebApplicationPolicies",
|
|
809
|
-
"label": "Mapped Web Application Policies",
|
|
810
|
-
"ctlType": "text",
|
|
811
|
-
"required": False,
|
|
812
|
-
"defaultValue": "false",
|
|
813
|
-
"value": "false",
|
|
814
|
-
"visible": False,
|
|
1075
|
+
"editable": False,
|
|
815
1076
|
},
|
|
816
1077
|
{
|
|
817
1078
|
"id": "TenantAdminSitesIncludeTypes",
|
|
@@ -822,6 +1083,7 @@ class AVTS(object):
|
|
|
822
1083
|
"defaultValue": "all",
|
|
823
1084
|
"value": "all",
|
|
824
1085
|
"visible": False,
|
|
1086
|
+
"editable": False,
|
|
825
1087
|
},
|
|
826
1088
|
{
|
|
827
1089
|
"id": "URLType",
|
|
@@ -830,7 +1092,7 @@ class AVTS(object):
|
|
|
830
1092
|
"description": 'The type of URL specified by "Sharepoint URL"',
|
|
831
1093
|
"required": True,
|
|
832
1094
|
"defaultValue": "",
|
|
833
|
-
"value":
|
|
1095
|
+
"value": sharepoint_url_type,
|
|
834
1096
|
"visible": True,
|
|
835
1097
|
"acceptedValues": [
|
|
836
1098
|
"WebApplication",
|
|
@@ -838,6 +1100,7 @@ class AVTS(object):
|
|
|
838
1100
|
"PersonalSiteCollection",
|
|
839
1101
|
"TenantAdmin",
|
|
840
1102
|
],
|
|
1103
|
+
"editable": False,
|
|
841
1104
|
},
|
|
842
1105
|
{
|
|
843
1106
|
"id": "IndexUserProfiles",
|
|
@@ -848,6 +1111,7 @@ class AVTS(object):
|
|
|
848
1111
|
"defaultValue": "false",
|
|
849
1112
|
"value": str(index_user_profiles).lower(),
|
|
850
1113
|
"visible": True,
|
|
1114
|
+
"editable": True,
|
|
851
1115
|
},
|
|
852
1116
|
{
|
|
853
1117
|
"id": "certificateFile",
|
|
@@ -856,8 +1120,9 @@ class AVTS(object):
|
|
|
856
1120
|
"description": 'Please upload a valid "*.pfx" certificate file',
|
|
857
1121
|
"required": True,
|
|
858
1122
|
"defaultValue": "",
|
|
859
|
-
"value": "C:\\fakepath\\certificate.pfx",
|
|
1123
|
+
"value": "C:\\fakepath\\certificate 1 3 (1).pfx",
|
|
860
1124
|
"visible": True,
|
|
1125
|
+
"editable": True,
|
|
861
1126
|
"fileDatabase64": f"data:application/x-pkcs12;base64,{certificate_file_content_base64}",
|
|
862
1127
|
},
|
|
863
1128
|
{
|
|
@@ -868,51 +1133,141 @@ class AVTS(object):
|
|
|
868
1133
|
"defaultValue": "",
|
|
869
1134
|
"value": certificate_password,
|
|
870
1135
|
"visible": True,
|
|
1136
|
+
"editable": True,
|
|
1137
|
+
},
|
|
1138
|
+
{
|
|
1139
|
+
"id": "proxy",
|
|
1140
|
+
"label": "Proxy Service",
|
|
1141
|
+
"ctlType": "boolean",
|
|
1142
|
+
"description": "",
|
|
1143
|
+
"required": False,
|
|
1144
|
+
"defaultValue": "false",
|
|
1145
|
+
"value": "true",
|
|
1146
|
+
"visible": True,
|
|
1147
|
+
"editable": True,
|
|
1148
|
+
},
|
|
1149
|
+
{
|
|
1150
|
+
"id": "proxyScheme",
|
|
1151
|
+
"label": "Proxy Scheme",
|
|
1152
|
+
"ctlType": "select",
|
|
1153
|
+
"description": "",
|
|
1154
|
+
"required": False,
|
|
1155
|
+
"defaultValue": "HTTP",
|
|
1156
|
+
"value": "HTTP",
|
|
1157
|
+
"visible": True,
|
|
1158
|
+
"acceptedValues": [
|
|
1159
|
+
"HTTP",
|
|
1160
|
+
"HTTPS",
|
|
1161
|
+
"SOCKS5",
|
|
1162
|
+
],
|
|
1163
|
+
"editable": True,
|
|
1164
|
+
},
|
|
1165
|
+
{
|
|
1166
|
+
"id": "proxyHost",
|
|
1167
|
+
"label": "Proxy Host",
|
|
1168
|
+
"ctlType": "text",
|
|
1169
|
+
"description": "",
|
|
1170
|
+
"required": False,
|
|
1171
|
+
"defaultValue": "",
|
|
1172
|
+
"value": "10.194.10.21",
|
|
1173
|
+
"visible": True,
|
|
1174
|
+
"editable": True,
|
|
1175
|
+
},
|
|
1176
|
+
{
|
|
1177
|
+
"id": "proxyPort",
|
|
1178
|
+
"label": "Proxy Port",
|
|
1179
|
+
"ctlType": "text",
|
|
1180
|
+
"description": "",
|
|
1181
|
+
"required": False,
|
|
1182
|
+
"defaultValue": "",
|
|
1183
|
+
"value": "3128",
|
|
1184
|
+
"visible": True,
|
|
1185
|
+
"editable": True,
|
|
1186
|
+
},
|
|
1187
|
+
{
|
|
1188
|
+
"id": "ProxyConfigService",
|
|
1189
|
+
"label": "Proxy Config Service",
|
|
1190
|
+
"ctlType": "text",
|
|
1191
|
+
"description": "",
|
|
1192
|
+
"required": False,
|
|
1193
|
+
"defaultValue": "",
|
|
1194
|
+
"value": "",
|
|
1195
|
+
"visible": False,
|
|
1196
|
+
"editable": True,
|
|
871
1197
|
},
|
|
872
1198
|
],
|
|
873
|
-
"
|
|
874
|
-
"
|
|
1199
|
+
"config": {
|
|
1200
|
+
"type": "nifi",
|
|
1201
|
+
"id": "SharePoint",
|
|
1202
|
+
"crawlConfig": {
|
|
1203
|
+
"name": "GetSharePointOData",
|
|
1204
|
+
"type": "idol.nifi.connector.GetSharePointOData",
|
|
1205
|
+
"group": "idol.nifi.connector",
|
|
1206
|
+
"artifact": "idol-nifi-connector-sharepointodata",
|
|
1207
|
+
"version": "25.1.0-nifi1",
|
|
1208
|
+
},
|
|
1209
|
+
"viewConfig": {
|
|
875
1210
|
"name": "ViewSharePointOData",
|
|
876
1211
|
"type": "idol.nifi.connector.ViewSharePointOData",
|
|
877
1212
|
"group": "idol.nifi.connector",
|
|
878
1213
|
"artifact": "idol-nifi-connector-sharepointodata",
|
|
879
|
-
"version":
|
|
1214
|
+
"version": "25.1.0-nifi1",
|
|
880
1215
|
},
|
|
881
|
-
"
|
|
882
|
-
"name": "
|
|
883
|
-
"type": "idol.nifi.connector.
|
|
1216
|
+
"omniConfig": {
|
|
1217
|
+
"name": "GetSharePointGroupsOData",
|
|
1218
|
+
"type": "idol.nifi.connector.GetSharePointGroupsOData",
|
|
884
1219
|
"group": "idol.nifi.connector",
|
|
885
1220
|
"artifact": "idol-nifi-connector-sharepointodata",
|
|
886
|
-
"version":
|
|
1221
|
+
"version": "25.1.0-nifi1",
|
|
1222
|
+
"repoName": "SharePoint",
|
|
887
1223
|
},
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
"view": {
|
|
1224
|
+
"crawlProps": {
|
|
1225
|
+
"META:SOURCE": "SharePoint Online",
|
|
891
1226
|
"SharepointUrl": "${sharePointUrl}",
|
|
1227
|
+
"MappedSecurity": "true",
|
|
892
1228
|
"Oauth2SiteName": "${OAuth2SiteName}",
|
|
893
1229
|
"Oauth2SitesFile": "${OAuth2SitesFile}",
|
|
894
1230
|
"SharepointOnline": "${sharePointOnline}",
|
|
1231
|
+
"IndexUserProfiles": "${IndexUserProfiles}",
|
|
895
1232
|
"SharepointUrlType": "${URLType}",
|
|
1233
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
896
1234
|
"SharepointAdminUrl": "${sharePointAdminUrl}",
|
|
897
1235
|
"SharepointMySiteUrl": "${sharePointMySiteUrl}",
|
|
1236
|
+
"RetrieveUserDetailsAs": "Title",
|
|
898
1237
|
"MappedWebApplicationPolicies": "${MappedWebApplicationPolicies}",
|
|
1238
|
+
"TenantAdminSitesIncludeTypes": "${TenantAdminSitesIncludeTypes}",
|
|
899
1239
|
},
|
|
900
|
-
"
|
|
901
|
-
"META:SOURCE": "SharePoint",
|
|
1240
|
+
"viewProps": {
|
|
902
1241
|
"SharepointUrl": "${sharePointUrl}",
|
|
903
1242
|
"Oauth2SiteName": "${OAuth2SiteName}",
|
|
904
1243
|
"Oauth2SitesFile": "${OAuth2SitesFile}",
|
|
905
1244
|
"SharepointOnline": "${sharePointOnline}",
|
|
906
|
-
"IndexUserProfiles": "${IndexUserProfiles}",
|
|
907
1245
|
"SharepointUrlType": "${URLType}",
|
|
1246
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
908
1247
|
"SharepointAdminUrl": "${sharePointAdminUrl}",
|
|
909
1248
|
"SharepointMySiteUrl": "${sharePointMySiteUrl}",
|
|
910
1249
|
"MappedWebApplicationPolicies": "${MappedWebApplicationPolicies}",
|
|
1250
|
+
},
|
|
1251
|
+
"omniProps": {
|
|
1252
|
+
"SharepointUrl": "${sharePointUrl}",
|
|
1253
|
+
"Oauth2SiteName": "${OAuth2SiteName}",
|
|
1254
|
+
"Oauth2SitesFile": "${OAuth2SitesFile}",
|
|
1255
|
+
"SharepointOnline": "true",
|
|
1256
|
+
"SharepointUrlType": "${URLType}",
|
|
1257
|
+
"ProxyConfigService": "${ProxyConfigService}",
|
|
1258
|
+
"SharepointAdminUrl": "${sharePointAdminUrl}",
|
|
1259
|
+
"SharepointMySiteUrl": "${sharePointMySiteUrl}",
|
|
1260
|
+
"MappedWebApplicationPolicies": "false",
|
|
911
1261
|
"TenantAdminSitesIncludeTypes": "${TenantAdminSitesIncludeTypes}",
|
|
912
1262
|
},
|
|
1263
|
+
"metadataFields": [
|
|
1264
|
+
"FILEREF",
|
|
1265
|
+
"FILEDIRREF",
|
|
1266
|
+
],
|
|
913
1267
|
},
|
|
914
|
-
"
|
|
915
|
-
"
|
|
1268
|
+
"name": name,
|
|
1269
|
+
"id": "SharePoint",
|
|
1270
|
+
"sourceId": "SharePoint",
|
|
916
1271
|
}
|
|
917
1272
|
|
|
918
1273
|
request_header = self.request_header()
|
|
@@ -925,9 +1280,11 @@ class AVTS(object):
|
|
|
925
1280
|
headers=request_header,
|
|
926
1281
|
timeout=None,
|
|
927
1282
|
failure_message="Failed to create repository -> '{}'".format(name),
|
|
1283
|
+
show_error=False,
|
|
928
1284
|
)
|
|
929
1285
|
|
|
930
1286
|
if response is None:
|
|
1287
|
+
self.logger.error("Failed to create repository -> %s", name)
|
|
931
1288
|
return None
|
|
932
1289
|
|
|
933
1290
|
self.repo_admin_consent(response["id"])
|
|
@@ -937,21 +1294,22 @@ class AVTS(object):
|
|
|
937
1294
|
# end method definition
|
|
938
1295
|
|
|
939
1296
|
def repo_admin_consent(self, repo_id: str) -> dict | None:
|
|
940
|
-
"""Send admin consent information for a repository
|
|
1297
|
+
"""Send admin consent information for a repository.
|
|
941
1298
|
|
|
942
1299
|
Args:
|
|
943
|
-
repo_id (str):
|
|
1300
|
+
repo_id (str):
|
|
1301
|
+
The ID of the repository.
|
|
944
1302
|
|
|
945
1303
|
Returns:
|
|
946
|
-
dict | None:
|
|
1304
|
+
dict | None:
|
|
1305
|
+
Parsed response object from the API or None in case of an error
|
|
1306
|
+
|
|
947
1307
|
"""
|
|
948
1308
|
|
|
949
1309
|
request_header = self.request_header()
|
|
950
1310
|
request_url = self.config()["repoUrl"]
|
|
951
1311
|
|
|
952
|
-
request_url = (
|
|
953
|
-
self.config()["repoUrl"] + "/" + repo_id + "/authorize?admin_consent=true"
|
|
954
|
-
)
|
|
1312
|
+
request_url = self.config()["repoUrl"] + "/" + repo_id + "/authorize?admin_consent=true"
|
|
955
1313
|
|
|
956
1314
|
return self.do_request(
|
|
957
1315
|
url=request_url,
|
|
@@ -959,22 +1317,26 @@ class AVTS(object):
|
|
|
959
1317
|
headers=request_header,
|
|
960
1318
|
timeout=None,
|
|
961
1319
|
failure_message="Failed to set admin_consent for repository -> '{}'".format(
|
|
962
|
-
repo_id
|
|
1320
|
+
repo_id,
|
|
963
1321
|
),
|
|
964
1322
|
)
|
|
965
1323
|
|
|
966
1324
|
# end method definition
|
|
967
1325
|
|
|
968
1326
|
def start_crawling(self, repo_name: str) -> list | None:
|
|
969
|
-
"""Start crawling of a repository
|
|
1327
|
+
"""Start crawling of a repository.
|
|
970
1328
|
|
|
971
1329
|
Args:
|
|
972
|
-
repo_name (str):
|
|
1330
|
+
repo_name (str):
|
|
1331
|
+
The name of the repository.
|
|
1332
|
+
|
|
973
1333
|
Returns:
|
|
974
|
-
list | None:
|
|
1334
|
+
list | None:
|
|
1335
|
+
Parsed response object from the API or None in case of an error
|
|
1336
|
+
|
|
975
1337
|
"""
|
|
976
1338
|
|
|
977
|
-
logger.info("Start crawling repository -> %s", repo_name)
|
|
1339
|
+
self.logger.info("Start crawling repository -> %s", repo_name)
|
|
978
1340
|
|
|
979
1341
|
repo = self.get_repo_by_name(name=repo_name)
|
|
980
1342
|
if repo is None:
|
|
@@ -989,19 +1351,23 @@ class AVTS(object):
|
|
|
989
1351
|
headers=request_header,
|
|
990
1352
|
timeout=None,
|
|
991
1353
|
failure_message="Failed to start crawling repository -> '{}'".format(
|
|
992
|
-
repo_name
|
|
1354
|
+
repo_name,
|
|
993
1355
|
),
|
|
994
1356
|
)
|
|
995
1357
|
|
|
996
1358
|
# end method definition
|
|
997
1359
|
|
|
998
1360
|
def stop_crawling(self, repo_name: str) -> list | None:
|
|
999
|
-
"""Stop the crawling of a repository
|
|
1361
|
+
"""Stop the crawling of a repository.
|
|
1000
1362
|
|
|
1001
1363
|
Args:
|
|
1002
|
-
repo_name (str):
|
|
1364
|
+
repo_name (str):
|
|
1365
|
+
The name of the repository.
|
|
1366
|
+
|
|
1003
1367
|
Returns:
|
|
1004
|
-
list | None:
|
|
1368
|
+
list | None:
|
|
1369
|
+
Parsed response object from the API or None in case of an error
|
|
1370
|
+
|
|
1005
1371
|
"""
|
|
1006
1372
|
|
|
1007
1373
|
repo = self.get_repo_by_name(name=repo_name)
|
|
@@ -1017,17 +1383,19 @@ class AVTS(object):
|
|
|
1017
1383
|
headers=request_header,
|
|
1018
1384
|
timeout=None,
|
|
1019
1385
|
failure_message="Failed to stop crawling repository -> '{}'".format(
|
|
1020
|
-
repo_name
|
|
1386
|
+
repo_name,
|
|
1021
1387
|
),
|
|
1022
1388
|
)
|
|
1023
1389
|
|
|
1024
1390
|
# end method definition
|
|
1025
1391
|
|
|
1026
1392
|
def get_repo_list(self) -> list | None:
|
|
1027
|
-
"""Get a list of all repositories
|
|
1393
|
+
"""Get a list of all repositories.
|
|
1028
1394
|
|
|
1029
1395
|
Returns:
|
|
1030
|
-
list | None:
|
|
1396
|
+
list | None:
|
|
1397
|
+
Parsed response object from the API listing all repositories or None in case of an error.
|
|
1398
|
+
|
|
1031
1399
|
"""
|
|
1032
1400
|
|
|
1033
1401
|
request_header = self.request_header()
|
|
@@ -1038,18 +1406,22 @@ class AVTS(object):
|
|
|
1038
1406
|
method="GET",
|
|
1039
1407
|
headers=request_header,
|
|
1040
1408
|
timeout=None,
|
|
1041
|
-
failure_message="Failed to get list of repositories to crawl
|
|
1409
|
+
failure_message="Failed to get list of repositories to crawl",
|
|
1042
1410
|
)
|
|
1043
1411
|
|
|
1044
1412
|
# end method definition
|
|
1045
1413
|
|
|
1046
1414
|
def get_repo_by_name(self, name: str) -> dict | None:
|
|
1047
|
-
"""Get a repository by name
|
|
1415
|
+
"""Get a repository by name.
|
|
1048
1416
|
|
|
1049
1417
|
Args:
|
|
1050
|
-
name (str):
|
|
1418
|
+
name (str):
|
|
1419
|
+
The name of the repository.
|
|
1420
|
+
|
|
1051
1421
|
Returns:
|
|
1052
|
-
dict | None:
|
|
1422
|
+
dict | None:
|
|
1423
|
+
ID of a repostiory by name or None in case of an error
|
|
1424
|
+
|
|
1053
1425
|
"""
|
|
1054
1426
|
|
|
1055
1427
|
repo_list = self.get_repo_list()
|
|
@@ -1063,3 +1435,58 @@ class AVTS(object):
|
|
|
1063
1435
|
)
|
|
1064
1436
|
|
|
1065
1437
|
# end method definition
|
|
1438
|
+
|
|
1439
|
+
def get_certificate_file_content_base64(self, filepath: str) -> str | None:
|
|
1440
|
+
"""Return the certificate as a base64 string.
|
|
1441
|
+
|
|
1442
|
+
In Kubernetes deploymnets the certificate is already mounted base64 encoded.
|
|
1443
|
+
|
|
1444
|
+
Args:
|
|
1445
|
+
filepath (str):
|
|
1446
|
+
The path to the certificate file.
|
|
1447
|
+
|
|
1448
|
+
Returns:
|
|
1449
|
+
str | None:
|
|
1450
|
+
Base64 encoded certificate file content.
|
|
1451
|
+
|
|
1452
|
+
"""
|
|
1453
|
+
|
|
1454
|
+
if not os.path.isfile(filepath):
|
|
1455
|
+
return None
|
|
1456
|
+
|
|
1457
|
+
file_ext = os.path.splitext(filepath)[1].lower()
|
|
1458
|
+
|
|
1459
|
+
if self.running_in_kubernetes_pod() and file_ext == ".pfx":
|
|
1460
|
+
# Return file directly as already base64 encoded
|
|
1461
|
+
self.logger.warning(
|
|
1462
|
+
"Detected a binary pfx file in Kubernetes environment, expecting it to be already base64 encoded",
|
|
1463
|
+
)
|
|
1464
|
+
with open(filepath, encoding="UTF-8") as file:
|
|
1465
|
+
return file.read().strip()
|
|
1466
|
+
|
|
1467
|
+
else:
|
|
1468
|
+
# Return file as base64 encoded
|
|
1469
|
+
with open(filepath, "rb") as file:
|
|
1470
|
+
# Read the content of the file
|
|
1471
|
+
file_content = file.read()
|
|
1472
|
+
# Convert the bytes to a base64 string
|
|
1473
|
+
return base64.b64encode(file_content).decode("utf-8")
|
|
1474
|
+
|
|
1475
|
+
# end method definition
|
|
1476
|
+
|
|
1477
|
+
def running_in_kubernetes_pod(self) -> bool:
|
|
1478
|
+
"""Check if the application is running inside a Kubernetes pod.
|
|
1479
|
+
|
|
1480
|
+
This function determines whether the process is running in a Kubernetes
|
|
1481
|
+
environment by checking for the presence of the `KUBERNETES_SERVICE_HOST`
|
|
1482
|
+
and `KUBERNETES_SERVICE_PORT` environment variables.
|
|
1483
|
+
|
|
1484
|
+
Returns:
|
|
1485
|
+
bool:
|
|
1486
|
+
True if running inside a Kubernetes pod, False otherwise.
|
|
1487
|
+
|
|
1488
|
+
"""
|
|
1489
|
+
|
|
1490
|
+
return bool(os.getenv("KUBERNETES_SERVICE_HOST") and os.getenv("KUBERNETES_SERVICE_PORT"))
|
|
1491
|
+
|
|
1492
|
+
# end method definition
|