pyxecm 1.6__py3-none-any.whl → 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of pyxecm might be problematic. Click here for more details.

Files changed (56) hide show
  1. pyxecm/__init__.py +6 -4
  2. pyxecm/avts.py +673 -246
  3. pyxecm/coreshare.py +686 -467
  4. pyxecm/customizer/__init__.py +16 -4
  5. pyxecm/customizer/__main__.py +58 -0
  6. pyxecm/customizer/api/__init__.py +5 -0
  7. pyxecm/customizer/api/__main__.py +6 -0
  8. pyxecm/customizer/api/app.py +914 -0
  9. pyxecm/customizer/api/auth.py +154 -0
  10. pyxecm/customizer/api/metrics.py +92 -0
  11. pyxecm/customizer/api/models.py +13 -0
  12. pyxecm/customizer/api/payload_list.py +865 -0
  13. pyxecm/customizer/api/settings.py +103 -0
  14. pyxecm/customizer/browser_automation.py +332 -139
  15. pyxecm/customizer/customizer.py +1007 -1130
  16. pyxecm/customizer/exceptions.py +35 -0
  17. pyxecm/customizer/guidewire.py +322 -0
  18. pyxecm/customizer/k8s.py +713 -378
  19. pyxecm/customizer/log.py +107 -0
  20. pyxecm/customizer/m365.py +2867 -909
  21. pyxecm/customizer/nhc.py +1169 -0
  22. pyxecm/customizer/openapi.py +258 -0
  23. pyxecm/customizer/payload.py +16817 -7467
  24. pyxecm/customizer/pht.py +699 -285
  25. pyxecm/customizer/salesforce.py +516 -342
  26. pyxecm/customizer/sap.py +58 -41
  27. pyxecm/customizer/servicenow.py +593 -371
  28. pyxecm/customizer/settings.py +442 -0
  29. pyxecm/customizer/successfactors.py +408 -346
  30. pyxecm/customizer/translate.py +83 -48
  31. pyxecm/helper/__init__.py +5 -2
  32. pyxecm/helper/assoc.py +83 -43
  33. pyxecm/helper/data.py +2406 -870
  34. pyxecm/helper/logadapter.py +27 -0
  35. pyxecm/helper/web.py +229 -101
  36. pyxecm/helper/xml.py +527 -171
  37. pyxecm/maintenance_page/__init__.py +5 -0
  38. pyxecm/maintenance_page/__main__.py +6 -0
  39. pyxecm/maintenance_page/app.py +51 -0
  40. pyxecm/maintenance_page/settings.py +28 -0
  41. pyxecm/maintenance_page/static/favicon.avif +0 -0
  42. pyxecm/maintenance_page/templates/maintenance.html +165 -0
  43. pyxecm/otac.py +234 -140
  44. pyxecm/otawp.py +1436 -557
  45. pyxecm/otcs.py +7716 -3161
  46. pyxecm/otds.py +2150 -919
  47. pyxecm/otiv.py +36 -21
  48. pyxecm/otmm.py +1272 -325
  49. pyxecm/otpd.py +231 -127
  50. pyxecm-2.0.0.dist-info/METADATA +145 -0
  51. pyxecm-2.0.0.dist-info/RECORD +54 -0
  52. {pyxecm-1.6.dist-info → pyxecm-2.0.0.dist-info}/WHEEL +1 -1
  53. pyxecm-1.6.dist-info/METADATA +0 -53
  54. pyxecm-1.6.dist-info/RECORD +0 -32
  55. {pyxecm-1.6.dist-info → pyxecm-2.0.0.dist-info/licenses}/LICENSE +0 -0
  56. {pyxecm-1.6.dist-info → pyxecm-2.0.0.dist-info}/top_level.txt +0 -0
pyxecm/avts.py CHANGED
@@ -1,35 +1,20 @@
1
- """
2
- AVTS stands for Aviator Search and is an OpenText offering for LLMM-based search across multiple repositories
3
-
4
- Class: AVTS
5
- Methods:
6
- __init__: class initializer
7
- request_header: Returns the request header used for Application calls.
8
- do_request: Call an Aviator Search REST API in a safe way
9
- parse_request_response: Converts the request response (JSon) to a Python list in a safe way
10
- authenticate: Authenticate at Search Aviator via oAuth authentication
11
- repo_create_extended_ecm: Create a new repository to crawl in Aviator Search
12
- start_crawling: Start crawling of a repository
13
- stop_crawling: Stop the crawling of a repository
14
- get_repo_list: Get a list of all repositories
15
- get_repo_by_name: Get a repository by name
16
- """
1
+ """AVTS stands for Aviator Search and is an OpenText offering for LLMM-based search across multiple repositories."""
17
2
 
18
3
  __author__ = "Dr. Marc Diefenbruch"
19
- __copyright__ = "Copyright 2024, OpenText"
4
+ __copyright__ = "Copyright (C) 2024-2025, OpenText"
20
5
  __credits__ = ["Kai-Philip Gatzweiler"]
21
6
  __maintainer__ = "Dr. Marc Diefenbruch"
22
7
  __email__ = "mdiefenb@opentext.com"
23
8
 
9
+ import base64
24
10
  import json
25
11
  import logging
26
- import time
27
12
  import os
28
- import base64
13
+ import time
29
14
 
30
15
  import requests
31
16
 
32
- logger = logging.getLogger("pyxecm.customizer.avts")
17
+ default_logger = logging.getLogger("pyxecm.customizer.avts")
33
18
 
34
19
  REQUEST_HEADERS = {"Accept": "application/json", "Content-Type": "application/json"}
35
20
 
@@ -38,8 +23,10 @@ REQUEST_RETRY_DELAY = 20
38
23
  REQUEST_MAX_RETRIES = 2
39
24
 
40
25
 
41
- class AVTS(object):
42
- """Used to configure and interact with Aviator Search"""
26
+ class AVTS:
27
+ """Configure and interact with Aviator Search REST API."""
28
+
29
+ logger: logging.Logger = default_logger
43
30
 
44
31
  _config: dict
45
32
  _session = None
@@ -52,18 +39,33 @@ class AVTS(object):
52
39
  base_url: str,
53
40
  username: str,
54
41
  password: str,
55
- ):
56
- """Initialize the AVTS object
42
+ logger: logging.Logger = default_logger,
43
+ ) -> None:
44
+ """Initialize the AVTS object.
57
45
 
58
46
  Args:
59
- otds_url (str): URL of the OTDS Server used by Aviator Search
60
- client_id (str): Client ID for the Aviator Search oAuth client
61
- client_secret (str): Client Secret for the Aviator Search oAuth client
62
- base_url (str): Aviator Search base URL
63
- username (str): User with administrative permissions in Aviator Search
64
- password (str): Password of the user with administrative permissions in Aviator Search
47
+ otds_url (str):
48
+ The URL of the OTDS Server used by Aviator Search.
49
+ client_id (str):
50
+ The client ID for the Aviator Search oAuth client.
51
+ client_secret (str):
52
+ The client secret for the Aviator Search oAuth client.
53
+ base_url (str):
54
+ The Aviator Search base URL.
55
+ username (str):
56
+ User with administrative permissions in Aviator Search.
57
+ password (str):
58
+ Password of the user with administrative permissions in Aviator Search.
59
+ logger (logging.Logger, optional):
60
+ The logging object to use for all log messages. Defaults to default_logger.
61
+
65
62
  """
66
63
 
64
+ if logger != default_logger:
65
+ self.logger = logger.getChild("avts")
66
+ for logfilter in logger.filters:
67
+ self.logger.addFilter(logfilter)
68
+
67
69
  avts_config = {}
68
70
 
69
71
  # Store the credentials and parameters in a config dictionary:
@@ -75,9 +77,7 @@ class AVTS(object):
75
77
  avts_config["password"] = password
76
78
 
77
79
  avts_config["tokenUrl"] = avts_config["otdsUrl"] + "/otdsws/oauth2/token"
78
- avts_config["repoUrl"] = (
79
- avts_config["baseUrl"] + "/aviator-gateway/avts-api/admin/v1/repo"
80
- )
80
+ avts_config["repoUrl"] = avts_config["baseUrl"] + "/aviator-gateway/avts-api/admin/v1/repo"
81
81
 
82
82
  self._config = avts_config
83
83
  self._accesstoken = None
@@ -87,23 +87,34 @@ class AVTS(object):
87
87
  # end method definition
88
88
 
89
89
  def config(self) -> dict:
90
- """Returns the configuration dictionary
90
+ """Return the configuration dictionary.
91
91
 
92
92
  Returns:
93
93
  dict: Configuration dictionary
94
+
94
95
  """
96
+
95
97
  return self._config
96
98
 
97
99
  # end method definition
98
100
 
99
101
  def request_header(self, content_type: str = "") -> dict:
100
- """Returns the request header used for Application calls.
101
- Consists of Bearer access token and Content Type
102
+ """Return the request header used for Application calls.
103
+
104
+ Consists of Bearer access token and Content Type
102
105
 
103
106
  Args:
104
- content_type (str, optional): custom content type for the request
105
- Return:
106
- dict: request header values
107
+ content_type (str, optional):
108
+ Custom content type for the request.
109
+ Typical values:
110
+ * application/json - Used for sending JSON-encoded data
111
+ * application/x-www-form-urlencoded - The default for HTML forms.
112
+ Data is sent as key-value pairs in the body of the request, similar to query parameters.
113
+ * multipart/form-data - Used for file uploads or when a form includes non-ASCII characters
114
+
115
+ Returns:
116
+ dict: The request header values.
117
+
107
118
  """
108
119
 
109
120
  request_header = {}
@@ -135,24 +146,40 @@ class AVTS(object):
135
146
  max_retries: int = REQUEST_MAX_RETRIES,
136
147
  retry_forever: bool = False,
137
148
  ) -> dict | None:
138
- """Call an Aviator Search REST API in a safe way
149
+ """Call an Aviator Search REST API in a safe way.
139
150
 
140
151
  Args:
141
- url (str): URL to send the request to.
142
- method (str, optional): HTTP method (GET, POST, etc.). Defaults to "GET".
143
- headers (dict | None, optional): Request Headers. Defaults to None.
144
- json (dict | None, optional): Request payload. Defaults to None.
145
- files (dict | None, optional): Dictionary of {"name": file-tuple} for multipart encoding upload.
146
- file-tuple can be a 2-tuple ("filename", fileobj) or a 3-tuple ("filename", fileobj, "content_type")
147
- timeout (int | None, optional): Timeout for the request in seconds. Defaults to REQUEST_TIMEOUT.
148
- show_error (bool, optional): Whether or not an error should be logged in case of a failed REST call.
149
- If False, then only a warning is logged. Defaults to True.
150
- failure_message (str, optional): Specific error message. Defaults to "".
151
- max_retries (int, optional): How many retries on Connection errors? Default is REQUEST_MAX_RETRIES.
152
- retry_forever (bool, optional): Eventually wait forever - without timeout. Defaults to False.
152
+ url (str):
153
+ URL to send the request to.
154
+ method (str, optional):
155
+ HTTP method (GET, POST, etc.). Defaults to "GET".
156
+ headers (dict | None, optional):
157
+ Request headers. Defaults to None.
158
+ data (dict | None, optional):
159
+ Request payload. Defaults to None.
160
+ json_data (dict | None, optional):
161
+ Request payload for the JSON parameter. Defaults to None.
162
+ files (dict | None, optional):
163
+ Dictionary of {"name": file-tuple} for multipart encoding upload.
164
+ The file-tuple can be a 2-tuple ("filename", fileobj) or a 3-tuple
165
+ ("filename", fileobj, "content_type").
166
+ timeout (int | None, optional):
167
+ Timeout for the request in seconds. Defaults to REQUEST_TIMEOUT.
168
+ show_error (bool, optional):
169
+ Whether or not an error should be logged in case of a failed REST call.
170
+ If False, then only a warning is logged. Defaults to True.
171
+ failure_message (str, optional):
172
+ Specific error message. Defaults to "".
173
+ success_message (str, optional):
174
+ Specific success message. Defaults to "".
175
+ max_retries (int, optional):
176
+ Number of retries on connection errors. Defaults to REQUEST_MAX_RETRIES.
177
+ retry_forever (bool, optional):
178
+ Whether to wait forever without timeout. Defaults to False.
153
179
 
154
180
  Returns:
155
181
  dict | None: Response of Aviator Search REST API or None in case of an error.
182
+
156
183
  """
157
184
 
158
185
  retries = 0
@@ -170,30 +197,27 @@ class AVTS(object):
170
197
 
171
198
  if response.ok:
172
199
  if success_message:
173
- logger.debug(success_message)
200
+ self.logger.debug(success_message)
174
201
  return self.parse_request_response(response)
175
202
  # Check if Session has expired - then re-authenticate and try once more
176
203
  elif response.status_code == 401 and retries == 0:
177
- logger.debug("Session has expired - try to re-authenticate...")
204
+ self.logger.debug("Session has expired - try to re-authenticate...")
178
205
  self.authenticate()
179
206
  retries += 1
180
207
  else:
181
208
  # Handle plain HTML responses to not pollute the logs
182
209
  content_type = response.headers.get("content-type", None)
183
- if content_type == "text/html":
184
- response_text = "HTML content (see debug log)"
185
- else:
186
- response_text = response.text
210
+ response_text = "HTML content (see debug log)" if content_type == "text/html" else response.text
187
211
 
188
212
  if show_error:
189
- logger.error(
213
+ self.logger.error(
190
214
  "%s; status -> %s; error -> %s",
191
215
  failure_message,
192
216
  response.status_code,
193
217
  response_text,
194
218
  )
195
219
  else:
196
- logger.warning(
220
+ self.logger.warning(
197
221
  "%s; status -> %s; warning -> %s",
198
222
  failure_message,
199
223
  response.status_code,
@@ -201,7 +225,7 @@ class AVTS(object):
201
225
  )
202
226
 
203
227
  if content_type == "text/html":
204
- logger.debug(
228
+ self.logger.debug(
205
229
  "%s; status -> %s; warning -> %s",
206
230
  failure_message,
207
231
  response.status_code,
@@ -211,39 +235,39 @@ class AVTS(object):
211
235
  return None
212
236
  except requests.exceptions.Timeout:
213
237
  if retries <= max_retries:
214
- logger.warning(
238
+ self.logger.warning(
215
239
  "Request timed out. Retrying in %s seconds...",
216
240
  str(REQUEST_RETRY_DELAY),
217
241
  )
218
242
  retries += 1
219
243
  time.sleep(REQUEST_RETRY_DELAY) # Add a delay before retrying
220
244
  else:
221
- logger.error(
222
- "%s; timeout error",
245
+ self.logger.error(
246
+ "%s; timeout error.",
223
247
  failure_message,
224
248
  )
225
249
  if retry_forever:
226
250
  # If it fails after REQUEST_MAX_RETRIES retries we let it wait forever
227
- logger.warning("Turn timeouts off and wait forever...")
251
+ self.logger.warning("Turn timeouts off and wait forever...")
228
252
  timeout = None
229
253
  else:
230
254
  return None
231
255
  except requests.exceptions.ConnectionError:
232
256
  if retries <= max_retries:
233
- logger.warning(
257
+ self.logger.warning(
234
258
  "Connection error. Retrying in %s seconds...",
235
259
  str(REQUEST_RETRY_DELAY),
236
260
  )
237
261
  retries += 1
238
262
  time.sleep(REQUEST_RETRY_DELAY) # Add a delay before retrying
239
263
  else:
240
- logger.error(
241
- "%s; connection error",
264
+ self.logger.error(
265
+ "%s; connection error.",
242
266
  failure_message,
243
267
  )
244
268
  if retry_forever:
245
269
  # If it fails after REQUEST_MAX_RETRIES retries we let it wait forever
246
- logger.warning("Turn timeouts off and wait forever...")
270
+ self.logger.warning("Turn timeouts off and wait forever...")
247
271
  timeout = None
248
272
  time.sleep(REQUEST_RETRY_DELAY) # Add a delay before retrying
249
273
  else:
@@ -257,43 +281,47 @@ class AVTS(object):
257
281
  additional_error_message: str = "",
258
282
  show_error: bool = True,
259
283
  ) -> list | None:
260
- """Converts the request response (JSon) to a Python list in a safe way
261
- that also handles exceptions. It first tries to load the response.text
262
- via json.loads() that produces a dict output. Only if response.text is
263
- not set or is empty it just converts the response_object to a dict using
264
- the vars() built-in method.
284
+ """Convert the request response (JSon) to a Python list in a safe way that also handles exceptions.
285
+
286
+ It first tries to load the response.text
287
+ via json.loads() that produces a dict output. Only if response.text is
288
+ not set or is empty it just converts the response_object to a dict using
289
+ the vars() built-in method.
265
290
 
266
291
  Args:
267
- response_object (object): this is reponse object delivered by the request call
268
- additional_error_message (str, optional): use a more specific error message
269
- in case of an error
270
- show_error (bool): True: write an error to the log file
271
- False: write a warning to the log file
292
+ response_object (object):
293
+ This is reponse object delivered by the request call.
294
+ additional_error_message (str, optional):
295
+ Use a more specific error message in case of an error.
296
+ show_error (bool, optional):
297
+ If True, write an error to the log file.
298
+ If False, write a warning to the log file.
299
+
272
300
  Returns:
273
- list: response information or None in case of an error
301
+ list | None:
302
+ The response information or None in case of an error.
303
+
274
304
  """
275
305
 
276
306
  if not response_object:
277
307
  return None
278
308
 
279
309
  try:
280
- if response_object.text:
281
- list_object = json.loads(response_object.text)
282
- else:
283
- list_object = vars(response_object)
310
+ list_object = json.loads(response_object.text) if response_object.text else vars(response_object)
284
311
  except json.JSONDecodeError as exception:
285
312
  if additional_error_message:
286
313
  message = "Cannot decode response as JSON. {}; error -> {}".format(
287
- additional_error_message, exception
314
+ additional_error_message,
315
+ exception,
288
316
  )
289
317
  else:
290
318
  message = "Cannot decode response as JSON; error -> {}".format(
291
- exception
319
+ exception,
292
320
  )
293
321
  if show_error:
294
- logger.error(message)
322
+ self.logger.error(message)
295
323
  else:
296
- logger.warning(message)
324
+ self.logger.warning(message)
297
325
  return None
298
326
  else:
299
327
  return list_object
@@ -301,7 +329,13 @@ class AVTS(object):
301
329
  # end method definition
302
330
 
303
331
  def authenticate(self) -> str | None:
304
- """Authenticate at Search Aviator via oAuth authentication."""
332
+ """Authenticate at Aviator Search via OAuth.
333
+
334
+ Returns:
335
+ str | None:
336
+ The access token or None in case of an error.
337
+
338
+ """
305
339
 
306
340
  if not self._session:
307
341
  self._session = requests.Session()
@@ -319,6 +353,7 @@ class AVTS(object):
319
353
  "client_secret": self.config()["clientSecret"],
320
354
  "username": self.config()["username"],
321
355
  "password": self.config()["password"],
356
+ "scope": "otds:roles",
322
357
  }
323
358
 
324
359
  response = self.do_request(
@@ -327,7 +362,10 @@ class AVTS(object):
327
362
  headers=request_header,
328
363
  data=request_payload,
329
364
  timeout=None,
330
- failure_message=f"Failed to authenticate to OTDS with username -> {self.config()['username']} and client_id -> {self.config()['clientId']}",
365
+ failure_message="Failed to authenticate to OTDS with username -> {} and client_id -> {}".format(
366
+ self.config()["username"],
367
+ self.config()["clientId"],
368
+ ),
331
369
  )
332
370
 
333
371
  if response is not None:
@@ -337,7 +375,7 @@ class AVTS(object):
337
375
 
338
376
  # end method definition
339
377
 
340
- def repo_create_extended_ecm(
378
+ def create_extended_ecm_repo(
341
379
  self,
342
380
  name: str,
343
381
  username: str,
@@ -345,128 +383,223 @@ class AVTS(object):
345
383
  otcs_url: str,
346
384
  otcs_api_url: str,
347
385
  node_id: int,
348
- version: str = "24.3.0",
349
386
  ) -> dict | None:
350
- """Create a new repository to crawl in Aviator Search
387
+ """Create a new Extended ECM repository to crawl with Aviator Search.
351
388
 
352
389
  Args:
353
- id (str): ID of the repository
354
- name (str): socName of the repository
355
- username (str): Username to use for crawling
356
- password (str): Password of the user used for crawling
357
- otcs_url (str): Base URL of Content Server e.g. https://otcs.base-url.tld/cs/cs
358
- node_id (int): Root Node ID for crawling
390
+ name (str):
391
+ The name of the repository.
392
+ username (str):
393
+ Username to use for crawling.
394
+ password (str):
395
+ Password of the user used for crawling.
396
+ otcs_url (str):
397
+ Base URL of Content Server e.g. https://otcs.base-url.tld/cs/cs
398
+ otcs_api_url (str):
399
+ The REST API URL of Content Server.
400
+ node_id (int):
401
+ Root Node ID for crawling
402
+ version (str, optional):
403
+ TODO: The version number of ???
359
404
 
360
405
  Returns:
361
- dict | None: Parsed response object from the API or None in case of an error
406
+ dict | None:
407
+ Parsed response object from the API or None in case of an error
408
+
362
409
  """
363
410
 
364
411
  payload = {
365
- "id": "xECM",
366
- "name": name,
367
- "metadataFields": ["NODE"],
368
- "socName": "xECM",
412
+ "authType": "Basic",
369
413
  "params": [
370
414
  {
371
415
  "id": "OpenTextApiUrl",
372
- "label": "xECM API URL",
416
+ "label": "Service URL",
373
417
  "ctlType": "text",
418
+ "description": "OpenText Content Management API URL",
374
419
  "required": True,
420
+ "defaultValue": "localhost",
421
+ "visible": True,
422
+ "editable": False,
375
423
  "value": otcs_api_url,
376
424
  },
377
425
  {
378
426
  "id": "Username",
379
- "label": "xECM username",
427
+ "label": "Username",
380
428
  "ctlType": "text",
429
+ "description": "OpenText Content Management Username",
381
430
  "required": True,
431
+ "defaultValue": "",
432
+ "visible": True,
433
+ "editable": True,
382
434
  "value": username,
383
435
  },
384
436
  {
385
437
  "id": "Password",
386
- "label": "xECM Password",
438
+ "label": "Password",
387
439
  "ctlType": "password",
440
+ "description": "OpenText Content Management password",
388
441
  "required": True,
442
+ "defaultValue": "",
443
+ "visible": True,
444
+ "editable": True,
389
445
  "value": password,
390
446
  },
391
447
  {
392
- "id": "RootNodeId",
393
- "label": "Root Node ID",
448
+ "id": "sourceLink",
449
+ "label": "Source Link",
394
450
  "ctlType": "text",
451
+ "description": "Example: <OpenText Content Management API URL>/app/nodes/${NODE}/metadata",
452
+ "required": False,
453
+ "defaultValue": otcs_url + "/app/nodes/${NODE}/metadata",
454
+ "visible": True,
455
+ "editable": True,
456
+ },
457
+ {
458
+ "id": "RootNodeIds",
459
+ "label": "Root Node ID's",
460
+ "ctlType": "text",
461
+ "description": "List of nodes to be crawled(comma seperated)",
395
462
  "required": True,
396
- "value": node_id,
463
+ "defaultValue": "",
464
+ "visible": True,
465
+ "editable": False,
466
+ "value": "2000",
397
467
  },
398
468
  {
399
- "id": "sourceLink",
400
- "label": "Source Link( ex:https://<xECM host>/cs/cs/app/nodes/${NODE}/metadata )",
469
+ "id": "proxy",
470
+ "label": "Proxy Service",
471
+ "ctlType": "boolean",
472
+ "description": "",
473
+ "required": False,
474
+ "defaultValue": "false",
475
+ "value": "false",
476
+ "visible": True,
477
+ "editable": True,
478
+ },
479
+ {
480
+ "id": "proxyScheme",
481
+ "label": "Proxy Scheme",
482
+ "ctlType": "select",
483
+ "description": "",
484
+ "required": False,
485
+ "defaultValue": "HTTP",
486
+ "value": "HTTP",
487
+ "visible": True,
488
+ "acceptedValues": ["HTTP", "HTTPS", "SOCKS5"],
489
+ "editable": True,
490
+ },
491
+ {
492
+ "id": "proxyHost",
493
+ "label": "Proxy Host",
401
494
  "ctlType": "text",
495
+ "description": "",
402
496
  "required": False,
403
- "defaultValue": otcs_url + "/app/nodes/${NODE}/metadata",
497
+ "defaultValue": "",
498
+ "value": "",
404
499
  "visible": True,
500
+ "editable": True,
501
+ },
502
+ {
503
+ "id": "proxyPort",
504
+ "label": "Proxy Port",
505
+ "ctlType": "text",
506
+ "description": "",
507
+ "required": False,
508
+ "defaultValue": "",
509
+ "value": "",
510
+ "visible": True,
511
+ "editable": True,
512
+ },
513
+ {
514
+ "id": "ProxyConfigService",
515
+ "label": "Proxy Config Service",
516
+ "ctlType": "text",
517
+ "description": "",
518
+ "required": False,
519
+ "defaultValue": "",
520
+ "value": "",
521
+ "visible": False,
522
+ "editable": True,
405
523
  },
406
524
  ],
407
- "idolConfig": {
408
- "view": {
409
- "name": "ViewOpenText",
410
- "type": "idol.nifi.connector.ViewOpenText",
525
+ "config": {
526
+ "type": "nifi",
527
+ "id": "xECM",
528
+ "crawlConfig": {
529
+ "name": "GetOpenText",
530
+ "type": "idol.nifi.connector.GetOpenText",
411
531
  "group": "idol.nifi.connector",
412
532
  "artifact": "idol-nifi-connector-opentext",
413
- "version": version,
533
+ "version": "25.1.0-nifi1",
414
534
  },
415
- "crawler": {
416
- "name": "GetOpenText",
417
- "type": "idol.nifi.connector.GetOpenText",
535
+ "viewConfig": {
536
+ "name": "ViewOpenText",
537
+ "type": "idol.nifi.connector.ViewOpenText",
418
538
  "group": "idol.nifi.connector",
419
539
  "artifact": "idol-nifi-connector-opentext",
420
- "version": version,
540
+ "version": "25.1.0-nifi1",
421
541
  },
422
- "omniGroup": {
542
+ "omniConfig": {
423
543
  "name": "GetOpenTextGroups",
424
544
  "type": "idol.nifi.connector.GetOpenTextGroups",
425
545
  "group": "idol.nifi.connector",
426
546
  "artifact": "idol-nifi-connector-opentext",
427
- "version": version,
547
+ "version": "25.1.0-nifi1",
548
+ "repoName": "ECM",
428
549
  },
429
- },
430
- "idolProperties": {
431
- "view": {
550
+ "crawlProps": {
432
551
  "Password": "${Password}",
433
552
  "Username": "${UserName}",
553
+ "META:SOURCE": "OPENTEXT",
554
+ "RootNodeIds": "${RootNodeIds}",
555
+ "MappedSecurity": "true",
434
556
  "OpenTextApiUrl": "${OpenTextApiUrl}",
557
+ "ProxyConfigService": "${ProxyConfigService}",
435
558
  },
436
- "crawler": {
559
+ "viewProps": {
437
560
  "Password": "${Password}",
438
561
  "Username": "${UserName}",
439
- "RootNodeId": "${RootNodeId}",
440
- "META:SOURCE": "OPENTEXT",
441
- "MappedSecurity": "true",
442
562
  "OpenTextApiUrl": "${OpenTextApiUrl}",
563
+ "ProxyConfigService": "${ProxyConfigService}",
443
564
  },
444
- "omniGroup": {
565
+ "omniProps": {
445
566
  "Password": "${Password}",
446
567
  "Username": "${UserName}",
447
568
  "OpenTextApiUrl": "${OpenTextApiUrl}",
569
+ "ProxyConfigService": "${ProxyConfigService}",
448
570
  "OpenTextApiPageSize": "10",
449
571
  },
572
+ "metadataFields": ["NODE"],
450
573
  },
574
+ "name": name,
575
+ "id": "xECM",
576
+ "sourceId": "xECM",
451
577
  }
452
578
 
453
579
  request_header = self.request_header()
454
580
  request_url = self.config()["repoUrl"]
455
581
 
456
- return self.do_request(
582
+ response = self.do_request(
457
583
  url=request_url,
458
584
  method="POST",
459
585
  json_data=payload,
460
586
  headers=request_header,
461
587
  timeout=None,
462
588
  failure_message="Failed to create repository -> '{}' ({})".format(
463
- name, node_id
589
+ name,
590
+ node_id,
464
591
  ),
592
+ show_error=False,
465
593
  )
466
594
 
595
+ if response is None:
596
+ self.logger.error("Failed to create repository -> %s (%s)", name, node_id)
597
+
598
+ return response
599
+
467
600
  # end method definition
468
601
 
469
- def repo_create_msteams(
602
+ def create_msteams_repo(
470
603
  self,
471
604
  name: str,
472
605
  client_id: str,
@@ -477,36 +610,44 @@ class AVTS(object):
477
610
  index_call_recordings: bool = True,
478
611
  index_message_replies: bool = True,
479
612
  index_user_chats: bool = True,
480
- oauth2_site_name: str = "AVTS",
481
- oauth2_sites_file: str = "",
482
- version: str = "24.3.0",
483
613
  ) -> dict | None:
484
- """Create a new repository to crawl in Aviator Search
614
+ """Create a new Microsoft Teams repository to crawl with Aviator Search.
485
615
 
486
616
  Args:
487
- id (str): ID of the repository
488
- name (str): socName of the repository
489
- #todo: add more params
617
+ name (str):
618
+ The name of the repository.
619
+ client_id (str):
620
+ The M365 client ID.
621
+ tenant_id (str):
622
+ The M365 tenant ID.
623
+ certificate_file (str):
624
+ The path to the certificate file.
625
+ certificate_password (str):
626
+ The password for the certificate.
627
+ index_attachments (bool, optional):
628
+ Whether or not to index / crawl attachments.
629
+ index_call_recordings (bool, optional):
630
+ Whether or not to index / crawl meeting recordings.
631
+ index_message_replies (bool, optional):
632
+ Whether or not to index / crawl message replies.
633
+ index_user_chats (bool, optional):
634
+ Whether or not to index / crawl user chats.
635
+ version(str, optional): default 24.3.0
636
+
637
+ # TODO: add more params
490
638
 
491
639
  Returns:
492
- dict | None: Parsed response object from the API or None in case of an error
640
+ dict | None:
641
+ Parsed response object from the API or None in case of an error
642
+
493
643
  """
494
644
 
495
- if os.path.isfile(certificate_file):
496
- # Open the file in binary mode
497
- with open(certificate_file, "rb") as file:
498
- # Read the content of the file
499
- certificate_file_content = file.read()
500
- # Convert the bytes to a base64 string
501
- certificate_file_content_base64 = base64.b64encode(
502
- certificate_file_content
503
- ).decode("utf-8")
645
+ certificate_file_content_base64 = self.get_certificate_file_content_base64(
646
+ certificate_file,
647
+ )
504
648
 
505
649
  payload = {
506
- "id": "MSTeams",
507
- "socName": "Microsoft Teams",
508
650
  "authType": "OAUTH",
509
- "name": name,
510
651
  "params": [
511
652
  {
512
653
  "id": "OAuth2SiteName",
@@ -516,6 +657,7 @@ class AVTS(object):
516
657
  "defaultValue": "AVTS",
517
658
  "value": "AVTS",
518
659
  "visible": False,
660
+ "editable": True,
519
661
  },
520
662
  {
521
663
  "id": "OAuth2SitesFile",
@@ -525,6 +667,7 @@ class AVTS(object):
525
667
  "defaultValue": "",
526
668
  "value": "",
527
669
  "visible": False,
670
+ "editable": True,
528
671
  },
529
672
  {
530
673
  "id": "sourceLink",
@@ -533,6 +676,7 @@ class AVTS(object):
533
676
  "required": False,
534
677
  "defaultValue": "",
535
678
  "visible": True,
679
+ "editable": True,
536
680
  },
537
681
  {
538
682
  "id": "clientID",
@@ -543,6 +687,7 @@ class AVTS(object):
543
687
  "defaultValue": "",
544
688
  "value": client_id,
545
689
  "visible": True,
690
+ "editable": True,
546
691
  },
547
692
  {
548
693
  "id": "tenant",
@@ -553,6 +698,7 @@ class AVTS(object):
553
698
  "defaultValue": "",
554
699
  "value": tenant_id,
555
700
  "visible": True,
701
+ "editable": False,
556
702
  },
557
703
  {
558
704
  "id": "IndexAttachments",
@@ -561,8 +707,9 @@ class AVTS(object):
561
707
  "description": "Specifies whether to index attachments",
562
708
  "required": False,
563
709
  "defaultValue": "true",
564
- "value": str(index_attachments).lower(),
565
- "visible": True,
710
+ "value": "true",
711
+ "visible": str(index_attachments).lower(),
712
+ "editable": True,
566
713
  },
567
714
  {
568
715
  "id": "IndexCallRecordings",
@@ -573,6 +720,7 @@ class AVTS(object):
573
720
  "defaultValue": "true",
574
721
  "value": str(index_call_recordings).lower(),
575
722
  "visible": True,
723
+ "editable": True,
576
724
  },
577
725
  {
578
726
  "id": "IndexMessageReplies",
@@ -583,6 +731,7 @@ class AVTS(object):
583
731
  "defaultValue": "true",
584
732
  "value": str(index_message_replies).lower(),
585
733
  "visible": True,
734
+ "editable": True,
586
735
  },
587
736
  {
588
737
  "id": "IndexUserChats",
@@ -593,6 +742,7 @@ class AVTS(object):
593
742
  "defaultValue": "true",
594
743
  "value": str(index_user_chats).lower(),
595
744
  "visible": True,
745
+ "editable": True,
596
746
  },
597
747
  {
598
748
  "id": "certificateFile",
@@ -601,8 +751,9 @@ class AVTS(object):
601
751
  "description": 'Please upload a valid "*.pfx" certificate file',
602
752
  "required": True,
603
753
  "defaultValue": "",
604
- "value": "C:\\fakepath\\certificate.pfx",
754
+ "value": "C:\\fakepath\\certificate 1 3 (1).pfx",
605
755
  "visible": True,
756
+ "editable": True,
606
757
  "fileDatabase64": f"data:application/x-pkcs12;base64,{certificate_file_content_base64}",
607
758
  },
608
759
  {
@@ -613,42 +764,126 @@ class AVTS(object):
613
764
  "defaultValue": "",
614
765
  "value": certificate_password,
615
766
  "visible": True,
767
+ "editable": True,
768
+ },
769
+ {
770
+ "id": "proxy",
771
+ "label": "Proxy Service",
772
+ "ctlType": "boolean",
773
+ "description": "",
774
+ "required": False,
775
+ "defaultValue": "false",
776
+ "value": "true",
777
+ "visible": True,
778
+ "editable": True,
779
+ },
780
+ {
781
+ "id": "proxyScheme",
782
+ "label": "Proxy Scheme",
783
+ "ctlType": "select",
784
+ "description": "",
785
+ "required": False,
786
+ "defaultValue": "HTTP",
787
+ "value": "HTTP",
788
+ "visible": True,
789
+ "acceptedValues": [
790
+ "HTTP",
791
+ "HTTPS",
792
+ "SOCKS5",
793
+ ],
794
+ "editable": True,
795
+ },
796
+ {
797
+ "id": "proxyHost",
798
+ "label": "Proxy Host",
799
+ "ctlType": "text",
800
+ "description": "",
801
+ "required": False,
802
+ "defaultValue": "",
803
+ "value": "10.194.10.21",
804
+ "visible": True,
805
+ "editable": True,
806
+ },
807
+ {
808
+ "id": "proxyPort",
809
+ "label": "Proxy Port",
810
+ "ctlType": "text",
811
+ "description": "",
812
+ "required": False,
813
+ "defaultValue": "",
814
+ "value": "3128",
815
+ "visible": True,
816
+ "editable": True,
817
+ },
818
+ {
819
+ "id": "ProxyConfigService",
820
+ "label": "Proxy Config Service",
821
+ "ctlType": "text",
822
+ "description": "",
823
+ "required": False,
824
+ "defaultValue": "",
825
+ "value": "",
826
+ "visible": False,
827
+ "editable": True,
616
828
  },
617
829
  ],
618
- "idolConfig": {
619
- "view": {
830
+ "config": {
831
+ "type": "nifi",
832
+ "id": "MSTeams",
833
+ "crawlConfig": {
834
+ "name": "GetMicrosoftTeams",
835
+ "type": "idol.nifi.connector.GetMicrosoftTeams",
836
+ "group": "idol.nifi.connector",
837
+ "artifact": "idol-nifi-connector-officeteams",
838
+ "version": "25.1.0-nifi1",
839
+ },
840
+ "viewConfig": {
620
841
  "name": "ViewMicrosoftTeams",
621
842
  "type": "idol.nifi.connector.ViewMicrosoftTeams",
622
843
  "group": "idol.nifi.connector",
623
844
  "artifact": "idol-nifi-connector-officeteams",
624
- "version": version,
845
+ "version": "25.1.0-nifi1",
625
846
  },
626
- "crawler": {
627
- "name": "GetMicrosoftTeams",
628
- "type": "idol.nifi.connector.GetMicrosoftTeams",
847
+ "omniConfig": {
848
+ "name": "GetMicrosoftTeamsGroups",
849
+ "type": "idol.nifi.connector.GetMicrosoftTeamsGroups",
629
850
  "group": "idol.nifi.connector",
630
851
  "artifact": "idol-nifi-connector-officeteams",
631
- "version": version,
852
+ "version": "25.1.0-nifi1",
853
+ "repoName": "OneDrive",
632
854
  },
633
- },
634
- "idolProperties": {
635
- "view": {
636
- "Oauth2SiteName": "${OAuth2SiteName}",
637
- "Oauth2SitesFile": "${OAuth2SitesFile}",
638
- "IndexCallRecordings": "true",
639
- },
640
- "crawler": {
641
- "META:SOURCE": "MSTeams",
855
+ "crawlProps": {
856
+ "META:SOURCE": "Microsoft Teams",
642
857
  "IndexUserChats": "${IndexUserChats}",
858
+ "MappedSecurity": "true",
643
859
  "Oauth2SiteName": "${OAuth2SiteName}",
644
860
  "Oauth2SitesFile": "${OAuth2SitesFile}",
645
861
  "IndexAttachments": "${IndexAttachments}",
862
+ "ProxyConfigService": "${ProxyConfigService}",
646
863
  "IndexCallRecordings": "${IndexCallRecordings}",
647
864
  "IndexMessageReplies": "${IndexMessageReplies}",
865
+ "ChatMessageGroupingSection": "chat",
866
+ "ChannelMessageGroupingSection": "channel",
867
+ "[chat]MessageGroupingInterval": "24 hour",
868
+ "[chat]MessageGroupingStrategy": "Interval",
869
+ "[channel]MessageGroupingInterval": "24 hour",
870
+ "[channel]MessageGroupingStrategy": "Interval",
871
+ },
872
+ "viewProps": {
873
+ "Oauth2SiteName": "${OAuth2SiteName}",
874
+ "Oauth2SitesFile": "${OAuth2SitesFile}",
875
+ "ProxyConfigService": "${ProxyConfigService}",
876
+ },
877
+ "omniProps": {
878
+ "Oauth2SiteName": "${OAuth2SiteName}",
879
+ "Oauth2SitesFile": "${OAuth2SitesFile}",
880
+ "ProxyConfigService": "${ProxyConfigService}",
648
881
  },
882
+ "metadataFields": [],
649
883
  },
650
- "authRedirect": "",
651
- "metadataFields": [],
884
+ "name": name,
885
+ "id": "MSTeams",
886
+ "sourceId": "MSTeams",
652
887
  }
653
888
 
654
889
  request_header = self.request_header()
@@ -661,9 +896,11 @@ class AVTS(object):
661
896
  headers=request_header,
662
897
  timeout=None,
663
898
  failure_message="Failed to create repository -> '{}'".format(name),
899
+ show_error=False,
664
900
  )
665
901
 
666
902
  if response is None:
903
+ self.logger.error("Failed to create repository -> %s", name)
667
904
  return None
668
905
 
669
906
  self.repo_admin_consent(response["id"])
@@ -672,7 +909,7 @@ class AVTS(object):
672
909
 
673
910
  # end method definition
674
911
 
675
- def repo_create_sharepoint(
912
+ def create_sharepoint_repo(
676
913
  self,
677
914
  name: str,
678
915
  client_id: str,
@@ -686,34 +923,49 @@ class AVTS(object):
686
923
  index_user_profiles: bool = True,
687
924
  oauth2_site_name: str = "AVTS",
688
925
  oauth2_sites_file: str = "",
689
- version: str = "24.3.0",
690
926
  ) -> dict | None:
691
- """Create a new repository to crawl in Aviator Search
927
+ """Create a new Microsoft SharePoint repository to crawl with Aviator Search.
692
928
 
693
929
  Args:
694
- id (str): ID of the repository
695
- name (str): socName of the repository
696
- #todo: add more params
930
+ name (str):
931
+ The name of the repository.
932
+ client_id (str):
933
+ The M365 client ID.
934
+ tenant_id (str):
935
+ The M365 tenant ID.
936
+ certificate_file (str):
937
+ TODO: _description_
938
+ certificate_password (int):
939
+ TODO: _description_
940
+ sharepoint_url (str):
941
+ The SharePoint URL.
942
+ sharepoint_url_type (str):
943
+ The SharePoint URL type.
944
+ sharepoint_mysite_url (str):
945
+ The SharePoint MySite URL.
946
+ sharepoint_admin_url (str):
947
+ The SharePoint administration URL.
948
+ index_user_profiles (bool, optional):
949
+ TODO: _description_. Defaults to True.
950
+ oauth2_site_name (str, optional):
951
+ TODO: _description_. Defaults to "AVTS".
952
+ oauth2_sites_file (str, optional):
953
+ TODO: _description_. Defaults to "".
954
+ version (str, optional):
955
+ TODO: _description_. Defaults to "24.3.0".
697
956
 
698
957
  Returns:
699
- dict | None: Parsed response object from the API or None in case of an error
958
+ dict | None:
959
+ Parsed response object from the API or None in case of an error
960
+
700
961
  """
701
962
 
702
- if os.path.isfile(certificate_file):
703
- # Open the file in binary mode
704
- with open(certificate_file, "rb") as file:
705
- # Read the content of the file
706
- certificate_file_content = file.read()
707
- # Convert the bytes to a base64 string
708
- certificate_file_content_base64 = base64.b64encode(
709
- certificate_file_content
710
- ).decode("utf-8")
963
+ certificate_file_content_base64 = self.get_certificate_file_content_base64(
964
+ certificate_file,
965
+ )
711
966
 
712
967
  payload = {
713
- "id": "SharePoint",
714
- "socName": "SharePoint Online",
715
968
  "authType": "OAUTH",
716
- "name": name,
717
969
  "params": [
718
970
  {
719
971
  "id": "OAuth2SiteName",
@@ -723,6 +975,7 @@ class AVTS(object):
723
975
  "defaultValue": "AVTS",
724
976
  "value": oauth2_site_name,
725
977
  "visible": False,
978
+ "editable": True,
726
979
  },
727
980
  {
728
981
  "id": "OAuth2SitesFile",
@@ -732,6 +985,7 @@ class AVTS(object):
732
985
  "defaultValue": "",
733
986
  "value": oauth2_sites_file,
734
987
  "visible": False,
988
+ "editable": True,
735
989
  },
736
990
  {
737
991
  "id": "sourceLink",
@@ -741,8 +995,8 @@ class AVTS(object):
741
995
  "required": False,
742
996
  "defaultValue": "",
743
997
  "visible": True,
744
- "value": sharepoint_url
745
- + "${FILEDIRREF}/Forms/AllItems.aspx?id=${FILEREF}&parent=${FILEDIRREF}",
998
+ "editable": True,
999
+ "value": sharepoint_url + "${FILEDIRREF}/Forms/AllItems.aspx?id=${FILEREF}&parent=${FILEDIRREF}",
746
1000
  },
747
1001
  {
748
1002
  "id": "clientID",
@@ -753,6 +1007,7 @@ class AVTS(object):
753
1007
  "defaultValue": "",
754
1008
  "value": client_id,
755
1009
  "visible": True,
1010
+ "editable": True,
756
1011
  },
757
1012
  {
758
1013
  "id": "tenant",
@@ -763,6 +1018,7 @@ class AVTS(object):
763
1018
  "defaultValue": "",
764
1019
  "value": tenant_id,
765
1020
  "visible": True,
1021
+ "editable": True,
766
1022
  },
767
1023
  {
768
1024
  "id": "sharePointUrl",
@@ -771,8 +1027,19 @@ class AVTS(object):
771
1027
  "description": 'The URL to start synchronizing from. Specify a URL that matches "SharePoint URL type"',
772
1028
  "required": True,
773
1029
  "defaultValue": "",
774
- "value": sharepoint_url + "/",
1030
+ "value": sharepoint_mysite_url,
1031
+ "visible": True,
1032
+ "editable": False,
1033
+ },
1034
+ {
1035
+ "id": "MappedWebApplicationPolicies",
1036
+ "label": "Mapped Web Application Policies",
1037
+ "ctlType": "boolean",
1038
+ "required": False,
1039
+ "defaultValue": "false",
1040
+ "value": "false",
775
1041
  "visible": True,
1042
+ "editable": False,
776
1043
  },
777
1044
  {
778
1045
  "id": "sharePointAdminUrl",
@@ -783,6 +1050,7 @@ class AVTS(object):
783
1050
  "defaultValue": "",
784
1051
  "value": sharepoint_admin_url,
785
1052
  "visible": True,
1053
+ "editable": False,
786
1054
  },
787
1055
  {
788
1056
  "id": "sharePointMySiteUrl",
@@ -793,6 +1061,7 @@ class AVTS(object):
793
1061
  "defaultValue": "",
794
1062
  "value": sharepoint_mysite_url,
795
1063
  "visible": True,
1064
+ "editable": False,
796
1065
  },
797
1066
  {
798
1067
  "id": "sharePointOnline",
@@ -803,15 +1072,7 @@ class AVTS(object):
803
1072
  "defaultValue": "true",
804
1073
  "value": "true",
805
1074
  "visible": False,
806
- },
807
- {
808
- "id": "MappedWebApplicationPolicies",
809
- "label": "Mapped Web Application Policies",
810
- "ctlType": "text",
811
- "required": False,
812
- "defaultValue": "false",
813
- "value": "false",
814
- "visible": False,
1075
+ "editable": False,
815
1076
  },
816
1077
  {
817
1078
  "id": "TenantAdminSitesIncludeTypes",
@@ -822,6 +1083,7 @@ class AVTS(object):
822
1083
  "defaultValue": "all",
823
1084
  "value": "all",
824
1085
  "visible": False,
1086
+ "editable": False,
825
1087
  },
826
1088
  {
827
1089
  "id": "URLType",
@@ -830,7 +1092,7 @@ class AVTS(object):
830
1092
  "description": 'The type of URL specified by "Sharepoint URL"',
831
1093
  "required": True,
832
1094
  "defaultValue": "",
833
- "value": "SiteCollection",
1095
+ "value": sharepoint_url_type,
834
1096
  "visible": True,
835
1097
  "acceptedValues": [
836
1098
  "WebApplication",
@@ -838,6 +1100,7 @@ class AVTS(object):
838
1100
  "PersonalSiteCollection",
839
1101
  "TenantAdmin",
840
1102
  ],
1103
+ "editable": False,
841
1104
  },
842
1105
  {
843
1106
  "id": "IndexUserProfiles",
@@ -848,6 +1111,7 @@ class AVTS(object):
848
1111
  "defaultValue": "false",
849
1112
  "value": str(index_user_profiles).lower(),
850
1113
  "visible": True,
1114
+ "editable": True,
851
1115
  },
852
1116
  {
853
1117
  "id": "certificateFile",
@@ -856,8 +1120,9 @@ class AVTS(object):
856
1120
  "description": 'Please upload a valid "*.pfx" certificate file',
857
1121
  "required": True,
858
1122
  "defaultValue": "",
859
- "value": "C:\\fakepath\\certificate.pfx",
1123
+ "value": "C:\\fakepath\\certificate 1 3 (1).pfx",
860
1124
  "visible": True,
1125
+ "editable": True,
861
1126
  "fileDatabase64": f"data:application/x-pkcs12;base64,{certificate_file_content_base64}",
862
1127
  },
863
1128
  {
@@ -868,51 +1133,141 @@ class AVTS(object):
868
1133
  "defaultValue": "",
869
1134
  "value": certificate_password,
870
1135
  "visible": True,
1136
+ "editable": True,
1137
+ },
1138
+ {
1139
+ "id": "proxy",
1140
+ "label": "Proxy Service",
1141
+ "ctlType": "boolean",
1142
+ "description": "",
1143
+ "required": False,
1144
+ "defaultValue": "false",
1145
+ "value": "true",
1146
+ "visible": True,
1147
+ "editable": True,
1148
+ },
1149
+ {
1150
+ "id": "proxyScheme",
1151
+ "label": "Proxy Scheme",
1152
+ "ctlType": "select",
1153
+ "description": "",
1154
+ "required": False,
1155
+ "defaultValue": "HTTP",
1156
+ "value": "HTTP",
1157
+ "visible": True,
1158
+ "acceptedValues": [
1159
+ "HTTP",
1160
+ "HTTPS",
1161
+ "SOCKS5",
1162
+ ],
1163
+ "editable": True,
1164
+ },
1165
+ {
1166
+ "id": "proxyHost",
1167
+ "label": "Proxy Host",
1168
+ "ctlType": "text",
1169
+ "description": "",
1170
+ "required": False,
1171
+ "defaultValue": "",
1172
+ "value": "10.194.10.21",
1173
+ "visible": True,
1174
+ "editable": True,
1175
+ },
1176
+ {
1177
+ "id": "proxyPort",
1178
+ "label": "Proxy Port",
1179
+ "ctlType": "text",
1180
+ "description": "",
1181
+ "required": False,
1182
+ "defaultValue": "",
1183
+ "value": "3128",
1184
+ "visible": True,
1185
+ "editable": True,
1186
+ },
1187
+ {
1188
+ "id": "ProxyConfigService",
1189
+ "label": "Proxy Config Service",
1190
+ "ctlType": "text",
1191
+ "description": "",
1192
+ "required": False,
1193
+ "defaultValue": "",
1194
+ "value": "",
1195
+ "visible": False,
1196
+ "editable": True,
871
1197
  },
872
1198
  ],
873
- "idolConfig": {
874
- "view": {
1199
+ "config": {
1200
+ "type": "nifi",
1201
+ "id": "SharePoint",
1202
+ "crawlConfig": {
1203
+ "name": "GetSharePointOData",
1204
+ "type": "idol.nifi.connector.GetSharePointOData",
1205
+ "group": "idol.nifi.connector",
1206
+ "artifact": "idol-nifi-connector-sharepointodata",
1207
+ "version": "25.1.0-nifi1",
1208
+ },
1209
+ "viewConfig": {
875
1210
  "name": "ViewSharePointOData",
876
1211
  "type": "idol.nifi.connector.ViewSharePointOData",
877
1212
  "group": "idol.nifi.connector",
878
1213
  "artifact": "idol-nifi-connector-sharepointodata",
879
- "version": version,
1214
+ "version": "25.1.0-nifi1",
880
1215
  },
881
- "crawler": {
882
- "name": "GetSharePointOData",
883
- "type": "idol.nifi.connector.GetSharePointOData",
1216
+ "omniConfig": {
1217
+ "name": "GetSharePointGroupsOData",
1218
+ "type": "idol.nifi.connector.GetSharePointGroupsOData",
884
1219
  "group": "idol.nifi.connector",
885
1220
  "artifact": "idol-nifi-connector-sharepointodata",
886
- "version": version,
1221
+ "version": "25.1.0-nifi1",
1222
+ "repoName": "SharePoint",
887
1223
  },
888
- },
889
- "idolProperties": {
890
- "view": {
1224
+ "crawlProps": {
1225
+ "META:SOURCE": "SharePoint Online",
891
1226
  "SharepointUrl": "${sharePointUrl}",
1227
+ "MappedSecurity": "true",
892
1228
  "Oauth2SiteName": "${OAuth2SiteName}",
893
1229
  "Oauth2SitesFile": "${OAuth2SitesFile}",
894
1230
  "SharepointOnline": "${sharePointOnline}",
1231
+ "IndexUserProfiles": "${IndexUserProfiles}",
895
1232
  "SharepointUrlType": "${URLType}",
1233
+ "ProxyConfigService": "${ProxyConfigService}",
896
1234
  "SharepointAdminUrl": "${sharePointAdminUrl}",
897
1235
  "SharepointMySiteUrl": "${sharePointMySiteUrl}",
1236
+ "RetrieveUserDetailsAs": "Title",
898
1237
  "MappedWebApplicationPolicies": "${MappedWebApplicationPolicies}",
1238
+ "TenantAdminSitesIncludeTypes": "${TenantAdminSitesIncludeTypes}",
899
1239
  },
900
- "crawler": {
901
- "META:SOURCE": "SharePoint",
1240
+ "viewProps": {
902
1241
  "SharepointUrl": "${sharePointUrl}",
903
1242
  "Oauth2SiteName": "${OAuth2SiteName}",
904
1243
  "Oauth2SitesFile": "${OAuth2SitesFile}",
905
1244
  "SharepointOnline": "${sharePointOnline}",
906
- "IndexUserProfiles": "${IndexUserProfiles}",
907
1245
  "SharepointUrlType": "${URLType}",
1246
+ "ProxyConfigService": "${ProxyConfigService}",
908
1247
  "SharepointAdminUrl": "${sharePointAdminUrl}",
909
1248
  "SharepointMySiteUrl": "${sharePointMySiteUrl}",
910
1249
  "MappedWebApplicationPolicies": "${MappedWebApplicationPolicies}",
1250
+ },
1251
+ "omniProps": {
1252
+ "SharepointUrl": "${sharePointUrl}",
1253
+ "Oauth2SiteName": "${OAuth2SiteName}",
1254
+ "Oauth2SitesFile": "${OAuth2SitesFile}",
1255
+ "SharepointOnline": "true",
1256
+ "SharepointUrlType": "${URLType}",
1257
+ "ProxyConfigService": "${ProxyConfigService}",
1258
+ "SharepointAdminUrl": "${sharePointAdminUrl}",
1259
+ "SharepointMySiteUrl": "${sharePointMySiteUrl}",
1260
+ "MappedWebApplicationPolicies": "false",
911
1261
  "TenantAdminSitesIncludeTypes": "${TenantAdminSitesIncludeTypes}",
912
1262
  },
1263
+ "metadataFields": [
1264
+ "FILEREF",
1265
+ "FILEDIRREF",
1266
+ ],
913
1267
  },
914
- "authRedirect": "",
915
- "metadataFields": ["FILEREF", "FILEDIRREF"],
1268
+ "name": name,
1269
+ "id": "SharePoint",
1270
+ "sourceId": "SharePoint",
916
1271
  }
917
1272
 
918
1273
  request_header = self.request_header()
@@ -925,9 +1280,11 @@ class AVTS(object):
925
1280
  headers=request_header,
926
1281
  timeout=None,
927
1282
  failure_message="Failed to create repository -> '{}'".format(name),
1283
+ show_error=False,
928
1284
  )
929
1285
 
930
1286
  if response is None:
1287
+ self.logger.error("Failed to create repository -> %s", name)
931
1288
  return None
932
1289
 
933
1290
  self.repo_admin_consent(response["id"])
@@ -937,21 +1294,22 @@ class AVTS(object):
937
1294
  # end method definition
938
1295
 
939
1296
  def repo_admin_consent(self, repo_id: str) -> dict | None:
940
- """Send admin consent information for a repository
1297
+ """Send admin consent information for a repository.
941
1298
 
942
1299
  Args:
943
- repo_id (str): id of the repository
1300
+ repo_id (str):
1301
+ The ID of the repository.
944
1302
 
945
1303
  Returns:
946
- dict | None: Parsed response object from the API or None in case of an error
1304
+ dict | None:
1305
+ Parsed response object from the API or None in case of an error
1306
+
947
1307
  """
948
1308
 
949
1309
  request_header = self.request_header()
950
1310
  request_url = self.config()["repoUrl"]
951
1311
 
952
- request_url = (
953
- self.config()["repoUrl"] + "/" + repo_id + "/authorize?admin_consent=true"
954
- )
1312
+ request_url = self.config()["repoUrl"] + "/" + repo_id + "/authorize?admin_consent=true"
955
1313
 
956
1314
  return self.do_request(
957
1315
  url=request_url,
@@ -959,22 +1317,26 @@ class AVTS(object):
959
1317
  headers=request_header,
960
1318
  timeout=None,
961
1319
  failure_message="Failed to set admin_consent for repository -> '{}'".format(
962
- repo_id
1320
+ repo_id,
963
1321
  ),
964
1322
  )
965
1323
 
966
1324
  # end method definition
967
1325
 
968
1326
  def start_crawling(self, repo_name: str) -> list | None:
969
- """Start crawling of a repository
1327
+ """Start crawling of a repository.
970
1328
 
971
1329
  Args:
972
- repo_name (str): name of the repository
1330
+ repo_name (str):
1331
+ The name of the repository.
1332
+
973
1333
  Returns:
974
- list | None: Parsed response object from the API or None in case of an error
1334
+ list | None:
1335
+ Parsed response object from the API or None in case of an error
1336
+
975
1337
  """
976
1338
 
977
- logger.info("Start crawling repository -> %s", repo_name)
1339
+ self.logger.info("Start crawling repository -> %s", repo_name)
978
1340
 
979
1341
  repo = self.get_repo_by_name(name=repo_name)
980
1342
  if repo is None:
@@ -989,19 +1351,23 @@ class AVTS(object):
989
1351
  headers=request_header,
990
1352
  timeout=None,
991
1353
  failure_message="Failed to start crawling repository -> '{}'".format(
992
- repo_name
1354
+ repo_name,
993
1355
  ),
994
1356
  )
995
1357
 
996
1358
  # end method definition
997
1359
 
998
1360
  def stop_crawling(self, repo_name: str) -> list | None:
999
- """Stop the crawling of a repository
1361
+ """Stop the crawling of a repository.
1000
1362
 
1001
1363
  Args:
1002
- repo_name (str): name of the repository
1364
+ repo_name (str):
1365
+ The name of the repository.
1366
+
1003
1367
  Returns:
1004
- list | None: Parsed response object from the API or None in case of an error
1368
+ list | None:
1369
+ Parsed response object from the API or None in case of an error
1370
+
1005
1371
  """
1006
1372
 
1007
1373
  repo = self.get_repo_by_name(name=repo_name)
@@ -1017,17 +1383,19 @@ class AVTS(object):
1017
1383
  headers=request_header,
1018
1384
  timeout=None,
1019
1385
  failure_message="Failed to stop crawling repository -> '{}'".format(
1020
- repo_name
1386
+ repo_name,
1021
1387
  ),
1022
1388
  )
1023
1389
 
1024
1390
  # end method definition
1025
1391
 
1026
1392
  def get_repo_list(self) -> list | None:
1027
- """Get a list of all repositories
1393
+ """Get a list of all repositories.
1028
1394
 
1029
1395
  Returns:
1030
- list | None: Parsed response object from the API listing all repositories or None in case of an error
1396
+ list | None:
1397
+ Parsed response object from the API listing all repositories or None in case of an error.
1398
+
1031
1399
  """
1032
1400
 
1033
1401
  request_header = self.request_header()
@@ -1038,18 +1406,22 @@ class AVTS(object):
1038
1406
  method="GET",
1039
1407
  headers=request_header,
1040
1408
  timeout=None,
1041
- failure_message="Failed to get list of repositories to crawl.",
1409
+ failure_message="Failed to get list of repositories to crawl",
1042
1410
  )
1043
1411
 
1044
1412
  # end method definition
1045
1413
 
1046
1414
  def get_repo_by_name(self, name: str) -> dict | None:
1047
- """Get a repository by name
1415
+ """Get a repository by name.
1048
1416
 
1049
1417
  Args:
1050
- name (str): name of the repository
1418
+ name (str):
1419
+ The name of the repository.
1420
+
1051
1421
  Returns:
1052
- dict | None: ID of a repostiory by name or None in case of an error
1422
+ dict | None:
1423
+ ID of a repostiory by name or None in case of an error
1424
+
1053
1425
  """
1054
1426
 
1055
1427
  repo_list = self.get_repo_list()
@@ -1063,3 +1435,58 @@ class AVTS(object):
1063
1435
  )
1064
1436
 
1065
1437
  # end method definition
1438
+
1439
+ def get_certificate_file_content_base64(self, filepath: str) -> str | None:
1440
+ """Return the certificate as a base64 string.
1441
+
1442
+ In Kubernetes deploymnets the certificate is already mounted base64 encoded.
1443
+
1444
+ Args:
1445
+ filepath (str):
1446
+ The path to the certificate file.
1447
+
1448
+ Returns:
1449
+ str | None:
1450
+ Base64 encoded certificate file content.
1451
+
1452
+ """
1453
+
1454
+ if not os.path.isfile(filepath):
1455
+ return None
1456
+
1457
+ file_ext = os.path.splitext(filepath)[1].lower()
1458
+
1459
+ if self.running_in_kubernetes_pod() and file_ext == ".pfx":
1460
+ # Return file directly as already base64 encoded
1461
+ self.logger.warning(
1462
+ "Detected a binary pfx file in Kubernetes environment, expecting it to be already base64 encoded",
1463
+ )
1464
+ with open(filepath, encoding="UTF-8") as file:
1465
+ return file.read().strip()
1466
+
1467
+ else:
1468
+ # Return file as base64 encoded
1469
+ with open(filepath, "rb") as file:
1470
+ # Read the content of the file
1471
+ file_content = file.read()
1472
+ # Convert the bytes to a base64 string
1473
+ return base64.b64encode(file_content).decode("utf-8")
1474
+
1475
+ # end method definition
1476
+
1477
+ def running_in_kubernetes_pod(self) -> bool:
1478
+ """Check if the application is running inside a Kubernetes pod.
1479
+
1480
+ This function determines whether the process is running in a Kubernetes
1481
+ environment by checking for the presence of the `KUBERNETES_SERVICE_HOST`
1482
+ and `KUBERNETES_SERVICE_PORT` environment variables.
1483
+
1484
+ Returns:
1485
+ bool:
1486
+ True if running inside a Kubernetes pod, False otherwise.
1487
+
1488
+ """
1489
+
1490
+ return bool(os.getenv("KUBERNETES_SERVICE_HOST") and os.getenv("KUBERNETES_SERVICE_PORT"))
1491
+
1492
+ # end method definition