illumio-pylo 0.3.10__tar.gz → 0.3.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/.github/workflows/make-binaries.yml +40 -2
  2. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/.github/workflows/python-publish.yml +2 -2
  3. {illumio_pylo-0.3.10/illumio_pylo.egg-info → illumio_pylo-0.3.12}/PKG-INFO +10 -3
  4. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/examples/extend_cli.py +1 -0
  5. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/APIConnector.py +145 -103
  6. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/CredentialsManager.py +38 -0
  7. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/Explorer.py +44 -0
  8. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/JsonPayloadTypes.py +39 -0
  9. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Helpers/exports.py +1 -1
  10. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/IPList.py +15 -8
  11. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/IPMap.py +9 -0
  12. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/__init__.py +1 -1
  13. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/__init__.py +1 -0
  14. illumio_pylo-0.3.12/illumio_pylo/cli/commands/credential_manager.py +591 -0
  15. illumio_pylo-0.3.12/illumio_pylo/cli/commands/label_delete_unused.py +79 -0
  16. illumio_pylo-0.3.12/illumio_pylo/cli/commands/ui/credential_manager_ui/app.js +449 -0
  17. illumio_pylo-0.3.12/illumio_pylo/cli/commands/ui/credential_manager_ui/index.html +168 -0
  18. illumio_pylo-0.3.12/illumio_pylo/cli/commands/ui/credential_manager_ui/styles.css +430 -0
  19. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/ven_duplicate_remover.py +145 -93
  20. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/cli.py +4 -1
  21. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/health_monitoring.py +5 -1
  22. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12/illumio_pylo.egg-info}/PKG-INFO +10 -3
  23. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo.egg-info/SOURCES.txt +4 -0
  24. illumio_pylo-0.3.10/requirements.txt → illumio_pylo-0.3.12/illumio_pylo.egg-info/requires.txt +3 -2
  25. illumio_pylo-0.3.10/illumio_pylo.egg-info/requires.txt → illumio_pylo-0.3.12/requirements.txt +2 -1
  26. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/setup.py +1 -1
  27. illumio_pylo-0.3.10/illumio_pylo/cli/commands/credential_manager.py +0 -216
  28. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/.devcontainer/Dockerfile +0 -0
  29. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/.devcontainer/devcontainer.json +0 -0
  30. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/.gitattributes +0 -0
  31. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/.github/workflows/doxygen-publish.yml +0 -0
  32. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/.gitignore +0 -0
  33. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/LICENSE +0 -0
  34. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/README.md +0 -0
  35. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/check_unique_hostnames.py +0 -0
  36. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/check_unique_services.py +0 -0
  37. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/delete_all_workloads.py +0 -0
  38. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/delete_unused_services.py +0 -0
  39. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/explorer_report_exporter.py +0 -0
  40. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/export_rules_to_firewall.py +0 -0
  41. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/generate-random-workloads.py +0 -0
  42. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/healthcheck_log.py +0 -0
  43. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/import-labels.py +0 -0
  44. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/import_workloads_placeholders.py +0 -0
  45. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/iplists_stats_duplicates_unused_finder.py +0 -0
  46. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/recalculate_explorer_logs.py +0 -0
  47. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/recalculate_explorer_logs_multithreaded.py +0 -0
  48. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/rules_exporter.py +0 -0
  49. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/rules_exporter_special.py +0 -0
  50. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/test.py +0 -0
  51. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/test_change_workload_desc.py +0 -0
  52. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/test_query.py +0 -0
  53. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/test_query2.py +0 -0
  54. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/test_securityprincipals.py +0 -0
  55. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/ven_idle_to_illumination.py +0 -0
  56. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/dev_playground/ven_reassign_pce.py +0 -0
  57. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/examples/explorer_query.py +0 -0
  58. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/AuditLog.py +0 -0
  59. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/ClusterHealth.py +0 -0
  60. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/RuleSearchQuery.py +0 -0
  61. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/API/__init__.py +0 -0
  62. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/AgentStore.py +0 -0
  63. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Exception.py +0 -0
  64. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Helpers/__init__.py +0 -0
  65. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Helpers/functions.py +0 -0
  66. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Label.py +0 -0
  67. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/LabelCommon.py +0 -0
  68. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/LabelGroup.py +0 -0
  69. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/LabelStore.py +0 -0
  70. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/LabeledObject.py +0 -0
  71. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Organization.py +0 -0
  72. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Query.py +0 -0
  73. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/ReferenceTracker.py +0 -0
  74. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Rule.py +0 -0
  75. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Ruleset.py +0 -0
  76. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/RulesetStore.py +0 -0
  77. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/SecurityPrincipal.py +0 -0
  78. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Service.py +0 -0
  79. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/SoftwareVersion.py +0 -0
  80. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/VirtualService.py +0 -0
  81. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/VirtualServiceStore.py +0 -0
  82. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/Workload.py +0 -0
  83. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/WorkloadStore.py +0 -0
  84. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/WorkloadStoreSubClasses.py +0 -0
  85. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/NativeParsers.py +0 -0
  86. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/__init__.py +0 -0
  87. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/__main__.py +0 -0
  88. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/iplist_analyzer.py +0 -0
  89. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/iplist_import_from_file.py +0 -0
  90. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/ruleset_export.py +0 -0
  91. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/update_pce_objects_cache.py +0 -0
  92. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/utils/LabelCreation.py +0 -0
  93. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/utils/__init__.py +0 -0
  94. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/utils/misc.py +0 -0
  95. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/ven_compatibility_report_export.py +0 -0
  96. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/ven_idle_to_visibility.py +0 -0
  97. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/ven_upgrader.py +0 -0
  98. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/workload_export.py +0 -0
  99. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/workload_import.py +0 -0
  100. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/workload_reset_names_to_null.py +0 -0
  101. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/workload_update.py +0 -0
  102. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/cli/commands/workload_used_in_rule_finder.py +0 -0
  103. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/docs/Doxygen +0 -0
  104. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/tmp.py +0 -0
  105. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/__init__.py +0 -0
  106. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/credentials.example.json +0 -0
  107. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/resources/iplists-import-example.csv +0 -0
  108. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/resources/iplists-import-example.xlsx +0 -0
  109. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/resources/workload-exporter-filter-example.csv +0 -0
  110. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/resources/workloads-import-example.csv +0 -0
  111. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo/utilities/resources/workloads-import-example.xlsx +0 -0
  112. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo.egg-info/dependency_links.txt +0 -0
  113. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/illumio_pylo.egg-info/top_level.txt +0 -0
  114. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/pyproject.toml +0 -0
  115. {illumio_pylo-0.3.10 → illumio_pylo-0.3.12}/setup.cfg +0 -0
@@ -20,6 +20,7 @@ jobs:
20
20
  - name: Change Pylo Version if this is a DEV build
21
21
  if: github.ref == 'refs/heads/dev' && github.event_name == 'push'
22
22
  run: |
23
+ pwd
23
24
  echo "Update version to append '-dev-' and current date ISO format:"
24
25
  # for double quoted strings version
25
26
  sed -i -E "s/__version__ = \"([0-9]+\.[0-9]+\.[0-9]+)\"/__version__ = '\\1-dev-$(date +'%Y%m%d')'/" illumio_pylo/__init__.py
@@ -28,12 +29,49 @@ jobs:
28
29
  grep __version__ illumio_pylo/__init__.py
29
30
 
30
31
 
32
+ - name: copy scripts and pkg to build folder
33
+ run: |
34
+ # build the package
35
+ pip install build
36
+ python -m build
37
+ echo "current working directory:"
38
+ pwd
39
+ # get package whl filename for later use, it is in dist/
40
+ package_filename=$(find dist/ -name "illumio_pylo-*.whl" | sed 's|dist/||')
41
+ echo "*** Package filename is: $package_filename"
42
+ mkdir cli-build
43
+ # move package to cli-build
44
+ mv "dist/$package_filename" cli-build/
45
+
46
+ # create requirements.txt that points to the local package
47
+ echo "illumio_pylo@file:./cli-build/$package_filename" > cli-build/requirements.txt
48
+ #echo "pyinstaller==6.16.0" >> cli-build/requirements.txt
49
+ echo "*** Created cli-build/requirements.txt with contents:"
50
+ cat cli-build/requirements.txt
51
+
52
+ # copy cli.py removing any sys.path.insert lines
53
+ grep -v 'sys.path.insert' illumio_pylo/utilities/cli.py > cli-build/cli.py
54
+ mv illumio_pylo/utilities/health_monitoring.py cli-build/
55
+
56
+ # delete illumio_pylo to avoid confusion
57
+ rm -rf illumio_pylo/
58
+
59
+ echo "*** Contents of cli-build/:"
60
+ find cli-build/
61
+
31
62
  - name: Make executables
32
63
  uses: cpainchaud/pyinstaller-action-windows@main
33
64
  with:
34
65
  path: ./
35
- spec: illumio_pylo/utilities/
36
- extra_python_paths: Z:\\github\\workspace\\;Z:\\github\\workspace\\pylo;C:\\Windows\\System32\\downlevel
66
+ spec: ./cli-build/
67
+ requirements: ./cli-build/requirements.txt
68
+ collect_data: illumio_pylo
69
+ extra_python_paths: Z:\\github\\workspace\\;C:\\Windows\\System32\\downlevel
70
+
71
+ - name: show spec files
72
+ run: |
73
+ echo "Showing the spec files created by PyInstaller:"
74
+ find ../ -name "*.spec" -exec cat {} \;
37
75
 
38
76
  - name: rename executables
39
77
  run: |
@@ -26,10 +26,10 @@ jobs:
26
26
  python-version: '3.11'
27
27
  - name: Install dependencies
28
28
  run: |
29
- python -m pip install --upgrade pip
29
+ python -m pip install --upgrade pip setuptools wheel twine pkginfo
30
30
  pip install build
31
31
  - name: Build package
32
32
  run: python -m build
33
33
  - name: Publish package
34
- uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
34
+ uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
35
35
 
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.4
2
2
  Name: illumio_pylo
3
- Version: 0.3.10
3
+ Version: 0.3.12
4
4
  Summary: A set of tools and library for working with Illumio PCE
5
5
  Home-page: https://github.com/cpainchaud/pylo
6
6
  Author: Christophe Painchaud
@@ -187,11 +187,18 @@ Requires-Python: >=3.11
187
187
  License-File: LICENSE
188
188
  Requires-Dist: click==8.1.7
189
189
  Requires-Dist: colorama~=0.4.4
190
- Requires-Dist: cryptography==43.0.1
190
+ Requires-Dist: cryptography==44.0.1
191
191
  Requires-Dist: openpyxl~=3.1.3
192
192
  Requires-Dist: paramiko~=3.4.0
193
193
  Requires-Dist: prettytable~=3.10.0
194
194
  Requires-Dist: requests~=2.32.0
195
195
  Requires-Dist: xlsxwriter~=3.2.0
196
+ Requires-Dist: flask~=2.2.0
197
+ Dynamic: author
198
+ Dynamic: author-email
199
+ Dynamic: description
200
+ Dynamic: home-page
201
+ Dynamic: license-file
202
+ Dynamic: requires-python
196
203
 
197
204
  README.md
@@ -23,6 +23,7 @@ class MyBuiltInParser: # optional, if you want to use built-in parsers
23
23
  label_type='env', # optional, it will ensure that selected labels are of a specified type
24
24
  is_required=False, allow_multiple=True)
25
25
 
26
+
26
27
  def fill_parser(parser: argparse.ArgumentParser):
27
28
  """ This function will be called by the CLI to fill the parser with the arguments of your command """
28
29
  parser.add_argument('--sort-by-name', '-s', action='store_true',
@@ -28,9 +28,13 @@ from typing import Union, Dict, Any, List, Optional, Literal
28
28
 
29
29
  requests.packages.urllib3.disable_warnings()
30
30
 
31
+ objects_types_strings = Literal[
32
+ 'workloads', 'virtual_services', 'labels', 'labelgroups', 'iplists', 'services',
33
+ 'rulesets', 'security_principals', 'label_dimensions']
34
+
31
35
  default_retry_count_if_api_call_limit_reached = 3
32
36
  default_retry_wait_time_if_api_call_limit_reached = 10
33
- default_max_objects_for_sync_calls = 99999
37
+ default_max_objects_for_sync_calls = 200000
34
38
 
35
39
 
36
40
  def get_field_or_die(field_name: str, data):
@@ -69,14 +73,14 @@ class APIConnector:
69
73
  if type(port) is int:
70
74
  port = str(port)
71
75
  self.port: int = port
72
- self._api_key: str = api_key
73
- self._decrypted_api_key: Optional[str] = None
74
76
  self.api_user: str = api_user
77
+ self._api_key: str = api_key
78
+ self._decrypted_api_key: Optional[str] = None # if api_key is encrypted, this will hold the decrypted value after first use
75
79
  self.org_id: int = org_id
76
80
  self.skipSSLCertCheck: bool = skip_ssl_cert_check
77
81
  self.version: Optional['pylo.SoftwareVersion'] = None
78
82
  self.version_string: str = "Not Defined"
79
- self._cached_session = requests.session()
83
+ self._cached_session = requests.sessions.Session()
80
84
 
81
85
  @property
82
86
  def api_key(self):
@@ -153,6 +157,33 @@ class APIConnector:
153
157
 
154
158
  return url
155
159
 
160
+ def _get_objects_auto_switch_async(self, path: str, data: Dict[str, Any], async_mode: bool, max_results: Optional[int]) -> Any:
161
+ # use a copy of data to not modify the original
162
+ data_copy = data.copy()
163
+
164
+ if async_mode:
165
+ if max_results is not None:
166
+ data_copy['max_results'] = max_results
167
+ return self.do_get_call(path=path, async_call=True, params=data_copy)
168
+
169
+ if max_results is not None:
170
+ data_copy['max_results'] = max_results
171
+ return self.do_get_call(path=path, async_call=False, params=data_copy)
172
+
173
+ # We will grab the maximum allowed with sync mode (variable default_max_objects_for_sync_calls) and switch to async if it's higher
174
+ data_copy['max_results'] = default_max_objects_for_sync_calls
175
+ results = self.do_get_call(path=path, async_call=False, params=data_copy, return_headers=True)
176
+ total_count = results[1].get('x-total-count')
177
+ if total_count is None:
178
+ raise pylo.PyloApiEx('API didnt provide field "x-total-count" in headers')
179
+ if not total_count.isdigit():
180
+ raise pylo.PyloApiEx('API returned invalid value for "x-total-count": {}'.format(total_count))
181
+ if int(total_count) > default_max_objects_for_sync_calls:
182
+ # remove the max_results from data
183
+ del data_copy['max_results']
184
+ return self.do_get_call(path=path, async_call=True, params=data_copy)
185
+ return results[0]
186
+
156
187
  def do_get_call(self, path, json_arguments=None, include_org_id=True, json_output_expected=True, async_call=False, params=None, skip_product_version_check=False,
157
188
  retry_count_if_api_call_limit_reached=default_retry_count_if_api_call_limit_reached,
158
189
  retry_wait_time_if_api_call_limit_reached=default_retry_wait_time_if_api_call_limit_reached,
@@ -275,7 +306,7 @@ class APIConnector:
275
306
  log.info("Job status is " + job_poll_status)
276
307
 
277
308
  log.info("Job is done, we will now download the resulting dataset")
278
- dataset = self.do_get_call(result_href, include_org_id=False)
309
+ dataset = self.do_get_call(result_href, include_org_id=False, return_headers=return_headers)
279
310
 
280
311
  return dataset
281
312
 
@@ -313,9 +344,6 @@ class APIConnector:
313
344
  raise pylo.PyloApiEx('API returned error status "' + str(req.status_code) + ' ' + req.reason
314
345
  + '" and error message: ' + req.text)
315
346
 
316
- if return_headers:
317
- return req.headers
318
-
319
347
  if json_output_expected:
320
348
  log.info("Parsing API answer to JSON (with a size of " + str(int(answer_size)) + "KB)")
321
349
  json_out = req.json()
@@ -325,8 +353,12 @@ class APIConnector:
325
353
  log.info(json.dumps(json_out, indent=2, sort_keys=True))
326
354
  else:
327
355
  log.info("Answer is too large to be printed")
356
+ if return_headers:
357
+ return json_out, req.headers
328
358
  return json_out
329
359
 
360
+ if return_headers:
361
+ return req.text, req.headers
330
362
  return req.text
331
363
 
332
364
  raise pylo.PyloApiEx("Unexpected API output or race condition")
@@ -339,33 +371,39 @@ class APIConnector:
339
371
  self.collect_pce_infos()
340
372
  return self.version_string
341
373
 
342
- def get_objects_count_by_type(self, object_type: str) -> int:
374
+ def get_objects_count_by_type(self, object_type: objects_types_strings) -> int:
343
375
 
344
376
  def extract_count(headers):
345
377
  count = headers.get('x-total-count')
346
378
  if count is None:
347
379
  raise pylo.PyloApiEx('API didnt provide field "x-total-count"')
348
380
 
381
+ # count should be an integer
382
+ if not count.isdigit():
383
+ raise pylo.PyloApiEx('API returned invalid value for "x-total-count": {}'.format(count))
384
+
349
385
  return int(count)
350
386
 
387
+ params = {'max_results': 1}
388
+
351
389
  if object_type == 'workloads':
352
- return extract_count(self.do_get_call('/workloads', async_call=False, return_headers=True))
390
+ return extract_count(self.do_get_call('/workloads', async_call=False, return_headers=True, params=params)[1])
353
391
  elif object_type == 'virtual_services':
354
- return extract_count(self.do_get_call('/sec_policy/draft/virtual_services', async_call=False, return_headers=True))
392
+ return extract_count(self.do_get_call('/sec_policy/draft/virtual_services', async_call=False, return_headers=True, params=params)[1])
355
393
  elif object_type == 'labels':
356
- return extract_count(self.do_get_call('/labels', async_call=False, return_headers=True))
394
+ return extract_count(self.do_get_call('/labels', async_call=False, return_headers=True, params=params)[1])
357
395
  elif object_type == 'labelgroups':
358
- return extract_count(self.do_get_call('/sec_policy/draft/label_groups', async_call=False, return_headers=True))
396
+ return extract_count(self.do_get_call('/sec_policy/draft/label_groups', async_call=False, return_headers=True, params=params)[1])
359
397
  elif object_type == 'iplists':
360
- return extract_count(self.do_get_call('/sec_policy/draft/ip_lists', async_call=False, return_headers=True))
398
+ return extract_count(self.do_get_call('/sec_policy/draft/ip_lists', async_call=False, return_headers=True, params=params)[1])
361
399
  elif object_type == 'services':
362
- return extract_count(self.do_get_call('/sec_policy/draft/services', async_call=False, return_headers=True))
400
+ return extract_count(self.do_get_call('/sec_policy/draft/services', async_call=False, return_headers=True, params=params)[1])
363
401
  elif object_type == 'rulesets':
364
- return extract_count(self.do_get_call('/sec_policy/draft/rule_sets', async_call=False, return_headers=True))
402
+ return extract_count(self.do_get_call('/sec_policy/draft/rule_sets', async_call=False, return_headers=True, params=params)[1])
365
403
  elif object_type == 'security_principals':
366
- return extract_count(self.do_get_call('/security_principals', async_call=False, return_headers=True))
404
+ return extract_count(self.do_get_call('/security_principals', async_call=False, return_headers=True, params=params)[1])
367
405
  elif object_type == 'label_dimensions':
368
- return extract_count(self.do_get_call('/label_dimensions', async_call=False, return_headers=True))
406
+ return extract_count(self.do_get_call('/label_dimensions', async_call=False, return_headers=True, params=params)[1])
369
407
  else:
370
408
  raise pylo.PyloEx("Unsupported object type '{}'".format(object_type))
371
409
 
@@ -403,57 +441,32 @@ class APIConnector:
403
441
  q.task_done()
404
442
  continue
405
443
  if object_type == 'workloads':
406
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls or force_async_mode:
407
- data['workloads'] = self.objects_workload_get(include_deleted=include_deleted_workloads)
408
- else:
409
- data['workloads'] = self.objects_workload_get(include_deleted=include_deleted_workloads, async_mode=False, max_results=default_max_objects_for_sync_calls)
444
+ data['workloads'] = self.objects_workload_get(include_deleted=include_deleted_workloads, async_mode=force_async_mode)
410
445
 
411
446
  elif object_type == 'virtual_services':
412
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
413
- data['virtual_services'] = self.objects_virtual_service_get()
414
- else:
415
- data['virtual_services'] = self.objects_virtual_service_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
447
+ data['virtual_services'] = self.objects_virtual_service_get(async_mode=force_async_mode)
416
448
 
417
449
  elif object_type == 'labels':
418
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
419
- data['labels'] = self.objects_label_get()
420
- else:
421
- data['labels'] = self.objects_label_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
450
+ data['labels'] = self.objects_label_get(async_mode=force_async_mode)
422
451
 
423
452
  elif object_type == 'labelgroups':
424
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
425
- data['labelgroups'] = self.objects_labelgroup_get()
426
- else:
427
- data['labelgroups'] = self.objects_labelgroup_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
453
+ data['labelgroups'] = self.objects_labelgroup_get(async_mode=force_async_mode)
428
454
 
429
455
  elif object_type == 'iplists':
430
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
431
- data['iplists'] = self.objects_iplist_get()
432
- else:
433
- data['iplists'] = self.objects_iplist_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
456
+ data['iplists'] = self.objects_iplist_get(async_mode=force_async_mode)
434
457
 
435
458
  elif object_type == 'services':
436
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
437
- data['services'] = self.objects_service_get()
438
- else:
439
- data['services'] = self.objects_service_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
459
+ data['services'] = self.objects_service_get(async_mode=force_async_mode)
440
460
 
441
461
  elif object_type == 'rulesets':
442
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
443
- data['rulesets'] = self.objects_ruleset_get()
444
- else:
445
- data['rulesets'] = self.objects_ruleset_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
462
+ data['rulesets'] = self.objects_ruleset_get(async_mode=force_async_mode)
446
463
 
447
464
  elif object_type == 'security_principals':
448
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
449
- data['security_principals'] = self.objects_securityprincipal_get()
450
- else:
451
- data['security_principals'] = self.objects_securityprincipal_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
465
+ data['security_principals'] = self.objects_securityprincipal_get(async_mode=force_async_mode)
466
+
452
467
  elif object_type == 'label_dimensions':
453
- if self.get_objects_count_by_type(object_type) > default_max_objects_for_sync_calls:
454
- data['label_dimensions'] = self.objects_label_dimension_get()
455
- else:
456
- data['label_dimensions'] = self.objects_label_dimension_get(async_mode=False, max_results=default_max_objects_for_sync_calls)
468
+ data['label_dimensions'] = self.objects_label_dimension_get()
469
+
457
470
  else:
458
471
  raise pylo.PyloEx("Unsupported object type '{}'".format(object_type))
459
472
  except Exception as e:
@@ -533,14 +546,17 @@ class APIConnector:
533
546
  params = {'include_deny_rules': include_boundary_rules}
534
547
  return self.do_post_call(path='/sec_policy/draft/rule_coverage', json_arguments=data, include_org_id=True, json_output_expected=True, async_call=False, params=params)
535
548
 
536
- def objects_label_get(self, max_results: int = None, async_mode=True) -> List[LabelObjectJsonStructure]:
549
+ def objects_label_get(self, max_results: int = None, async_mode=False, get_usage: bool = False, get_deleted: bool = False) -> List[LabelObjectJsonStructure]:
537
550
  path = '/labels'
538
551
  data = {}
539
552
 
540
- if max_results is not None:
541
- data['max_results'] = max_results
553
+ if get_usage:
554
+ data['usage'] = 'true'
555
+
556
+ if get_deleted:
557
+ data['includeDeleted'] = 'true'
542
558
 
543
- return self.do_get_call(path=path, async_call=async_mode, params=data)
559
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
544
560
 
545
561
  def objects_label_update(self, href: str, data: LabelObjectUpdateJsonStructure):
546
562
  path = href
@@ -553,19 +569,70 @@ class APIConnector:
553
569
 
554
570
  return self.do_delete_call(path=path, json_output_expected=False, include_org_id=False)
555
571
 
572
+ class LabelMultiDeleteTracker:
573
+ _errors: Dict[str, str]
574
+ _hrefs: Dict[str, bool]
575
+ _labels: Dict[str, 'pylo.Label'] # dict of workloads by HREF
576
+ connector: 'pylo.APIConnector'
577
+
578
+ def __init__(self, connector: 'pylo.APIConnector'):
579
+ self.connector = connector
580
+ self._errors: Dict[str, Union[bool, str]] = {}
581
+ self._hrefs: Dict[str, str] = {}
582
+ self._labels: Dict[str, 'pylo.Label'] = {}
583
+
584
+ def add_label(self, label_or_href: Union['pylo.Label', str]):
585
+ if type(label_or_href) is str:
586
+ self._hrefs[label_or_href] = True
587
+ return
588
+ self._labels[label_or_href.href] = label_or_href
589
+ self._hrefs[label_or_href.href] = True
590
+
591
+ def execute_deletion(self):
592
+ for href in self._hrefs.keys():
593
+ try:
594
+ self.connector.objects_label_delete(href)
595
+ self._errors[href] = False
596
+ except Exception as e:
597
+ self._errors[href] = str(e)
598
+
599
+ return self._errors
600
+
601
+ def has_errors(self) -> bool:
602
+ for href in self._errors.keys():
603
+ if self._errors[href] is not False:
604
+ return True
605
+ return False
606
+
607
+ def get_errors_count(self) -> int:
608
+ count = 0
609
+ for href in self._errors.keys():
610
+ if self._errors[href] is not False:
611
+ count += 1
612
+ return count
613
+
614
+ def get_error(self, label_or_href: Union['pylo.Label', str]) -> Optional[str]:
615
+ href = label_or_href
616
+ if type(label_or_href) is pylo.Label:
617
+ href = label_or_href.href
618
+ error = self._errors.get(href, None)
619
+ if error is None or error is False:
620
+ return None
621
+ return error
622
+
623
+ def new_tracker_for_label_multi_deletion(self) -> 'pylo.APIConnector.LabelMultiDeleteTracker':
624
+ return pylo.APIConnector.LabelMultiDeleteTracker(self)
625
+
556
626
  def objects_label_create(self, label_name: str, label_type: str):
557
627
  path = '/labels'
558
628
  data: LabelObjectCreationJsonStructure = {'key': label_type, 'value': label_name}
559
629
  return self.do_post_call(path=path, json_arguments=data)
560
630
 
561
- def objects_labelgroup_get(self, max_results: int = None, async_mode=True) -> List[LabelGroupObjectJsonStructure]:
631
+ def objects_labelgroup_get(self, max_results: int = None, async_mode=False) -> List[LabelGroupObjectJsonStructure]:
562
632
  path = '/sec_policy/draft/label_groups'
563
633
  data = {}
564
634
 
565
- if max_results is not None:
566
- data['max_results'] = max_results
567
-
568
- return self.do_get_call(path=path, async_call=async_mode, params=data)
635
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
569
636
 
570
637
  def objects_labelgroup_update(self, href: str, data: LabelGroupObjectUpdateJsonStructure):
571
638
  path = href
@@ -575,34 +642,26 @@ class APIConnector:
575
642
  path = '/label_dimensions'
576
643
  data = {}
577
644
 
578
- if max_results is not None:
579
- data['max_results'] = max_results
580
- return self.do_get_call(path=path, async_call=async_mode, params=data)
645
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
581
646
 
582
- def objects_virtual_service_get(self, max_results: int = None, async_mode=True) -> List[VirtualServiceObjectJsonStructure]:
647
+ def objects_virtual_service_get(self, max_results: int = None, async_mode=False) -> List[VirtualServiceObjectJsonStructure]:
583
648
  path = '/sec_policy/draft/virtual_services'
584
649
  data = {}
585
650
 
586
- if max_results is not None:
587
- data['max_results'] = max_results
588
-
589
- results = self.do_get_call(path=path, async_call=async_mode, params=data)
651
+ results = self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
590
652
  # check type
591
653
  if type(results) is list:
592
654
  return results
593
655
  raise pylo.PyloEx("Unexpected result type '{}' while expecting an array of Virtual Service objects".format(type(results)), results)
594
656
 
595
- def objects_iplist_get(self, max_results: int = None, async_mode=True, search_name: str = None) -> List[IPListObjectJsonStructure]:
657
+ def objects_iplist_get(self, max_results: int = None, async_mode=False, search_name: str = None) -> List[IPListObjectJsonStructure]:
596
658
  path = '/sec_policy/draft/ip_lists'
597
659
  data = {}
598
660
 
599
661
  if search_name is not None:
600
662
  data['name'] = search_name
601
663
 
602
- if max_results is not None:
603
- data['max_results'] = max_results
604
-
605
- results: List[IPListObjectJsonStructure] = self.do_get_call(path=path, async_call=async_mode, params=data)
664
+ results: List[IPListObjectJsonStructure] = self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
606
665
  # check type
607
666
  if type(results) is list:
608
667
  return results
@@ -654,10 +713,7 @@ class APIConnector:
654
713
  if representation is not None:
655
714
  data['representation'] = representation
656
715
 
657
- if max_results is not None:
658
- data['max_results'] = max_results
659
-
660
- return self.do_get_call(path=path, async_call=async_mode, params=data)
716
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
661
717
 
662
718
  def objects_workload_get(self,
663
719
  include_deleted=False,
@@ -667,7 +723,7 @@ class APIConnector:
667
723
  filter_by_managed: bool = None,
668
724
  filer_by_policy_health: Literal['active', 'warning', 'error'] = None,
669
725
  max_results: int = None,
670
- async_mode=True) -> List[WorkloadObjectJsonStructure]:
726
+ async_mode=False) -> List[WorkloadObjectJsonStructure]:
671
727
  path = '/workloads'
672
728
  data = {}
673
729
 
@@ -690,10 +746,8 @@ class APIConnector:
690
746
  if filer_by_policy_health is not None:
691
747
  data['policy_health'] = filer_by_policy_health
692
748
 
693
- if max_results is not None:
694
- data['max_results'] = max_results
749
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
695
750
 
696
- return self.do_get_call(path=path, async_call=async_mode, params=data)
697
751
 
698
752
  def objects_workload_agent_upgrade(self, workload_href: str, target_version: str):
699
753
  path = '{}/upgrade'.format(workload_href)
@@ -941,14 +995,11 @@ class APIConnector:
941
995
  path = '/workloads/bulk_create'
942
996
  return self.do_put_call(path=path, json_arguments=workloads_json_payload)
943
997
 
944
- def objects_service_get(self, max_results: int = None, async_mode=True):
998
+ def objects_service_get(self, max_results: int = None, async_mode=False):
945
999
  path = '/sec_policy/draft/services'
946
1000
  data = {}
947
1001
 
948
- if max_results is not None:
949
- data['max_results'] = max_results
950
-
951
- return self.do_get_call(path=path, async_call=async_mode, params=data)
1002
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
952
1003
 
953
1004
  def objects_service_delete(self, href):
954
1005
  """
@@ -990,14 +1041,11 @@ class APIConnector:
990
1041
 
991
1042
  return self.do_post_call(path=path, async_call=False, include_org_id=False, json_arguments=data, json_output_expected=True)
992
1043
 
993
- def objects_ruleset_get(self, max_results: int = None, async_mode=True) -> List[RulesetObjectJsonStructure]:
1044
+ def objects_ruleset_get(self, max_results: int = None, async_mode=False) -> List[RulesetObjectJsonStructure]:
994
1045
  path = '/sec_policy/draft/rule_sets'
995
1046
  data = {}
996
1047
 
997
- if max_results is not None:
998
- data['max_results'] = max_results
999
-
1000
- return self.do_get_call(path=path, async_call=async_mode, params=data)
1048
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
1001
1049
 
1002
1050
  def objects_ruleset_create(self, name: str,
1003
1051
  scope_app: 'pylo.Label' = None,
@@ -1116,14 +1164,11 @@ class APIConnector:
1116
1164
 
1117
1165
  return self.do_post_call(path, json_arguments=data, json_output_expected=True, include_org_id=False)
1118
1166
 
1119
- def objects_securityprincipal_get(self, max_results: int = None, async_mode=True) -> List[SecurityPrincipalObjectJsonStructure]:
1167
+ def objects_securityprincipal_get(self, max_results: int = None, async_mode=False) -> List[SecurityPrincipalObjectJsonStructure]:
1120
1168
  path = '/security_principals'
1121
1169
  data = {}
1122
1170
 
1123
- if max_results is not None:
1124
- data['max_results'] = max_results
1125
-
1126
- return self.do_get_call(path=path, async_call=async_mode, params=data)
1171
+ return self._get_objects_auto_switch_async(path=path, data=data, async_mode=async_mode, max_results=max_results)
1127
1172
 
1128
1173
  def objects_securityprincipal_create(self, name: str = None, sid: str = None, json_data=None) -> str:
1129
1174
  """
@@ -1378,7 +1423,4 @@ class APIConnector:
1378
1423
  def get_pce_ui_workload_url(self, href: str) -> str:
1379
1424
  # extract UUID from workload HREF:
1380
1425
  uuid = href.split('/')[-1]
1381
- return self._make_base_url('/#/workloads/' + uuid )
1382
-
1383
-
1384
-
1426
+ return self._make_base_url('/#/workloads/' + uuid)
@@ -199,6 +199,44 @@ def create_credential_in_file(file_full_path: str, data: CredentialFileEntry, ov
199
199
  return file_full_path
200
200
 
201
201
 
202
+ def delete_credential_from_file(profile_name: str, file_path: str) -> bool:
203
+ """
204
+ Delete a credential from a file by profile name.
205
+ :param profile_name: Name of the profile to delete
206
+ :param file_path: Path to the credential file
207
+ :return: True if deleted successfully
208
+ """
209
+ if not os.path.exists(file_path):
210
+ raise PyloEx("Credential file does not exist: {}".format(file_path))
211
+
212
+ with open(file_path, 'r') as f:
213
+ credentials: CredentialsFileType = json.load(f)
214
+
215
+ if isinstance(credentials, list):
216
+ original_len = len(credentials)
217
+ credentials = [c for c in credentials if c['name'].lower() != profile_name.lower()]
218
+ if len(credentials) == original_len:
219
+ raise PyloEx("Profile '{}' not found in file '{}'".format(profile_name, file_path))
220
+
221
+ if len(credentials) == 0:
222
+ # If no credentials left, delete the file
223
+ os.remove(file_path)
224
+ return True
225
+ else:
226
+ if credentials['name'].lower() == profile_name.lower():
227
+ # Single credential in file, delete the file
228
+ os.remove(file_path)
229
+ return True
230
+ else:
231
+ raise PyloEx("Profile '{}' not found in file '{}'".format(profile_name, file_path))
232
+
233
+ # Write the updated credentials back to the file
234
+ with open(file_path, 'w') as f:
235
+ json.dump(credentials, f, indent=4)
236
+
237
+ return True
238
+
239
+
202
240
  def create_credential_in_default_file(data: CredentialFileEntry) -> str:
203
241
  """
204
242
  Create a credential in the default credential file and return the full path to the file