foundry-dev-tools 2.1.14__tar.gz → 2.1.16__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/PKG-INFO +1 -1
  2. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/pyproject.toml +1 -1
  3. foundry_dev_tools-2.1.16/src/foundry_dev_tools/__about__.py +1 -0
  4. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/api_client.py +7 -5
  5. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/context_client.py +10 -18
  6. foundry_dev_tools-2.1.16/src/foundry_dev_tools/clients/magritte_coordinator.py +467 -0
  7. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/multipass.py +39 -0
  8. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/s3_client.py +2 -1
  9. foundry_dev_tools-2.1.16/src/foundry_dev_tools/clients/scheduler.py +54 -0
  10. foundry_dev_tools-2.1.16/src/foundry_dev_tools/clients/tables.py +145 -0
  11. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/config/config.py +5 -0
  12. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/config/context.py +38 -2
  13. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/resources/dataset.py +3 -1
  14. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/api_types.py +15 -0
  15. foundry_dev_tools-2.1.14/src/foundry_dev_tools/__about__.py +0 -1
  16. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/README.md +0 -0
  17. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/__init__.py +0 -0
  18. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/_optional/__init__.py +0 -0
  19. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/_optional/pandas.py +0 -0
  20. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/_optional/polars.py +0 -0
  21. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/_optional/pyarrow.py +0 -0
  22. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/_optional/pyspark.py +0 -0
  23. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
  24. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/__init__.py +0 -0
  25. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/build.py +0 -0
  26. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/config.py +0 -0
  27. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/git.py +0 -0
  28. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/git_credential_foundry.py +0 -0
  29. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/info.py +0 -0
  30. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/main.py +0 -0
  31. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/s3.py +0 -0
  32. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/cli/stubs.py +0 -0
  33. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/__init__.py +0 -0
  34. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/build2.py +0 -0
  35. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/catalog.py +0 -0
  36. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/compass.py +0 -0
  37. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/data_proxy.py +0 -0
  38. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/foundry_sql_server.py +0 -0
  39. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/foundry_stats.py +0 -0
  40. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/jemma.py +0 -0
  41. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/metadata.py +0 -0
  42. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/public_ontologies_client.py +0 -0
  43. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/clients/schema_inference.py +0 -0
  44. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/config/__init__.py +0 -0
  45. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/config/config_types.py +0 -0
  46. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/config/token_provider.py +0 -0
  47. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/__init__.py +0 -0
  48. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/compass.py +0 -0
  49. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/config.py +0 -0
  50. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/dataset.py +0 -0
  51. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/handling.py +0 -0
  52. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/meta.py +0 -0
  53. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/multipass.py +0 -0
  54. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/errors/sql.py +0 -0
  55. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/foundry_api_client.py +0 -0
  56. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/helpers/__init__.py +0 -0
  57. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/helpers/multipass/__init__.py +0 -0
  58. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/helpers/multipass/group.py +0 -0
  59. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/helpers/multipass/principal.py +0 -0
  60. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/helpers/multipass/user.py +0 -0
  61. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/py.typed +0 -0
  62. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/resources/__init__.py +0 -0
  63. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/resources/folder.py +0 -0
  64. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/resources/resource.py +0 -0
  65. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/__init__.py +0 -0
  66. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/async_s3.py +0 -0
  67. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  68. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  69. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  70. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/cli.py +0 -0
  71. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/clients.py +0 -0
  72. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/compat.py +0 -0
  73. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/config.py +0 -0
  74. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  75. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  76. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/misc.py +0 -0
  77. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/public_api_types.py +0 -0
  78. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/repo.py +0 -0
  79. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/s3.py +0 -0
  80. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.16}/src/foundry_dev_tools/utils/spark.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 2.1.14
3
+ Version: 2.1.16
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-Email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -41,7 +41,7 @@ dependencies = [
41
41
  "tomli_w",
42
42
  ]
43
43
  dynamic = []
44
- version = "2.1.14"
44
+ version = "2.1.16"
45
45
 
46
46
  [project.license]
47
47
  text = "Apache-2.0"
@@ -0,0 +1 @@
1
+ __version__ = '2.1.16'
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  from typing import TYPE_CHECKING, ClassVar, Literal
6
6
 
7
+ from foundry_dev_tools.errors.handling import ErrorHandlingConfig, raise_foundry_api_error
7
8
  from foundry_dev_tools.utils.clients import build_api_url, build_public_api_url
8
9
 
9
10
  if TYPE_CHECKING:
@@ -23,7 +24,6 @@ if TYPE_CHECKING:
23
24
  )
24
25
 
25
26
  from foundry_dev_tools.config.context import FoundryContext
26
- from foundry_dev_tools.errors.handling import ErrorHandlingConfig
27
27
 
28
28
 
29
29
  class APIClient:
@@ -93,7 +93,7 @@ class APIClient:
93
93
  else:
94
94
  headers = {"content-type": "application/json"}
95
95
 
96
- return self.context.client.request(
96
+ response = self.context.client.request(
97
97
  method=method,
98
98
  url=self.api_url(api_path),
99
99
  params=params,
@@ -110,8 +110,9 @@ class APIClient:
110
110
  verify=verify,
111
111
  cert=cert,
112
112
  json=json,
113
- error_handling=error_handling,
114
113
  )
114
+ raise_foundry_api_error(response, error_handling)
115
+ return response
115
116
 
116
117
 
117
118
  class PublicAPIClient:
@@ -196,7 +197,7 @@ class PublicAPIClient:
196
197
  params["preview"] = "true"
197
198
  else:
198
199
  params = {"preview": "true"}
199
- return self.context.client.request(
200
+ response = self.context.client.request(
200
201
  method=method,
201
202
  url=self.api_url(api_path, version=api_version),
202
203
  params=params,
@@ -213,5 +214,6 @@ class PublicAPIClient:
213
214
  verify=verify,
214
215
  cert=cert,
215
216
  json=json,
216
- error_handling=error_handling,
217
217
  )
218
+ raise_foundry_api_error(response, error_handling)
219
+ return response
@@ -4,17 +4,15 @@ from __future__ import annotations
4
4
 
5
5
  import logging
6
6
  import numbers
7
- import os
8
7
  import time
9
8
  import typing
10
- from typing import TYPE_CHECKING, Literal
9
+ from typing import TYPE_CHECKING
11
10
 
12
11
  import requests
13
12
 
14
- from foundry_dev_tools.__about__ import __version__
15
- from foundry_dev_tools.errors.handling import ErrorHandlingConfig, raise_foundry_api_error
16
-
17
13
  if TYPE_CHECKING:
14
+ from os import PathLike
15
+
18
16
  from requests import Response
19
17
  from requests.cookies import RequestsCookieJar
20
18
  from requests.sessions import ( # type: ignore[attr-defined]
@@ -31,7 +29,6 @@ if TYPE_CHECKING:
31
29
  _Verify,
32
30
  )
33
31
 
34
- from foundry_dev_tools.config.context import FoundryContext
35
32
 
36
33
  DEFAULT_TIMEOUT = (60, None)
37
34
  LOGGER = logging.getLogger(__name__)
@@ -73,14 +70,12 @@ def retry(times: int, exceptions: tuple[Exception]) -> typing.Callable:
73
70
  class ContextHTTPClient(requests.Session):
74
71
  """Requests Session with config and authentication applied."""
75
72
 
76
- def __init__(self, context: FoundryContext) -> None:
77
- self.context = context
73
+ def __init__(self, debug: bool = False, requests_ca_bundle: PathLike[str] | None = None) -> None:
74
+ self.debug = debug
75
+ self.requests_ca_bundle = requests_ca_bundle
78
76
  super().__init__()
79
- if self.context.config.requests_ca_bundle:
80
- self.verify = os.fspath(self.context.config.requests_ca_bundle)
81
- self.auth = lambda r: self.context.token_provider.requests_auth_handler(r)
77
+
82
78
  self._counter = 0
83
- self.headers = {"User-Agent": f"foundry-dev-tools/{__version__}/python-requests"}
84
79
 
85
80
  @retry(times=3, exceptions=requests.exceptions.ConnectionError)
86
81
  def request(
@@ -101,7 +96,6 @@ class ContextHTTPClient(requests.Session):
101
96
  verify: _Verify | None = None,
102
97
  cert: _Cert | None = None,
103
98
  json: Incomplete | None = None,
104
- error_handling: ErrorHandlingConfig | Literal[False] | None = None,
105
99
  ) -> Response:
106
100
  """Make an authenticated HTTP request.
107
101
 
@@ -122,11 +116,10 @@ class ContextHTTPClient(requests.Session):
122
116
  verify: see :py:meth:`requests.Session.request`
123
117
  cert: see :py:meth:`requests.Session.request`
124
118
  json: see :py:meth:`requests.Session.request`
125
- error_handling: error handling config; if set to False, errors won't be automatically handled
126
119
  """
127
- if verify is None and (rcab := self.context.config.requests_ca_bundle) is not None:
120
+ if verify is None and (rcab := self.requests_ca_bundle) is not None:
128
121
  verify = rcab
129
- if self.context.config.debug:
122
+ if self.debug:
130
123
  self._counter = count = self._counter + 1
131
124
  LOGGER.debug(f"(r{count}) Making {method!s} request to {url!s}") # noqa: G004
132
125
 
@@ -155,12 +148,11 @@ class ContextHTTPClient(requests.Session):
155
148
  cert=cert,
156
149
  json=json,
157
150
  )
158
- if self.context.config.debug:
151
+ if self.debug:
159
152
  LOGGER.debug(
160
153
  f"(r{count}) Got response status={response.status_code}, " # noqa: G004
161
154
  f"content_type={response.headers.get('content-type')}, "
162
155
  f"content_length={response.headers.get('content-length')}, "
163
156
  f"Server-Timing={response.headers.get('Server-Timing')}",
164
157
  )
165
- raise_foundry_api_error(response, error_handling)
166
158
  return response
@@ -0,0 +1,467 @@
1
+ """Implementation of the foundry-stats API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from foundry_dev_tools.clients.api_client import APIClient
8
+
9
+ if TYPE_CHECKING:
10
+ import requests
11
+
12
+ from foundry_dev_tools.utils.api_types import (
13
+ CodeResourceType,
14
+ FolderRid,
15
+ MarkingId,
16
+ NetworkEgressPolicyRid,
17
+ Rid,
18
+ SourceRid,
19
+ )
20
+
21
+
22
+ class MagritteCoordinatorClient(APIClient):
23
+ """The mothership that controls remote Magritte agents and cloud runtimes."""
24
+
25
+ api_name = "magritte-coordinator"
26
+
27
+ def api_get_oidc_metadata(self, **kwargs) -> requests.Response:
28
+ """Returns the OIDC Metadata of the Foundry Stack.
29
+
30
+ Args:
31
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
32
+ Returns:
33
+ dict: {'issuer': 'https://pltroidcpublic<id>.s3.<region>.amazonaws.com/foundry'}
34
+ """
35
+ return self.api_request(
36
+ "POST",
37
+ "oidc/metadata",
38
+ **kwargs,
39
+ )
40
+
41
+ def get_oidc_issuer(self) -> str:
42
+ """Returns the OIDC issuer of the Foundry Stack."""
43
+ return self.api_get_oidc_metadata().json()["issuer"]
44
+
45
+ def api_add_source_v3(
46
+ self, config: dict, description: dict, runtime_platform_request: dict, parent_rid: FolderRid, **kwargs
47
+ ) -> requests.Response:
48
+ """Low level method to add a new source to Data Connection.
49
+
50
+ Use a high level method like :py:meth:`MagritteCoordinatorClient.create_s3_direct_source`
51
+ for easier usage.
52
+
53
+ Args:
54
+ config: Source Config, individual format for each supported Source.
55
+ description: Textual description field.
56
+ runtime_platform_request: For Direct Connect { "cloud": {"networkEgresses": []}, "type": "cloud" }
57
+ parent_rid: Rid of the parent Compass folder.
58
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
59
+ """
60
+ return self.api_request(
61
+ "POST",
62
+ "source-store/source/v3",
63
+ json={
64
+ "config": config,
65
+ "description": description,
66
+ "runtimePlatformRequest": runtime_platform_request,
67
+ "parentRid": parent_rid,
68
+ },
69
+ **kwargs,
70
+ )
71
+
72
+ def api_update_source_v2(
73
+ self,
74
+ source_rid: Rid,
75
+ updated_source_config: dict | None = None,
76
+ patch_cloud_runtime_platform: dict | None = None,
77
+ **kwargs,
78
+ ) -> requests.Response:
79
+ """Updates a Source.
80
+
81
+ Args:
82
+ source_rid: The Rid of the Magritte Source
83
+ updated_source_config: example {"type":"s3-direct","url":"s3://bucket1"}
84
+ patch_cloud_runtime_platform: example
85
+ {
86
+ "secretsToUpdate": {},
87
+ "removedSecrets": [],
88
+ "patchCloudIdentity": {},
89
+ "patchOidcRuntime": {},
90
+ }
91
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
92
+
93
+ Returns:
94
+ requests.Response: dict with top level key updatedSource
95
+ {
96
+ "updatedSource": {
97
+ "sourceConfig": {
98
+ "type": "s3-direct",
99
+ "url": "s3://bucket"
100
+ },
101
+ "connections": {},
102
+ "additionalSecrets": {},
103
+ "mavenCoordinates": {
104
+ "userProvided": [],
105
+ "foundryProvided": []
106
+ }
107
+ }
108
+ }
109
+ """
110
+ payload = {}
111
+ if updated_source_config:
112
+ payload["patchSourceConfigRequest"] = {}
113
+ payload["patchSourceConfigRequest"]["updatedSourceConfig"] = updated_source_config
114
+ if patch_cloud_runtime_platform:
115
+ payload["patchRuntimePlatformRequest"] = {}
116
+ payload["patchRuntimePlatformRequest"]["patchCloudRuntimePlatform"] = patch_cloud_runtime_platform
117
+ payload["patchRuntimePlatformRequest"]["type"] = "patchCloudRuntimePlatform"
118
+ return self.api_request(
119
+ "PUT",
120
+ f"source-store/v2/sources/{source_rid}",
121
+ json=payload,
122
+ **kwargs,
123
+ )
124
+
125
+ def create_s3_direct_source(
126
+ self,
127
+ name: str,
128
+ parent_rid: FolderRid,
129
+ url: str,
130
+ description: str | None = "",
131
+ region: str | None = None,
132
+ catalog: dict | None = None,
133
+ sts_role_configuration: dict | None = None,
134
+ network_egress_policies: set[Rid] | None = None,
135
+ **source_kwargs,
136
+ ) -> str:
137
+ """Create a new Source of type s3-direct. Only supports Direct Connect sources.
138
+
139
+ Args:
140
+ name: Name of the Source in Compass
141
+ parent_rid: Rid of the parent Compass folder.
142
+ url: S3 Url, e.g. s3://bucket-name
143
+ description: Textual description field.
144
+ region: AWS Region of the bucket.
145
+ catalog: Catalog Integration for Iceberg Tables.
146
+ sts_role_configuration: sts role configuration for Assuming a role to connect to the bucket.
147
+ network_egress_policies: Network Egress Policies required to connect to the bucket.
148
+ source_kwargs: Any other arguments get added to the "source" key of the payload.
149
+ """
150
+ payload = {
151
+ "source": {
152
+ "type": "s3-direct",
153
+ "url": url,
154
+ **({"region": region} if region else {}),
155
+ **({"catalog": catalog} if catalog else {}),
156
+ **({"stsRoleConfiguration": sts_role_configuration} if sts_role_configuration else {}),
157
+ **source_kwargs,
158
+ }
159
+ }
160
+ network_egress_policies = network_egress_policies or set()
161
+
162
+ response = self.api_add_source_v3(
163
+ config=payload,
164
+ description={"name": name, "description": description},
165
+ runtime_platform_request={
166
+ "cloud": {"networkEgresses": list(network_egress_policies)},
167
+ "type": "cloud",
168
+ },
169
+ parent_rid=parent_rid,
170
+ )
171
+ return response.json()
172
+
173
+ def get_source_config(self, source_rid: SourceRid) -> dict:
174
+ """Returns the configuration of a Source."""
175
+ return self.api_bulk_get_source_config(source_rids={source_rid}).json()[source_rid]
176
+
177
+ def api_bulk_get_source_config(self, source_rids: set[SourceRid], **kwargs) -> requests.Response:
178
+ """Returns the configuration of a multiple sources."""
179
+ return self.api_request(
180
+ "POST",
181
+ "source-store/sources/config/bulk",
182
+ json=list(source_rids),
183
+ **kwargs,
184
+ )
185
+
186
+ def get_source_description(self, source_rid: SourceRid) -> dict:
187
+ """Returns the description of a Source.
188
+
189
+ Returns:
190
+ dictionary with example payload:
191
+ {'apiName': None, 'description': 'the desc.', 'name': 'source-name', 'type': 's3-direct'}
192
+ """
193
+ return self.api_bulk_get_source_description(source_rids={source_rid}).json()[source_rid]
194
+
195
+ def api_bulk_get_source_description(self, source_rids: set[SourceRid], **kwargs) -> requests.Response:
196
+ """Returns the description of multiple Sources."""
197
+ return self.api_request(
198
+ "POST",
199
+ "source-store/sources/description/bulk",
200
+ json=list(source_rids),
201
+ **kwargs,
202
+ )
203
+
204
+ def api_get_runtime_platform(self, source_rid: SourceRid, **kwargs) -> requests.Response:
205
+ """Get runtime platform information for a source.
206
+
207
+ Returns:
208
+ response: example format {
209
+ "type": "cloud",
210
+ "cloud": {
211
+ "credential": {
212
+ "credentialId": "ri.credential..credential.7ddaaa49-1337-1337-a0c0-8b621fa30364",
213
+ "secretNames": []
214
+ },
215
+ "networkEgressPolicies": {
216
+ "networkEgressPolicies": []
217
+ },
218
+ "cloudIdentity": null,
219
+ "oidcRuntime": null
220
+ }
221
+ }
222
+
223
+ """
224
+ return self.api_request(
225
+ "GET",
226
+ f"source-store/v2/source/{source_rid}/runtimePlatform",
227
+ **kwargs,
228
+ )
229
+
230
+ def set_network_egress_policies(self, source_rid: SourceRid, network_egress_policies: set[NetworkEgressPolicyRid]):
231
+ """Sets the network egress policies for a source, overwrites all pre-existing egress policies."""
232
+ _ = self.api_update_source_v2(
233
+ source_rid=source_rid,
234
+ patch_cloud_runtime_platform={"updatedNetworkEgresses": list(network_egress_policies)},
235
+ )
236
+
237
+ def get_network_egress_policies(self, source_rid: SourceRid) -> list[NetworkEgressPolicyRid]:
238
+ """Returns list of network egress policies for a source."""
239
+ runtime_platform = self.api_get_runtime_platform(source_rid=source_rid)
240
+ return runtime_platform.json()["cloud"]["networkEgressPolicies"]["networkEgressPolicies"]
241
+
242
+ def add_network_egress_policy(self, source_rid: SourceRid, network_egress_policy_rid: NetworkEgressPolicyRid):
243
+ """Adds a single egress policy to a source. Additive operation."""
244
+ existing_egress_policies = self.get_network_egress_policies(source_rid=source_rid)
245
+ existing_egress_policies.append(network_egress_policy_rid)
246
+ self.set_network_egress_policies(source_rid=source_rid, network_egress_policies=set(existing_egress_policies))
247
+
248
+ def delete_network_egress_policy(self, source_rid: SourceRid, network_egress_policy_rid: NetworkEgressPolicyRid):
249
+ """Deletes a single egress policy from a source."""
250
+ existing_egress_policies = self.get_network_egress_policies(source_rid=source_rid)
251
+ existing_egress_policies.remove(network_egress_policy_rid)
252
+ self.set_network_egress_policies(source_rid=source_rid, network_egress_policies=set(existing_egress_policies))
253
+
254
+ def enable_s3_oidc_runtime(self, source_rid: SourceRid):
255
+ """Enables OpenID Connect (OIDC) in (S3) Source."""
256
+ _ = self.api_update_source_v2(
257
+ source_rid=source_rid,
258
+ patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": {"audience": "sts.amazonaws.com"}}},
259
+ )
260
+
261
+ def disable_s3_oidc_runtime(self, source_rid: SourceRid):
262
+ """Disables OpenID Connect (OIDC) in (S3) Source."""
263
+ _ = self.api_update_source_v2(
264
+ source_rid=source_rid,
265
+ patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": None}},
266
+ )
267
+
268
+ def api_modify_source_usage_restrictions(self, source_rid: SourceRid, payload: dict, **kwargs) -> requests.Response:
269
+ """Update usage restrictions for a source.
270
+
271
+ Args:
272
+ source_rid: the source to modify usage restrictions for
273
+ payload: the payload to modify the usage restrictions for, example:
274
+ {"toRestrict": [], "toEnable": [{"type": "stemmaRepository", "stemmaRepository": {}}]}
275
+ kwargs: any additional keyword arguments are passed to the requests library.
276
+
277
+ """
278
+ return self.api_request(
279
+ "PUT",
280
+ f"code-resource-source-import/sources/{source_rid}/usage-restrictions",
281
+ json=payload,
282
+ **kwargs,
283
+ )
284
+
285
+ def enable_code_imports(self, source_rid: SourceRid, to_enable: list[CodeResourceType]):
286
+ """Enable code import on Magritte Source.
287
+
288
+ Args:
289
+ source_rid: Magritte Source Rid.
290
+ to_enable: Pick from "stemmaRepository", "computeModule", "eddiePipeline"
291
+ """
292
+ base_payload = {"toRestrict": [], "toEnable": []}
293
+ for entry in to_enable:
294
+ base_payload["toEnable"].append({"type": entry, entry: {}})
295
+ self.api_modify_source_usage_restrictions(source_rid=source_rid, payload=base_payload)
296
+
297
+ def restrict_code_imports(self, source_rid: SourceRid, to_restrict: list[CodeResourceType]):
298
+ """Restrict code import on Magritte Source.
299
+
300
+ Args:
301
+ source_rid: Magritte Source Rid.
302
+ to_restrict: Pick from "stemmaRepository", "computeModule", "eddiePipeline"
303
+ """
304
+ base_payload = {"toRestrict": [], "toEnable": []}
305
+ for entry in to_restrict:
306
+ base_payload["toRestrict"].append({"type": entry, entry: {}})
307
+ self.api_modify_source_usage_restrictions(source_rid=source_rid, payload=base_payload)
308
+
309
+ def api_bulk_get_usage_restrictions_for_source(self, source_rids: list[SourceRid], **kwargs) -> requests.Response:
310
+ """Get information about which types of code resources allow the import of the requested sources."""
311
+ return self.api_request(
312
+ "PUT",
313
+ "code-resource-source-import/sources/usage-restrictions/bulk",
314
+ json={"sourceRids": source_rids},
315
+ **kwargs,
316
+ )
317
+
318
+ # Enable Export
319
+ def api_get_export_state_for_source(self, source_rid: SourceRid, **kwargs) -> requests.Response:
320
+ """Returns the current export config of a source.
321
+
322
+ Returns:
323
+ example format {
324
+ "isEnabled": True,
325
+ "exportableMarkings": ["marking_id"],
326
+ "isEnabledWithoutMarkingsValidation": False
327
+ }
328
+ """
329
+ return self.api_request(
330
+ "GET",
331
+ f"export-control/state/{source_rid}",
332
+ **kwargs,
333
+ )
334
+
335
+ def api_update_export_state_for_source(
336
+ self,
337
+ source_rid: SourceRid,
338
+ is_enabled: bool,
339
+ is_enabled_without_markings_validation: bool | None = None,
340
+ **kwargs,
341
+ ) -> requests.Response:
342
+ """Updates Export config for a source.
343
+
344
+ Args:
345
+ source_rid: Magritte Source Rid.
346
+ is_enabled: Whether the export is allowed or not.
347
+ is_enabled_without_markings_validation: Whether export without marking validation is allowed or not.
348
+ kwargs: any additional keyword arguments are passed to the requests library.
349
+ """
350
+ payload = {"isEnabled": is_enabled}
351
+ if is_enabled_without_markings_validation is not None:
352
+ payload["isEnabledWithoutMarkingsValidation"] = is_enabled_without_markings_validation
353
+ return self.api_request(
354
+ "POST",
355
+ f"export-control/state/{source_rid}",
356
+ json=payload,
357
+ **kwargs,
358
+ )
359
+
360
+ def api_add_exportable_markings_for_source(
361
+ self, source_rid: SourceRid, exportable_markings: list[MarkingId], **kwargs
362
+ ) -> requests.Response:
363
+ """Adds exportable markings for a source."""
364
+ return self.api_request(
365
+ "POST",
366
+ f"export-control/markings/{source_rid}",
367
+ json={"exportableMarkings": exportable_markings},
368
+ **kwargs,
369
+ )
370
+
371
+ def api_remove_exportable_markings_for_source(
372
+ self, source_rid: SourceRid, markings_to_remove: list[MarkingId], **kwargs
373
+ ) -> requests.Response:
374
+ """Remove exportable markings for a source."""
375
+ return self.api_request(
376
+ "DELETE",
377
+ f"export-control/markings/{source_rid}",
378
+ json={"markingsToRemove": markings_to_remove},
379
+ **kwargs,
380
+ )
381
+
382
+ def create_snowflake_source(
383
+ self,
384
+ name: str,
385
+ parent_rid: FolderRid,
386
+ account_identifier: str,
387
+ description: str | None = "",
388
+ role: str | None = None,
389
+ db: str | None = None,
390
+ schema: str | None = None,
391
+ warehouse: str | None = None,
392
+ network_egress_policies: set[Rid] | None = None,
393
+ ) -> str:
394
+ """Create a new Source of type snowflake. Only supports Direct Connect sources.
395
+
396
+ Args:
397
+ name: Name of the Source in Compass
398
+ parent_rid: Rid of the parent Compass folder.
399
+ account_identifier: https://docs.snowflake.com/en/user-guide/admin-account-identifier.html
400
+ description: Textual description field.
401
+ role: The default access control role to use in the Snowflake session initiated by the driver
402
+ db: The default database to use once connected
403
+ schema: The default schema to use once connected
404
+ warehouse: The virtual warehouse to use once connected
405
+ network_egress_policies: Network Egress Policies required to connect to the Snowflake Account.
406
+ Front-door plus internal stage bucket is usually required.
407
+
408
+ """
409
+ payload = {
410
+ "source": {
411
+ "type": "snowflake",
412
+ "config": {
413
+ "connection": {
414
+ "accountIdentifier": account_identifier,
415
+ **({"role": role} if role else {}),
416
+ **({"db": db} if db else {}),
417
+ **({"schema": schema} if schema else {}),
418
+ **({"warehouse": warehouse} if warehouse else {}),
419
+ },
420
+ "auth": {
421
+ "basic": {"username": "", "password": "{{SNOWFLAKE_BASIC_AUTH_PASSWORD}}"},
422
+ "type": "basic",
423
+ },
424
+ },
425
+ }
426
+ }
427
+ network_egress_policies = network_egress_policies or set()
428
+
429
+ response = self.api_add_source_v3(
430
+ config=payload,
431
+ description={"name": name, "description": description},
432
+ runtime_platform_request={
433
+ "cloud": {"networkEgresses": list(network_egress_policies)},
434
+ "type": "cloud",
435
+ },
436
+ parent_rid=parent_rid,
437
+ )
438
+ return response.json()
439
+
440
+ def enable_snowflake_external_oauth(self, source_rid: SourceRid):
441
+ """Enables external OAuth (OIDC) in (Snowflake) Source."""
442
+ config = self.get_source_config(source_rid=source_rid)["source"]
443
+
444
+ account_identifier = config["config"]["connection"]["accountIdentifier"]
445
+ config["config"]["auth"] = {"externalOauth": {}, "type": "externalOauth"}
446
+ _ = self.api_update_source_v2(
447
+ source_rid=source_rid,
448
+ updated_source_config=config,
449
+ patch_cloud_runtime_platform={
450
+ "patchOidcRuntime": {
451
+ "oidcRuntime": {"audience": f"https://{account_identifier}.snowflakecomputing.com"}
452
+ }
453
+ },
454
+ )
455
+
456
+ def disable_snowflake_external_oauth(self, source_rid: SourceRid):
457
+ """Disables external OAuth (OIDC) in (Snowflake) Source."""
458
+ config = self.get_source_config(source_rid=source_rid)["source"]
459
+ config["config"]["auth"] = {
460
+ "basic": {"username": "", "password": "{{SNOWFLAKE_BASIC_AUTH_PASSWORD}}"},
461
+ "type": "basic",
462
+ }
463
+ _ = self.api_update_source_v2(
464
+ source_rid=source_rid,
465
+ updated_source_config=config,
466
+ patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": None}},
467
+ )
@@ -95,6 +95,32 @@ class MultipassClient(APIClient):
95
95
  **kwargs,
96
96
  )
97
97
 
98
+ def search(
99
+ self, query: str, principal_types: set[api_types.PrincipalTypes] | None = None, **kwargs
100
+ ) -> Iterator[dict]:
101
+ """Searches for multipass principals based on a text.
102
+
103
+ Args:
104
+ query: the text string to search for
105
+ principal_types: set of principal types to search in, e.g. "GROUP"
106
+ Default is GROUP and USER
107
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
108
+ """
109
+ if principal_types is None:
110
+ principal_types = {"USER", "GROUP"}
111
+
112
+ json = {"attributeFilters": {}, "principalTypes": list(principal_types), "query": query}
113
+
114
+ next_start = None
115
+ while True:
116
+ if next_start:
117
+ json["pageStart"] = next_start
118
+
119
+ response_as_json = self.api_request("POST", "search/v2/search", json=json, **kwargs).json()
120
+ yield from response_as_json["values"]
121
+ if (next_start := response_as_json["nextPageToken"]) is None:
122
+ break
123
+
98
124
  def api_get_groups_of_user(self, **kwargs) -> requests.Response:
99
125
  """Returns all groups for which the user is immediate or indirect member of the group.
100
126
 
@@ -333,6 +359,19 @@ class MultipassClient(APIClient):
333
359
  **kwargs,
334
360
  )
335
361
 
362
+ def api_delete_user(self, user_id: api_types.UserId, **kwargs) -> requests.Response:
363
+ """Deletes multipass user.
364
+
365
+ Args:
366
+ user_id: the identifier of the group which to delete
367
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
368
+ """
369
+ return self.api_request(
370
+ "DELETE",
371
+ f"administration/users/{user_id}",
372
+ **kwargs,
373
+ )
374
+
336
375
  def api_get_group_manager_managers(
337
376
  self,
338
377
  group_id: api_types.GroupId,
@@ -177,7 +177,8 @@ class S3Client:
177
177
  expiration_duration: seconds the credentials should be valid, defaults to 3600 (the upper bound)
178
178
  """
179
179
  # does not call the api_request method, as this is not a regular api
180
- resp = self.context.client.post(
180
+ resp = self.context.client.request(
181
+ "POST",
181
182
  self.get_url(),
182
183
  params={
183
184
  "Action": "AssumeRoleWithWebIdentity",