foundry-dev-tools 2.1.14__tar.gz → 2.1.15__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/PKG-INFO +1 -1
  2. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/pyproject.toml +1 -1
  3. foundry_dev_tools-2.1.15/src/foundry_dev_tools/__about__.py +1 -0
  4. foundry_dev_tools-2.1.15/src/foundry_dev_tools/clients/magritte_coordinator.py +467 -0
  5. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/multipass.py +26 -0
  6. foundry_dev_tools-2.1.15/src/foundry_dev_tools/clients/scheduler.py +54 -0
  7. foundry_dev_tools-2.1.15/src/foundry_dev_tools/clients/tables.py +145 -0
  8. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/context.py +18 -0
  9. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/dataset.py +3 -1
  10. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/api_types.py +15 -0
  11. foundry_dev_tools-2.1.14/src/foundry_dev_tools/__about__.py +0 -1
  12. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/README.md +0 -0
  13. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/__init__.py +0 -0
  14. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/__init__.py +0 -0
  15. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pandas.py +0 -0
  16. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/polars.py +0 -0
  17. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pyarrow.py +0 -0
  18. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pyspark.py +0 -0
  19. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
  20. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/__init__.py +0 -0
  21. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/build.py +0 -0
  22. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/config.py +0 -0
  23. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/git.py +0 -0
  24. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/git_credential_foundry.py +0 -0
  25. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/info.py +0 -0
  26. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/main.py +0 -0
  27. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/s3.py +0 -0
  28. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/stubs.py +0 -0
  29. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/__init__.py +0 -0
  30. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/api_client.py +0 -0
  31. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/build2.py +0 -0
  32. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/catalog.py +0 -0
  33. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/compass.py +0 -0
  34. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/context_client.py +0 -0
  35. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/data_proxy.py +0 -0
  36. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/foundry_sql_server.py +0 -0
  37. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/foundry_stats.py +0 -0
  38. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/jemma.py +0 -0
  39. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/metadata.py +0 -0
  40. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/public_ontologies_client.py +0 -0
  41. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/s3_client.py +0 -0
  42. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/schema_inference.py +0 -0
  43. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/__init__.py +0 -0
  44. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/config.py +0 -0
  45. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/config_types.py +0 -0
  46. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/token_provider.py +0 -0
  47. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/__init__.py +0 -0
  48. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/compass.py +0 -0
  49. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/config.py +0 -0
  50. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/dataset.py +0 -0
  51. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/handling.py +0 -0
  52. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/meta.py +0 -0
  53. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/multipass.py +0 -0
  54. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/sql.py +0 -0
  55. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/foundry_api_client.py +0 -0
  56. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/__init__.py +0 -0
  57. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/__init__.py +0 -0
  58. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/group.py +0 -0
  59. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/principal.py +0 -0
  60. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/user.py +0 -0
  61. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/py.typed +0 -0
  62. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/__init__.py +0 -0
  63. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/folder.py +0 -0
  64. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/resource.py +0 -0
  65. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/__init__.py +0 -0
  66. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/async_s3.py +0 -0
  67. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  68. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  69. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  70. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/cli.py +0 -0
  71. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/clients.py +0 -0
  72. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/compat.py +0 -0
  73. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/config.py +0 -0
  74. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  75. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  76. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/misc.py +0 -0
  77. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/public_api_types.py +0 -0
  78. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/repo.py +0 -0
  79. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/s3.py +0 -0
  80. {foundry_dev_tools-2.1.14 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/spark.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 2.1.14
3
+ Version: 2.1.15
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-Email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -41,7 +41,7 @@ dependencies = [
41
41
  "tomli_w",
42
42
  ]
43
43
  dynamic = []
44
- version = "2.1.14"
44
+ version = "2.1.15"
45
45
 
46
46
  [project.license]
47
47
  text = "Apache-2.0"
@@ -0,0 +1 @@
1
+ __version__ = '2.1.15'
@@ -0,0 +1,467 @@
1
+ """Implementation of the foundry-stats API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from foundry_dev_tools.clients.api_client import APIClient
8
+
9
+ if TYPE_CHECKING:
10
+ import requests
11
+
12
+ from foundry_dev_tools.utils.api_types import (
13
+ CodeResourceType,
14
+ FolderRid,
15
+ MarkingId,
16
+ NetworkEgressPolicyRid,
17
+ Rid,
18
+ SourceRid,
19
+ )
20
+
21
+
22
+ class MagritteCoordinatorClient(APIClient):
23
+ """The mothership that controls remote Magritte agents and cloud runtimes."""
24
+
25
+ api_name = "magritte-coordinator"
26
+
27
+ def api_get_oidc_metadata(self, **kwargs) -> requests.Response:
28
+ """Returns the OIDC Metadata of the Foundry Stack.
29
+
30
+ Args:
31
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
32
+ Returns:
33
+ dict: {'issuer': 'https://pltroidcpublic<id>.s3.<region>.amazonaws.com/foundry'}
34
+ """
35
+ return self.api_request(
36
+ "POST",
37
+ "oidc/metadata",
38
+ **kwargs,
39
+ )
40
+
41
+ def get_oidc_issuer(self) -> str:
42
+ """Returns the OIDC issuer of the Foundry Stack."""
43
+ return self.api_get_oidc_metadata().json()["issuer"]
44
+
45
+ def api_add_source_v3(
46
+ self, config: dict, description: dict, runtime_platform_request: dict, parent_rid: FolderRid, **kwargs
47
+ ) -> requests.Response:
48
+ """Low level method to add a new source to Data Connection.
49
+
50
+ Use a high level method like :py:meth:`MagritteCoordinatorClient.create_s3_direct_source`
51
+ for easier usage.
52
+
53
+ Args:
54
+ config: Source Config, individual format for each supported Source.
55
+ description: Textual description field.
56
+ runtime_platform_request: For Direct Connect { "cloud": {"networkEgresses": []}, "type": "cloud" }
57
+ parent_rid: Rid of the parent Compass folder.
58
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
59
+ """
60
+ return self.api_request(
61
+ "POST",
62
+ "source-store/source/v3",
63
+ json={
64
+ "config": config,
65
+ "description": description,
66
+ "runtimePlatformRequest": runtime_platform_request,
67
+ "parentRid": parent_rid,
68
+ },
69
+ **kwargs,
70
+ )
71
+
72
+ def api_update_source_v2(
73
+ self,
74
+ source_rid: Rid,
75
+ updated_source_config: dict | None = None,
76
+ patch_cloud_runtime_platform: dict | None = None,
77
+ **kwargs,
78
+ ) -> requests.Response:
79
+ """Updates a Source.
80
+
81
+ Args:
82
+ source_rid: The Rid of the Magritte Source
83
+ updated_source_config: example {"type":"s3-direct","url":"s3://bucket1"}
84
+ patch_cloud_runtime_platform: example
85
+ {
86
+ "secretsToUpdate": {},
87
+ "removedSecrets": [],
88
+ "patchCloudIdentity": {},
89
+ "patchOidcRuntime": {},
90
+ }
91
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
92
+
93
+ Returns:
94
+ requests.Response: dict with top level key updatedSource
95
+ {
96
+ "updatedSource": {
97
+ "sourceConfig": {
98
+ "type": "s3-direct",
99
+ "url": "s3://bucket"
100
+ },
101
+ "connections": {},
102
+ "additionalSecrets": {},
103
+ "mavenCoordinates": {
104
+ "userProvided": [],
105
+ "foundryProvided": []
106
+ }
107
+ }
108
+ }
109
+ """
110
+ payload = {}
111
+ if updated_source_config:
112
+ payload["patchSourceConfigRequest"] = {}
113
+ payload["patchSourceConfigRequest"]["updatedSourceConfig"] = updated_source_config
114
+ if patch_cloud_runtime_platform:
115
+ payload["patchRuntimePlatformRequest"] = {}
116
+ payload["patchRuntimePlatformRequest"]["patchCloudRuntimePlatform"] = patch_cloud_runtime_platform
117
+ payload["patchRuntimePlatformRequest"]["type"] = "patchCloudRuntimePlatform"
118
+ return self.api_request(
119
+ "PUT",
120
+ f"source-store/v2/sources/{source_rid}",
121
+ json=payload,
122
+ **kwargs,
123
+ )
124
+
125
+ def create_s3_direct_source(
126
+ self,
127
+ name: str,
128
+ parent_rid: FolderRid,
129
+ url: str,
130
+ description: str | None = "",
131
+ region: str | None = None,
132
+ catalog: dict | None = None,
133
+ sts_role_configuration: dict | None = None,
134
+ network_egress_policies: set[Rid] | None = None,
135
+ **source_kwargs,
136
+ ) -> str:
137
+ """Create a new Source of type s3-direct. Only supports Direct Connect sources.
138
+
139
+ Args:
140
+ name: Name of the Source in Compass
141
+ parent_rid: Rid of the parent Compass folder.
142
+ url: S3 Url, e.g. s3://bucket-name
143
+ description: Textual description field.
144
+ region: AWS Region of the bucket.
145
+ catalog: Catalog Integration for Iceberg Tables.
146
+ sts_role_configuration: sts role configuration for Assuming a role to connect to the bucket.
147
+ network_egress_policies: Network Egress Policies required to connect to the bucket.
148
+ source_kwargs: Any other arguments get added to the "source" key of the payload.
149
+ """
150
+ payload = {
151
+ "source": {
152
+ "type": "s3-direct",
153
+ "url": url,
154
+ **({"region": region} if region else {}),
155
+ **({"catalog": catalog} if catalog else {}),
156
+ **({"stsRoleConfiguration": sts_role_configuration} if sts_role_configuration else {}),
157
+ **source_kwargs,
158
+ }
159
+ }
160
+ network_egress_policies = network_egress_policies or set()
161
+
162
+ response = self.api_add_source_v3(
163
+ config=payload,
164
+ description={"name": name, "description": description},
165
+ runtime_platform_request={
166
+ "cloud": {"networkEgresses": list(network_egress_policies)},
167
+ "type": "cloud",
168
+ },
169
+ parent_rid=parent_rid,
170
+ )
171
+ return response.json()
172
+
173
+ def get_source_config(self, source_rid: SourceRid) -> dict:
174
+ """Returns the configuration of a Source."""
175
+ return self.api_bulk_get_source_config(source_rids={source_rid}).json()[source_rid]
176
+
177
+ def api_bulk_get_source_config(self, source_rids: set[SourceRid], **kwargs) -> requests.Response:
178
+ """Returns the configuration of a multiple sources."""
179
+ return self.api_request(
180
+ "POST",
181
+ "source-store/sources/config/bulk",
182
+ json=list(source_rids),
183
+ **kwargs,
184
+ )
185
+
186
+ def get_source_description(self, source_rid: SourceRid) -> dict:
187
+ """Returns the description of a Source.
188
+
189
+ Returns:
190
+ dictionary with example payload:
191
+ {'apiName': None, 'description': 'the desc.', 'name': 'source-name', 'type': 's3-direct'}
192
+ """
193
+ return self.api_bulk_get_source_description(source_rids={source_rid}).json()[source_rid]
194
+
195
+ def api_bulk_get_source_description(self, source_rids: set[SourceRid], **kwargs) -> requests.Response:
196
+ """Returns the description of multiple Sources."""
197
+ return self.api_request(
198
+ "POST",
199
+ "source-store/sources/description/bulk",
200
+ json=list(source_rids),
201
+ **kwargs,
202
+ )
203
+
204
+ def api_get_runtime_platform(self, source_rid: SourceRid, **kwargs) -> requests.Response:
205
+ """Get runtime platform information for a source.
206
+
207
+ Returns:
208
+ response: example format {
209
+ "type": "cloud",
210
+ "cloud": {
211
+ "credential": {
212
+ "credentialId": "ri.credential..credential.7ddaaa49-1337-1337-a0c0-8b621fa30364",
213
+ "secretNames": []
214
+ },
215
+ "networkEgressPolicies": {
216
+ "networkEgressPolicies": []
217
+ },
218
+ "cloudIdentity": null,
219
+ "oidcRuntime": null
220
+ }
221
+ }
222
+
223
+ """
224
+ return self.api_request(
225
+ "GET",
226
+ f"source-store/v2/source/{source_rid}/runtimePlatform",
227
+ **kwargs,
228
+ )
229
+
230
+ def set_network_egress_policies(self, source_rid: SourceRid, network_egress_policies: set[NetworkEgressPolicyRid]):
231
+ """Sets the network egress policies for a source, overwrites all pre-existing egress policies."""
232
+ _ = self.api_update_source_v2(
233
+ source_rid=source_rid,
234
+ patch_cloud_runtime_platform={"updatedNetworkEgresses": list(network_egress_policies)},
235
+ )
236
+
237
+ def get_network_egress_policies(self, source_rid: SourceRid) -> list[NetworkEgressPolicyRid]:
238
+ """Returns list of network egress policies for a source."""
239
+ runtime_platform = self.api_get_runtime_platform(source_rid=source_rid)
240
+ return runtime_platform.json()["cloud"]["networkEgressPolicies"]["networkEgressPolicies"]
241
+
242
+ def add_network_egress_policy(self, source_rid: SourceRid, network_egress_policy_rid: NetworkEgressPolicyRid):
243
+ """Adds a single egress policy to a source. Additive operation."""
244
+ existing_egress_policies = self.get_network_egress_policies(source_rid=source_rid)
245
+ existing_egress_policies.append(network_egress_policy_rid)
246
+ self.set_network_egress_policies(source_rid=source_rid, network_egress_policies=set(existing_egress_policies))
247
+
248
+ def delete_network_egress_policy(self, source_rid: SourceRid, network_egress_policy_rid: NetworkEgressPolicyRid):
249
+ """Deletes a single egress policy from a source."""
250
+ existing_egress_policies = self.get_network_egress_policies(source_rid=source_rid)
251
+ existing_egress_policies.remove(network_egress_policy_rid)
252
+ self.set_network_egress_policies(source_rid=source_rid, network_egress_policies=set(existing_egress_policies))
253
+
254
+ def enable_s3_oidc_runtime(self, source_rid: SourceRid):
255
+ """Enables OpenID Connect (OIDC) in (S3) Source."""
256
+ _ = self.api_update_source_v2(
257
+ source_rid=source_rid,
258
+ patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": {"audience": "sts.amazonaws.com"}}},
259
+ )
260
+
261
+ def disable_s3_oidc_runtime(self, source_rid: SourceRid):
262
+ """Disables OpenID Connect (OIDC) in (S3) Source."""
263
+ _ = self.api_update_source_v2(
264
+ source_rid=source_rid,
265
+ patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": None}},
266
+ )
267
+
268
+ def api_modify_source_usage_restrictions(self, source_rid: SourceRid, payload: dict, **kwargs) -> requests.Response:
269
+ """Update usage restrictions for a source.
270
+
271
+ Args:
272
+ source_rid: the source to modify usage restrictions for
273
+ payload: the payload to modify the usage restrictions for, example:
274
+ {"toRestrict": [], "toEnable": [{"type": "stemmaRepository", "stemmaRepository": {}}]}
275
+ kwargs: any additional keyword arguments are passed to the requests library.
276
+
277
+ """
278
+ return self.api_request(
279
+ "PUT",
280
+ f"code-resource-source-import/sources/{source_rid}/usage-restrictions",
281
+ json=payload,
282
+ **kwargs,
283
+ )
284
+
285
+ def enable_code_imports(self, source_rid: SourceRid, to_enable: list[CodeResourceType]):
286
+ """Enable code import on Magritte Source.
287
+
288
+ Args:
289
+ source_rid: Magritte Source Rid.
290
+ to_enable: Pick from "stemmaRepository", "computeModule", "eddiePipeline"
291
+ """
292
+ base_payload = {"toRestrict": [], "toEnable": []}
293
+ for entry in to_enable:
294
+ base_payload["toEnable"].append({"type": entry, entry: {}})
295
+ self.api_modify_source_usage_restrictions(source_rid=source_rid, payload=base_payload)
296
+
297
+ def restrict_code_imports(self, source_rid: SourceRid, to_restrict: list[CodeResourceType]):
298
+ """Restrict code import on Magritte Source.
299
+
300
+ Args:
301
+ source_rid: Magritte Source Rid.
302
+ to_restrict: Pick from "stemmaRepository", "computeModule", "eddiePipeline"
303
+ """
304
+ base_payload = {"toRestrict": [], "toEnable": []}
305
+ for entry in to_restrict:
306
+ base_payload["toRestrict"].append({"type": entry, entry: {}})
307
+ self.api_modify_source_usage_restrictions(source_rid=source_rid, payload=base_payload)
308
+
309
+ def api_bulk_get_usage_restrictions_for_source(self, source_rids: list[SourceRid], **kwargs) -> requests.Response:
310
+ """Get information about which types of code resources allow the import of the requested sources."""
311
+ return self.api_request(
312
+ "PUT",
313
+ "code-resource-source-import/sources/usage-restrictions/bulk",
314
+ json={"sourceRids": source_rids},
315
+ **kwargs,
316
+ )
317
+
318
+ # Enable Export
319
+ def api_get_export_state_for_source(self, source_rid: SourceRid, **kwargs) -> requests.Response:
320
+ """Returns the current export config of a source.
321
+
322
+ Returns:
323
+ example format {
324
+ "isEnabled": True,
325
+ "exportableMarkings": ["marking_id"],
326
+ "isEnabledWithoutMarkingsValidation": False
327
+ }
328
+ """
329
+ return self.api_request(
330
+ "GET",
331
+ f"export-control/state/{source_rid}",
332
+ **kwargs,
333
+ )
334
+
335
+ def api_update_export_state_for_source(
336
+ self,
337
+ source_rid: SourceRid,
338
+ is_enabled: bool,
339
+ is_enabled_without_markings_validation: bool | None = None,
340
+ **kwargs,
341
+ ) -> requests.Response:
342
+ """Updates Export config for a source.
343
+
344
+ Args:
345
+ source_rid: Magritte Source Rid.
346
+ is_enabled: Whether the export is allowed or not.
347
+ is_enabled_without_markings_validation: Whether export without marking validation is allowed or not.
348
+ kwargs: any additional keyword arguments are passed to the requests library.
349
+ """
350
+ payload = {"isEnabled": is_enabled}
351
+ if is_enabled_without_markings_validation is not None:
352
+ payload["isEnabledWithoutMarkingsValidation"] = is_enabled_without_markings_validation
353
+ return self.api_request(
354
+ "POST",
355
+ f"export-control/state/{source_rid}",
356
+ json=payload,
357
+ **kwargs,
358
+ )
359
+
360
+ def api_add_exportable_markings_for_source(
361
+ self, source_rid: SourceRid, exportable_markings: list[MarkingId], **kwargs
362
+ ) -> requests.Response:
363
+ """Adds exportable markings for a source."""
364
+ return self.api_request(
365
+ "POST",
366
+ f"export-control/markings/{source_rid}",
367
+ json={"exportableMarkings": exportable_markings},
368
+ **kwargs,
369
+ )
370
+
371
+ def api_remove_exportable_markings_for_source(
372
+ self, source_rid: SourceRid, markings_to_remove: list[MarkingId], **kwargs
373
+ ) -> requests.Response:
374
+ """Remove exportable markings for a source."""
375
+ return self.api_request(
376
+ "DELETE",
377
+ f"export-control/markings/{source_rid}",
378
+ json={"markingsToRemove": markings_to_remove},
379
+ **kwargs,
380
+ )
381
+
382
+ def create_snowflake_source(
383
+ self,
384
+ name: str,
385
+ parent_rid: FolderRid,
386
+ account_identifier: str,
387
+ description: str | None = "",
388
+ role: str | None = None,
389
+ db: str | None = None,
390
+ schema: str | None = None,
391
+ warehouse: str | None = None,
392
+ network_egress_policies: set[Rid] | None = None,
393
+ ) -> str:
394
+ """Create a new Source of type snowflake. Only supports Direct Connect sources.
395
+
396
+ Args:
397
+ name: Name of the Source in Compass
398
+ parent_rid: Rid of the parent Compass folder.
399
+ account_identifier: https://docs.snowflake.com/en/user-guide/admin-account-identifier.html
400
+ description: Textual description field.
401
+ role: The default access control role to use in the Snowflake session initiated by the driver
402
+ db: The default database to use once connected
403
+ schema: The default schema to use once connected
404
+ warehouse: The virtual warehouse to use once connected
405
+ network_egress_policies: Network Egress Policies required to connect to the Snowflake Account.
406
+ Front-door plus internal stage bucket is usually required.
407
+
408
+ """
409
+ payload = {
410
+ "source": {
411
+ "type": "snowflake",
412
+ "config": {
413
+ "connection": {
414
+ "accountIdentifier": account_identifier,
415
+ **({"role": role} if role else {}),
416
+ **({"db": db} if db else {}),
417
+ **({"schema": schema} if schema else {}),
418
+ **({"warehouse": warehouse} if warehouse else {}),
419
+ },
420
+ "auth": {
421
+ "basic": {"username": "", "password": "{{SNOWFLAKE_BASIC_AUTH_PASSWORD}}"},
422
+ "type": "basic",
423
+ },
424
+ },
425
+ }
426
+ }
427
+ network_egress_policies = network_egress_policies or set()
428
+
429
+ response = self.api_add_source_v3(
430
+ config=payload,
431
+ description={"name": name, "description": description},
432
+ runtime_platform_request={
433
+ "cloud": {"networkEgresses": list(network_egress_policies)},
434
+ "type": "cloud",
435
+ },
436
+ parent_rid=parent_rid,
437
+ )
438
+ return response.json()
439
+
440
+ def enable_snowflake_external_oauth(self, source_rid: SourceRid):
441
+ """Enables external OAuth (OIDC) in (Snowflake) Source."""
442
+ config = self.get_source_config(source_rid=source_rid)["source"]
443
+
444
+ account_identifier = config["config"]["connection"]["accountIdentifier"]
445
+ config["config"]["auth"] = {"externalOauth": {}, "type": "externalOauth"}
446
+ _ = self.api_update_source_v2(
447
+ source_rid=source_rid,
448
+ updated_source_config=config,
449
+ patch_cloud_runtime_platform={
450
+ "patchOidcRuntime": {
451
+ "oidcRuntime": {"audience": f"https://{account_identifier}.snowflakecomputing.com"}
452
+ }
453
+ },
454
+ )
455
+
456
+ def disable_snowflake_external_oauth(self, source_rid: SourceRid):
457
+ """Disables external OAuth (OIDC) in (Snowflake) Source."""
458
+ config = self.get_source_config(source_rid=source_rid)["source"]
459
+ config["config"]["auth"] = {
460
+ "basic": {"username": "", "password": "{{SNOWFLAKE_BASIC_AUTH_PASSWORD}}"},
461
+ "type": "basic",
462
+ }
463
+ _ = self.api_update_source_v2(
464
+ source_rid=source_rid,
465
+ updated_source_config=config,
466
+ patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": None}},
467
+ )
@@ -95,6 +95,32 @@ class MultipassClient(APIClient):
95
95
  **kwargs,
96
96
  )
97
97
 
98
+ def search(
99
+ self, query: str, principal_types: set[api_types.PrincipalTypes] | None = None, **kwargs
100
+ ) -> Iterator[dict]:
101
+ """Searches for multipass principals based on a text.
102
+
103
+ Args:
104
+ query: the text string to search for
105
+ principal_types: set of principal types to search in, e.g. "GROUP"
106
+ Default is GROUP and USER
107
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
108
+ """
109
+ if principal_types is None:
110
+ principal_types = {"USER", "GROUP"}
111
+
112
+ json = {"attributeFilters": {}, "principalTypes": list(principal_types), "query": query}
113
+
114
+ next_start = None
115
+ while True:
116
+ if next_start:
117
+ json["pageStart"] = next_start
118
+
119
+ response_as_json = self.api_request("POST", "search/v2/search", json=json, **kwargs).json()
120
+ yield from response_as_json["values"]
121
+ if (next_start := response_as_json["nextPageToken"]) is None:
122
+ break
123
+
98
124
  def api_get_groups_of_user(self, **kwargs) -> requests.Response:
99
125
  """Returns all groups for which the user is immediate or indirect member of the group.
100
126
 
@@ -0,0 +1,54 @@
1
+ """Implementation of the scheduler API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from foundry_dev_tools.clients.api_client import APIClient
8
+ from foundry_dev_tools.errors.compass import ResourceNotFoundError
9
+ from foundry_dev_tools.errors.handling import ErrorHandlingConfig
10
+
11
+ if TYPE_CHECKING:
12
+ import requests
13
+
14
+ from foundry_dev_tools.utils import api_types
15
+
16
+
17
+ class SchedulerClient(APIClient):
18
+ """SchedulerClient class that implements methods from the 'scheduler' API."""
19
+
20
+ api_name = "scheduler"
21
+
22
+ def get_schedules_by_run_build(
23
+ self, rid: api_types.Rid, branch: api_types.DatasetBranch = "master", **kwargs
24
+ ) -> requests.Response:
25
+ """Gets the schedules of the resource dataset.
26
+
27
+ Args:
28
+ rid: the identifier of the datset
29
+ branch: the branch name of the dataset
30
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
31
+ """
32
+ params = {"datasetRids": [rid], "branch": branch}
33
+
34
+ return self.api_request(
35
+ "POST",
36
+ "scheduler/get-schedules-by-run-build-dataset",
37
+ json=params,
38
+ error_handling=ErrorHandlingConfig(api_error_mapping={204: ResourceNotFoundError}),
39
+ **kwargs,
40
+ )
41
+
42
+ def get_schedule(self, rid: api_types.Rid, **kwargs) -> requests.Response:
43
+ """Gets the schedule details of a given schedule.
44
+
45
+ Args:
46
+ rid: the identifier of the scheduler
47
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
48
+ """
49
+ return self.api_request(
50
+ "GET",
51
+ f"scheduler/schedules/{rid}",
52
+ error_handling=ErrorHandlingConfig(api_error_mapping={204: ResourceNotFoundError}),
53
+ **kwargs,
54
+ )
@@ -0,0 +1,145 @@
1
+ """Implementation of the tables API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING
6
+
7
+ from foundry_dev_tools.clients.api_client import APIClient
8
+
9
+ if TYPE_CHECKING:
10
+ import requests
11
+
12
+ from foundry_dev_tools.utils.api_types import FolderRid, SourceRid
13
+
14
+
15
+ class TablesClient(APIClient):
16
+ """TablesClient class that implements the 'tables' api."""
17
+
18
+ api_name = "tables"
19
+
20
+ def api_create_table(
21
+ self, name: str, parent_rid: FolderRid, upstream_config: dict, skip_validation: bool = False, **kwargs
22
+ ) -> requests.Response:
23
+ """Low level method to create a virtual table.
24
+
25
+ Use a high level method like :py:meth:`TablesClient.create_glue_table`
26
+ for easier usage.
27
+
28
+ Args:
29
+ name: The name of the table. This will be the name of the compass resource.
30
+ parent_rid: The parent Rid of the compass resource.
31
+ upstream_config: Individual table config.
32
+ skip_validation: If true, skips the validation of the table. Default is False.
33
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
34
+
35
+ """
36
+ return self.api_request(
37
+ "POST",
38
+ api_path="tables",
39
+ json={"name": name, "parentRid": parent_rid, "upstream": upstream_config},
40
+ params={"skipValidation": skip_validation},
41
+ **kwargs,
42
+ )
43
+
44
+ def create_glue_table(
45
+ self,
46
+ name: str,
47
+ parent_rid: FolderRid,
48
+ connection_rid: SourceRid,
49
+ table: str,
50
+ database: str,
51
+ skip_validation: bool = False,
52
+ ) -> dict:
53
+ """Creates a Virtual Table backed by the Glue Catalog.
54
+
55
+ Args:
56
+ name: The name of the table. This will be the name of the compass resource.
57
+ parent_rid: The parent Rid of the compass resource.
58
+ connection_rid: The rid of the compass Source.
59
+ table: The name of the table in glue.
60
+ database: The name of the database in glue.
61
+ skip_validation: If true, skips the validation of the table. Default is False.
62
+
63
+ """
64
+ upstream_config = {
65
+ "type": "glue",
66
+ "glue": {"connectionRid": connection_rid, "relation": {"table": table, "database": database}},
67
+ }
68
+ return self.api_create_table(
69
+ name=name, parent_rid=parent_rid, upstream_config=upstream_config, skip_validation=skip_validation
70
+ ).json()
71
+
72
+ def create_snowflake_table(
73
+ self,
74
+ name: str,
75
+ parent_rid: FolderRid,
76
+ connection_rid: SourceRid,
77
+ database: str,
78
+ schema: str,
79
+ table: str,
80
+ skip_validation: bool = False,
81
+ ) -> dict:
82
+ """Creates a Virtual Table on a Snowflake Source.
83
+
84
+ Args:
85
+ name: The name of the table. This will be the name of the compass resource.
86
+ parent_rid: The parent Rid of the compass resource.
87
+ connection_rid: The rid of the compass Source.
88
+ table: The name of the table in snowflake.
89
+ database: The name of the database Snowflake.
90
+ schema: The name of the schema in Snowflake.
91
+ skip_validation: If true, skips the validation of the table. Default is False.
92
+
93
+ """
94
+ upstream_config = {
95
+ "type": "snowflake",
96
+ "snowflake": {
97
+ "connectionRid": connection_rid,
98
+ "relation": {"table": table, "database": database, "schema": schema},
99
+ },
100
+ }
101
+ return self.api_create_table(
102
+ name=name, parent_rid=parent_rid, upstream_config=upstream_config, skip_validation=skip_validation
103
+ ).json()
104
+
105
+ def api_list_tables(
106
+ self, connection_rid: SourceRid, limit: int | None = 1000, page_token: str | None = None, **kwargs
107
+ ) -> requests.Response:
108
+ """List Virtual Tables for a source."""
109
+ return self.api_request(
110
+ "GET",
111
+ api_path="tables",
112
+ params={
113
+ "connectionRid": connection_rid,
114
+ **({"limit": limit} if limit else {}),
115
+ **({"pageToken": page_token} if page_token else {}),
116
+ },
117
+ **kwargs,
118
+ )
119
+
120
+ def list_tables(self, connection_rid: SourceRid) -> list[dict]:
121
+ """Returns all tables for a connection/source by handling pagination.
122
+
123
+ Args:
124
+ connection_rid: The rid of the compass Source/Connection.
125
+
126
+ Returns:
127
+ a list of dicts with the following format:
128
+ {
129
+ 'rid': 'ri.tables.main.table.<...>',
130
+ 'upstream': {
131
+ 'type': '<type>',
132
+ '<type>': {
133
+ 'connectionRid': 'ri.magritte..source.<...>',
134
+ 'relation': {'database': 'test', 'schema': 'test', 'table': 'test2'}
135
+ }
136
+ }
137
+ }
138
+ """
139
+ tables_response = self.api_list_tables(connection_rid=connection_rid, limit=1000)
140
+ response_json = tables_response.json()
141
+ tables = response_json["tables"]
142
+ while next_page_token := response_json.get("nextPageToken"):
143
+ response_json = self.api_list_tables(connection_rid=connection_rid, page_token=next_page_token).json()
144
+ tables.extend(response_json["tables"])
145
+ return tables
@@ -14,11 +14,14 @@ from foundry_dev_tools.clients import (
14
14
  foundry_sql_server,
15
15
  foundry_stats,
16
16
  jemma,
17
+ magritte_coordinator,
17
18
  metadata,
18
19
  multipass,
19
20
  public_ontologies_client,
20
21
  s3_client,
22
+ scheduler,
21
23
  schema_inference,
24
+ tables,
22
25
  )
23
26
  from foundry_dev_tools.config.config import Config, get_config_dict, parse_credentials_config, parse_general_config
24
27
  from foundry_dev_tools.helpers.multipass import Group, User
@@ -85,6 +88,21 @@ class FoundryContext:
85
88
  """Returns :py:class:`foundry_dev_tools.clients.jemma.JemmaClient`."""
86
89
  return jemma.JemmaClient(self)
87
90
 
91
+ @cached_property
92
+ def tables(self) -> tables.TablesClient:
93
+ """Returns :py:class:`foundry_dev_tools.clients.tables.TablesClient`."""
94
+ return tables.TablesClient(self)
95
+
96
+ @cached_property
97
+ def magritte_coordinator(self) -> magritte_coordinator.MagritteCoordinatorClient:
98
+ """Returns :py:class:`foundry_dev_tools.clients.magritte_coordinator.MagritteCoordinatorClient`."""
99
+ return magritte_coordinator.MagritteCoordinatorClient(self)
100
+
101
+ @cached_property
102
+ def scheduler(self) -> scheduler.SchedulerClient:
103
+ """Returns :py:class:`foundry_dev_tools.clients.scheduler.SchedulerClient`."""
104
+ return scheduler.SchedulerClient(self)
105
+
88
106
  @cached_property
89
107
  def metadata(self) -> metadata.MetadataClient:
90
108
  """Returns :py:class:`foundry_dev_tools.clients.metadata.MetadataClient`."""
@@ -381,7 +381,9 @@ class Dataset(resource.Resource):
381
381
  """
382
382
  kwargs.setdefault("transaction_type", "UPDATE")
383
383
  self.upload_files(
384
- {str(f.relative_to(folder_path)): f for f in folder_path.rglob("*") if f.is_file()}, max_workers=max_workers
384
+ {str(f.relative_to(folder_path)): f for f in folder_path.rglob("*") if f.is_file()},
385
+ max_workers=max_workers,
386
+ **kwargs,
385
387
  )
386
388
  return self
387
389
 
@@ -68,6 +68,15 @@ OrganizationRid = str
68
68
  ProjectRid = str
69
69
  """A compass project root folder resource identifier."""
70
70
 
71
+ SourceRid = str
72
+ """A magritte source resource identifier."""
73
+
74
+ TableRid = str
75
+ """A virtual table resource identifier."""
76
+
77
+ NetworkEgressPolicyRid = str
78
+ """A virtual table resource identifier."""
79
+
71
80
  # TODO further typing?
72
81
  FoundrySchema = dict[str, Any]
73
82
 
@@ -166,6 +175,12 @@ TokenType = Literal["USER_GENERATED"]
166
175
  UserId = str
167
176
  """An identifier for an user on foundry."""
168
177
 
178
+ CodeResourceType = Literal["stemmaRepository", "computeModule", "eddiePipeline"]
179
+ """Code Resource type."""
180
+
181
+ PrincipalTypes = Literal["USER", "GROUP"]
182
+ """A type for a principle on foundry."""
183
+
169
184
 
170
185
  class Attribution(TypedDict):
171
186
  """User and timestamp for resource creation."""
@@ -1 +0,0 @@
1
- __version__ = '2.1.14'