foundry-dev-tools 2.1.13__tar.gz → 2.1.15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/PKG-INFO +1 -1
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/pyproject.toml +1 -1
- foundry_dev_tools-2.1.15/src/foundry_dev_tools/__about__.py +1 -0
- foundry_dev_tools-2.1.15/src/foundry_dev_tools/clients/magritte_coordinator.py +467 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/multipass.py +26 -0
- foundry_dev_tools-2.1.15/src/foundry_dev_tools/clients/scheduler.py +54 -0
- foundry_dev_tools-2.1.15/src/foundry_dev_tools/clients/tables.py +145 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/context.py +18 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/dataset.py +8 -2
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/api_types.py +15 -0
- foundry_dev_tools-2.1.13/src/foundry_dev_tools/__about__.py +0 -1
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/README.md +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pandas.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/polars.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pyarrow.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pyspark.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/build.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/config.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/git.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/git_credential_foundry.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/info.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/main.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/s3.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cli/stubs.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/api_client.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/build2.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/catalog.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/compass.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/context_client.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/data_proxy.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/foundry_sql_server.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/foundry_stats.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/jemma.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/metadata.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/public_ontologies_client.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/s3_client.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/schema_inference.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/config.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/config_types.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/token_provider.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/compass.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/config.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/dataset.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/handling.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/meta.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/multipass.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/sql.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/foundry_api_client.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/group.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/principal.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/multipass/user.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/py.typed +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/folder.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/resource.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/async_s3.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/cli.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/clients.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/compat.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/config.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/misc.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/public_api_types.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/repo.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/s3.py +0 -0
- {foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/spark.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: foundry-dev-tools
|
|
3
|
-
Version: 2.1.
|
|
3
|
+
Version: 2.1.15
|
|
4
4
|
Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
|
|
5
5
|
Author-Email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.1.15'
|
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
"""Implementation of the foundry-stats API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from foundry_dev_tools.clients.api_client import APIClient
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from foundry_dev_tools.utils.api_types import (
|
|
13
|
+
CodeResourceType,
|
|
14
|
+
FolderRid,
|
|
15
|
+
MarkingId,
|
|
16
|
+
NetworkEgressPolicyRid,
|
|
17
|
+
Rid,
|
|
18
|
+
SourceRid,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class MagritteCoordinatorClient(APIClient):
|
|
23
|
+
"""The mothership that controls remote Magritte agents and cloud runtimes."""
|
|
24
|
+
|
|
25
|
+
api_name = "magritte-coordinator"
|
|
26
|
+
|
|
27
|
+
def api_get_oidc_metadata(self, **kwargs) -> requests.Response:
|
|
28
|
+
"""Returns the OIDC Metadata of the Foundry Stack.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
**kwargs: gets passed to :py:meth:`APIClient.api_request`
|
|
32
|
+
Returns:
|
|
33
|
+
dict: {'issuer': 'https://pltroidcpublic<id>.s3.<region>.amazonaws.com/foundry'}
|
|
34
|
+
"""
|
|
35
|
+
return self.api_request(
|
|
36
|
+
"POST",
|
|
37
|
+
"oidc/metadata",
|
|
38
|
+
**kwargs,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
def get_oidc_issuer(self) -> str:
|
|
42
|
+
"""Returns the OIDC issuer of the Foundry Stack."""
|
|
43
|
+
return self.api_get_oidc_metadata().json()["issuer"]
|
|
44
|
+
|
|
45
|
+
def api_add_source_v3(
|
|
46
|
+
self, config: dict, description: dict, runtime_platform_request: dict, parent_rid: FolderRid, **kwargs
|
|
47
|
+
) -> requests.Response:
|
|
48
|
+
"""Low level method to add a new source to Data Connection.
|
|
49
|
+
|
|
50
|
+
Use a high level method like :py:meth:`MagritteCoordinatorClient.create_s3_direct_source`
|
|
51
|
+
for easier usage.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
config: Source Config, individual format for each supported Source.
|
|
55
|
+
description: Textual description field.
|
|
56
|
+
runtime_platform_request: For Direct Connect { "cloud": {"networkEgresses": []}, "type": "cloud" }
|
|
57
|
+
parent_rid: Rid of the parent Compass folder.
|
|
58
|
+
**kwargs: gets passed to :py:meth:`APIClient.api_request`
|
|
59
|
+
"""
|
|
60
|
+
return self.api_request(
|
|
61
|
+
"POST",
|
|
62
|
+
"source-store/source/v3",
|
|
63
|
+
json={
|
|
64
|
+
"config": config,
|
|
65
|
+
"description": description,
|
|
66
|
+
"runtimePlatformRequest": runtime_platform_request,
|
|
67
|
+
"parentRid": parent_rid,
|
|
68
|
+
},
|
|
69
|
+
**kwargs,
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def api_update_source_v2(
|
|
73
|
+
self,
|
|
74
|
+
source_rid: Rid,
|
|
75
|
+
updated_source_config: dict | None = None,
|
|
76
|
+
patch_cloud_runtime_platform: dict | None = None,
|
|
77
|
+
**kwargs,
|
|
78
|
+
) -> requests.Response:
|
|
79
|
+
"""Updates a Source.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
source_rid: The Rid of the Magritte Source
|
|
83
|
+
updated_source_config: example {"type":"s3-direct","url":"s3://bucket1"}
|
|
84
|
+
patch_cloud_runtime_platform: example
|
|
85
|
+
{
|
|
86
|
+
"secretsToUpdate": {},
|
|
87
|
+
"removedSecrets": [],
|
|
88
|
+
"patchCloudIdentity": {},
|
|
89
|
+
"patchOidcRuntime": {},
|
|
90
|
+
}
|
|
91
|
+
**kwargs: gets passed to :py:meth:`APIClient.api_request`
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
requests.Response: dict with top level key updatedSource
|
|
95
|
+
{
|
|
96
|
+
"updatedSource": {
|
|
97
|
+
"sourceConfig": {
|
|
98
|
+
"type": "s3-direct",
|
|
99
|
+
"url": "s3://bucket"
|
|
100
|
+
},
|
|
101
|
+
"connections": {},
|
|
102
|
+
"additionalSecrets": {},
|
|
103
|
+
"mavenCoordinates": {
|
|
104
|
+
"userProvided": [],
|
|
105
|
+
"foundryProvided": []
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
"""
|
|
110
|
+
payload = {}
|
|
111
|
+
if updated_source_config:
|
|
112
|
+
payload["patchSourceConfigRequest"] = {}
|
|
113
|
+
payload["patchSourceConfigRequest"]["updatedSourceConfig"] = updated_source_config
|
|
114
|
+
if patch_cloud_runtime_platform:
|
|
115
|
+
payload["patchRuntimePlatformRequest"] = {}
|
|
116
|
+
payload["patchRuntimePlatformRequest"]["patchCloudRuntimePlatform"] = patch_cloud_runtime_platform
|
|
117
|
+
payload["patchRuntimePlatformRequest"]["type"] = "patchCloudRuntimePlatform"
|
|
118
|
+
return self.api_request(
|
|
119
|
+
"PUT",
|
|
120
|
+
f"source-store/v2/sources/{source_rid}",
|
|
121
|
+
json=payload,
|
|
122
|
+
**kwargs,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
def create_s3_direct_source(
|
|
126
|
+
self,
|
|
127
|
+
name: str,
|
|
128
|
+
parent_rid: FolderRid,
|
|
129
|
+
url: str,
|
|
130
|
+
description: str | None = "",
|
|
131
|
+
region: str | None = None,
|
|
132
|
+
catalog: dict | None = None,
|
|
133
|
+
sts_role_configuration: dict | None = None,
|
|
134
|
+
network_egress_policies: set[Rid] | None = None,
|
|
135
|
+
**source_kwargs,
|
|
136
|
+
) -> str:
|
|
137
|
+
"""Create a new Source of type s3-direct. Only supports Direct Connect sources.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
name: Name of the Source in Compass
|
|
141
|
+
parent_rid: Rid of the parent Compass folder.
|
|
142
|
+
url: S3 Url, e.g. s3://bucket-name
|
|
143
|
+
description: Textual description field.
|
|
144
|
+
region: AWS Region of the bucket.
|
|
145
|
+
catalog: Catalog Integration for Iceberg Tables.
|
|
146
|
+
sts_role_configuration: sts role configuration for Assuming a role to connect to the bucket.
|
|
147
|
+
network_egress_policies: Network Egress Policies required to connect to the bucket.
|
|
148
|
+
source_kwargs: Any other arguments get added to the "source" key of the payload.
|
|
149
|
+
"""
|
|
150
|
+
payload = {
|
|
151
|
+
"source": {
|
|
152
|
+
"type": "s3-direct",
|
|
153
|
+
"url": url,
|
|
154
|
+
**({"region": region} if region else {}),
|
|
155
|
+
**({"catalog": catalog} if catalog else {}),
|
|
156
|
+
**({"stsRoleConfiguration": sts_role_configuration} if sts_role_configuration else {}),
|
|
157
|
+
**source_kwargs,
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
network_egress_policies = network_egress_policies or set()
|
|
161
|
+
|
|
162
|
+
response = self.api_add_source_v3(
|
|
163
|
+
config=payload,
|
|
164
|
+
description={"name": name, "description": description},
|
|
165
|
+
runtime_platform_request={
|
|
166
|
+
"cloud": {"networkEgresses": list(network_egress_policies)},
|
|
167
|
+
"type": "cloud",
|
|
168
|
+
},
|
|
169
|
+
parent_rid=parent_rid,
|
|
170
|
+
)
|
|
171
|
+
return response.json()
|
|
172
|
+
|
|
173
|
+
def get_source_config(self, source_rid: SourceRid) -> dict:
|
|
174
|
+
"""Returns the configuration of a Source."""
|
|
175
|
+
return self.api_bulk_get_source_config(source_rids={source_rid}).json()[source_rid]
|
|
176
|
+
|
|
177
|
+
def api_bulk_get_source_config(self, source_rids: set[SourceRid], **kwargs) -> requests.Response:
|
|
178
|
+
"""Returns the configuration of a multiple sources."""
|
|
179
|
+
return self.api_request(
|
|
180
|
+
"POST",
|
|
181
|
+
"source-store/sources/config/bulk",
|
|
182
|
+
json=list(source_rids),
|
|
183
|
+
**kwargs,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
def get_source_description(self, source_rid: SourceRid) -> dict:
|
|
187
|
+
"""Returns the description of a Source.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
dictionary with example payload:
|
|
191
|
+
{'apiName': None, 'description': 'the desc.', 'name': 'source-name', 'type': 's3-direct'}
|
|
192
|
+
"""
|
|
193
|
+
return self.api_bulk_get_source_description(source_rids={source_rid}).json()[source_rid]
|
|
194
|
+
|
|
195
|
+
def api_bulk_get_source_description(self, source_rids: set[SourceRid], **kwargs) -> requests.Response:
|
|
196
|
+
"""Returns the description of multiple Sources."""
|
|
197
|
+
return self.api_request(
|
|
198
|
+
"POST",
|
|
199
|
+
"source-store/sources/description/bulk",
|
|
200
|
+
json=list(source_rids),
|
|
201
|
+
**kwargs,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
def api_get_runtime_platform(self, source_rid: SourceRid, **kwargs) -> requests.Response:
|
|
205
|
+
"""Get runtime platform information for a source.
|
|
206
|
+
|
|
207
|
+
Returns:
|
|
208
|
+
response: example format {
|
|
209
|
+
"type": "cloud",
|
|
210
|
+
"cloud": {
|
|
211
|
+
"credential": {
|
|
212
|
+
"credentialId": "ri.credential..credential.7ddaaa49-1337-1337-a0c0-8b621fa30364",
|
|
213
|
+
"secretNames": []
|
|
214
|
+
},
|
|
215
|
+
"networkEgressPolicies": {
|
|
216
|
+
"networkEgressPolicies": []
|
|
217
|
+
},
|
|
218
|
+
"cloudIdentity": null,
|
|
219
|
+
"oidcRuntime": null
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
"""
|
|
224
|
+
return self.api_request(
|
|
225
|
+
"GET",
|
|
226
|
+
f"source-store/v2/source/{source_rid}/runtimePlatform",
|
|
227
|
+
**kwargs,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def set_network_egress_policies(self, source_rid: SourceRid, network_egress_policies: set[NetworkEgressPolicyRid]):
|
|
231
|
+
"""Sets the network egress policies for a source, overwrites all pre-existing egress policies."""
|
|
232
|
+
_ = self.api_update_source_v2(
|
|
233
|
+
source_rid=source_rid,
|
|
234
|
+
patch_cloud_runtime_platform={"updatedNetworkEgresses": list(network_egress_policies)},
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
def get_network_egress_policies(self, source_rid: SourceRid) -> list[NetworkEgressPolicyRid]:
|
|
238
|
+
"""Returns list of network egress policies for a source."""
|
|
239
|
+
runtime_platform = self.api_get_runtime_platform(source_rid=source_rid)
|
|
240
|
+
return runtime_platform.json()["cloud"]["networkEgressPolicies"]["networkEgressPolicies"]
|
|
241
|
+
|
|
242
|
+
def add_network_egress_policy(self, source_rid: SourceRid, network_egress_policy_rid: NetworkEgressPolicyRid):
|
|
243
|
+
"""Adds a single egress policy to a source. Additive operation."""
|
|
244
|
+
existing_egress_policies = self.get_network_egress_policies(source_rid=source_rid)
|
|
245
|
+
existing_egress_policies.append(network_egress_policy_rid)
|
|
246
|
+
self.set_network_egress_policies(source_rid=source_rid, network_egress_policies=set(existing_egress_policies))
|
|
247
|
+
|
|
248
|
+
def delete_network_egress_policy(self, source_rid: SourceRid, network_egress_policy_rid: NetworkEgressPolicyRid):
|
|
249
|
+
"""Deletes a single egress policy from a source."""
|
|
250
|
+
existing_egress_policies = self.get_network_egress_policies(source_rid=source_rid)
|
|
251
|
+
existing_egress_policies.remove(network_egress_policy_rid)
|
|
252
|
+
self.set_network_egress_policies(source_rid=source_rid, network_egress_policies=set(existing_egress_policies))
|
|
253
|
+
|
|
254
|
+
def enable_s3_oidc_runtime(self, source_rid: SourceRid):
|
|
255
|
+
"""Enables OpenID Connect (OIDC) in (S3) Source."""
|
|
256
|
+
_ = self.api_update_source_v2(
|
|
257
|
+
source_rid=source_rid,
|
|
258
|
+
patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": {"audience": "sts.amazonaws.com"}}},
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
def disable_s3_oidc_runtime(self, source_rid: SourceRid):
|
|
262
|
+
"""Disables OpenID Connect (OIDC) in (S3) Source."""
|
|
263
|
+
_ = self.api_update_source_v2(
|
|
264
|
+
source_rid=source_rid,
|
|
265
|
+
patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": None}},
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
def api_modify_source_usage_restrictions(self, source_rid: SourceRid, payload: dict, **kwargs) -> requests.Response:
|
|
269
|
+
"""Update usage restrictions for a source.
|
|
270
|
+
|
|
271
|
+
Args:
|
|
272
|
+
source_rid: the source to modify usage restrictions for
|
|
273
|
+
payload: the payload to modify the usage restrictions for, example:
|
|
274
|
+
{"toRestrict": [], "toEnable": [{"type": "stemmaRepository", "stemmaRepository": {}}]}
|
|
275
|
+
kwargs: any additional keyword arguments are passed to the requests library.
|
|
276
|
+
|
|
277
|
+
"""
|
|
278
|
+
return self.api_request(
|
|
279
|
+
"PUT",
|
|
280
|
+
f"code-resource-source-import/sources/{source_rid}/usage-restrictions",
|
|
281
|
+
json=payload,
|
|
282
|
+
**kwargs,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
def enable_code_imports(self, source_rid: SourceRid, to_enable: list[CodeResourceType]):
|
|
286
|
+
"""Enable code import on Magritte Source.
|
|
287
|
+
|
|
288
|
+
Args:
|
|
289
|
+
source_rid: Magritte Source Rid.
|
|
290
|
+
to_enable: Pick from "stemmaRepository", "computeModule", "eddiePipeline"
|
|
291
|
+
"""
|
|
292
|
+
base_payload = {"toRestrict": [], "toEnable": []}
|
|
293
|
+
for entry in to_enable:
|
|
294
|
+
base_payload["toEnable"].append({"type": entry, entry: {}})
|
|
295
|
+
self.api_modify_source_usage_restrictions(source_rid=source_rid, payload=base_payload)
|
|
296
|
+
|
|
297
|
+
def restrict_code_imports(self, source_rid: SourceRid, to_restrict: list[CodeResourceType]):
|
|
298
|
+
"""Restrict code import on Magritte Source.
|
|
299
|
+
|
|
300
|
+
Args:
|
|
301
|
+
source_rid: Magritte Source Rid.
|
|
302
|
+
to_restrict: Pick from "stemmaRepository", "computeModule", "eddiePipeline"
|
|
303
|
+
"""
|
|
304
|
+
base_payload = {"toRestrict": [], "toEnable": []}
|
|
305
|
+
for entry in to_restrict:
|
|
306
|
+
base_payload["toRestrict"].append({"type": entry, entry: {}})
|
|
307
|
+
self.api_modify_source_usage_restrictions(source_rid=source_rid, payload=base_payload)
|
|
308
|
+
|
|
309
|
+
def api_bulk_get_usage_restrictions_for_source(self, source_rids: list[SourceRid], **kwargs) -> requests.Response:
|
|
310
|
+
"""Get information about which types of code resources allow the import of the requested sources."""
|
|
311
|
+
return self.api_request(
|
|
312
|
+
"PUT",
|
|
313
|
+
"code-resource-source-import/sources/usage-restrictions/bulk",
|
|
314
|
+
json={"sourceRids": source_rids},
|
|
315
|
+
**kwargs,
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# Enable Export
|
|
319
|
+
def api_get_export_state_for_source(self, source_rid: SourceRid, **kwargs) -> requests.Response:
|
|
320
|
+
"""Returns the current export config of a source.
|
|
321
|
+
|
|
322
|
+
Returns:
|
|
323
|
+
example format {
|
|
324
|
+
"isEnabled": True,
|
|
325
|
+
"exportableMarkings": ["marking_id"],
|
|
326
|
+
"isEnabledWithoutMarkingsValidation": False
|
|
327
|
+
}
|
|
328
|
+
"""
|
|
329
|
+
return self.api_request(
|
|
330
|
+
"GET",
|
|
331
|
+
f"export-control/state/{source_rid}",
|
|
332
|
+
**kwargs,
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
def api_update_export_state_for_source(
|
|
336
|
+
self,
|
|
337
|
+
source_rid: SourceRid,
|
|
338
|
+
is_enabled: bool,
|
|
339
|
+
is_enabled_without_markings_validation: bool | None = None,
|
|
340
|
+
**kwargs,
|
|
341
|
+
) -> requests.Response:
|
|
342
|
+
"""Updates Export config for a source.
|
|
343
|
+
|
|
344
|
+
Args:
|
|
345
|
+
source_rid: Magritte Source Rid.
|
|
346
|
+
is_enabled: Whether the export is allowed or not.
|
|
347
|
+
is_enabled_without_markings_validation: Whether export without marking validation is allowed or not.
|
|
348
|
+
kwargs: any additional keyword arguments are passed to the requests library.
|
|
349
|
+
"""
|
|
350
|
+
payload = {"isEnabled": is_enabled}
|
|
351
|
+
if is_enabled_without_markings_validation is not None:
|
|
352
|
+
payload["isEnabledWithoutMarkingsValidation"] = is_enabled_without_markings_validation
|
|
353
|
+
return self.api_request(
|
|
354
|
+
"POST",
|
|
355
|
+
f"export-control/state/{source_rid}",
|
|
356
|
+
json=payload,
|
|
357
|
+
**kwargs,
|
|
358
|
+
)
|
|
359
|
+
|
|
360
|
+
def api_add_exportable_markings_for_source(
|
|
361
|
+
self, source_rid: SourceRid, exportable_markings: list[MarkingId], **kwargs
|
|
362
|
+
) -> requests.Response:
|
|
363
|
+
"""Adds exportable markings for a source."""
|
|
364
|
+
return self.api_request(
|
|
365
|
+
"POST",
|
|
366
|
+
f"export-control/markings/{source_rid}",
|
|
367
|
+
json={"exportableMarkings": exportable_markings},
|
|
368
|
+
**kwargs,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
def api_remove_exportable_markings_for_source(
|
|
372
|
+
self, source_rid: SourceRid, markings_to_remove: list[MarkingId], **kwargs
|
|
373
|
+
) -> requests.Response:
|
|
374
|
+
"""Remove exportable markings for a source."""
|
|
375
|
+
return self.api_request(
|
|
376
|
+
"DELETE",
|
|
377
|
+
f"export-control/markings/{source_rid}",
|
|
378
|
+
json={"markingsToRemove": markings_to_remove},
|
|
379
|
+
**kwargs,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
def create_snowflake_source(
|
|
383
|
+
self,
|
|
384
|
+
name: str,
|
|
385
|
+
parent_rid: FolderRid,
|
|
386
|
+
account_identifier: str,
|
|
387
|
+
description: str | None = "",
|
|
388
|
+
role: str | None = None,
|
|
389
|
+
db: str | None = None,
|
|
390
|
+
schema: str | None = None,
|
|
391
|
+
warehouse: str | None = None,
|
|
392
|
+
network_egress_policies: set[Rid] | None = None,
|
|
393
|
+
) -> str:
|
|
394
|
+
"""Create a new Source of type snowflake. Only supports Direct Connect sources.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
name: Name of the Source in Compass
|
|
398
|
+
parent_rid: Rid of the parent Compass folder.
|
|
399
|
+
account_identifier: https://docs.snowflake.com/en/user-guide/admin-account-identifier.html
|
|
400
|
+
description: Textual description field.
|
|
401
|
+
role: The default access control role to use in the Snowflake session initiated by the driver
|
|
402
|
+
db: The default database to use once connected
|
|
403
|
+
schema: The default schema to use once connected
|
|
404
|
+
warehouse: The virtual warehouse to use once connected
|
|
405
|
+
network_egress_policies: Network Egress Policies required to connect to the Snowflake Account.
|
|
406
|
+
Front-door plus internal stage bucket is usually required.
|
|
407
|
+
|
|
408
|
+
"""
|
|
409
|
+
payload = {
|
|
410
|
+
"source": {
|
|
411
|
+
"type": "snowflake",
|
|
412
|
+
"config": {
|
|
413
|
+
"connection": {
|
|
414
|
+
"accountIdentifier": account_identifier,
|
|
415
|
+
**({"role": role} if role else {}),
|
|
416
|
+
**({"db": db} if db else {}),
|
|
417
|
+
**({"schema": schema} if schema else {}),
|
|
418
|
+
**({"warehouse": warehouse} if warehouse else {}),
|
|
419
|
+
},
|
|
420
|
+
"auth": {
|
|
421
|
+
"basic": {"username": "", "password": "{{SNOWFLAKE_BASIC_AUTH_PASSWORD}}"},
|
|
422
|
+
"type": "basic",
|
|
423
|
+
},
|
|
424
|
+
},
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
network_egress_policies = network_egress_policies or set()
|
|
428
|
+
|
|
429
|
+
response = self.api_add_source_v3(
|
|
430
|
+
config=payload,
|
|
431
|
+
description={"name": name, "description": description},
|
|
432
|
+
runtime_platform_request={
|
|
433
|
+
"cloud": {"networkEgresses": list(network_egress_policies)},
|
|
434
|
+
"type": "cloud",
|
|
435
|
+
},
|
|
436
|
+
parent_rid=parent_rid,
|
|
437
|
+
)
|
|
438
|
+
return response.json()
|
|
439
|
+
|
|
440
|
+
def enable_snowflake_external_oauth(self, source_rid: SourceRid):
|
|
441
|
+
"""Enables external OAuth (OIDC) in (Snowflake) Source."""
|
|
442
|
+
config = self.get_source_config(source_rid=source_rid)["source"]
|
|
443
|
+
|
|
444
|
+
account_identifier = config["config"]["connection"]["accountIdentifier"]
|
|
445
|
+
config["config"]["auth"] = {"externalOauth": {}, "type": "externalOauth"}
|
|
446
|
+
_ = self.api_update_source_v2(
|
|
447
|
+
source_rid=source_rid,
|
|
448
|
+
updated_source_config=config,
|
|
449
|
+
patch_cloud_runtime_platform={
|
|
450
|
+
"patchOidcRuntime": {
|
|
451
|
+
"oidcRuntime": {"audience": f"https://{account_identifier}.snowflakecomputing.com"}
|
|
452
|
+
}
|
|
453
|
+
},
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
def disable_snowflake_external_oauth(self, source_rid: SourceRid):
|
|
457
|
+
"""Disables external OAuth (OIDC) in (Snowflake) Source."""
|
|
458
|
+
config = self.get_source_config(source_rid=source_rid)["source"]
|
|
459
|
+
config["config"]["auth"] = {
|
|
460
|
+
"basic": {"username": "", "password": "{{SNOWFLAKE_BASIC_AUTH_PASSWORD}}"},
|
|
461
|
+
"type": "basic",
|
|
462
|
+
}
|
|
463
|
+
_ = self.api_update_source_v2(
|
|
464
|
+
source_rid=source_rid,
|
|
465
|
+
updated_source_config=config,
|
|
466
|
+
patch_cloud_runtime_platform={"patchOidcRuntime": {"oidcRuntime": None}},
|
|
467
|
+
)
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/multipass.py
RENAMED
|
@@ -95,6 +95,32 @@ class MultipassClient(APIClient):
|
|
|
95
95
|
**kwargs,
|
|
96
96
|
)
|
|
97
97
|
|
|
98
|
+
def search(
|
|
99
|
+
self, query: str, principal_types: set[api_types.PrincipalTypes] | None = None, **kwargs
|
|
100
|
+
) -> Iterator[dict]:
|
|
101
|
+
"""Searches for multipass principals based on a text.
|
|
102
|
+
|
|
103
|
+
Args:
|
|
104
|
+
query: the text string to search for
|
|
105
|
+
principal_types: set of principal types to search in, e.g. "GROUP"
|
|
106
|
+
Default is GROUP and USER
|
|
107
|
+
**kwargs: gets passed to :py:meth:`APIClient.api_request`
|
|
108
|
+
"""
|
|
109
|
+
if principal_types is None:
|
|
110
|
+
principal_types = {"USER", "GROUP"}
|
|
111
|
+
|
|
112
|
+
json = {"attributeFilters": {}, "principalTypes": list(principal_types), "query": query}
|
|
113
|
+
|
|
114
|
+
next_start = None
|
|
115
|
+
while True:
|
|
116
|
+
if next_start:
|
|
117
|
+
json["pageStart"] = next_start
|
|
118
|
+
|
|
119
|
+
response_as_json = self.api_request("POST", "search/v2/search", json=json, **kwargs).json()
|
|
120
|
+
yield from response_as_json["values"]
|
|
121
|
+
if (next_start := response_as_json["nextPageToken"]) is None:
|
|
122
|
+
break
|
|
123
|
+
|
|
98
124
|
def api_get_groups_of_user(self, **kwargs) -> requests.Response:
|
|
99
125
|
"""Returns all groups for which the user is immediate or indirect member of the group.
|
|
100
126
|
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Implementation of the scheduler API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from foundry_dev_tools.clients.api_client import APIClient
|
|
8
|
+
from foundry_dev_tools.errors.compass import ResourceNotFoundError
|
|
9
|
+
from foundry_dev_tools.errors.handling import ErrorHandlingConfig
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
import requests
|
|
13
|
+
|
|
14
|
+
from foundry_dev_tools.utils import api_types
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SchedulerClient(APIClient):
|
|
18
|
+
"""SchedulerClient class that implements methods from the 'scheduler' API."""
|
|
19
|
+
|
|
20
|
+
api_name = "scheduler"
|
|
21
|
+
|
|
22
|
+
def get_schedules_by_run_build(
|
|
23
|
+
self, rid: api_types.Rid, branch: api_types.DatasetBranch = "master", **kwargs
|
|
24
|
+
) -> requests.Response:
|
|
25
|
+
"""Gets the schedules of the resource dataset.
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
rid: the identifier of the datset
|
|
29
|
+
branch: the branch name of the dataset
|
|
30
|
+
**kwargs: gets passed to :py:meth:`APIClient.api_request`
|
|
31
|
+
"""
|
|
32
|
+
params = {"datasetRids": [rid], "branch": branch}
|
|
33
|
+
|
|
34
|
+
return self.api_request(
|
|
35
|
+
"POST",
|
|
36
|
+
"scheduler/get-schedules-by-run-build-dataset",
|
|
37
|
+
json=params,
|
|
38
|
+
error_handling=ErrorHandlingConfig(api_error_mapping={204: ResourceNotFoundError}),
|
|
39
|
+
**kwargs,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
def get_schedule(self, rid: api_types.Rid, **kwargs) -> requests.Response:
|
|
43
|
+
"""Gets the schedule details of a given schedule.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
rid: the identifier of the scheduler
|
|
47
|
+
**kwargs: gets passed to :py:meth:`APIClient.api_request`
|
|
48
|
+
"""
|
|
49
|
+
return self.api_request(
|
|
50
|
+
"GET",
|
|
51
|
+
f"scheduler/schedules/{rid}",
|
|
52
|
+
error_handling=ErrorHandlingConfig(api_error_mapping={204: ResourceNotFoundError}),
|
|
53
|
+
**kwargs,
|
|
54
|
+
)
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""Implementation of the tables API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
|
+
|
|
7
|
+
from foundry_dev_tools.clients.api_client import APIClient
|
|
8
|
+
|
|
9
|
+
if TYPE_CHECKING:
|
|
10
|
+
import requests
|
|
11
|
+
|
|
12
|
+
from foundry_dev_tools.utils.api_types import FolderRid, SourceRid
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TablesClient(APIClient):
|
|
16
|
+
"""TablesClient class that implements the 'tables' api."""
|
|
17
|
+
|
|
18
|
+
api_name = "tables"
|
|
19
|
+
|
|
20
|
+
def api_create_table(
|
|
21
|
+
self, name: str, parent_rid: FolderRid, upstream_config: dict, skip_validation: bool = False, **kwargs
|
|
22
|
+
) -> requests.Response:
|
|
23
|
+
"""Low level method to create a virtual table.
|
|
24
|
+
|
|
25
|
+
Use a high level method like :py:meth:`TablesClient.create_glue_table`
|
|
26
|
+
for easier usage.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
name: The name of the table. This will be the name of the compass resource.
|
|
30
|
+
parent_rid: The parent Rid of the compass resource.
|
|
31
|
+
upstream_config: Individual table config.
|
|
32
|
+
skip_validation: If true, skips the validation of the table. Default is False.
|
|
33
|
+
**kwargs: gets passed to :py:meth:`APIClient.api_request`
|
|
34
|
+
|
|
35
|
+
"""
|
|
36
|
+
return self.api_request(
|
|
37
|
+
"POST",
|
|
38
|
+
api_path="tables",
|
|
39
|
+
json={"name": name, "parentRid": parent_rid, "upstream": upstream_config},
|
|
40
|
+
params={"skipValidation": skip_validation},
|
|
41
|
+
**kwargs,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def create_glue_table(
|
|
45
|
+
self,
|
|
46
|
+
name: str,
|
|
47
|
+
parent_rid: FolderRid,
|
|
48
|
+
connection_rid: SourceRid,
|
|
49
|
+
table: str,
|
|
50
|
+
database: str,
|
|
51
|
+
skip_validation: bool = False,
|
|
52
|
+
) -> dict:
|
|
53
|
+
"""Creates a Virtual Table backed by the Glue Catalog.
|
|
54
|
+
|
|
55
|
+
Args:
|
|
56
|
+
name: The name of the table. This will be the name of the compass resource.
|
|
57
|
+
parent_rid: The parent Rid of the compass resource.
|
|
58
|
+
connection_rid: The rid of the compass Source.
|
|
59
|
+
table: The name of the table in glue.
|
|
60
|
+
database: The name of the database in glue.
|
|
61
|
+
skip_validation: If true, skips the validation of the table. Default is False.
|
|
62
|
+
|
|
63
|
+
"""
|
|
64
|
+
upstream_config = {
|
|
65
|
+
"type": "glue",
|
|
66
|
+
"glue": {"connectionRid": connection_rid, "relation": {"table": table, "database": database}},
|
|
67
|
+
}
|
|
68
|
+
return self.api_create_table(
|
|
69
|
+
name=name, parent_rid=parent_rid, upstream_config=upstream_config, skip_validation=skip_validation
|
|
70
|
+
).json()
|
|
71
|
+
|
|
72
|
+
def create_snowflake_table(
|
|
73
|
+
self,
|
|
74
|
+
name: str,
|
|
75
|
+
parent_rid: FolderRid,
|
|
76
|
+
connection_rid: SourceRid,
|
|
77
|
+
database: str,
|
|
78
|
+
schema: str,
|
|
79
|
+
table: str,
|
|
80
|
+
skip_validation: bool = False,
|
|
81
|
+
) -> dict:
|
|
82
|
+
"""Creates a Virtual Table on a Snowflake Source.
|
|
83
|
+
|
|
84
|
+
Args:
|
|
85
|
+
name: The name of the table. This will be the name of the compass resource.
|
|
86
|
+
parent_rid: The parent Rid of the compass resource.
|
|
87
|
+
connection_rid: The rid of the compass Source.
|
|
88
|
+
table: The name of the table in snowflake.
|
|
89
|
+
database: The name of the database Snowflake.
|
|
90
|
+
schema: The name of the schema in Snowflake.
|
|
91
|
+
skip_validation: If true, skips the validation of the table. Default is False.
|
|
92
|
+
|
|
93
|
+
"""
|
|
94
|
+
upstream_config = {
|
|
95
|
+
"type": "snowflake",
|
|
96
|
+
"snowflake": {
|
|
97
|
+
"connectionRid": connection_rid,
|
|
98
|
+
"relation": {"table": table, "database": database, "schema": schema},
|
|
99
|
+
},
|
|
100
|
+
}
|
|
101
|
+
return self.api_create_table(
|
|
102
|
+
name=name, parent_rid=parent_rid, upstream_config=upstream_config, skip_validation=skip_validation
|
|
103
|
+
).json()
|
|
104
|
+
|
|
105
|
+
def api_list_tables(
|
|
106
|
+
self, connection_rid: SourceRid, limit: int | None = 1000, page_token: str | None = None, **kwargs
|
|
107
|
+
) -> requests.Response:
|
|
108
|
+
"""List Virtual Tables for a source."""
|
|
109
|
+
return self.api_request(
|
|
110
|
+
"GET",
|
|
111
|
+
api_path="tables",
|
|
112
|
+
params={
|
|
113
|
+
"connectionRid": connection_rid,
|
|
114
|
+
**({"limit": limit} if limit else {}),
|
|
115
|
+
**({"pageToken": page_token} if page_token else {}),
|
|
116
|
+
},
|
|
117
|
+
**kwargs,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
def list_tables(self, connection_rid: SourceRid) -> list[dict]:
|
|
121
|
+
"""Returns all tables for a connection/source by handling pagination.
|
|
122
|
+
|
|
123
|
+
Args:
|
|
124
|
+
connection_rid: The rid of the compass Source/Connection.
|
|
125
|
+
|
|
126
|
+
Returns:
|
|
127
|
+
a list of dicts with the following format:
|
|
128
|
+
{
|
|
129
|
+
'rid': 'ri.tables.main.table.<...>',
|
|
130
|
+
'upstream': {
|
|
131
|
+
'type': '<type>',
|
|
132
|
+
'<type>': {
|
|
133
|
+
'connectionRid': 'ri.magritte..source.<...>',
|
|
134
|
+
'relation': {'database': 'test', 'schema': 'test', 'table': 'test2'}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
"""
|
|
139
|
+
tables_response = self.api_list_tables(connection_rid=connection_rid, limit=1000)
|
|
140
|
+
response_json = tables_response.json()
|
|
141
|
+
tables = response_json["tables"]
|
|
142
|
+
while next_page_token := response_json.get("nextPageToken"):
|
|
143
|
+
response_json = self.api_list_tables(connection_rid=connection_rid, page_token=next_page_token).json()
|
|
144
|
+
tables.extend(response_json["tables"])
|
|
145
|
+
return tables
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/context.py
RENAMED
|
@@ -14,11 +14,14 @@ from foundry_dev_tools.clients import (
|
|
|
14
14
|
foundry_sql_server,
|
|
15
15
|
foundry_stats,
|
|
16
16
|
jemma,
|
|
17
|
+
magritte_coordinator,
|
|
17
18
|
metadata,
|
|
18
19
|
multipass,
|
|
19
20
|
public_ontologies_client,
|
|
20
21
|
s3_client,
|
|
22
|
+
scheduler,
|
|
21
23
|
schema_inference,
|
|
24
|
+
tables,
|
|
22
25
|
)
|
|
23
26
|
from foundry_dev_tools.config.config import Config, get_config_dict, parse_credentials_config, parse_general_config
|
|
24
27
|
from foundry_dev_tools.helpers.multipass import Group, User
|
|
@@ -85,6 +88,21 @@ class FoundryContext:
|
|
|
85
88
|
"""Returns :py:class:`foundry_dev_tools.clients.jemma.JemmaClient`."""
|
|
86
89
|
return jemma.JemmaClient(self)
|
|
87
90
|
|
|
91
|
+
@cached_property
|
|
92
|
+
def tables(self) -> tables.TablesClient:
|
|
93
|
+
"""Returns :py:class:`foundry_dev_tools.clients.tables.TablesClient`."""
|
|
94
|
+
return tables.TablesClient(self)
|
|
95
|
+
|
|
96
|
+
@cached_property
|
|
97
|
+
def magritte_coordinator(self) -> magritte_coordinator.MagritteCoordinatorClient:
|
|
98
|
+
"""Returns :py:class:`foundry_dev_tools.clients.magritte_coordinator.MagritteCoordinatorClient`."""
|
|
99
|
+
return magritte_coordinator.MagritteCoordinatorClient(self)
|
|
100
|
+
|
|
101
|
+
@cached_property
|
|
102
|
+
def scheduler(self) -> scheduler.SchedulerClient:
|
|
103
|
+
"""Returns :py:class:`foundry_dev_tools.clients.scheduler.SchedulerClient`."""
|
|
104
|
+
return scheduler.SchedulerClient(self)
|
|
105
|
+
|
|
88
106
|
@cached_property
|
|
89
107
|
def metadata(self) -> metadata.MetadataClient:
|
|
90
108
|
"""Returns :py:class:`foundry_dev_tools.clients.metadata.MetadataClient`."""
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/dataset.py
RENAMED
|
@@ -381,7 +381,9 @@ class Dataset(resource.Resource):
|
|
|
381
381
|
"""
|
|
382
382
|
kwargs.setdefault("transaction_type", "UPDATE")
|
|
383
383
|
self.upload_files(
|
|
384
|
-
{str(f.relative_to(folder_path)): f for f in folder_path.rglob("*") if f.is_file()},
|
|
384
|
+
{str(f.relative_to(folder_path)): f for f in folder_path.rglob("*") if f.is_file()},
|
|
385
|
+
max_workers=max_workers,
|
|
386
|
+
**kwargs,
|
|
385
387
|
)
|
|
386
388
|
return self
|
|
387
389
|
|
|
@@ -748,8 +750,12 @@ class Dataset(resource.Resource):
|
|
|
748
750
|
>>> ds = ctx.get_dataset_by_path("/path/to/dataset")
|
|
749
751
|
>>> ds.query_foundry_sql("SELECT * WHERE a=1")
|
|
750
752
|
""" # noqa: E501
|
|
753
|
+
# This is a quick fix. Foundry Sql Server seems to decide that
|
|
754
|
+
# "FROM `rid` SELECT *" is not direct read eligible.
|
|
755
|
+
# tracking: ri.issues.main.issue.da8272ca-9e78-4100-af3f-6ac40aaecf51
|
|
756
|
+
query_string = f"SELECT * FROM `{self.rid}`" if query == "SELECT *" else f"FROM `{self.rid}` {query}" # noqa: S608
|
|
751
757
|
return self._context.foundry_sql_server.query_foundry_sql(
|
|
752
|
-
|
|
758
|
+
query=query_string,
|
|
753
759
|
return_type=return_type,
|
|
754
760
|
branch=self.branch["id"],
|
|
755
761
|
sql_dialect=sql_dialect,
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/api_types.py
RENAMED
|
@@ -68,6 +68,15 @@ OrganizationRid = str
|
|
|
68
68
|
ProjectRid = str
|
|
69
69
|
"""A compass project root folder resource identifier."""
|
|
70
70
|
|
|
71
|
+
SourceRid = str
|
|
72
|
+
"""A magritte source resource identifier."""
|
|
73
|
+
|
|
74
|
+
TableRid = str
|
|
75
|
+
"""A virtual table resource identifier."""
|
|
76
|
+
|
|
77
|
+
NetworkEgressPolicyRid = str
|
|
78
|
+
"""A virtual table resource identifier."""
|
|
79
|
+
|
|
71
80
|
# TODO further typing?
|
|
72
81
|
FoundrySchema = dict[str, Any]
|
|
73
82
|
|
|
@@ -166,6 +175,12 @@ TokenType = Literal["USER_GENERATED"]
|
|
|
166
175
|
UserId = str
|
|
167
176
|
"""An identifier for an user on foundry."""
|
|
168
177
|
|
|
178
|
+
CodeResourceType = Literal["stemmaRepository", "computeModule", "eddiePipeline"]
|
|
179
|
+
"""Code Resource type."""
|
|
180
|
+
|
|
181
|
+
PrincipalTypes = Literal["USER", "GROUP"]
|
|
182
|
+
"""A type for a principle on foundry."""
|
|
183
|
+
|
|
169
184
|
|
|
170
185
|
class Attribution(TypedDict):
|
|
171
186
|
"""User and timestamp for resource creation."""
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = '2.1.13'
|
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/__init__.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pandas.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/polars.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pyarrow.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/_optional/pyspark.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/cached_foundry_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/__init__.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/api_client.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/build2.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/catalog.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/compass.py
RENAMED
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/data_proxy.py
RENAMED
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/foundry_stats.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/jemma.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/metadata.py
RENAMED
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/clients/s3_client.py
RENAMED
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/__init__.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/config.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/config_types.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/config/token_provider.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/__init__.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/compass.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/config.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/dataset.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/handling.py
RENAMED
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/errors/multipass.py
RENAMED
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/foundry_api_client.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/helpers/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/__init__.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/folder.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/resources/resource.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/__init__.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/async_s3.py
RENAMED
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/caches/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{foundry_dev_tools-2.1.13 → foundry_dev_tools-2.1.15}/src/foundry_dev_tools/utils/clients.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|