foundry-dev-tools 2.1.1__tar.gz → 2.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/PKG-INFO +1 -1
  2. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/pyproject.toml +1 -1
  3. foundry_dev_tools-2.1.3/src/foundry_dev_tools/__about__.py +1 -0
  4. foundry_dev_tools-2.1.3/src/foundry_dev_tools/clients/build2.py +239 -0
  5. foundry_dev_tools-2.1.3/src/foundry_dev_tools/clients/multipass.py +1375 -0
  6. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/config/config.py +3 -1
  7. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/config/context.py +22 -0
  8. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/errors/handling.py +2 -0
  9. foundry_dev_tools-2.1.3/src/foundry_dev_tools/errors/multipass.py +9 -0
  10. foundry_dev_tools-2.1.3/src/foundry_dev_tools/helpers/__init__.py +1 -0
  11. foundry_dev_tools-2.1.3/src/foundry_dev_tools/helpers/multipass/__init__.py +7 -0
  12. foundry_dev_tools-2.1.3/src/foundry_dev_tools/helpers/multipass/group.py +264 -0
  13. foundry_dev_tools-2.1.3/src/foundry_dev_tools/helpers/multipass/principal.py +63 -0
  14. foundry_dev_tools-2.1.3/src/foundry_dev_tools/helpers/multipass/user.py +59 -0
  15. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/api_types.py +100 -1
  16. foundry_dev_tools-2.1.1/src/foundry_dev_tools/__about__.py +0 -1
  17. foundry_dev_tools-2.1.1/src/foundry_dev_tools/clients/build2.py +0 -50
  18. foundry_dev_tools-2.1.1/src/foundry_dev_tools/clients/multipass.py +0 -560
  19. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/README.md +0 -0
  20. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/__init__.py +0 -0
  21. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/_optional/__init__.py +0 -0
  22. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/_optional/pandas.py +0 -0
  23. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/_optional/polars.py +0 -0
  24. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/_optional/pyarrow.py +0 -0
  25. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/_optional/pyspark.py +0 -0
  26. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
  27. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/__init__.py +0 -0
  28. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/build.py +0 -0
  29. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/config.py +0 -0
  30. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/git.py +0 -0
  31. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/git_credential_foundry.py +0 -0
  32. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/info.py +0 -0
  33. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/main.py +0 -0
  34. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/s3.py +0 -0
  35. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/cli/stubs.py +0 -0
  36. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/__init__.py +0 -0
  37. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/api_client.py +0 -0
  38. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/catalog.py +0 -0
  39. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/compass.py +0 -0
  40. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/context_client.py +0 -0
  41. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/data_proxy.py +0 -0
  42. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/foundry_sql_server.py +0 -0
  43. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/foundry_stats.py +0 -0
  44. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/jemma.py +0 -0
  45. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/metadata.py +0 -0
  46. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/public_ontologies_client.py +0 -0
  47. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/s3_client.py +0 -0
  48. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/clients/schema_inference.py +0 -0
  49. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/config/__init__.py +0 -0
  50. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/config/config_types.py +0 -0
  51. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/config/token_provider.py +0 -0
  52. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/errors/__init__.py +0 -0
  53. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/errors/compass.py +0 -0
  54. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/errors/config.py +0 -0
  55. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/errors/dataset.py +0 -0
  56. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/errors/meta.py +0 -0
  57. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/errors/sql.py +0 -0
  58. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/foundry_api_client.py +0 -0
  59. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/py.typed +0 -0
  60. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/resources/__init__.py +0 -0
  61. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/resources/dataset.py +0 -0
  62. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/resources/folder.py +0 -0
  63. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/resources/resource.py +0 -0
  64. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/__init__.py +0 -0
  65. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/async_s3.py +0 -0
  66. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  67. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  68. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  69. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/cli.py +0 -0
  70. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/clients.py +0 -0
  71. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/compat.py +0 -0
  72. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/config.py +0 -0
  73. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  74. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  75. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/misc.py +0 -0
  76. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/public_api_types.py +0 -0
  77. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/repo.py +0 -0
  78. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/s3.py +0 -0
  79. {foundry_dev_tools-2.1.1 → foundry_dev_tools-2.1.3}/src/foundry_dev_tools/utils/spark.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 2.1.1
3
+ Version: 2.1.3
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-Email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -41,7 +41,7 @@ dependencies = [
41
41
  "tomli_w",
42
42
  ]
43
43
  dynamic = []
44
- version = "2.1.1"
44
+ version = "2.1.3"
45
45
 
46
46
  [project.license]
47
47
  text = "Apache-2.0"
@@ -0,0 +1 @@
1
+ __version__ = '2.1.3'
@@ -0,0 +1,239 @@
1
+ """Implementation of the build2 API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import TYPE_CHECKING, Any
6
+
7
+ from foundry_dev_tools.clients.api_client import APIClient
8
+ from foundry_dev_tools.errors.compass import ResourceNotFoundError
9
+ from foundry_dev_tools.errors.handling import ErrorHandlingConfig
10
+ from foundry_dev_tools.utils import api_types
11
+ from foundry_dev_tools.utils.api_types import assert_in_literal
12
+
13
+ if TYPE_CHECKING:
14
+ import requests
15
+
16
+ JOB_SPEC_SELECTION_TYPES = {
17
+ api_types.JobSpecJobSpecsSelection: "jobSpecs",
18
+ api_types.UpstreamJobSpecSelection: "upstream",
19
+ api_types.ConnectingJobSpecSelection: "connecting",
20
+ api_types.DatasetsJobSpecSelection: "datasets",
21
+ }
22
+
23
+
24
+ def _convert_job_spec_selections(
25
+ job_spec_selections: list[
26
+ api_types.DatasetsJobSpecSelection
27
+ | api_types.JobSpecJobSpecsSelection
28
+ | api_types.UpstreamJobSpecSelection
29
+ | api_types.ConnectingJobSpecSelection
30
+ ],
31
+ ) -> list:
32
+ """Converts existing job spec selections based on the provided 'job_spec_selection_type'."""
33
+ result = []
34
+
35
+ for job_spec_selection in job_spec_selections:
36
+ for job_spec_selection_class, job_spec_selection_type in JOB_SPEC_SELECTION_TYPES.items():
37
+ if set(job_spec_selection.keys()) == set(job_spec_selection_class.__annotations__.keys()):
38
+ result.append(
39
+ {
40
+ job_spec_selection_type: job_spec_selection,
41
+ "type": job_spec_selection_type,
42
+ }
43
+ )
44
+ continue
45
+
46
+ return result
47
+
48
+
49
+ class Build2Client(APIClient):
50
+ """Build2Client class implements the 'build2' API."""
51
+
52
+ api_name = "build2"
53
+
54
+ def api_get_build_report(self, build_rid: api_types.Rid, **kwargs) -> requests.Response:
55
+ """Returns the build report.
56
+
57
+ Args:
58
+ build_rid: the build RID
59
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
60
+
61
+ """
62
+ return self.api_request(
63
+ "GET",
64
+ "info/builds2/" + build_rid,
65
+ error_handling=ErrorHandlingConfig({204: ResourceNotFoundError}),
66
+ **kwargs,
67
+ )
68
+
69
+ def api_get_job_report(self, job_rid: api_types.Rid, **kwargs) -> requests.Response:
70
+ """Returns the report for a job.
71
+
72
+ Args:
73
+ job_rid: the job RID
74
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
75
+
76
+ """
77
+ return self.api_request(
78
+ "GET",
79
+ "info/jobs3/" + job_rid,
80
+ error_handling=ErrorHandlingConfig({204: ResourceNotFoundError}),
81
+ **kwargs,
82
+ )
83
+
84
+ def api_submit_build(
85
+ self,
86
+ job_spec_selections: list[
87
+ api_types.DatasetsJobSpecSelection
88
+ | api_types.JobSpecJobSpecsSelection
89
+ | api_types.UpstreamJobSpecSelection
90
+ | api_types.ConnectingJobSpecSelection
91
+ ],
92
+ submission_id: str | None = None,
93
+ build_group_rid: str | None = None,
94
+ branch: api_types.DatasetBranch = "master",
95
+ branch_fallbacks: set[api_types.DatasetBranch] | None = None,
96
+ force_build: bool | None = None,
97
+ ignore_build_policy: bool | None = False,
98
+ finish_on_failure: bool | None = None,
99
+ num_run_job_attempts: int | None = None,
100
+ force_retry: bool | None = None,
101
+ retry_backoff_duration: int | None = None,
102
+ build_parameters: dict[str, Any] | None = None,
103
+ suppress_notifications: bool | None = None,
104
+ exceeded_duration_mode: api_types.ExceededDurationMode | None = None,
105
+ input_failure_strategies: list[api_types.InputStrategy] | None = None,
106
+ output_queue_strategy: api_types.OutputQueueStrategy | None = None,
107
+ **kwargs,
108
+ ) -> requests.Response:
109
+ """Request to submit a build.
110
+
111
+ Args:
112
+ job_spec_selections: A set of job spec selection for which the build service will determine the job specs
113
+ to run based off of the information provided in the selection.
114
+ There are several types of job spec selections to choose from:
115
+ (a) :py:class:`~foundry_dev_tools.utils.api_types.DatasetsJobSpecSelection`: The build service
116
+ will determine the job specs to run, based on the dataset rids provided
117
+ (b) :py:class:`~foundry_dev_tools.utils.api_types.JobSpecJobSpecsSelection`: The build service
118
+ will directly run the specified job spec rids
119
+ (c) :py:class:`~foundry_dev_tools.utils.api_types.UpstreamJobSpecSelection`:
120
+ A list of downstream dataset rids can be defined to start from searching for job specs.
121
+ A separate list of dataset rids to ignore can be specified as well in order
122
+ to prevent further exploration of the dependency graph on the given dataset rids
123
+ (d) :py:class:`~foundry_dev_tools.utils.api_types.ConnectingJobSpecSelection`: The build service
124
+ will determine the job specs in between the list of upstream and downstream dataset rids.
125
+ Additionally, a list of dataset rids to ignore can be passed along
126
+ on which to prevent further exploration of the dependency graph
127
+ submission_id: Optional submission identifier to uniquely identify a new build submission
128
+ build_group_rid: Specify in order to join a build group which serves as a collector
129
+ acting as if all the builds sharing the same build group were submitted as a single build
130
+ branch: The branch that all jobs in this build are run on. Defaults to `master` if not specified
131
+ branch_fallbacks: Fallback branches to use when resolving dataset properties for input specs
132
+ force_build: When set to 'True', will ignore staleness checking when running this build
133
+ and runs all job specs regardless of whether they are stale
134
+ ignore_build_policy: When set to 'True', will allow for jobs to be run
135
+ even if their build policy is not satisfied
136
+ finish_on_failure: When set to 'True', will abort all other jobs and finish the build
137
+ if any single job fails
138
+ num_run_job_attempts: The maximum number of attempts a job should be run when it fails
139
+ due to a retryable error. If a value <= 1 is specified, jobs will never be retried.
140
+ force_retry: When set to 'True', all errors will be treated as retryable.
141
+ The maximum number of retry attempts is strictly given by `num_run_job_attempts`
142
+ retry_backoff_duration: The duration in seconds to wait before attempting to run a job again.
143
+ If not present, jobs will be retried immediately
144
+ build_parameters: Parameters to apply to every job in the build
145
+ which typically serve as "configuration" for the workers
146
+ suppress_notifications: When set to 'True', the user will not receive a notification
147
+ when the build is complete
148
+ exceeded_duration_mode: The action taken when job with custom expiration has expired. Defaults to 'CANCEL'
149
+ if not provided at all
150
+ input_failure_strategies: Overwrites the strategy for each input dataset in case its build fails.
151
+ output_queue_strategy: Defines the output queueing behaviour of the jobs that are created by this build.
152
+ Defaults to 'QUEUE_UP' for builds with BatchVariant job specs and 'SUPERSEDE'
153
+ for builds with LongRunningVariant job specs.
154
+ **kwargs: gets passed to :py:meth:`APIClient.api_request`
155
+
156
+ Returns:
157
+ requests.Response:
158
+ the response contains a json which returns information to the build submitted previously.
159
+ """
160
+ if exceeded_duration_mode:
161
+ assert_in_literal(exceeded_duration_mode, api_types.ExceededDurationMode, "exceeded_duration_mode")
162
+
163
+ for input_strategy in input_failure_strategies or {}:
164
+ assert_in_literal(
165
+ input_strategy["failureStrategy"],
166
+ api_types.InputFailureStrategy,
167
+ "input_failure_strategies['failureStrategy']",
168
+ )
169
+
170
+ if output_queue_strategy:
171
+ assert_in_literal(output_queue_strategy, api_types.OutputQueueStrategy, "output_queue_strategy")
172
+
173
+ job_spec_selections = _convert_job_spec_selections(job_spec_selections)
174
+
175
+ body = {
176
+ "jobSpecSelections": job_spec_selections,
177
+ "submissionId": submission_id,
178
+ "buildGroupRid": build_group_rid,
179
+ "branch": branch,
180
+ "branchFallbacks": {"branches": list(branch_fallbacks) if branch_fallbacks else []},
181
+ "forceBuild": force_build,
182
+ "ignoreBuildPolicy": ignore_build_policy,
183
+ "finishOnFailures": finish_on_failure,
184
+ "numRunJobAttempts": num_run_job_attempts,
185
+ "forceRetry": force_retry,
186
+ "retryBackoffDuration": retry_backoff_duration,
187
+ "buildParameters": build_parameters or {},
188
+ "suppressNotifications": suppress_notifications,
189
+ "exceededDurationMode": exceeded_duration_mode,
190
+ "inputFailureStrategies": list(input_failure_strategies) if input_failure_strategies else [],
191
+ "outputQueueStrategy": output_queue_strategy,
192
+ }
193
+
194
+ return self.api_request(
195
+ "POST",
196
+ "manager/submitBuild",
197
+ json=body,
198
+ **kwargs,
199
+ )
200
+
201
+ def submit_dataset_build(
202
+ self,
203
+ dataset_rid: api_types.DatasetRid,
204
+ branch: api_types.DatasetBranch = "master",
205
+ force_build: bool | None = False,
206
+ ) -> dict:
207
+ """Submit a dataset build.
208
+
209
+ Args:
210
+ dataset_rid: The resource identifier of the dataset to build
211
+ branch: The branch that all jobs in this build are run on. Defaults to `master` if not specified
212
+ force_build: When set to 'True', will ignore staleness checking when running this build
213
+ and runs all job specs regardless of whether they are stale. Defaults to 'False'
214
+
215
+ Returns:
216
+ dict:
217
+
218
+ .. code-block:: python
219
+
220
+ {
221
+ "isNewSubmission": "<submission-id>",
222
+ "buildRid": "<...>",
223
+ "buildGroupRid": "<build-group-rid>",
224
+ "jobsCreated": {
225
+ "<job-spec-rid>": "<job-rid>",
226
+ ...
227
+ },
228
+ "jobsInOtherBuilds": {
229
+ "<job-spec-rid>": "<job-rid>",
230
+ ...
231
+ }
232
+ }
233
+
234
+ """
235
+ datasets_job_spec_selections = [{"datasetRids": [dataset_rid], "isRequired": True}]
236
+
237
+ return self.api_submit_build(
238
+ job_spec_selections=datasets_job_spec_selections, branch=branch, force_build=force_build
239
+ ).json()