foundry-dev-tools-transforms 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- foundry_dev_tools_transforms-2.0.0.dist-info/METADATA +125 -0
- foundry_dev_tools_transforms-2.0.0.dist-info/RECORD +12 -0
- foundry_dev_tools_transforms-2.0.0.dist-info/WHEEL +4 -0
- transforms/__about__.py +1 -0
- transforms/__init__.py +5 -0
- transforms/api/__init__.py +33 -0
- transforms/api/_configure.py +27 -0
- transforms/api/_dataset.py +348 -0
- transforms/api/_decorators.py +275 -0
- transforms/api/_transform.py +604 -0
- transforms/errors.py +17 -0
- transforms/py.typed +0 -0
|
@@ -0,0 +1,604 @@
|
|
|
1
|
+
"""The exposed Function definitions and docstrings is Copyright © 2023 Palantir Technologies Inc. and/or affiliates (“Palantir”). All rights reserved.
|
|
2
|
+
|
|
3
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/
|
|
4
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-classes/
|
|
5
|
+
|
|
6
|
+
""" # noqa: E501
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import inspect
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
from functools import cached_property
|
|
14
|
+
from os import PathLike
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple
|
|
17
|
+
|
|
18
|
+
import fs
|
|
19
|
+
import pyspark
|
|
20
|
+
|
|
21
|
+
from foundry_dev_tools.config.context import FoundryContext
|
|
22
|
+
from foundry_dev_tools.utils.spark import get_spark_session
|
|
23
|
+
from transforms.api._dataset import Input, Output
|
|
24
|
+
|
|
25
|
+
DECORATOR_TYPE = Literal["spark", "pandas", "transform", "lightweight", "lightweight-pandas", "lightweight-polars"]
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from collections.abc import Callable, Iterator
|
|
29
|
+
|
|
30
|
+
import pandas as pd
|
|
31
|
+
import polars as pl
|
|
32
|
+
import pyarrow as pa
|
|
33
|
+
|
|
34
|
+
from foundry_dev_tools.utils import api_types
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class Transform:
|
|
38
|
+
"""A Python Transform.
|
|
39
|
+
|
|
40
|
+
Wraps a function and holds the Input and Output context.
|
|
41
|
+
When the function is called, the Input and Output parameters
|
|
42
|
+
are automatically passed.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
compute_func: Callable,
|
|
48
|
+
outputs: dict[str, Output] | None = None,
|
|
49
|
+
inputs: dict[str, Input] | None = None,
|
|
50
|
+
decorator: DECORATOR_TYPE = "spark",
|
|
51
|
+
):
|
|
52
|
+
"""Initialize the `Transform`.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
compute_func (Callable): The compute function to wrap.
|
|
56
|
+
inputs (Dict[str, Input]): A dictionary mapping input names to :class:`Input` specs.
|
|
57
|
+
outputs (Dict[str, Output]): A dictionary mapping output names to :class:`Output` specs.
|
|
58
|
+
decorator (str): decorator
|
|
59
|
+
"""
|
|
60
|
+
# Wrap the compute function and store its properties
|
|
61
|
+
self._compute_func = compute_func
|
|
62
|
+
|
|
63
|
+
self.inputs = inputs or {}
|
|
64
|
+
self.outputs = outputs or {}
|
|
65
|
+
|
|
66
|
+
self._bound_transform = True
|
|
67
|
+
self._type = decorator
|
|
68
|
+
|
|
69
|
+
for name, tinput in self.inputs.items():
|
|
70
|
+
if not isinstance(tinput, Input):
|
|
71
|
+
msg = f"Input '{name}' to transform {self} is not a transforms.api.Input"
|
|
72
|
+
raise TypeError(msg)
|
|
73
|
+
for toutput in self.outputs.values():
|
|
74
|
+
if not isinstance(toutput, Output):
|
|
75
|
+
msg = f"Output '{compute_func.__name__}' of transform {self} is not a transforms.api.Output"
|
|
76
|
+
raise TypeError(
|
|
77
|
+
msg,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
self._use_context = "ctx" in inspect.getfullargspec(compute_func).args
|
|
81
|
+
|
|
82
|
+
def __call__(self, *args, **kwargs):
|
|
83
|
+
"""Passthrough call to the underlying compute function."""
|
|
84
|
+
try:
|
|
85
|
+
return self._compute_func(*args, **kwargs)
|
|
86
|
+
except Exception as exc:
|
|
87
|
+
setattr(exc, "__transform_compute_error", True)
|
|
88
|
+
raise
|
|
89
|
+
|
|
90
|
+
def compute(self, context: FoundryContext | None = None): # noqa: ANN202
|
|
91
|
+
"""Execute the wrapped transform function."""
|
|
92
|
+
handlers: dict[DECORATOR_TYPE, Callable[[FoundryContext], Any]] = {
|
|
93
|
+
"spark": self._compute_spark,
|
|
94
|
+
"pandas": self._compute_pandas,
|
|
95
|
+
"transform": self._compute_transform,
|
|
96
|
+
"lightweight": self._compute_lightweight,
|
|
97
|
+
"lightweight-pandas": self._compute_lightweight_pandas,
|
|
98
|
+
"lightweight-polars": self._compute_lightweight_polars,
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
return handlers[self._type](context or FoundryContext())
|
|
102
|
+
|
|
103
|
+
def _compute_spark(
|
|
104
|
+
self,
|
|
105
|
+
context: FoundryContext,
|
|
106
|
+
) -> pyspark.sql.DataFrame:
|
|
107
|
+
"""Execute the wrapped transform function.
|
|
108
|
+
|
|
109
|
+
Returns:
|
|
110
|
+
:external+spark:py:class:`~pyspark.sql.DataFrame`:
|
|
111
|
+
the result of the transforms function
|
|
112
|
+
|
|
113
|
+
"""
|
|
114
|
+
kwargs = {name: i.init_input(context).dataframe() for name, i in self.inputs.items()}
|
|
115
|
+
if self._use_context:
|
|
116
|
+
kwargs["ctx"] = TransformContext()
|
|
117
|
+
|
|
118
|
+
output_df = self(**kwargs)
|
|
119
|
+
|
|
120
|
+
if not isinstance(output_df, pyspark.sql.DataFrame):
|
|
121
|
+
msg = f"Expected {self} to return a pyspark.sql.DataFrame, instead got {output_df}"
|
|
122
|
+
raise TypeError(msg)
|
|
123
|
+
|
|
124
|
+
return output_df
|
|
125
|
+
|
|
126
|
+
def _compute_pandas(
|
|
127
|
+
self,
|
|
128
|
+
context: FoundryContext,
|
|
129
|
+
) -> pd.core.frame.DataFrame:
|
|
130
|
+
"""Execute the wrapped transform function.
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
:external+spark:py:class:`~pd.core.frame.DataFrame`:
|
|
134
|
+
the result of the transforms function
|
|
135
|
+
|
|
136
|
+
"""
|
|
137
|
+
kwargs = {name: i.init_input(context).dataframe().toPandas() for name, i in self.inputs.items()}
|
|
138
|
+
if self._use_context:
|
|
139
|
+
kwargs["ctx"] = TransformContext()
|
|
140
|
+
|
|
141
|
+
output_df = self(**kwargs)
|
|
142
|
+
from foundry_dev_tools._optional.pandas import pd
|
|
143
|
+
|
|
144
|
+
if not isinstance(output_df, pd.DataFrame):
|
|
145
|
+
msg = f"Expected {self} to return a pandas.DataFrame, instead got {output_df}"
|
|
146
|
+
raise TypeError(msg)
|
|
147
|
+
|
|
148
|
+
return output_df
|
|
149
|
+
|
|
150
|
+
def _compute_transform( # noqa: ANN202
|
|
151
|
+
self,
|
|
152
|
+
context: FoundryContext,
|
|
153
|
+
):
|
|
154
|
+
# prepare Transform
|
|
155
|
+
inputs = {argument_name: TransformInput(i, context) for argument_name, i in self.inputs.items()}
|
|
156
|
+
outputs = {
|
|
157
|
+
argument_name: TransformOutput(o, argument_name, context) for argument_name, o in self.outputs.items()
|
|
158
|
+
}
|
|
159
|
+
kwargs = {**inputs, **outputs}
|
|
160
|
+
|
|
161
|
+
if self._use_context:
|
|
162
|
+
kwargs["ctx"] = TransformContext()
|
|
163
|
+
|
|
164
|
+
self(**kwargs)
|
|
165
|
+
|
|
166
|
+
return {name: i.dataframe() for name, i in outputs.items()}
|
|
167
|
+
|
|
168
|
+
def _compute_lightweight( # noqa: ANN202
|
|
169
|
+
self,
|
|
170
|
+
context: FoundryContext,
|
|
171
|
+
):
|
|
172
|
+
if self._use_context:
|
|
173
|
+
msg = "Lightweight transforms do not support the context argument."
|
|
174
|
+
raise ValueError(msg)
|
|
175
|
+
|
|
176
|
+
inputs = {argument_name: LightweightTransformInput(i, context) for argument_name, i in self.inputs.items()}
|
|
177
|
+
outputs = {
|
|
178
|
+
argument_name: LightweightTransformOutput(o, argument_name, context)
|
|
179
|
+
for argument_name, o in self.outputs.items()
|
|
180
|
+
}
|
|
181
|
+
kwargs = {**inputs, **outputs}
|
|
182
|
+
|
|
183
|
+
self(**kwargs)
|
|
184
|
+
|
|
185
|
+
return {name: i.df for name, i in outputs.items()}
|
|
186
|
+
|
|
187
|
+
def _compute_lightweight_pandas(
|
|
188
|
+
self,
|
|
189
|
+
context: FoundryContext,
|
|
190
|
+
) -> pd.core.frame.DataFrame:
|
|
191
|
+
if self._use_context:
|
|
192
|
+
msg = "Lightweight transforms do not support the context argument."
|
|
193
|
+
raise ValueError(msg)
|
|
194
|
+
|
|
195
|
+
inputs = {
|
|
196
|
+
argument_name: LightweightTransformInput(i, context).pandas() for argument_name, i in self.inputs.items()
|
|
197
|
+
}
|
|
198
|
+
return self(**inputs)
|
|
199
|
+
|
|
200
|
+
def _compute_lightweight_polars(
|
|
201
|
+
self,
|
|
202
|
+
context: FoundryContext,
|
|
203
|
+
) -> pl.DataFrame:
|
|
204
|
+
if self._use_context:
|
|
205
|
+
msg = "Lightweight transforms do not support the context argument."
|
|
206
|
+
raise ValueError(msg)
|
|
207
|
+
|
|
208
|
+
inputs = {
|
|
209
|
+
argument_name: LightweightTransformInput(i, context).polars() for argument_name, i in self.inputs.items()
|
|
210
|
+
}
|
|
211
|
+
return self(**inputs)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
class TransformContext:
|
|
215
|
+
"""The TransformContext is passed to the transform function if ctx is the first argument."""
|
|
216
|
+
|
|
217
|
+
def __init__(self):
|
|
218
|
+
pass
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def spark_session(self) -> pyspark.sql.SparkSession:
|
|
222
|
+
"""Return the spark session.
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
:external+spark:py:class:`~pyspark.sql.SparkSession`
|
|
226
|
+
|
|
227
|
+
"""
|
|
228
|
+
return get_spark_session()
|
|
229
|
+
|
|
230
|
+
@property
|
|
231
|
+
def is_incremental(self) -> bool:
|
|
232
|
+
"""Not implemented.
|
|
233
|
+
|
|
234
|
+
Returns:
|
|
235
|
+
bool:
|
|
236
|
+
false, as it is not implemented
|
|
237
|
+
|
|
238
|
+
"""
|
|
239
|
+
import warnings
|
|
240
|
+
|
|
241
|
+
warnings.warn("is_incremental functionality not implemented in Foundry DevTools")
|
|
242
|
+
return False
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
class TransformInput:
|
|
246
|
+
"""TransformInput class, passed when using @transform decorator."""
|
|
247
|
+
|
|
248
|
+
def __init__(self, input_arg: Input, context: FoundryContext):
|
|
249
|
+
self._input_arg = input_arg
|
|
250
|
+
self._context = context
|
|
251
|
+
|
|
252
|
+
@cached_property
|
|
253
|
+
def branch(self) -> str:
|
|
254
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
255
|
+
self._input_arg.init_input(self._context)
|
|
256
|
+
return self._input_arg.branch
|
|
257
|
+
|
|
258
|
+
@cached_property
|
|
259
|
+
def _dataset_identity(self) -> api_types.DatasetIdentity:
|
|
260
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
261
|
+
self._input_arg.init_input(self._context)
|
|
262
|
+
return self._input_arg.get_dataset_identity()
|
|
263
|
+
|
|
264
|
+
@cached_property
|
|
265
|
+
def rid(self) -> api_types.Rid:
|
|
266
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
267
|
+
self._input_arg.init_input(self._context)
|
|
268
|
+
return self._dataset_identity["dataset_rid"]
|
|
269
|
+
|
|
270
|
+
@cached_property
|
|
271
|
+
def path(self) -> api_types.FoundryPath:
|
|
272
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
273
|
+
self._input_arg.init_input(self._context)
|
|
274
|
+
return self._dataset_identity["dataset_path"]
|
|
275
|
+
|
|
276
|
+
def dataframe(self) -> pyspark.sql.DataFrame:
|
|
277
|
+
"""Returns the dataframe of this input.
|
|
278
|
+
|
|
279
|
+
Returns:
|
|
280
|
+
:external+spark:py:class:`~pyspark.sql.DataFrame`:
|
|
281
|
+
The dataframe for the dataset.
|
|
282
|
+
"""
|
|
283
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
284
|
+
self._input_arg.init_input(self._context)
|
|
285
|
+
return self._input_arg.dataframe()
|
|
286
|
+
|
|
287
|
+
def pandas(self) -> pd.core.frame.DataFrame:
|
|
288
|
+
"""Returns the pandas dataframe of this transform input.
|
|
289
|
+
|
|
290
|
+
Returns:
|
|
291
|
+
:external+pandas:py:class:`~pandas.DataFrame`:
|
|
292
|
+
the pandas dataframe of this transform input.
|
|
293
|
+
|
|
294
|
+
"""
|
|
295
|
+
return self.dataframe().toPandas()
|
|
296
|
+
|
|
297
|
+
def filesystem(self) -> FileSystem:
|
|
298
|
+
"""Returns a read-only filesystem object, that has read and write functions.
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
FileSystem:
|
|
302
|
+
A `FileSystem` object for reading from `Foundry`.
|
|
303
|
+
"""
|
|
304
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
305
|
+
self._input_arg.init_input(self._context)
|
|
306
|
+
return FileSystem(base_path=self._input_arg.get_local_path_to_dataset())
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class LightweightTransformInput(TransformInput):
|
|
310
|
+
"""LightweightTransformInput class, passed when using @lightweight decorator."""
|
|
311
|
+
|
|
312
|
+
def __init__(self, input_arg: Input, context: FoundryContext):
|
|
313
|
+
super().__init__(input_arg, context)
|
|
314
|
+
|
|
315
|
+
@cached_property
|
|
316
|
+
def rid(self) -> api_types.Rid:
|
|
317
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
318
|
+
self._input_arg.init_input(self._context)
|
|
319
|
+
return self._input_arg.get_dataset_identity()["dataset_rid"]
|
|
320
|
+
|
|
321
|
+
@cached_property
|
|
322
|
+
def _local_path(self) -> str:
|
|
323
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
324
|
+
self._input_arg.init_input(self._context)
|
|
325
|
+
return self._input_arg.get_local_path_to_dataset()
|
|
326
|
+
|
|
327
|
+
@cached_property
|
|
328
|
+
def branch(self) -> str:
|
|
329
|
+
"""The branch of the Input."""
|
|
330
|
+
if not self._input_arg._initialized: # noqa: SLF001
|
|
331
|
+
self._input_arg.init_input(self._context)
|
|
332
|
+
return self._input_arg.branch
|
|
333
|
+
|
|
334
|
+
@cached_property
|
|
335
|
+
def _parquet_files(self) -> list[Path]:
|
|
336
|
+
return list(Path(self._local_path).glob("**/*.parquet"))
|
|
337
|
+
|
|
338
|
+
@cached_property
|
|
339
|
+
def _csv_files(self) -> list[Path]:
|
|
340
|
+
return list(Path(self._local_path).glob("**/*.csv"))
|
|
341
|
+
|
|
342
|
+
def dataframe(self):
|
|
343
|
+
msg = "Lightweight transforms do not support the dataframe method."
|
|
344
|
+
raise NotImplementedError(msg)
|
|
345
|
+
|
|
346
|
+
@property
|
|
347
|
+
def path(self) -> str:
|
|
348
|
+
"""Download the dataset's underlying files and return a Path to them."""
|
|
349
|
+
return self._local_path
|
|
350
|
+
|
|
351
|
+
def pandas(self) -> pd.DataFrame:
|
|
352
|
+
"""A Pandas DataFrame containing the full view of the dataset."""
|
|
353
|
+
from foundry_dev_tools._optional.pandas import pd
|
|
354
|
+
|
|
355
|
+
return (
|
|
356
|
+
pd.concat(map(pd.read_parquet, self._parquet_files), ignore_index=True)
|
|
357
|
+
if self._parquet_files
|
|
358
|
+
else pd.concat(map(pd.read_csv, self._csv_files), ignore_index=True)
|
|
359
|
+
)
|
|
360
|
+
|
|
361
|
+
def arrow(self) -> pa.Table:
|
|
362
|
+
"""A PyArrow table containing the full view of the dataset."""
|
|
363
|
+
from foundry_dev_tools._optional.pyarrow import pa, pq
|
|
364
|
+
|
|
365
|
+
return (
|
|
366
|
+
pq.ParquetDataset(str(self._local_path) + "/", use_legacy_dataset=False).read()
|
|
367
|
+
if self._parquet_files
|
|
368
|
+
else pa.concat_tables([pa.csv.read_csv(p) for p in self._csv_files])
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
def polars(self, lazy: bool | None = False) -> pl.DataFrame | pl.LazyFrame:
|
|
372
|
+
"""A Polars DataFrame or LazyFrame containing the full view of the dataset.
|
|
373
|
+
|
|
374
|
+
Args:
|
|
375
|
+
lazy (bool, optional): Whether to return a LazyFrame or DataFrame. Defaults to False.
|
|
376
|
+
"""
|
|
377
|
+
from foundry_dev_tools._optional.polars import pl
|
|
378
|
+
|
|
379
|
+
if lazy:
|
|
380
|
+
return (
|
|
381
|
+
pl.scan_parquet(
|
|
382
|
+
f"{self._local_path}/**/*.parquet",
|
|
383
|
+
rechunk=False,
|
|
384
|
+
low_memory=True,
|
|
385
|
+
cache=False,
|
|
386
|
+
)
|
|
387
|
+
if self._parquet_files
|
|
388
|
+
else pl.scan_csv(
|
|
389
|
+
f"{self._local_path}/**/*.csv",
|
|
390
|
+
rechunk=False,
|
|
391
|
+
low_memory=True,
|
|
392
|
+
cache=False,
|
|
393
|
+
)
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
return (
|
|
397
|
+
pl.read_parquet(
|
|
398
|
+
f"{self._local_path}/**/*.parquet",
|
|
399
|
+
rechunk=False,
|
|
400
|
+
low_memory=True,
|
|
401
|
+
)
|
|
402
|
+
if self._parquet_files
|
|
403
|
+
else pl.read_csv(f"{self._local_path}/**/*.csv", rechunk=False, low_memory=True)
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
class TransformOutput:
|
|
408
|
+
"""The output object passed into Transform objects at runtime."""
|
|
409
|
+
|
|
410
|
+
def __init__(self, output: Output, argument_name: str, context: FoundryContext):
|
|
411
|
+
"""Initialize the TransformOutput.
|
|
412
|
+
|
|
413
|
+
Args:
|
|
414
|
+
output (Output): the output
|
|
415
|
+
argument_name (str): the argument name
|
|
416
|
+
context: a FoundryContext
|
|
417
|
+
"""
|
|
418
|
+
self._fs = None
|
|
419
|
+
self._df = None
|
|
420
|
+
self._context = context
|
|
421
|
+
self._argument_name = argument_name
|
|
422
|
+
self.branch = "no-implemented-in-foundry-dev-tools"
|
|
423
|
+
self.path = output.alias
|
|
424
|
+
self.rid = "no-implemented-in-foundry-dev-tools"
|
|
425
|
+
|
|
426
|
+
def dataframe(self) -> pyspark.sql.DataFrame | None:
|
|
427
|
+
"""Returns pyspark DataFrame.
|
|
428
|
+
|
|
429
|
+
Returns:
|
|
430
|
+
:external+spark:py:class:`~pyspark.sql.DataFrame`:
|
|
431
|
+
the dataframe of this output or None
|
|
432
|
+
|
|
433
|
+
"""
|
|
434
|
+
return self._df
|
|
435
|
+
|
|
436
|
+
def write_dataframe(self, df: pyspark.sql.DataFrame, **kwargs): # noqa: ARG002
|
|
437
|
+
"""Storing dataframe as variable.
|
|
438
|
+
|
|
439
|
+
Args:
|
|
440
|
+
df (:external+spark:py:class:`~pyspark.sql.DataFrame`): spark dataframe
|
|
441
|
+
**kwargs: unused
|
|
442
|
+
|
|
443
|
+
"""
|
|
444
|
+
self._df = df
|
|
445
|
+
|
|
446
|
+
def write_pandas(self, pandas_df: pd.core.frame.DataFrame):
|
|
447
|
+
"""Write the given :class:`pandas.DataFrame` to the dataset.
|
|
448
|
+
|
|
449
|
+
Args:
|
|
450
|
+
pandas_df (pandas.DataFrame): The dataframe to write.
|
|
451
|
+
"""
|
|
452
|
+
self.write_dataframe(get_spark_session().createDataFrame(pandas_df))
|
|
453
|
+
|
|
454
|
+
def _make_filesystem(self) -> FileSystem:
|
|
455
|
+
if self._context.config.transforms_output_folder is not None:
|
|
456
|
+
base_path = self._context.config.transforms_output_folder.joinpath(
|
|
457
|
+
self._argument_name,
|
|
458
|
+
)
|
|
459
|
+
base_path.mkdir(parents=True, exist_ok=True)
|
|
460
|
+
return FileSystem(base_path=base_path)
|
|
461
|
+
return FileSystem()
|
|
462
|
+
|
|
463
|
+
def filesystem(self) -> FileSystem:
|
|
464
|
+
"""Returns a temporary filesystem for the output that can be written to.
|
|
465
|
+
|
|
466
|
+
Returns:
|
|
467
|
+
an object with read write methods
|
|
468
|
+
"""
|
|
469
|
+
if self._fs is None:
|
|
470
|
+
self._fs = self._make_filesystem()
|
|
471
|
+
return self._fs
|
|
472
|
+
|
|
473
|
+
def set_mode(self, mode): # noqa: ANN001
|
|
474
|
+
"""Not implemented in Foundry DevTools.
|
|
475
|
+
|
|
476
|
+
Args:
|
|
477
|
+
mode: write mode
|
|
478
|
+
"""
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
class LightweightTransformOutput(TransformOutput):
|
|
482
|
+
"""The output object passed to the user code at runtime.
|
|
483
|
+
|
|
484
|
+
Its aim is to mimic a subset of the API of `TransformOutput`.
|
|
485
|
+
"""
|
|
486
|
+
|
|
487
|
+
def __init__(self, output: Output, argument_name: str, context: FoundryContext):
|
|
488
|
+
super().__init__(output, argument_name, context)
|
|
489
|
+
self.branch = "no-implemented-in-foundry-dev-tools"
|
|
490
|
+
self.path = output.alias
|
|
491
|
+
self.rid = "no-implemented-in-foundry-dev-tools"
|
|
492
|
+
self.df = None
|
|
493
|
+
|
|
494
|
+
def write_pandas(
|
|
495
|
+
self,
|
|
496
|
+
pandas_df: pd.DataFrame,
|
|
497
|
+
*args,
|
|
498
|
+
**kwargs,
|
|
499
|
+
) -> None:
|
|
500
|
+
"""Write the given :class:`pandas.DataFrame` to the dataset."""
|
|
501
|
+
self.write_table(pandas_df, *args, **kwargs)
|
|
502
|
+
|
|
503
|
+
def write_table(
|
|
504
|
+
self,
|
|
505
|
+
df: pd.core.frame.DataFrame | pa.Table | pl.DataFrame | pl.LazyFrame,
|
|
506
|
+
*args, # noqa: ARG002
|
|
507
|
+
**kwargs, # noqa: ARG002
|
|
508
|
+
) -> None:
|
|
509
|
+
"""Write a Pandas DataFrame, Arrow Table, Polars DataFrame or LazyFrame, to a Foundry Dataset.
|
|
510
|
+
|
|
511
|
+
Note:
|
|
512
|
+
In case a path is specified, it must match the return value of ``path_for_write_table``.
|
|
513
|
+
|
|
514
|
+
Args:
|
|
515
|
+
df: pd.DataFrame, pa.Table, pl.DataFrame, pl.LazyFrame, or pathlib.Path with the data to upload
|
|
516
|
+
*args: not implemented in Foundry DevTools
|
|
517
|
+
**kwargs: not implemented in Foundry DevTools
|
|
518
|
+
|
|
519
|
+
Returns:
|
|
520
|
+
None
|
|
521
|
+
"""
|
|
522
|
+
from foundry_dev_tools._optional.polars import pl
|
|
523
|
+
|
|
524
|
+
if not pl.__fake__ and isinstance(df, pl.LazyFrame):
|
|
525
|
+
df = df.collect() # noqa: PD901
|
|
526
|
+
|
|
527
|
+
if isinstance(df, str | Path):
|
|
528
|
+
if df != self.path_for_write_table:
|
|
529
|
+
msg = f"Path '{df}' does not match expected path '{self.path_for_write_table}'"
|
|
530
|
+
raise ValueError(msg)
|
|
531
|
+
|
|
532
|
+
from foundry_dev_tools._optional.pandas import pd
|
|
533
|
+
|
|
534
|
+
self.df = pd.read_parquet(df) # make df pretty printable
|
|
535
|
+
else:
|
|
536
|
+
self.df = df
|
|
537
|
+
|
|
538
|
+
@property
|
|
539
|
+
def path_for_write_table(self) -> str | None:
|
|
540
|
+
"""Return the path for the dataset's files to be used with write_table."""
|
|
541
|
+
return self.path
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
class FileStatus(NamedTuple):
|
|
545
|
+
path: str
|
|
546
|
+
size: str
|
|
547
|
+
modified: str
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
"""A :class:`collections.namedtuple` capturing details about a `FoundryFS` file."""
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
class FileSystem:
|
|
554
|
+
"""File System for TransformOutput and TransformInput."""
|
|
555
|
+
|
|
556
|
+
def __init__(self, base_path: str | PathLike | None = None):
|
|
557
|
+
if base_path:
|
|
558
|
+
self._fs = fs.open_fs(os.fspath(base_path))
|
|
559
|
+
else:
|
|
560
|
+
self._fs = fs.open_fs("mem://")
|
|
561
|
+
|
|
562
|
+
def ls(self, glob: str | None = None, regex: str = ".*", show_hidden: bool = False) -> Iterator[FileStatus]:
|
|
563
|
+
"""Recurses through all directories and lists all files matching the given patterns.
|
|
564
|
+
|
|
565
|
+
Starting from the root directory of the dataset.
|
|
566
|
+
|
|
567
|
+
Args:
|
|
568
|
+
glob (str | None): A unix file matching pattern. Also supports globstar.
|
|
569
|
+
regex (str | None): A regex pattern against which to match filenames.
|
|
570
|
+
show_hidden (bool): Include hidden files, those prefixed with '.' or '_'.
|
|
571
|
+
|
|
572
|
+
Yields:
|
|
573
|
+
:class:`~transforms.api.FileStatus`:
|
|
574
|
+
The logical path, file size (bytes),
|
|
575
|
+
modified timestamp (ms since January 1, 1970 UTC)
|
|
576
|
+
"""
|
|
577
|
+
if glob:
|
|
578
|
+
# need to remove trailing slash
|
|
579
|
+
result = [glob.path[1:] for glob in self._fs.glob(glob)]
|
|
580
|
+
else:
|
|
581
|
+
result = [file[1:] for file in self._fs.walk.files(path="/")]
|
|
582
|
+
pattern = re.compile(regex)
|
|
583
|
+
result_after_regex_match = [s for s in result if pattern.match(s)]
|
|
584
|
+
if show_hidden:
|
|
585
|
+
import warnings
|
|
586
|
+
|
|
587
|
+
warnings.warn("argument 'show_hidden' not implemented in foundry_dev_tools")
|
|
588
|
+
for result_path in result_after_regex_match:
|
|
589
|
+
yield FileStatus(result_path, "size not implemented", "modified not implemented")
|
|
590
|
+
|
|
591
|
+
def open(self, path: PathLike[str] | str, mode: str = "w", **kwargs) -> IO:
|
|
592
|
+
"""Open file in this filesystem.
|
|
593
|
+
|
|
594
|
+
Args:
|
|
595
|
+
path (str): the path to the file
|
|
596
|
+
mode (str): usual file modes
|
|
597
|
+
**kwargs: pass through
|
|
598
|
+
|
|
599
|
+
Returns:
|
|
600
|
+
IO:
|
|
601
|
+
|
|
602
|
+
"""
|
|
603
|
+
self._fs.makedirs(os.path.dirname(path), recreate=True) # noqa: PTH120
|
|
604
|
+
return self._fs.open(os.fspath(path), mode, **kwargs)
|
transforms/errors.py
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Transforms specific FoundryDevToolsError classes."""
|
|
2
|
+
|
|
3
|
+
from foundry_dev_tools.errors.meta import FoundryDevToolsError
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class FoundryTransformsError(FoundryDevToolsError):
|
|
7
|
+
"""Errors related to transforms."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class UninitializedInputError(FoundryTransformsError):
|
|
11
|
+
"""Raised when the Input has not been initialized with a FoundryContext."""
|
|
12
|
+
|
|
13
|
+
message = (
|
|
14
|
+
"This Input has not been initialized, this is normally done by the transforms library automatically."
|
|
15
|
+
"If you've created the Input manually and wanted to use it outside of the local transforms implementation "
|
|
16
|
+
"you'll need to call the init_input method on it with a FoundryContext."
|
|
17
|
+
)
|
transforms/py.typed
ADDED
|
File without changes
|