foundry-dev-tools-transforms 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,604 @@
1
+ """The exposed Function definitions and docstrings is Copyright © 2023 Palantir Technologies Inc. and/or affiliates (“Palantir”). All rights reserved.
2
+
3
+ https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/
4
+ https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-classes/
5
+
6
+ """ # noqa: E501
7
+
8
+ from __future__ import annotations
9
+
10
+ import inspect
11
+ import os
12
+ import re
13
+ from functools import cached_property
14
+ from os import PathLike
15
+ from pathlib import Path
16
+ from typing import IO, TYPE_CHECKING, Any, Literal, NamedTuple
17
+
18
+ import fs
19
+ import pyspark
20
+
21
+ from foundry_dev_tools.config.context import FoundryContext
22
+ from foundry_dev_tools.utils.spark import get_spark_session
23
+ from transforms.api._dataset import Input, Output
24
+
25
+ DECORATOR_TYPE = Literal["spark", "pandas", "transform", "lightweight", "lightweight-pandas", "lightweight-polars"]
26
+
27
+ if TYPE_CHECKING:
28
+ from collections.abc import Callable, Iterator
29
+
30
+ import pandas as pd
31
+ import polars as pl
32
+ import pyarrow as pa
33
+
34
+ from foundry_dev_tools.utils import api_types
35
+
36
+
37
+ class Transform:
38
+ """A Python Transform.
39
+
40
+ Wraps a function and holds the Input and Output context.
41
+ When the function is called, the Input and Output parameters
42
+ are automatically passed.
43
+ """
44
+
45
+ def __init__(
46
+ self,
47
+ compute_func: Callable,
48
+ outputs: dict[str, Output] | None = None,
49
+ inputs: dict[str, Input] | None = None,
50
+ decorator: DECORATOR_TYPE = "spark",
51
+ ):
52
+ """Initialize the `Transform`.
53
+
54
+ Args:
55
+ compute_func (Callable): The compute function to wrap.
56
+ inputs (Dict[str, Input]): A dictionary mapping input names to :class:`Input` specs.
57
+ outputs (Dict[str, Output]): A dictionary mapping output names to :class:`Output` specs.
58
+ decorator (str): decorator
59
+ """
60
+ # Wrap the compute function and store its properties
61
+ self._compute_func = compute_func
62
+
63
+ self.inputs = inputs or {}
64
+ self.outputs = outputs or {}
65
+
66
+ self._bound_transform = True
67
+ self._type = decorator
68
+
69
+ for name, tinput in self.inputs.items():
70
+ if not isinstance(tinput, Input):
71
+ msg = f"Input '{name}' to transform {self} is not a transforms.api.Input"
72
+ raise TypeError(msg)
73
+ for toutput in self.outputs.values():
74
+ if not isinstance(toutput, Output):
75
+ msg = f"Output '{compute_func.__name__}' of transform {self} is not a transforms.api.Output"
76
+ raise TypeError(
77
+ msg,
78
+ )
79
+
80
+ self._use_context = "ctx" in inspect.getfullargspec(compute_func).args
81
+
82
+ def __call__(self, *args, **kwargs):
83
+ """Passthrough call to the underlying compute function."""
84
+ try:
85
+ return self._compute_func(*args, **kwargs)
86
+ except Exception as exc:
87
+ setattr(exc, "__transform_compute_error", True)
88
+ raise
89
+
90
+ def compute(self, context: FoundryContext | None = None): # noqa: ANN202
91
+ """Execute the wrapped transform function."""
92
+ handlers: dict[DECORATOR_TYPE, Callable[[FoundryContext], Any]] = {
93
+ "spark": self._compute_spark,
94
+ "pandas": self._compute_pandas,
95
+ "transform": self._compute_transform,
96
+ "lightweight": self._compute_lightweight,
97
+ "lightweight-pandas": self._compute_lightweight_pandas,
98
+ "lightweight-polars": self._compute_lightweight_polars,
99
+ }
100
+
101
+ return handlers[self._type](context or FoundryContext())
102
+
103
+ def _compute_spark(
104
+ self,
105
+ context: FoundryContext,
106
+ ) -> pyspark.sql.DataFrame:
107
+ """Execute the wrapped transform function.
108
+
109
+ Returns:
110
+ :external+spark:py:class:`~pyspark.sql.DataFrame`:
111
+ the result of the transforms function
112
+
113
+ """
114
+ kwargs = {name: i.init_input(context).dataframe() for name, i in self.inputs.items()}
115
+ if self._use_context:
116
+ kwargs["ctx"] = TransformContext()
117
+
118
+ output_df = self(**kwargs)
119
+
120
+ if not isinstance(output_df, pyspark.sql.DataFrame):
121
+ msg = f"Expected {self} to return a pyspark.sql.DataFrame, instead got {output_df}"
122
+ raise TypeError(msg)
123
+
124
+ return output_df
125
+
126
+ def _compute_pandas(
127
+ self,
128
+ context: FoundryContext,
129
+ ) -> pd.core.frame.DataFrame:
130
+ """Execute the wrapped transform function.
131
+
132
+ Returns:
133
+ :external+spark:py:class:`~pd.core.frame.DataFrame`:
134
+ the result of the transforms function
135
+
136
+ """
137
+ kwargs = {name: i.init_input(context).dataframe().toPandas() for name, i in self.inputs.items()}
138
+ if self._use_context:
139
+ kwargs["ctx"] = TransformContext()
140
+
141
+ output_df = self(**kwargs)
142
+ from foundry_dev_tools._optional.pandas import pd
143
+
144
+ if not isinstance(output_df, pd.DataFrame):
145
+ msg = f"Expected {self} to return a pandas.DataFrame, instead got {output_df}"
146
+ raise TypeError(msg)
147
+
148
+ return output_df
149
+
150
+ def _compute_transform( # noqa: ANN202
151
+ self,
152
+ context: FoundryContext,
153
+ ):
154
+ # prepare Transform
155
+ inputs = {argument_name: TransformInput(i, context) for argument_name, i in self.inputs.items()}
156
+ outputs = {
157
+ argument_name: TransformOutput(o, argument_name, context) for argument_name, o in self.outputs.items()
158
+ }
159
+ kwargs = {**inputs, **outputs}
160
+
161
+ if self._use_context:
162
+ kwargs["ctx"] = TransformContext()
163
+
164
+ self(**kwargs)
165
+
166
+ return {name: i.dataframe() for name, i in outputs.items()}
167
+
168
+ def _compute_lightweight( # noqa: ANN202
169
+ self,
170
+ context: FoundryContext,
171
+ ):
172
+ if self._use_context:
173
+ msg = "Lightweight transforms do not support the context argument."
174
+ raise ValueError(msg)
175
+
176
+ inputs = {argument_name: LightweightTransformInput(i, context) for argument_name, i in self.inputs.items()}
177
+ outputs = {
178
+ argument_name: LightweightTransformOutput(o, argument_name, context)
179
+ for argument_name, o in self.outputs.items()
180
+ }
181
+ kwargs = {**inputs, **outputs}
182
+
183
+ self(**kwargs)
184
+
185
+ return {name: i.df for name, i in outputs.items()}
186
+
187
+ def _compute_lightweight_pandas(
188
+ self,
189
+ context: FoundryContext,
190
+ ) -> pd.core.frame.DataFrame:
191
+ if self._use_context:
192
+ msg = "Lightweight transforms do not support the context argument."
193
+ raise ValueError(msg)
194
+
195
+ inputs = {
196
+ argument_name: LightweightTransformInput(i, context).pandas() for argument_name, i in self.inputs.items()
197
+ }
198
+ return self(**inputs)
199
+
200
+ def _compute_lightweight_polars(
201
+ self,
202
+ context: FoundryContext,
203
+ ) -> pl.DataFrame:
204
+ if self._use_context:
205
+ msg = "Lightweight transforms do not support the context argument."
206
+ raise ValueError(msg)
207
+
208
+ inputs = {
209
+ argument_name: LightweightTransformInput(i, context).polars() for argument_name, i in self.inputs.items()
210
+ }
211
+ return self(**inputs)
212
+
213
+
214
+ class TransformContext:
215
+ """The TransformContext is passed to the transform function if ctx is the first argument."""
216
+
217
+ def __init__(self):
218
+ pass
219
+
220
+ @property
221
+ def spark_session(self) -> pyspark.sql.SparkSession:
222
+ """Return the spark session.
223
+
224
+ Returns:
225
+ :external+spark:py:class:`~pyspark.sql.SparkSession`
226
+
227
+ """
228
+ return get_spark_session()
229
+
230
+ @property
231
+ def is_incremental(self) -> bool:
232
+ """Not implemented.
233
+
234
+ Returns:
235
+ bool:
236
+ false, as it is not implemented
237
+
238
+ """
239
+ import warnings
240
+
241
+ warnings.warn("is_incremental functionality not implemented in Foundry DevTools")
242
+ return False
243
+
244
+
245
+ class TransformInput:
246
+ """TransformInput class, passed when using @transform decorator."""
247
+
248
+ def __init__(self, input_arg: Input, context: FoundryContext):
249
+ self._input_arg = input_arg
250
+ self._context = context
251
+
252
+ @cached_property
253
+ def branch(self) -> str:
254
+ if not self._input_arg._initialized: # noqa: SLF001
255
+ self._input_arg.init_input(self._context)
256
+ return self._input_arg.branch
257
+
258
+ @cached_property
259
+ def _dataset_identity(self) -> api_types.DatasetIdentity:
260
+ if not self._input_arg._initialized: # noqa: SLF001
261
+ self._input_arg.init_input(self._context)
262
+ return self._input_arg.get_dataset_identity()
263
+
264
+ @cached_property
265
+ def rid(self) -> api_types.Rid:
266
+ if not self._input_arg._initialized: # noqa: SLF001
267
+ self._input_arg.init_input(self._context)
268
+ return self._dataset_identity["dataset_rid"]
269
+
270
+ @cached_property
271
+ def path(self) -> api_types.FoundryPath:
272
+ if not self._input_arg._initialized: # noqa: SLF001
273
+ self._input_arg.init_input(self._context)
274
+ return self._dataset_identity["dataset_path"]
275
+
276
+ def dataframe(self) -> pyspark.sql.DataFrame:
277
+ """Returns the dataframe of this input.
278
+
279
+ Returns:
280
+ :external+spark:py:class:`~pyspark.sql.DataFrame`:
281
+ The dataframe for the dataset.
282
+ """
283
+ if not self._input_arg._initialized: # noqa: SLF001
284
+ self._input_arg.init_input(self._context)
285
+ return self._input_arg.dataframe()
286
+
287
+ def pandas(self) -> pd.core.frame.DataFrame:
288
+ """Returns the pandas dataframe of this transform input.
289
+
290
+ Returns:
291
+ :external+pandas:py:class:`~pandas.DataFrame`:
292
+ the pandas dataframe of this transform input.
293
+
294
+ """
295
+ return self.dataframe().toPandas()
296
+
297
+ def filesystem(self) -> FileSystem:
298
+ """Returns a read-only filesystem object, that has read and write functions.
299
+
300
+ Returns:
301
+ FileSystem:
302
+ A `FileSystem` object for reading from `Foundry`.
303
+ """
304
+ if not self._input_arg._initialized: # noqa: SLF001
305
+ self._input_arg.init_input(self._context)
306
+ return FileSystem(base_path=self._input_arg.get_local_path_to_dataset())
307
+
308
+
309
+ class LightweightTransformInput(TransformInput):
310
+ """LightweightTransformInput class, passed when using @lightweight decorator."""
311
+
312
+ def __init__(self, input_arg: Input, context: FoundryContext):
313
+ super().__init__(input_arg, context)
314
+
315
+ @cached_property
316
+ def rid(self) -> api_types.Rid:
317
+ if not self._input_arg._initialized: # noqa: SLF001
318
+ self._input_arg.init_input(self._context)
319
+ return self._input_arg.get_dataset_identity()["dataset_rid"]
320
+
321
+ @cached_property
322
+ def _local_path(self) -> str:
323
+ if not self._input_arg._initialized: # noqa: SLF001
324
+ self._input_arg.init_input(self._context)
325
+ return self._input_arg.get_local_path_to_dataset()
326
+
327
+ @cached_property
328
+ def branch(self) -> str:
329
+ """The branch of the Input."""
330
+ if not self._input_arg._initialized: # noqa: SLF001
331
+ self._input_arg.init_input(self._context)
332
+ return self._input_arg.branch
333
+
334
+ @cached_property
335
+ def _parquet_files(self) -> list[Path]:
336
+ return list(Path(self._local_path).glob("**/*.parquet"))
337
+
338
+ @cached_property
339
+ def _csv_files(self) -> list[Path]:
340
+ return list(Path(self._local_path).glob("**/*.csv"))
341
+
342
+ def dataframe(self):
343
+ msg = "Lightweight transforms do not support the dataframe method."
344
+ raise NotImplementedError(msg)
345
+
346
+ @property
347
+ def path(self) -> str:
348
+ """Download the dataset's underlying files and return a Path to them."""
349
+ return self._local_path
350
+
351
+ def pandas(self) -> pd.DataFrame:
352
+ """A Pandas DataFrame containing the full view of the dataset."""
353
+ from foundry_dev_tools._optional.pandas import pd
354
+
355
+ return (
356
+ pd.concat(map(pd.read_parquet, self._parquet_files), ignore_index=True)
357
+ if self._parquet_files
358
+ else pd.concat(map(pd.read_csv, self._csv_files), ignore_index=True)
359
+ )
360
+
361
+ def arrow(self) -> pa.Table:
362
+ """A PyArrow table containing the full view of the dataset."""
363
+ from foundry_dev_tools._optional.pyarrow import pa, pq
364
+
365
+ return (
366
+ pq.ParquetDataset(str(self._local_path) + "/", use_legacy_dataset=False).read()
367
+ if self._parquet_files
368
+ else pa.concat_tables([pa.csv.read_csv(p) for p in self._csv_files])
369
+ )
370
+
371
+ def polars(self, lazy: bool | None = False) -> pl.DataFrame | pl.LazyFrame:
372
+ """A Polars DataFrame or LazyFrame containing the full view of the dataset.
373
+
374
+ Args:
375
+ lazy (bool, optional): Whether to return a LazyFrame or DataFrame. Defaults to False.
376
+ """
377
+ from foundry_dev_tools._optional.polars import pl
378
+
379
+ if lazy:
380
+ return (
381
+ pl.scan_parquet(
382
+ f"{self._local_path}/**/*.parquet",
383
+ rechunk=False,
384
+ low_memory=True,
385
+ cache=False,
386
+ )
387
+ if self._parquet_files
388
+ else pl.scan_csv(
389
+ f"{self._local_path}/**/*.csv",
390
+ rechunk=False,
391
+ low_memory=True,
392
+ cache=False,
393
+ )
394
+ )
395
+
396
+ return (
397
+ pl.read_parquet(
398
+ f"{self._local_path}/**/*.parquet",
399
+ rechunk=False,
400
+ low_memory=True,
401
+ )
402
+ if self._parquet_files
403
+ else pl.read_csv(f"{self._local_path}/**/*.csv", rechunk=False, low_memory=True)
404
+ )
405
+
406
+
407
+ class TransformOutput:
408
+ """The output object passed into Transform objects at runtime."""
409
+
410
+ def __init__(self, output: Output, argument_name: str, context: FoundryContext):
411
+ """Initialize the TransformOutput.
412
+
413
+ Args:
414
+ output (Output): the output
415
+ argument_name (str): the argument name
416
+ context: a FoundryContext
417
+ """
418
+ self._fs = None
419
+ self._df = None
420
+ self._context = context
421
+ self._argument_name = argument_name
422
+ self.branch = "no-implemented-in-foundry-dev-tools"
423
+ self.path = output.alias
424
+ self.rid = "no-implemented-in-foundry-dev-tools"
425
+
426
+ def dataframe(self) -> pyspark.sql.DataFrame | None:
427
+ """Returns pyspark DataFrame.
428
+
429
+ Returns:
430
+ :external+spark:py:class:`~pyspark.sql.DataFrame`:
431
+ the dataframe of this output or None
432
+
433
+ """
434
+ return self._df
435
+
436
+ def write_dataframe(self, df: pyspark.sql.DataFrame, **kwargs): # noqa: ARG002
437
+ """Storing dataframe as variable.
438
+
439
+ Args:
440
+ df (:external+spark:py:class:`~pyspark.sql.DataFrame`): spark dataframe
441
+ **kwargs: unused
442
+
443
+ """
444
+ self._df = df
445
+
446
+ def write_pandas(self, pandas_df: pd.core.frame.DataFrame):
447
+ """Write the given :class:`pandas.DataFrame` to the dataset.
448
+
449
+ Args:
450
+ pandas_df (pandas.DataFrame): The dataframe to write.
451
+ """
452
+ self.write_dataframe(get_spark_session().createDataFrame(pandas_df))
453
+
454
+ def _make_filesystem(self) -> FileSystem:
455
+ if self._context.config.transforms_output_folder is not None:
456
+ base_path = self._context.config.transforms_output_folder.joinpath(
457
+ self._argument_name,
458
+ )
459
+ base_path.mkdir(parents=True, exist_ok=True)
460
+ return FileSystem(base_path=base_path)
461
+ return FileSystem()
462
+
463
+ def filesystem(self) -> FileSystem:
464
+ """Returns a temporary filesystem for the output that can be written to.
465
+
466
+ Returns:
467
+ an object with read write methods
468
+ """
469
+ if self._fs is None:
470
+ self._fs = self._make_filesystem()
471
+ return self._fs
472
+
473
+ def set_mode(self, mode): # noqa: ANN001
474
+ """Not implemented in Foundry DevTools.
475
+
476
+ Args:
477
+ mode: write mode
478
+ """
479
+
480
+
481
+ class LightweightTransformOutput(TransformOutput):
482
+ """The output object passed to the user code at runtime.
483
+
484
+ Its aim is to mimic a subset of the API of `TransformOutput`.
485
+ """
486
+
487
+ def __init__(self, output: Output, argument_name: str, context: FoundryContext):
488
+ super().__init__(output, argument_name, context)
489
+ self.branch = "no-implemented-in-foundry-dev-tools"
490
+ self.path = output.alias
491
+ self.rid = "no-implemented-in-foundry-dev-tools"
492
+ self.df = None
493
+
494
+ def write_pandas(
495
+ self,
496
+ pandas_df: pd.DataFrame,
497
+ *args,
498
+ **kwargs,
499
+ ) -> None:
500
+ """Write the given :class:`pandas.DataFrame` to the dataset."""
501
+ self.write_table(pandas_df, *args, **kwargs)
502
+
503
+ def write_table(
504
+ self,
505
+ df: pd.core.frame.DataFrame | pa.Table | pl.DataFrame | pl.LazyFrame,
506
+ *args, # noqa: ARG002
507
+ **kwargs, # noqa: ARG002
508
+ ) -> None:
509
+ """Write a Pandas DataFrame, Arrow Table, Polars DataFrame or LazyFrame, to a Foundry Dataset.
510
+
511
+ Note:
512
+ In case a path is specified, it must match the return value of ``path_for_write_table``.
513
+
514
+ Args:
515
+ df: pd.DataFrame, pa.Table, pl.DataFrame, pl.LazyFrame, or pathlib.Path with the data to upload
516
+ *args: not implemented in Foundry DevTools
517
+ **kwargs: not implemented in Foundry DevTools
518
+
519
+ Returns:
520
+ None
521
+ """
522
+ from foundry_dev_tools._optional.polars import pl
523
+
524
+ if not pl.__fake__ and isinstance(df, pl.LazyFrame):
525
+ df = df.collect() # noqa: PD901
526
+
527
+ if isinstance(df, str | Path):
528
+ if df != self.path_for_write_table:
529
+ msg = f"Path '{df}' does not match expected path '{self.path_for_write_table}'"
530
+ raise ValueError(msg)
531
+
532
+ from foundry_dev_tools._optional.pandas import pd
533
+
534
+ self.df = pd.read_parquet(df) # make df pretty printable
535
+ else:
536
+ self.df = df
537
+
538
+ @property
539
+ def path_for_write_table(self) -> str | None:
540
+ """Return the path for the dataset's files to be used with write_table."""
541
+ return self.path
542
+
543
+
544
+ class FileStatus(NamedTuple):
545
+ path: str
546
+ size: str
547
+ modified: str
548
+
549
+
550
+ """A :class:`collections.namedtuple` capturing details about a `FoundryFS` file."""
551
+
552
+
553
+ class FileSystem:
554
+ """File System for TransformOutput and TransformInput."""
555
+
556
+ def __init__(self, base_path: str | PathLike | None = None):
557
+ if base_path:
558
+ self._fs = fs.open_fs(os.fspath(base_path))
559
+ else:
560
+ self._fs = fs.open_fs("mem://")
561
+
562
+ def ls(self, glob: str | None = None, regex: str = ".*", show_hidden: bool = False) -> Iterator[FileStatus]:
563
+ """Recurses through all directories and lists all files matching the given patterns.
564
+
565
+ Starting from the root directory of the dataset.
566
+
567
+ Args:
568
+ glob (str | None): A unix file matching pattern. Also supports globstar.
569
+ regex (str | None): A regex pattern against which to match filenames.
570
+ show_hidden (bool): Include hidden files, those prefixed with '.' or '_'.
571
+
572
+ Yields:
573
+ :class:`~transforms.api.FileStatus`:
574
+ The logical path, file size (bytes),
575
+ modified timestamp (ms since January 1, 1970 UTC)
576
+ """
577
+ if glob:
578
+ # need to remove trailing slash
579
+ result = [glob.path[1:] for glob in self._fs.glob(glob)]
580
+ else:
581
+ result = [file[1:] for file in self._fs.walk.files(path="/")]
582
+ pattern = re.compile(regex)
583
+ result_after_regex_match = [s for s in result if pattern.match(s)]
584
+ if show_hidden:
585
+ import warnings
586
+
587
+ warnings.warn("argument 'show_hidden' not implemented in foundry_dev_tools")
588
+ for result_path in result_after_regex_match:
589
+ yield FileStatus(result_path, "size not implemented", "modified not implemented")
590
+
591
+ def open(self, path: PathLike[str] | str, mode: str = "w", **kwargs) -> IO:
592
+ """Open file in this filesystem.
593
+
594
+ Args:
595
+ path (str): the path to the file
596
+ mode (str): usual file modes
597
+ **kwargs: pass through
598
+
599
+ Returns:
600
+ IO:
601
+
602
+ """
603
+ self._fs.makedirs(os.path.dirname(path), recreate=True) # noqa: PTH120
604
+ return self._fs.open(os.fspath(path), mode, **kwargs)
transforms/errors.py ADDED
@@ -0,0 +1,17 @@
1
+ """Transforms specific FoundryDevToolsError classes."""
2
+
3
+ from foundry_dev_tools.errors.meta import FoundryDevToolsError
4
+
5
+
6
+ class FoundryTransformsError(FoundryDevToolsError):
7
+ """Errors related to transforms."""
8
+
9
+
10
+ class UninitializedInputError(FoundryTransformsError):
11
+ """Raised when the Input has not been initialized with a FoundryContext."""
12
+
13
+ message = (
14
+ "This Input has not been initialized, this is normally done by the transforms library automatically."
15
+ "If you've created the Input manually and wanted to use it outside of the local transforms implementation "
16
+ "you'll need to call the init_input method on it with a FoundryContext."
17
+ )
transforms/py.typed ADDED
File without changes