foundry-dev-tools-transforms 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- foundry_dev_tools_transforms-2.0.0.dist-info/METADATA +125 -0
- foundry_dev_tools_transforms-2.0.0.dist-info/RECORD +12 -0
- foundry_dev_tools_transforms-2.0.0.dist-info/WHEEL +4 -0
- transforms/__about__.py +1 -0
- transforms/__init__.py +5 -0
- transforms/api/__init__.py +33 -0
- transforms/api/_configure.py +27 -0
- transforms/api/_dataset.py +348 -0
- transforms/api/_decorators.py +275 -0
- transforms/api/_transform.py +604 -0
- transforms/errors.py +17 -0
- transforms/py.typed +0 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: foundry-dev-tools-transforms
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
|
|
5
|
+
Author-Email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
8
|
+
Classifier: Programming Language :: Python
|
|
9
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
15
|
+
Classifier: Operating System :: MacOS
|
|
16
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
18
|
+
Project-URL: Homepage, https://emdgroup.github.io/foundry-dev-tools
|
|
19
|
+
Project-URL: Documentation, https://emdgroup.github.io/foundry-dev-tools
|
|
20
|
+
Project-URL: Source, https://github.com/emdgroup/foundry-dev-tools
|
|
21
|
+
Project-URL: Tracker, https://github.com/emdgroup/foundry-dev-tools/issues
|
|
22
|
+
Project-URL: Changelog, https://emdgroup.github.io/foundry-dev-tools/changelog.html
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: foundry-dev-tools
|
|
25
|
+
Requires-Dist: typing_extensions; python_version < "3.11"
|
|
26
|
+
Requires-Dist: pyspark>=3.0.0
|
|
27
|
+
Requires-Dist: numpy<2
|
|
28
|
+
Requires-Dist: fs
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
<div align="center">
|
|
32
|
+
<br/>
|
|
33
|
+
|
|
34
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/ci.yml?style=flat-square"/></img>
|
|
35
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/docs.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/docs.yml?style=flat-square"/></img>
|
|
36
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square&label=Supported%20Python%20versions&color=%23ffb86c"/></a>
|
|
37
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square&label=PyPI%20version&color=%23bd93f9"/></a>
|
|
38
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/vn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Version&color=%23bd93f9" alt="Conda Version"/></a>
|
|
39
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/dm/foundry-dev-tools?label=PyPI%20Downloads&style=flat-square&color=%236272a4"/></a>
|
|
40
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/dn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Downloads&color=%236272a4" alt="Conda Downloads"/></a>
|
|
41
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
42
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
43
|
+
<a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square&color=%234c1"/></a>
|
|
44
|
+
|
|
45
|
+
<p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
|
|
46
|
+
|
|
47
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/getting_started/index.html">Getting Started / Usage<a/>
|
|
48
|
+
•
|
|
49
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/examples/api.html">Examples<a/>
|
|
50
|
+
•
|
|
51
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/dev/contribute.html">Development/Contribute<a/>
|
|
52
|
+
|
|
53
|
+
</div>
|
|
54
|
+
|
|
55
|
+
# Foundry DevTools
|
|
56
|
+
|
|
57
|
+
Seamlessly run your Palantir Foundry Repository transforms code and more on your local machine.
|
|
58
|
+
Foundry DevTools is a set of useful libraries to interact with the Foundry APIs.
|
|
59
|
+
It consists of two parts:
|
|
60
|
+
|
|
61
|
+
- The [transforms](https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/) implementation
|
|
62
|
+
|
|
63
|
+
- An implementation of the Foundry `transforms` package that internally uses the `CachedFoundryClient`.
|
|
64
|
+
This allows you to seamlessly run your Palantir Foundry Code Repository transforms code on your local machine.
|
|
65
|
+
Foundry DevTools does not cover all of Foundry's features, more on this [here](https://emdgroup.github.io/foundry-dev-tools/dev/architecture.html#known-limitations-contributions-welcome).
|
|
66
|
+
|
|
67
|
+
- API clients
|
|
68
|
+
|
|
69
|
+
We implemented multiple clients for many foundry APIs like compass, catalog or foundry-sql-server.
|
|
70
|
+
|
|
71
|
+
- For example:
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from foundry_dev_tools import FoundryContext
|
|
75
|
+
|
|
76
|
+
# the context, that contains your credentials and configuration
|
|
77
|
+
ctx = FoundryContext()
|
|
78
|
+
|
|
79
|
+
df = ctx.foundry_sql_server.query_foundry_sql("SELECT * FROM `/Global/Foundry Training and Resources/Example Data/Aviation Ontology/airlines`", branch='master')
|
|
80
|
+
df.shape
|
|
81
|
+
# Out[2]: (17, 10)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Quickstart
|
|
85
|
+
|
|
86
|
+
With pip:
|
|
87
|
+
|
|
88
|
+
```shell
|
|
89
|
+
pip install foundry-dev-tools
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
With conda or mamba on the conda-forge channel:
|
|
93
|
+
|
|
94
|
+
```shell
|
|
95
|
+
conda install -c conda-forge foundry-dev-tools
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/getting_started/installation.html) can be found in our documentation.
|
|
99
|
+
|
|
100
|
+
## Why did we build this?
|
|
101
|
+
|
|
102
|
+
- Local development experience in your favorite IDE (PyCharm, VSCode, ...)
|
|
103
|
+
- Access to modern developer tools and workflows such as ruff, mypy, pylint, black, pre-commit hooks etc.
|
|
104
|
+
- Quicker turnaround time when making changes
|
|
105
|
+
- Debug, change code and run in a matter of seconds instead of minutes
|
|
106
|
+
- No accidental or auto commits
|
|
107
|
+
- Keep your git history clean
|
|
108
|
+
|
|
109
|
+
# License
|
|
110
|
+
|
|
111
|
+
Copyright (c) 2024 Merck KGaA, Darmstadt, Germany
|
|
112
|
+
|
|
113
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
114
|
+
you may not use this file except in compliance with the License.
|
|
115
|
+
You may obtain a copy of the License at
|
|
116
|
+
|
|
117
|
+
https://www.apache.org/licenses/LICENSE-2.0
|
|
118
|
+
|
|
119
|
+
Unless required by applicable law or agreed to in writing, software
|
|
120
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
121
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
122
|
+
See the License for the specific language governing permissions and
|
|
123
|
+
limitations under the License.
|
|
124
|
+
|
|
125
|
+
The full text of the license can be found in the [LICENSE](https://github.com/emdgroup/foundry-dev-tools/blob/main/LICENSE) file in the repository root directory.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
foundry_dev_tools_transforms-2.0.0.dist-info/METADATA,sha256=yvekXPhQ-4Xz5GxfSChkPZisHTyWNXM2O1T8x696ATs,6589
|
|
2
|
+
foundry_dev_tools_transforms-2.0.0.dist-info/WHEEL,sha256=rSwsxJWe3vzyR5HCwjWXQruDgschpei4h_giTm0dJVE,90
|
|
3
|
+
transforms/__about__.py,sha256=8sGE31KO-DzdioQZ2J7tZ-GG2VwZjYHpfIX2_sBl6H0,21
|
|
4
|
+
transforms/__init__.py,sha256=Q6MosvguLOMURW_loIlGp1aREbGbM1sW6742kXJrEXw,132
|
|
5
|
+
transforms/api/__init__.py,sha256=ZEQp8qTeTV9RW5OCT5w-i-uCxkL04YEN-c1K13X7WnQ,914
|
|
6
|
+
transforms/api/_configure.py,sha256=d2NBsA8IhjPmzk2P2GmMYCzpIP-DxFiqc7ROrIr4QmA,783
|
|
7
|
+
transforms/api/_dataset.py,sha256=B_b13pY1kjw1gzw_TI9EY2TczA4NOSklg0Ywgk2G4xQ,13315
|
|
8
|
+
transforms/api/_decorators.py,sha256=HylpbvXXzTeGwqbe4fRBWFU0yqaYlgyWrem8JoSBJ5s,11617
|
|
9
|
+
transforms/api/_transform.py,sha256=e8KoF4_v9sWE_v_MxrCrT5j-Cva385R3JVp6cWZJRSU,20496
|
|
10
|
+
transforms/errors.py,sha256=rT-y4cGKpRkaR6Spr38Zg2pKnAM4SVMLV1DeH24KciI,681
|
|
11
|
+
transforms/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
foundry_dev_tools_transforms-2.0.0.dist-info/RECORD,,
|
transforms/__about__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.0.0'
|
transforms/__init__.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""The exposed Function definitions and docstrings is Copyright © 2023 Palantir Technologies Inc. and/or affiliates (“Palantir”). All rights reserved.
|
|
2
|
+
|
|
3
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/
|
|
4
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-classes/
|
|
5
|
+
|
|
6
|
+
""" # noqa: E501
|
|
7
|
+
|
|
8
|
+
from transforms.api._configure import configure
|
|
9
|
+
from transforms.api._dataset import Input, Markings, OrgMarkings, Output
|
|
10
|
+
from transforms.api._decorators import (
|
|
11
|
+
incremental,
|
|
12
|
+
lightweight,
|
|
13
|
+
transform,
|
|
14
|
+
transform_df,
|
|
15
|
+
transform_pandas,
|
|
16
|
+
transform_polars,
|
|
17
|
+
)
|
|
18
|
+
from transforms.api._transform import TransformContext
|
|
19
|
+
|
|
20
|
+
__all__ = (
|
|
21
|
+
"Input",
|
|
22
|
+
"Output",
|
|
23
|
+
"Markings",
|
|
24
|
+
"OrgMarkings",
|
|
25
|
+
"transform_df",
|
|
26
|
+
"transform_pandas",
|
|
27
|
+
"transform",
|
|
28
|
+
"lightweight",
|
|
29
|
+
"transform_polars",
|
|
30
|
+
"TransformContext",
|
|
31
|
+
"incremental",
|
|
32
|
+
"configure",
|
|
33
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""The exposed Function definitions and docstrings is Copyright © 2023 Palantir Technologies Inc. and/or affiliates (“Palantir”). All rights reserved.
|
|
2
|
+
|
|
3
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/
|
|
4
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-classes/
|
|
5
|
+
|
|
6
|
+
""" # noqa: E501
|
|
7
|
+
|
|
8
|
+
import warnings
|
|
9
|
+
from collections.abc import Callable
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def configure(*args, **kwargs) -> Callable: # noqa: ARG001
|
|
13
|
+
"""Not implemented in local.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
args: unused
|
|
17
|
+
kwargs: unused
|
|
18
|
+
|
|
19
|
+
Returns:
|
|
20
|
+
the compute_func
|
|
21
|
+
"""
|
|
22
|
+
warnings.warn("@configure functionality not implemented in Foundry DevTools")
|
|
23
|
+
|
|
24
|
+
def _configure(compute_func: Callable) -> Callable:
|
|
25
|
+
return compute_func
|
|
26
|
+
|
|
27
|
+
return _configure
|
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
"""The exposed Function definitions and docstrings is Copyright © 2023 Palantir Technologies Inc. and/or affiliates (“Palantir”). All rights reserved.
|
|
2
|
+
|
|
3
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/
|
|
4
|
+
https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-classes/
|
|
5
|
+
|
|
6
|
+
""" # noqa: E501
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import inspect
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import warnings
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import TYPE_CHECKING
|
|
16
|
+
|
|
17
|
+
from foundry_dev_tools.errors.dataset import BranchNotFoundError, DatasetHasNoSchemaError, DatasetHasNoTransactionsError
|
|
18
|
+
from foundry_dev_tools.utils.misc import is_dataset_a_view
|
|
19
|
+
from foundry_dev_tools.utils.repo import get_branch
|
|
20
|
+
from transforms.errors import UninitializedInputError
|
|
21
|
+
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from typing import Any, Self
|
|
24
|
+
|
|
25
|
+
import pyspark
|
|
26
|
+
import pyspark.sql
|
|
27
|
+
|
|
28
|
+
from foundry_dev_tools.config.context import FoundryContext
|
|
29
|
+
from foundry_dev_tools.utils import api_types
|
|
30
|
+
|
|
31
|
+
LOGGER = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _as_list(list_or_single_item: list[Any] | Any | None) -> list[Any]: # noqa: ANN401
|
|
35
|
+
"""Helper function turning single values or None into lists.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
list_or_single_item (List[Any] | Any | None): item or list to return as a list
|
|
39
|
+
|
|
40
|
+
Returns:
|
|
41
|
+
list:
|
|
42
|
+
either the single item as a list, or the list passed in list_or_single_item
|
|
43
|
+
|
|
44
|
+
"""
|
|
45
|
+
if not list_or_single_item:
|
|
46
|
+
return []
|
|
47
|
+
|
|
48
|
+
return list_or_single_item if isinstance(list_or_single_item, list) else [list_or_single_item]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class Input:
|
|
52
|
+
"""Specification of a transform dataset input.
|
|
53
|
+
|
|
54
|
+
Some API requests may be sent when the Input class is constructed. However, the actual download
|
|
55
|
+
is only initiated when dataframe() or get_local_path_to_dataset() is called.
|
|
56
|
+
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(
|
|
60
|
+
self,
|
|
61
|
+
alias: api_types.DatasetRid | api_types.FoundryPath,
|
|
62
|
+
branch: api_types.DatasetBranch | None = None,
|
|
63
|
+
description: str | None = None, # noqa: ARG002
|
|
64
|
+
stop_propagating=None, # noqa: ARG002,ANN001
|
|
65
|
+
stop_requiring=None, # noqa: ARG002,ANN001
|
|
66
|
+
checks=None, # noqa: ARG002,ANN001
|
|
67
|
+
):
|
|
68
|
+
"""Specification of a transform dataset input.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
alias (str | None): Dataset rid or the absolute Compass path of the dataset.
|
|
72
|
+
If not specified, parameter is unbound.
|
|
73
|
+
branch (str | None): Branch name to resolve the input dataset to.
|
|
74
|
+
If not specified, resolved at build-time.
|
|
75
|
+
stop_propagating (Markings | None): not implemented in Foundry DevTools
|
|
76
|
+
stop_requiring (OrgMarkings | None): not implemented in Foundry DevTools
|
|
77
|
+
checks (List[Check], Check): not implemented in foundry-dev-tools
|
|
78
|
+
description (str): not implemented in foundry-dev-tools
|
|
79
|
+
|
|
80
|
+
"""
|
|
81
|
+
# extract caller filename to retrieve git information
|
|
82
|
+
caller_filename = inspect.stack()[1].filename
|
|
83
|
+
LOGGER.debug("Input instantiated from %s", caller_filename)
|
|
84
|
+
self._spark_df = None
|
|
85
|
+
if branch is None:
|
|
86
|
+
branch = get_branch(Path(caller_filename))
|
|
87
|
+
self._alias = alias
|
|
88
|
+
self.branch = branch
|
|
89
|
+
self._initialized = False
|
|
90
|
+
|
|
91
|
+
def init_input(self, context: FoundryContext) -> Self:
|
|
92
|
+
self.context = context
|
|
93
|
+
if self._is_online:
|
|
94
|
+
(
|
|
95
|
+
self._is_spark_df_retrievable,
|
|
96
|
+
self._dataset_identity,
|
|
97
|
+
self.branch,
|
|
98
|
+
) = self._online(self._alias, self.branch)
|
|
99
|
+
else:
|
|
100
|
+
(
|
|
101
|
+
self._is_spark_df_retrievable,
|
|
102
|
+
self._dataset_identity,
|
|
103
|
+
self.branch,
|
|
104
|
+
) = self._offline(self._alias, self.branch)
|
|
105
|
+
self._initialized = True
|
|
106
|
+
return self
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def _is_online(self) -> bool:
|
|
110
|
+
return not self.context.config.transforms_freeze_cache
|
|
111
|
+
|
|
112
|
+
def _online(
|
|
113
|
+
self,
|
|
114
|
+
alias: api_types.DatasetRid | api_types.FoundryPath,
|
|
115
|
+
branch: api_types.DatasetBranch,
|
|
116
|
+
) -> tuple[bool, api_types.DatasetIdentity, api_types.DatasetBranch]:
|
|
117
|
+
try:
|
|
118
|
+
dataset_identity = self.context.foundry_rest_client.get_dataset_identity(alias, branch)
|
|
119
|
+
except BranchNotFoundError:
|
|
120
|
+
LOGGER.debug(
|
|
121
|
+
"Dataset %s not found on branch %s, falling back to dataset from master.",
|
|
122
|
+
alias,
|
|
123
|
+
branch,
|
|
124
|
+
)
|
|
125
|
+
branch = "master"
|
|
126
|
+
dataset_identity = self.context.foundry_rest_client.get_dataset_identity(alias, branch)
|
|
127
|
+
if dataset_identity["last_transaction_rid"] is None:
|
|
128
|
+
raise DatasetHasNoTransactionsError(dataset=alias)
|
|
129
|
+
if self._dataset_has_schema(dataset_identity, branch):
|
|
130
|
+
return (
|
|
131
|
+
True,
|
|
132
|
+
dataset_identity,
|
|
133
|
+
branch,
|
|
134
|
+
)
|
|
135
|
+
LOGGER.debug(
|
|
136
|
+
"Dataset rid: %s, path: %s on branch %s has no schema, "
|
|
137
|
+
"falling back to file download. "
|
|
138
|
+
"Only filesystem() is supported with this dataset.",
|
|
139
|
+
dataset_identity["dataset_rid"],
|
|
140
|
+
dataset_identity["dataset_path"],
|
|
141
|
+
branch,
|
|
142
|
+
)
|
|
143
|
+
return False, dataset_identity, branch
|
|
144
|
+
|
|
145
|
+
def _offline(self, alias: str, branch: str) -> tuple[bool, api_types.DatasetIdentity, str]:
|
|
146
|
+
dataset_identity = self.context.cached_foundry_client.cache.get_dataset_identity_not_branch_aware(
|
|
147
|
+
alias,
|
|
148
|
+
)
|
|
149
|
+
if self.context.cached_foundry_client.cache.dataset_has_schema(dataset_identity):
|
|
150
|
+
return True, dataset_identity, branch
|
|
151
|
+
return False, dataset_identity, branch
|
|
152
|
+
|
|
153
|
+
def _dataset_has_schema(
|
|
154
|
+
self,
|
|
155
|
+
dataset_identity: api_types.DatasetIdentity,
|
|
156
|
+
branch: api_types.DatasetBranch,
|
|
157
|
+
) -> bool | None:
|
|
158
|
+
try:
|
|
159
|
+
self.context.foundry_rest_client.get_dataset_schema(
|
|
160
|
+
dataset_identity["dataset_rid"],
|
|
161
|
+
dataset_identity["last_transaction_rid"],
|
|
162
|
+
branch,
|
|
163
|
+
)
|
|
164
|
+
except DatasetHasNoSchemaError:
|
|
165
|
+
return False
|
|
166
|
+
else:
|
|
167
|
+
return True
|
|
168
|
+
|
|
169
|
+
def _retrieve_spark_df(
|
|
170
|
+
self,
|
|
171
|
+
dataset_identity: api_types.DatasetIdentity,
|
|
172
|
+
branch: api_types.DatasetBranch,
|
|
173
|
+
) -> pyspark.sql.DataFrame:
|
|
174
|
+
if dataset_identity in list(self.context.cached_foundry_client.cache.keys()):
|
|
175
|
+
return self._retrieve_from_cache(dataset_identity, branch)
|
|
176
|
+
return self._retrieve_from_foundry_and_cache(dataset_identity, branch)
|
|
177
|
+
|
|
178
|
+
def _retrieve_from_cache(
|
|
179
|
+
self,
|
|
180
|
+
dataset_identity: api_types.DatasetIdentity,
|
|
181
|
+
branch: api_types.DatasetBranch,
|
|
182
|
+
) -> pyspark.sql.DataFrame:
|
|
183
|
+
LOGGER.debug("Returning data for %s on branch %s from cache", dataset_identity, branch)
|
|
184
|
+
return self.context.cached_foundry_client.cache[dataset_identity]
|
|
185
|
+
|
|
186
|
+
def _read_spark_df_with_sql_query(
|
|
187
|
+
self,
|
|
188
|
+
dataset_path: api_types.FoundryPath,
|
|
189
|
+
branch: api_types.DatasetBranch = "master",
|
|
190
|
+
) -> pyspark.sql.DataFrame:
|
|
191
|
+
query = f"SELECT * FROM `{dataset_path}`" # noqa: S608
|
|
192
|
+
if self.context.config.transforms_sql_sample_select_random:
|
|
193
|
+
query = query + " ORDER BY RAND()"
|
|
194
|
+
query = query + f" LIMIT {self.context.config.transforms_sql_sample_row_limit}"
|
|
195
|
+
LOGGER.debug("Executing Foundry/SparkSQL Query: %s \n on branch %s", query, branch)
|
|
196
|
+
return self.context.foundry_rest_client.query_foundry_sql(
|
|
197
|
+
query,
|
|
198
|
+
branch=branch,
|
|
199
|
+
return_type="spark",
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
def _retrieve_from_foundry_and_cache(
|
|
203
|
+
self,
|
|
204
|
+
dataset_identity: api_types.DatasetIdentity,
|
|
205
|
+
branch: str,
|
|
206
|
+
) -> pyspark.sql.DataFrame:
|
|
207
|
+
LOGGER.debug("Caching data for %s on branch %s", dataset_identity, branch)
|
|
208
|
+
transaction = dataset_identity["last_transaction"]["transaction"]
|
|
209
|
+
if is_dataset_a_view(transaction):
|
|
210
|
+
foundry_stats = self.context.foundry_rest_client.foundry_stats(
|
|
211
|
+
dataset_identity["dataset_rid"],
|
|
212
|
+
dataset_identity["last_transaction"]["rid"],
|
|
213
|
+
)
|
|
214
|
+
size_in_bytes = int(foundry_stats["computedDatasetStats"]["sizeInBytes"])
|
|
215
|
+
else:
|
|
216
|
+
size_in_bytes = transaction["metadata"]["totalFileSize"]
|
|
217
|
+
size_in_mega_bytes = size_in_bytes / 1024 / 1024
|
|
218
|
+
size_in_mega_bytes_rounded = round(size_in_mega_bytes, ndigits=2)
|
|
219
|
+
LOGGER.debug("Dataset has size of %s MegaBytes.", size_in_mega_bytes_rounded)
|
|
220
|
+
if (self.context.config.transforms_force_full_dataset_download) or (
|
|
221
|
+
size_in_mega_bytes < self.context.config.transforms_sql_dataset_size_threshold
|
|
222
|
+
):
|
|
223
|
+
spark_df = self.context.cached_foundry_client.load_dataset(
|
|
224
|
+
dataset_identity["dataset_rid"],
|
|
225
|
+
branch,
|
|
226
|
+
)
|
|
227
|
+
else:
|
|
228
|
+
dataset_name = dataset_identity["dataset_path"].split("/")[-1]
|
|
229
|
+
warnings.warn(
|
|
230
|
+
f"Retrieving subset ({self.context.config.transforms_sql_sample_row_limit} rows)"
|
|
231
|
+
f" of dataset '{dataset_name}'"
|
|
232
|
+
f" with rid '{dataset_identity['dataset_rid']}' "
|
|
233
|
+
f"because dataset size {size_in_mega_bytes_rounded} megabytes >= "
|
|
234
|
+
f"{self.context.config.transforms_sql_dataset_size_threshold} megabytes "
|
|
235
|
+
f"(as defined in config['transforms_sql_dataset_size_threshold']).",
|
|
236
|
+
)
|
|
237
|
+
spark_df = self._read_spark_df_with_sql_query(dataset_identity["dataset_path"], branch)
|
|
238
|
+
self.context.cached_foundry_client.cache[dataset_identity] = spark_df
|
|
239
|
+
return spark_df
|
|
240
|
+
|
|
241
|
+
def dataframe(self) -> pyspark.sql.DataFrame | None:
|
|
242
|
+
"""Get the cached :external+spark:py:class:`~pyspark.sql.DataFrame` of this Input.
|
|
243
|
+
|
|
244
|
+
Only available if the input has a schema. The Spark DataFrame will get loaded the first
|
|
245
|
+
time this method is invoked.
|
|
246
|
+
|
|
247
|
+
Returns:
|
|
248
|
+
:external+spark:py:class:`~pyspark.sql.DataFrame`: The cached DataFrame of this Input
|
|
249
|
+
|
|
250
|
+
"""
|
|
251
|
+
if not self._initialized:
|
|
252
|
+
raise UninitializedInputError
|
|
253
|
+
if self._is_spark_df_retrievable and self._spark_df is None:
|
|
254
|
+
if self._is_online:
|
|
255
|
+
self._spark_df = self._retrieve_spark_df(self._dataset_identity, self.branch)
|
|
256
|
+
else:
|
|
257
|
+
self._spark_df = self.context.cached_foundry_client.cache[self._dataset_identity]
|
|
258
|
+
|
|
259
|
+
return self._spark_df
|
|
260
|
+
|
|
261
|
+
def get_dataset_identity(self) -> api_types.DatasetIdentity:
|
|
262
|
+
"""Returns identity of this Input.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
dict:
|
|
266
|
+
with the keys dataset_path, dataset_rid, last_transaction_rid
|
|
267
|
+
|
|
268
|
+
"""
|
|
269
|
+
if not self._initialized:
|
|
270
|
+
raise UninitializedInputError
|
|
271
|
+
return self._dataset_identity
|
|
272
|
+
|
|
273
|
+
def get_local_path_to_dataset(self) -> str:
|
|
274
|
+
"""Returns path to the dataset's files on disk.
|
|
275
|
+
|
|
276
|
+
Calling this method for the first time may trigger downloading the dataset files.
|
|
277
|
+
|
|
278
|
+
Returns:
|
|
279
|
+
str:
|
|
280
|
+
path to the dataset's files on disk
|
|
281
|
+
|
|
282
|
+
"""
|
|
283
|
+
if not self._initialized:
|
|
284
|
+
raise UninitializedInputError
|
|
285
|
+
return os.fspath(
|
|
286
|
+
self.context.cached_foundry_client._fetch_dataset( # noqa: SLF001
|
|
287
|
+
self._dataset_identity,
|
|
288
|
+
self.branch,
|
|
289
|
+
)
|
|
290
|
+
if self._is_online
|
|
291
|
+
else self.context.cached_foundry_client.cache.get_path_to_local_dataset(
|
|
292
|
+
self._dataset_identity,
|
|
293
|
+
),
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
class Output:
|
|
298
|
+
"""Specification of a transform dataset output.
|
|
299
|
+
|
|
300
|
+
Writing the Output back to Foundry is not implemented.
|
|
301
|
+
|
|
302
|
+
"""
|
|
303
|
+
|
|
304
|
+
def __init__(
|
|
305
|
+
self,
|
|
306
|
+
alias: str | None = None,
|
|
307
|
+
sever_permissions: bool | None = False, # noqa: ARG002
|
|
308
|
+
description: str | None = None, # noqa: ARG002
|
|
309
|
+
checks=None, # noqa: ANN001,ARG002
|
|
310
|
+
):
|
|
311
|
+
"""Specification of a transform output.
|
|
312
|
+
|
|
313
|
+
Args:
|
|
314
|
+
alias (str | None): Dataset rid or the absolute Compass path of the dataset.
|
|
315
|
+
If not specified, parameter is unbound.
|
|
316
|
+
sever_permissions (bool | None): not implemented in foundry-dev-tools
|
|
317
|
+
description (str | None): not implemented in foundry-dev-tools
|
|
318
|
+
checks (List[Check], Check): not implemented in foundry-dev-tools
|
|
319
|
+
"""
|
|
320
|
+
self.alias = alias
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class UnmarkingDef:
|
|
324
|
+
"""Base class for unmarking datasets configuration."""
|
|
325
|
+
|
|
326
|
+
def __init__(self, marking_ids: list[str] | str, on_branches: list[str] | str | None):
|
|
327
|
+
"""Default constructor.
|
|
328
|
+
|
|
329
|
+
Args:
|
|
330
|
+
marking_ids (List[str], str): List of marking identifiers or single marking identifier.
|
|
331
|
+
on_branches (List[str], str): Branch on which to apply unmarking.
|
|
332
|
+
"""
|
|
333
|
+
self.marking_ids = _as_list(marking_ids)
|
|
334
|
+
self.branches = _as_list(on_branches)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
class Markings(UnmarkingDef):
|
|
338
|
+
"""Specification of a marking that stops propagating from input.
|
|
339
|
+
|
|
340
|
+
The actual marking removal is not implemented.
|
|
341
|
+
"""
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class OrgMarkings(UnmarkingDef):
|
|
345
|
+
"""Specification of a marking that is no longer required on the output.
|
|
346
|
+
|
|
347
|
+
The actual marking requirement check is not implemented.
|
|
348
|
+
"""
|