foundry-dev-tools 1.0.6__tar.gz → 1.0.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/workflows/ci.yml +3 -3
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/workflows/docs.yml +1 -1
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/PKG-INFO +16 -7
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/README.md +14 -6
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/changelog.md +37 -1
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/installation.md +7 -1
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/usage_and_examples.md +2 -2
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/setup.cfg +1 -1
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/cached_foundry_client.py +5 -4
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/foundry_api_client.py +0 -23
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/fsspec_impl.py +8 -2
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/PKG-INFO +16 -7
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_transform.py +1 -1
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_fsspec_impl.py +40 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.coveragerc +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/blank_issue.yml +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/pull_request_template.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.gitignore +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.pre-commit-config.yaml +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/LICENSE +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/Configuration_usage.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/FoundryFileSystem_usage.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/FoundryRestClient_usage.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/SSO_usage.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/_static/.gitignore +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/architecture.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/conf.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/contributors.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/develop.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/index.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/license.md +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-already-cached-dark.svg +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-already-cached-light.svg +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-new-transaction-dark.svg +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-new-transaction-light.svg +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/tpa_config.png +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/requirements.txt +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/pyproject.toml +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/setup.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/config.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/importer.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/spark.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/token_provider/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/token_provider/foundry_token_provider.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/SOURCES.txt +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/dependency_links.txt +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/entry_points.txt +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/not-zip-safe +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/requires.txt +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/top_level.txt +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_configure.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_dataset.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_decorators.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/__init__.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/conftest.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/foundry_mock_client.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_cached_foundry_client.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_cached_foundry_client_integration.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_config.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_data/binary_dataset/bin +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_data/iris/iris.csv +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_api.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_local_deprecation.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_mock.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_spark_converters.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_sql_client.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_importer.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_multipass_tpa.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_spark_caches.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_token_provider.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_transforms.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_transforms_integration.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/utils.py +0 -0
- {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tox.ini +0 -0
|
@@ -28,7 +28,7 @@ jobs:
|
|
|
28
28
|
with: {fetch-depth: 0} # deep clone for setuptools-scm
|
|
29
29
|
- uses: actions/setup-python@v4
|
|
30
30
|
id: setup-python
|
|
31
|
-
with: {python-version: "3.
|
|
31
|
+
with: {python-version: "3.11"}
|
|
32
32
|
- name: Run static analysis and format checkers
|
|
33
33
|
run: |
|
|
34
34
|
pip install .[testing,transforms]
|
|
@@ -55,7 +55,7 @@ jobs:
|
|
|
55
55
|
matrix:
|
|
56
56
|
python:
|
|
57
57
|
- "3.8"
|
|
58
|
-
- "3.
|
|
58
|
+
- "3.11" # newest Python that is supported by pyspark
|
|
59
59
|
runs-on: ubuntu-latest
|
|
60
60
|
steps:
|
|
61
61
|
- uses: actions/checkout@v3
|
|
@@ -79,7 +79,7 @@ jobs:
|
|
|
79
79
|
steps:
|
|
80
80
|
- uses: actions/checkout@v3
|
|
81
81
|
- uses: actions/setup-python@v4
|
|
82
|
-
with: {python-version: "3.
|
|
82
|
+
with: {python-version: "3.11"}
|
|
83
83
|
- name: Retrieve pre-built distribution files
|
|
84
84
|
uses: actions/download-artifact@v3
|
|
85
85
|
with: {name: python-distribution-files, path: dist/}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: foundry-dev-tools
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.8
|
|
4
4
|
Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
|
|
5
5
|
Home-page: https://github.com/emdgroup/foundry-dev-tools
|
|
6
6
|
Author: Nicolas Renkamp, Jonas Wunderlich
|
|
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.8
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.9
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
22
|
Classifier: Intended Audience :: Developers
|
|
22
23
|
Classifier: Operating System :: POSIX :: Linux
|
|
23
24
|
Classifier: Operating System :: MacOS
|
|
@@ -35,12 +36,14 @@ License-File: LICENSE
|
|
|
35
36
|
|
|
36
37
|
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/ci.yml?style=flat-square"/></img>
|
|
37
38
|
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/docs.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/docs.yml?style=flat-square"/></img>
|
|
38
|
-
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/
|
|
39
|
-
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/
|
|
40
|
-
<a href="
|
|
41
|
-
<a href="https://
|
|
42
|
-
<a href="https://
|
|
43
|
-
|
|
39
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square&label=Supported%20Python%20versions&color=%23ffb86c"/></a>
|
|
40
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square&label=PyPI%20version&color=%23bd93f9"/></a>
|
|
41
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/vn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Version&color=%23bd93f9" alt="Conda Version"/></a>
|
|
42
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/dm/foundry-dev-tools?label=PyPI%20Downloads&style=flat-square&color=%236272a4"/></a>
|
|
43
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/dn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Downloads&color=%236272a4" alt="Conda Downloads"/></a>
|
|
44
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
45
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
46
|
+
<a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square&color=%234c1"/></a>
|
|
44
47
|
<p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
|
|
45
48
|
|
|
46
49
|
<a href="https://emdgroup.github.io/foundry-dev-tools/installation.html">Installation<a/>
|
|
@@ -97,10 +100,16 @@ high level entrypoints to Foundry DevTools:
|
|
|
97
100
|
|
|
98
101
|
## Quickstart
|
|
99
102
|
|
|
103
|
+
With pip:
|
|
100
104
|
```shell
|
|
101
105
|
pip install foundry-dev-tools
|
|
102
106
|
```
|
|
103
107
|
|
|
108
|
+
With conda or mamba on the conda-forge channel:
|
|
109
|
+
```shell
|
|
110
|
+
conda install -c conda-forge foundry-dev-tools
|
|
111
|
+
```
|
|
112
|
+
|
|
104
113
|
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
|
|
105
114
|
|
|
106
115
|
## Why did we build this?
|
|
@@ -3,12 +3,14 @@
|
|
|
3
3
|
|
|
4
4
|
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/ci.yml?style=flat-square"/></img>
|
|
5
5
|
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/docs.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/docs.yml?style=flat-square"/></img>
|
|
6
|
-
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/
|
|
7
|
-
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/
|
|
8
|
-
<a href="
|
|
9
|
-
<a href="https://
|
|
10
|
-
<a href="https://
|
|
11
|
-
|
|
6
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square&label=Supported%20Python%20versions&color=%23ffb86c"/></a>
|
|
7
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square&label=PyPI%20version&color=%23bd93f9"/></a>
|
|
8
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/vn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Version&color=%23bd93f9" alt="Conda Version"/></a>
|
|
9
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/dm/foundry-dev-tools?label=PyPI%20Downloads&style=flat-square&color=%236272a4"/></a>
|
|
10
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/dn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Downloads&color=%236272a4" alt="Conda Downloads"/></a>
|
|
11
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
12
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
13
|
+
<a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square&color=%234c1"/></a>
|
|
12
14
|
<p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
|
|
13
15
|
|
|
14
16
|
<a href="https://emdgroup.github.io/foundry-dev-tools/installation.html">Installation<a/>
|
|
@@ -65,10 +67,16 @@ high level entrypoints to Foundry DevTools:
|
|
|
65
67
|
|
|
66
68
|
## Quickstart
|
|
67
69
|
|
|
70
|
+
With pip:
|
|
68
71
|
```shell
|
|
69
72
|
pip install foundry-dev-tools
|
|
70
73
|
```
|
|
71
74
|
|
|
75
|
+
With conda or mamba on the conda-forge channel:
|
|
76
|
+
```shell
|
|
77
|
+
conda install -c conda-forge foundry-dev-tools
|
|
78
|
+
```
|
|
79
|
+
|
|
72
80
|
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
|
|
73
81
|
|
|
74
82
|
## Why did we build this?
|
|
@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog],
|
|
6
6
|
and this project adheres to [Semantic Versioning].
|
|
7
7
|
|
|
8
|
+
## [1.0.8] - 2023-04-17
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- python 3.11 support, as pyspark 3.4 gained support for it (#13)
|
|
13
|
+
- conda-forge badges to the README (#13)
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- typo in docs (#12)
|
|
18
|
+
|
|
19
|
+
### Removed
|
|
20
|
+
|
|
21
|
+
- pandas<2 restriction, as pyspark 3.4 supports it (#13)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
## [1.0.7] - 2023-04-05
|
|
25
|
+
|
|
26
|
+
### Added
|
|
27
|
+
|
|
28
|
+
- skip_instance_cache kwarg to FoundryFileSystem, which gets passed to fsspec
|
|
29
|
+
it should be set to True in a multithreaded environment (e.g. Streamlit),
|
|
30
|
+
as the same filesystem instance gets reused by default,
|
|
31
|
+
which resulted in weird behaviour. (#11)
|
|
32
|
+
- an extra check, which may prevent sending a bad request to foundry
|
|
33
|
+
when skip_instance_cache is not used (#11)
|
|
34
|
+
|
|
35
|
+
### Changed
|
|
36
|
+
|
|
37
|
+
- remove pandas from top level import, which should speed up the initial import (#11)
|
|
38
|
+
- restrict pandas to version less than 2,
|
|
39
|
+
as pyspark is currently not compatible with version 2 (#11)
|
|
40
|
+
|
|
8
41
|
## [1.0.6] - 2023-03-31
|
|
9
42
|
|
|
10
43
|
### Changed
|
|
@@ -63,9 +96,12 @@ and this project adheres to [Semantic Versioning].
|
|
|
63
96
|
|
|
64
97
|
[Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
|
|
65
98
|
[Semantic Versioning]: https://semver.org/spec/v2.0.0.html
|
|
99
|
+
[1.0.8]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.7...v1.0.8
|
|
100
|
+
[1.0.7]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.6...v1.0.7
|
|
101
|
+
[1.0.6]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.5...v1.0.6
|
|
66
102
|
[1.0.5]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.4...v1.0.5
|
|
67
103
|
[1.0.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.3...v1.0.4
|
|
68
104
|
[1.0.3]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.2...v1.0.3
|
|
69
105
|
[1.0.2]: https://github.com/emdgroup/foundry-dev-tools/releases/tag/v1.0.2
|
|
70
106
|
[1.0.1]: https://github.com/emdgroup/foundry-dev-tools
|
|
71
|
-
[1.0]: https://github.com/emdgroup/foundry-dev-tools
|
|
107
|
+
[1.0]: https://github.com/emdgroup/foundry-dev-tools
|
|
@@ -33,10 +33,16 @@ jwt=<paste your foundry token here>
|
|
|
33
33
|
We recommend using a new [conda environment] or [python environment],
|
|
34
34
|
after you activated it, just run:
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
With pip:
|
|
37
|
+
```shell
|
|
37
38
|
pip install foundry-dev-tools
|
|
38
39
|
```
|
|
39
40
|
|
|
41
|
+
With conda or mamba on the conda-forge channel:
|
|
42
|
+
```shell
|
|
43
|
+
conda install -c conda-forge foundry-dev-tools
|
|
44
|
+
```
|
|
45
|
+
|
|
40
46
|
or, if you have the repository locally available and want to tinker with the source code
|
|
41
47
|
you could install it in "editable"/"develop mode". Run the following command in
|
|
42
48
|
the Foundry DevTools root directory:
|
|
@@ -51,8 +51,8 @@ for package name and package version:
|
|
|
51
51
|
```diff
|
|
52
52
|
- name=os.environ['PKG_NAME'],
|
|
53
53
|
+ name=os.getenv('PKG_NAME', 'your-package-name'),
|
|
54
|
-
-
|
|
55
|
-
+
|
|
54
|
+
- version=os.environ['PKG_VERSION'],
|
|
55
|
+
+ version=os.getenv('PKG_VERSION', 'your-package-version'),
|
|
56
56
|
```
|
|
57
57
|
|
|
58
58
|
Now you can install the repository into your local environment, when in directory `transform-python/src`:
|
|
@@ -23,6 +23,7 @@ classifiers =
|
|
|
23
23
|
Programming Language :: Python :: 3.8
|
|
24
24
|
Programming Language :: Python :: 3.9
|
|
25
25
|
Programming Language :: Python :: 3.10
|
|
26
|
+
Programming Language :: Python :: 3.11
|
|
26
27
|
Intended Audience :: Developers
|
|
27
28
|
Operating System :: POSIX :: Linux
|
|
28
29
|
Operating System :: MacOS
|
|
@@ -75,7 +76,6 @@ extras = True
|
|
|
75
76
|
|
|
76
77
|
[tool:pytest]
|
|
77
78
|
addopts =
|
|
78
|
-
--cov=src --cov-report term --cov-report xml:coverage.xml --cov-report html:htmlcov
|
|
79
79
|
--verbose
|
|
80
80
|
norecursedirs =
|
|
81
81
|
dist
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/cached_foundry_client.py
RENAMED
|
@@ -6,8 +6,6 @@ import tempfile
|
|
|
6
6
|
import time
|
|
7
7
|
from typing import Tuple, Union
|
|
8
8
|
|
|
9
|
-
import pandas as pd
|
|
10
|
-
|
|
11
9
|
import foundry_dev_tools
|
|
12
10
|
from foundry_dev_tools.foundry_api_client import (
|
|
13
11
|
BranchNotFoundError,
|
|
@@ -175,13 +173,13 @@ class CachedFoundryClient:
|
|
|
175
173
|
|
|
176
174
|
def save_dataset(
|
|
177
175
|
self,
|
|
178
|
-
df: Union[pd.DataFrame, "pyspark.sql.DataFrame"],
|
|
176
|
+
df: Union["pd.DataFrame", "pyspark.sql.DataFrame"],
|
|
179
177
|
dataset_path_or_rid: str,
|
|
180
178
|
branch: str = "master",
|
|
181
179
|
exists_ok: bool = False,
|
|
182
180
|
mode: str = "SNAPSHOT",
|
|
183
181
|
) -> Tuple[str, str]:
|
|
184
|
-
# pylint: disable=invalid-name,too-many-arguments
|
|
182
|
+
# pylint: disable=invalid-name,too-many-arguments,too-many-locals
|
|
185
183
|
"""Saves a dataframe to Foundry. If the dataset in Foundry does not exist it is created.
|
|
186
184
|
|
|
187
185
|
If the branch does not exist, it is created. If the dataset exists, an exception is thrown.
|
|
@@ -218,6 +216,9 @@ class CachedFoundryClient:
|
|
|
218
216
|
raise ValueError("Please provide a dataset branch with parameter 'branch'")
|
|
219
217
|
|
|
220
218
|
with tempfile.TemporaryDirectory() as path:
|
|
219
|
+
# pylint: disable=import-outside-toplevel
|
|
220
|
+
import pandas as pd
|
|
221
|
+
|
|
221
222
|
if isinstance(df, pd.DataFrame):
|
|
222
223
|
df.to_parquet(
|
|
223
224
|
os.sep.join([path + "/dataset.parquet"]),
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/foundry_api_client.py
RENAMED
|
@@ -1324,8 +1324,6 @@ class FoundryRestClient:
|
|
|
1324
1324
|
|
|
1325
1325
|
"""
|
|
1326
1326
|
assert return_type in {"pandas", "arrow", "spark"}
|
|
1327
|
-
_assert_pyarrow_packages_available()
|
|
1328
|
-
_assert_pandas_packages_available()
|
|
1329
1327
|
foundry_sql_client = FoundrySqlClient(config=self._config, branch=branch)
|
|
1330
1328
|
try:
|
|
1331
1329
|
return foundry_sql_client.query(query=query, return_type=return_type)
|
|
@@ -1696,7 +1694,6 @@ class FoundrySqlClient:
|
|
|
1696
1694
|
branch (str): default = master, all queries will be executed against this default branch
|
|
1697
1695
|
|
|
1698
1696
|
"""
|
|
1699
|
-
_assert_pyarrow_packages_available()
|
|
1700
1697
|
self._config = foundry_dev_tools.Configuration.get_config(config)
|
|
1701
1698
|
self._requests_verify_value = _determine_requests_verify_value(self._config)
|
|
1702
1699
|
self.foundry_sql_server_api = (
|
|
@@ -1899,26 +1896,6 @@ def _transform_bad_request_response_to_exception(response):
|
|
|
1899
1896
|
raise DatasetNotFoundError("SQL")
|
|
1900
1897
|
|
|
1901
1898
|
|
|
1902
|
-
def _assert_pyarrow_packages_available():
|
|
1903
|
-
try:
|
|
1904
|
-
# pylint: disable=import-outside-toplevel,unused-import
|
|
1905
|
-
import pyarrow
|
|
1906
|
-
except ImportError as err:
|
|
1907
|
-
raise ValueError(
|
|
1908
|
-
"Please install package 'pyarrow' to use SQL functionality."
|
|
1909
|
-
) from err
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
def _assert_pandas_packages_available():
|
|
1913
|
-
try:
|
|
1914
|
-
# pylint: disable=import-outside-toplevel,unused-import
|
|
1915
|
-
import pandas as pd
|
|
1916
|
-
except ImportError as err:
|
|
1917
|
-
raise ValueError(
|
|
1918
|
-
"Please install package 'pandas' to use SQL functionality."
|
|
1919
|
-
) from err
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
1899
|
def _is_palantir_oauth_client_installed():
|
|
1923
1900
|
try:
|
|
1924
1901
|
# pylint: disable=import-outside-toplevel,unused-import,import-error
|
|
@@ -25,6 +25,8 @@ class FoundryFileSystem(AbstractFileSystem): # noqa
|
|
|
25
25
|
or using SSO.
|
|
26
26
|
transaction_backoff (bool): if FoundryFileSystem should retry to open a transaction, in case one is already
|
|
27
27
|
open. The default value is 'True' and FoundryFileSystem will retry 60 seconds to open a transaction.
|
|
28
|
+
skip_instance_cache (bool): gets passed to FoundryFileSystem,
|
|
29
|
+
in multithreaded environments you want to set this to true.
|
|
28
30
|
**kwargs: passed to underlying fsspec.AbstractFileSystem
|
|
29
31
|
"""
|
|
30
32
|
|
|
@@ -36,9 +38,11 @@ class FoundryFileSystem(AbstractFileSystem): # noqa
|
|
|
36
38
|
branch: str = DEFAULT_BRANCH,
|
|
37
39
|
token: str = None,
|
|
38
40
|
transaction_backoff: bool = True,
|
|
41
|
+
skip_instance_cache: bool = False,
|
|
39
42
|
**kwargs,
|
|
40
43
|
):
|
|
41
|
-
|
|
44
|
+
# pylint: disable=too-many-arguments
|
|
45
|
+
super().__init__(skip_instance_cache=skip_instance_cache, **kwargs)
|
|
42
46
|
self.token = token
|
|
43
47
|
self.dataset_identity = self.api.get_dataset_identity(
|
|
44
48
|
dataset_path_or_rid=dataset, branch=branch
|
|
@@ -310,7 +314,9 @@ class FoundryFile(AbstractBufferedFile):
|
|
|
310
314
|
"""Internal function to add a chunk of data to a started upload."""
|
|
311
315
|
assert final is True, "chunked uploading not supported"
|
|
312
316
|
self.buffer.seek(0)
|
|
313
|
-
|
|
317
|
+
assert (
|
|
318
|
+
self.fs._transaction is not None # pylint: disable=protected-access
|
|
319
|
+
), "multiple threads, same dataset? use skip_instance_cache"
|
|
314
320
|
self.fs.api.upload_dataset_file(
|
|
315
321
|
dataset_rid=self.fs.dataset_identity["dataset_rid"],
|
|
316
322
|
transaction_rid=self.fs._transaction.transaction_rid, # pylint: disable=protected-access
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: foundry-dev-tools
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.8
|
|
4
4
|
Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
|
|
5
5
|
Home-page: https://github.com/emdgroup/foundry-dev-tools
|
|
6
6
|
Author: Nicolas Renkamp, Jonas Wunderlich
|
|
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.8
|
|
19
19
|
Classifier: Programming Language :: Python :: 3.9
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
22
|
Classifier: Intended Audience :: Developers
|
|
22
23
|
Classifier: Operating System :: POSIX :: Linux
|
|
23
24
|
Classifier: Operating System :: MacOS
|
|
@@ -35,12 +36,14 @@ License-File: LICENSE
|
|
|
35
36
|
|
|
36
37
|
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/ci.yml?style=flat-square"/></img>
|
|
37
38
|
<a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/docs.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/docs.yml?style=flat-square"/></img>
|
|
38
|
-
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/
|
|
39
|
-
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/
|
|
40
|
-
<a href="
|
|
41
|
-
<a href="https://
|
|
42
|
-
<a href="https://
|
|
43
|
-
|
|
39
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square&label=Supported%20Python%20versions&color=%23ffb86c"/></a>
|
|
40
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square&label=PyPI%20version&color=%23bd93f9"/></a>
|
|
41
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/vn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Version&color=%23bd93f9" alt="Conda Version"/></a>
|
|
42
|
+
<a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/dm/foundry-dev-tools?label=PyPI%20Downloads&style=flat-square&color=%236272a4"/></a>
|
|
43
|
+
<a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/dn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Downloads&color=%236272a4" alt="Conda Downloads"/></a>
|
|
44
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
45
|
+
<a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
|
|
46
|
+
<a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square&color=%234c1"/></a>
|
|
44
47
|
<p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
|
|
45
48
|
|
|
46
49
|
<a href="https://emdgroup.github.io/foundry-dev-tools/installation.html">Installation<a/>
|
|
@@ -97,10 +100,16 @@ high level entrypoints to Foundry DevTools:
|
|
|
97
100
|
|
|
98
101
|
## Quickstart
|
|
99
102
|
|
|
103
|
+
With pip:
|
|
100
104
|
```shell
|
|
101
105
|
pip install foundry-dev-tools
|
|
102
106
|
```
|
|
103
107
|
|
|
108
|
+
With conda or mamba on the conda-forge channel:
|
|
109
|
+
```shell
|
|
110
|
+
conda install -c conda-forge foundry-dev-tools
|
|
111
|
+
```
|
|
112
|
+
|
|
104
113
|
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
|
|
105
114
|
|
|
106
115
|
## Why did we build this?
|
|
@@ -12,7 +12,6 @@ import re
|
|
|
12
12
|
from typing import Callable, Dict, Optional
|
|
13
13
|
|
|
14
14
|
import fs
|
|
15
|
-
import pandas as pd
|
|
16
15
|
import pyspark
|
|
17
16
|
|
|
18
17
|
from transforms.api import Input, Output
|
|
@@ -121,6 +120,7 @@ class Transform:
|
|
|
121
120
|
kwargs["ctx"] = TransformContext()
|
|
122
121
|
|
|
123
122
|
output_df = self(**kwargs)
|
|
123
|
+
import pandas as pd # pylint: disable=import-outside-toplevel
|
|
124
124
|
|
|
125
125
|
if not isinstance(output_df, pd.DataFrame):
|
|
126
126
|
raise ValueError(
|
|
@@ -971,3 +971,43 @@ def test_ls_trailing_slash_empty_folder(random_file, fsspec_write_test_folder):
|
|
|
971
971
|
ls_result_no_slash = fs.ls(random_folder, detail=False)
|
|
972
972
|
ls_result_with_slash = fs.ls(random_folder + "/", detail=False)
|
|
973
973
|
assert ls_result_no_slash == ls_result_with_slash
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
@patch(
|
|
977
|
+
"foundry_dev_tools.FoundryRestClient.get_dataset_identity",
|
|
978
|
+
MagicMock(),
|
|
979
|
+
)
|
|
980
|
+
def test_skip_instance_cache():
|
|
981
|
+
fs = FoundryFileSystem(
|
|
982
|
+
dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
|
|
983
|
+
branch="master",
|
|
984
|
+
token="super-secret-token",
|
|
985
|
+
skip_instance_cache=True,
|
|
986
|
+
)
|
|
987
|
+
fs2 = FoundryFileSystem(
|
|
988
|
+
dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
|
|
989
|
+
branch="master",
|
|
990
|
+
token="super-secret-token",
|
|
991
|
+
skip_instance_cache=True,
|
|
992
|
+
)
|
|
993
|
+
|
|
994
|
+
fs3 = FoundryFileSystem(
|
|
995
|
+
dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
|
|
996
|
+
branch="master",
|
|
997
|
+
token="super-secret-token",
|
|
998
|
+
skip_instance_cache=False,
|
|
999
|
+
)
|
|
1000
|
+
fs4 = FoundryFileSystem(
|
|
1001
|
+
dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
|
|
1002
|
+
branch="master",
|
|
1003
|
+
token="super-secret-token",
|
|
1004
|
+
skip_instance_cache=False,
|
|
1005
|
+
)
|
|
1006
|
+
fs.x = 123
|
|
1007
|
+
fs2.x = 4567
|
|
1008
|
+
fs3.x = 8901
|
|
1009
|
+
fs4.x = 2345
|
|
1010
|
+
assert fs.x == 123
|
|
1011
|
+
assert fs2.x == 4567
|
|
1012
|
+
assert fs3.x == 2345
|
|
1013
|
+
assert fs4.x == 2345
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/feature_request.yml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/caches/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/SOURCES.txt
RENAMED
|
File without changes
|
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/entry_points.txt
RENAMED
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/not-zip-safe
RENAMED
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/requires.txt
RENAMED
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/top_level.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_cached_foundry_client_integration.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|