foundry-dev-tools 1.0.6__tar.gz → 1.0.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/workflows/ci.yml +3 -3
  2. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/workflows/docs.yml +1 -1
  3. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/PKG-INFO +16 -7
  4. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/README.md +14 -6
  5. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/changelog.md +37 -1
  6. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/installation.md +7 -1
  7. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/usage_and_examples.md +2 -2
  8. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/setup.cfg +1 -1
  9. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/cached_foundry_client.py +5 -4
  10. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/foundry_api_client.py +0 -23
  11. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/fsspec_impl.py +8 -2
  12. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/PKG-INFO +16 -7
  13. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_transform.py +1 -1
  14. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_fsspec_impl.py +40 -0
  15. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.coveragerc +0 -0
  16. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/blank_issue.yml +0 -0
  17. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  18. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  19. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  20. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.github/pull_request_template.md +0 -0
  21. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.gitignore +0 -0
  22. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/.pre-commit-config.yaml +0 -0
  23. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/LICENSE +0 -0
  24. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/Configuration_usage.md +0 -0
  25. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/FoundryFileSystem_usage.md +0 -0
  26. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/FoundryRestClient_usage.md +0 -0
  27. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/SSO_usage.md +0 -0
  28. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/_static/.gitignore +0 -0
  29. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/architecture.md +0 -0
  30. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/conf.py +0 -0
  31. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/contributors.md +0 -0
  32. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/develop.md +0 -0
  33. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/index.md +0 -0
  34. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/license.md +0 -0
  35. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-already-cached-dark.svg +0 -0
  36. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-already-cached-light.svg +0 -0
  37. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-new-transaction-dark.svg +0 -0
  38. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/mermaid-diagram-new-transaction-light.svg +0 -0
  39. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/pictures/tpa_config.png +0 -0
  40. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/docs/requirements.txt +0 -0
  41. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/pyproject.toml +0 -0
  42. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/setup.py +0 -0
  43. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/__init__.py +0 -0
  44. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/__init__.py +0 -0
  45. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/config.py +0 -0
  46. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/__init__.py +0 -0
  47. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  48. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  49. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  50. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  51. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  52. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/importer.py +0 -0
  53. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/spark.py +0 -0
  54. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/token_provider/__init__.py +0 -0
  55. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools/utils/token_provider/foundry_token_provider.py +0 -0
  56. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/SOURCES.txt +0 -0
  57. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/dependency_links.txt +0 -0
  58. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/entry_points.txt +0 -0
  59. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/not-zip-safe +0 -0
  60. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/requires.txt +0 -0
  61. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/foundry_dev_tools.egg-info/top_level.txt +0 -0
  62. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/__init__.py +0 -0
  63. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/__init__.py +0 -0
  64. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_configure.py +0 -0
  65. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_dataset.py +0 -0
  66. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/src/transforms/api/_decorators.py +0 -0
  67. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/__init__.py +0 -0
  68. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/conftest.py +0 -0
  69. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/foundry_mock_client.py +0 -0
  70. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_cached_foundry_client.py +0 -0
  71. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_cached_foundry_client_integration.py +0 -0
  72. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_config.py +0 -0
  73. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_data/binary_dataset/bin +0 -0
  74. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_data/iris/iris.csv +0 -0
  75. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_api.py +0 -0
  76. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_local_deprecation.py +0 -0
  77. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_mock.py +0 -0
  78. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_spark_converters.py +0 -0
  79. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_foundry_sql_client.py +0 -0
  80. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_importer.py +0 -0
  81. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_multipass_tpa.py +0 -0
  82. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_spark_caches.py +0 -0
  83. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_token_provider.py +0 -0
  84. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_transforms.py +0 -0
  85. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/test_transforms_integration.py +0 -0
  86. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tests/utils.py +0 -0
  87. {foundry-dev-tools-1.0.6 → foundry-dev-tools-1.0.8}/tox.ini +0 -0
@@ -28,7 +28,7 @@ jobs:
28
28
  with: {fetch-depth: 0} # deep clone for setuptools-scm
29
29
  - uses: actions/setup-python@v4
30
30
  id: setup-python
31
- with: {python-version: "3.10"}
31
+ with: {python-version: "3.11"}
32
32
  - name: Run static analysis and format checkers
33
33
  run: |
34
34
  pip install .[testing,transforms]
@@ -55,7 +55,7 @@ jobs:
55
55
  matrix:
56
56
  python:
57
57
  - "3.8"
58
- - "3.10" # newest Python that is supported by pyspark
58
+ - "3.11" # newest Python that is supported by pyspark
59
59
  runs-on: ubuntu-latest
60
60
  steps:
61
61
  - uses: actions/checkout@v3
@@ -79,7 +79,7 @@ jobs:
79
79
  steps:
80
80
  - uses: actions/checkout@v3
81
81
  - uses: actions/setup-python@v4
82
- with: {python-version: "3.10"}
82
+ with: {python-version: "3.11"}
83
83
  - name: Retrieve pre-built distribution files
84
84
  uses: actions/download-artifact@v3
85
85
  with: {name: python-distribution-files, path: dist/}
@@ -22,7 +22,7 @@ jobs:
22
22
  steps:
23
23
  - uses: actions/checkout@v3
24
24
  - uses: actions/setup-python@v4
25
- with: {python-version: "3.10"}
25
+ with: {python-version: "3.11"}
26
26
  - name: Build Docs
27
27
  run: >-
28
28
  pipx run tox -e docs
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.0.6
3
+ Version: 1.0.8
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Home-page: https://github.com/emdgroup/foundry-dev-tools
6
6
  Author: Nicolas Renkamp, Jonas Wunderlich
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
18
18
  Classifier: Programming Language :: Python :: 3.8
19
19
  Classifier: Programming Language :: Python :: 3.9
20
20
  Classifier: Programming Language :: Python :: 3.10
21
+ Classifier: Programming Language :: Python :: 3.11
21
22
  Classifier: Intended Audience :: Developers
22
23
  Classifier: Operating System :: POSIX :: Linux
23
24
  Classifier: Operating System :: MacOS
@@ -35,12 +36,14 @@ License-File: LICENSE
35
36
 
36
37
  <a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/ci.yml?style=flat-square"/></img>
37
38
  <a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/docs.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/docs.yml?style=flat-square"/></img>
38
- <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square"/></a>
39
- <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square"/></a>
40
- <a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square"/></a>
41
- <a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?color=important&style=flat-square"/></a>
42
- <a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?color=blueviolet&style=flat-square"/></a>
43
-
39
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square&label=Supported%20Python%20versions&color=%23ffb86c"/></a>
40
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square&label=PyPI%20version&color=%23bd93f9"/></a>
41
+ <a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/vn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Version&color=%23bd93f9" alt="Conda Version"/></a>
42
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/dm/foundry-dev-tools?label=PyPI%20Downloads&style=flat-square&color=%236272a4"/></a>
43
+ <a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/dn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Downloads&color=%236272a4" alt="Conda Downloads"/></a>
44
+ <a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
45
+ <a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
46
+ <a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square&color=%234c1"/></a>
44
47
  <p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
45
48
 
46
49
  <a href="https://emdgroup.github.io/foundry-dev-tools/installation.html">Installation<a/>
@@ -97,10 +100,16 @@ high level entrypoints to Foundry DevTools:
97
100
 
98
101
  ## Quickstart
99
102
 
103
+ With pip:
100
104
  ```shell
101
105
  pip install foundry-dev-tools
102
106
  ```
103
107
 
108
+ With conda or mamba on the conda-forge channel:
109
+ ```shell
110
+ conda install -c conda-forge foundry-dev-tools
111
+ ```
112
+
104
113
  [Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
105
114
 
106
115
  ## Why did we build this?
@@ -3,12 +3,14 @@
3
3
 
4
4
  <a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/ci.yml?style=flat-square"/></img>
5
5
  <a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/docs.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/docs.yml?style=flat-square"/></img>
6
- <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square"/></a>
7
- <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square"/></a>
8
- <a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square"/></a>
9
- <a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?color=important&style=flat-square"/></a>
10
- <a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?color=blueviolet&style=flat-square"/></a>
11
-
6
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square&label=Supported%20Python%20versions&color=%23ffb86c"/></a>
7
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square&label=PyPI%20version&color=%23bd93f9"/></a>
8
+ <a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/vn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Version&color=%23bd93f9" alt="Conda Version"/></a>
9
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/dm/foundry-dev-tools?label=PyPI%20Downloads&style=flat-square&color=%236272a4"/></a>
10
+ <a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/dn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Downloads&color=%236272a4" alt="Conda Downloads"/></a>
11
+ <a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
12
+ <a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
13
+ <a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square&color=%234c1"/></a>
12
14
  <p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
13
15
 
14
16
  <a href="https://emdgroup.github.io/foundry-dev-tools/installation.html">Installation<a/>
@@ -65,10 +67,16 @@ high level entrypoints to Foundry DevTools:
65
67
 
66
68
  ## Quickstart
67
69
 
70
+ With pip:
68
71
  ```shell
69
72
  pip install foundry-dev-tools
70
73
  ```
71
74
 
75
+ With conda or mamba on the conda-forge channel:
76
+ ```shell
77
+ conda install -c conda-forge foundry-dev-tools
78
+ ```
79
+
72
80
  [Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
73
81
 
74
82
  ## Why did we build this?
@@ -5,6 +5,39 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog],
6
6
  and this project adheres to [Semantic Versioning].
7
7
 
8
+ ## [1.0.8] - 2023-04-17
9
+
10
+ ### Added
11
+
12
+ - python 3.11 support, as pyspark 3.4 gained support for it (#13)
13
+ - conda-forge badges to the README (#13)
14
+
15
+ ### Fixed
16
+
17
+ - typo in docs (#12)
18
+
19
+ ### Removed
20
+
21
+ - pandas<2 restriction, as pyspark 3.4 supports it (#13)
22
+
23
+
24
+ ## [1.0.7] - 2023-04-05
25
+
26
+ ### Added
27
+
28
+ - skip_instance_cache kwarg to FoundryFileSystem, which gets passed to fsspec
29
+ it should be set to True in a multithreaded environment (e.g. Streamlit),
30
+ as the same filesystem instance gets reused by default,
31
+ which resulted in weird behaviour. (#11)
32
+ - an extra check, which may prevent sending a bad request to foundry
33
+ when skip_instance_cache is not used (#11)
34
+
35
+ ### Changed
36
+
37
+ - remove pandas from top level import, which should speed up the initial import (#11)
38
+ - restrict pandas to version less than 2,
39
+ as pyspark is currently not compatible with version 2 (#11)
40
+
8
41
  ## [1.0.6] - 2023-03-31
9
42
 
10
43
  ### Changed
@@ -63,9 +96,12 @@ and this project adheres to [Semantic Versioning].
63
96
 
64
97
  [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
65
98
  [Semantic Versioning]: https://semver.org/spec/v2.0.0.html
99
+ [1.0.8]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.7...v1.0.8
100
+ [1.0.7]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.6...v1.0.7
101
+ [1.0.6]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.5...v1.0.6
66
102
  [1.0.5]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.4...v1.0.5
67
103
  [1.0.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.3...v1.0.4
68
104
  [1.0.3]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.0.2...v1.0.3
69
105
  [1.0.2]: https://github.com/emdgroup/foundry-dev-tools/releases/tag/v1.0.2
70
106
  [1.0.1]: https://github.com/emdgroup/foundry-dev-tools
71
- [1.0]: https://github.com/emdgroup/foundry-dev-tools
107
+ [1.0]: https://github.com/emdgroup/foundry-dev-tools
@@ -33,10 +33,16 @@ jwt=<paste your foundry token here>
33
33
  We recommend using a new [conda environment] or [python environment],
34
34
  after you activated it, just run:
35
35
 
36
- ```bash
36
+ With pip:
37
+ ```shell
37
38
  pip install foundry-dev-tools
38
39
  ```
39
40
 
41
+ With conda or mamba on the conda-forge channel:
42
+ ```shell
43
+ conda install -c conda-forge foundry-dev-tools
44
+ ```
45
+
40
46
  or, if you have the repository locally available and want to tinker with the source code
41
47
  you could install it in "editable"/"develop mode". Run the following command in
42
48
  the Foundry DevTools root directory:
@@ -51,8 +51,8 @@ for package name and package version:
51
51
  ```diff
52
52
  - name=os.environ['PKG_NAME'],
53
53
  + name=os.getenv('PKG_NAME', 'your-package-name'),
54
- - name=os.environ['PKG_VERSION'],
55
- + name=os.getenv('PKG_VERSION', 'your-package-version'),
54
+ - version=os.environ['PKG_VERSION'],
55
+ + version=os.getenv('PKG_VERSION', 'your-package-version'),
56
56
  ```
57
57
 
58
58
  Now you can install the repository into your local environment, when in directory `transform-python/src`:
@@ -23,6 +23,7 @@ classifiers =
23
23
  Programming Language :: Python :: 3.8
24
24
  Programming Language :: Python :: 3.9
25
25
  Programming Language :: Python :: 3.10
26
+ Programming Language :: Python :: 3.11
26
27
  Intended Audience :: Developers
27
28
  Operating System :: POSIX :: Linux
28
29
  Operating System :: MacOS
@@ -75,7 +76,6 @@ extras = True
75
76
 
76
77
  [tool:pytest]
77
78
  addopts =
78
- --cov=src --cov-report term --cov-report xml:coverage.xml --cov-report html:htmlcov
79
79
  --verbose
80
80
  norecursedirs =
81
81
  dist
@@ -6,8 +6,6 @@ import tempfile
6
6
  import time
7
7
  from typing import Tuple, Union
8
8
 
9
- import pandas as pd
10
-
11
9
  import foundry_dev_tools
12
10
  from foundry_dev_tools.foundry_api_client import (
13
11
  BranchNotFoundError,
@@ -175,13 +173,13 @@ class CachedFoundryClient:
175
173
 
176
174
  def save_dataset(
177
175
  self,
178
- df: Union[pd.DataFrame, "pyspark.sql.DataFrame"],
176
+ df: Union["pd.DataFrame", "pyspark.sql.DataFrame"],
179
177
  dataset_path_or_rid: str,
180
178
  branch: str = "master",
181
179
  exists_ok: bool = False,
182
180
  mode: str = "SNAPSHOT",
183
181
  ) -> Tuple[str, str]:
184
- # pylint: disable=invalid-name,too-many-arguments
182
+ # pylint: disable=invalid-name,too-many-arguments,too-many-locals
185
183
  """Saves a dataframe to Foundry. If the dataset in Foundry does not exist it is created.
186
184
 
187
185
  If the branch does not exist, it is created. If the dataset exists, an exception is thrown.
@@ -218,6 +216,9 @@ class CachedFoundryClient:
218
216
  raise ValueError("Please provide a dataset branch with parameter 'branch'")
219
217
 
220
218
  with tempfile.TemporaryDirectory() as path:
219
+ # pylint: disable=import-outside-toplevel
220
+ import pandas as pd
221
+
221
222
  if isinstance(df, pd.DataFrame):
222
223
  df.to_parquet(
223
224
  os.sep.join([path + "/dataset.parquet"]),
@@ -1324,8 +1324,6 @@ class FoundryRestClient:
1324
1324
 
1325
1325
  """
1326
1326
  assert return_type in {"pandas", "arrow", "spark"}
1327
- _assert_pyarrow_packages_available()
1328
- _assert_pandas_packages_available()
1329
1327
  foundry_sql_client = FoundrySqlClient(config=self._config, branch=branch)
1330
1328
  try:
1331
1329
  return foundry_sql_client.query(query=query, return_type=return_type)
@@ -1696,7 +1694,6 @@ class FoundrySqlClient:
1696
1694
  branch (str): default = master, all queries will be executed against this default branch
1697
1695
 
1698
1696
  """
1699
- _assert_pyarrow_packages_available()
1700
1697
  self._config = foundry_dev_tools.Configuration.get_config(config)
1701
1698
  self._requests_verify_value = _determine_requests_verify_value(self._config)
1702
1699
  self.foundry_sql_server_api = (
@@ -1899,26 +1896,6 @@ def _transform_bad_request_response_to_exception(response):
1899
1896
  raise DatasetNotFoundError("SQL")
1900
1897
 
1901
1898
 
1902
- def _assert_pyarrow_packages_available():
1903
- try:
1904
- # pylint: disable=import-outside-toplevel,unused-import
1905
- import pyarrow
1906
- except ImportError as err:
1907
- raise ValueError(
1908
- "Please install package 'pyarrow' to use SQL functionality."
1909
- ) from err
1910
-
1911
-
1912
- def _assert_pandas_packages_available():
1913
- try:
1914
- # pylint: disable=import-outside-toplevel,unused-import
1915
- import pandas as pd
1916
- except ImportError as err:
1917
- raise ValueError(
1918
- "Please install package 'pandas' to use SQL functionality."
1919
- ) from err
1920
-
1921
-
1922
1899
  def _is_palantir_oauth_client_installed():
1923
1900
  try:
1924
1901
  # pylint: disable=import-outside-toplevel,unused-import,import-error
@@ -25,6 +25,8 @@ class FoundryFileSystem(AbstractFileSystem): # noqa
25
25
  or using SSO.
26
26
  transaction_backoff (bool): if FoundryFileSystem should retry to open a transaction, in case one is already
27
27
  open. The default value is 'True' and FoundryFileSystem will retry 60 seconds to open a transaction.
28
+ skip_instance_cache (bool): gets passed to FoundryFileSystem,
29
+ in multithreaded environments you want to set this to true.
28
30
  **kwargs: passed to underlying fsspec.AbstractFileSystem
29
31
  """
30
32
 
@@ -36,9 +38,11 @@ class FoundryFileSystem(AbstractFileSystem): # noqa
36
38
  branch: str = DEFAULT_BRANCH,
37
39
  token: str = None,
38
40
  transaction_backoff: bool = True,
41
+ skip_instance_cache: bool = False,
39
42
  **kwargs,
40
43
  ):
41
- super().__init__(**kwargs)
44
+ # pylint: disable=too-many-arguments
45
+ super().__init__(skip_instance_cache=skip_instance_cache, **kwargs)
42
46
  self.token = token
43
47
  self.dataset_identity = self.api.get_dataset_identity(
44
48
  dataset_path_or_rid=dataset, branch=branch
@@ -310,7 +314,9 @@ class FoundryFile(AbstractBufferedFile):
310
314
  """Internal function to add a chunk of data to a started upload."""
311
315
  assert final is True, "chunked uploading not supported"
312
316
  self.buffer.seek(0)
313
-
317
+ assert (
318
+ self.fs._transaction is not None # pylint: disable=protected-access
319
+ ), "multiple threads, same dataset? use skip_instance_cache"
314
320
  self.fs.api.upload_dataset_file(
315
321
  dataset_rid=self.fs.dataset_identity["dataset_rid"],
316
322
  transaction_rid=self.fs._transaction.transaction_rid, # pylint: disable=protected-access
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.0.6
3
+ Version: 1.0.8
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Home-page: https://github.com/emdgroup/foundry-dev-tools
6
6
  Author: Nicolas Renkamp, Jonas Wunderlich
@@ -18,6 +18,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
18
18
  Classifier: Programming Language :: Python :: 3.8
19
19
  Classifier: Programming Language :: Python :: 3.9
20
20
  Classifier: Programming Language :: Python :: 3.10
21
+ Classifier: Programming Language :: Python :: 3.11
21
22
  Classifier: Intended Audience :: Developers
22
23
  Classifier: Operating System :: POSIX :: Linux
23
24
  Classifier: Operating System :: MacOS
@@ -35,12 +36,14 @@ License-File: LICENSE
35
36
 
36
37
  <a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/ci.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/ci.yml?style=flat-square"/></img>
37
38
  <a href="https://github.com/emdgroup/foundry-dev-tools/actions/workflows/docs.yml"><img src="https://img.shields.io/github/actions/workflow/status/emdgroup/foundry-dev-tools/docs.yml?style=flat-square"/></img>
38
- <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square"/></a>
39
- <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square"/></a>
40
- <a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square"/></a>
41
- <a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?color=important&style=flat-square"/></a>
42
- <a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?color=blueviolet&style=flat-square"/></a>
43
-
39
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/pyversions/foundry-dev-tools?style=flat-square&label=Supported%20Python%20versions&color=%23ffb86c"/></a>
40
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/v/foundry-dev-tools.svg?style=flat-square&label=PyPI%20version&color=%23bd93f9"/></a>
41
+ <a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/vn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Version&color=%23bd93f9" alt="Conda Version"/></a>
42
+ <a href="https://pypi.org/project/foundry-dev-tools/"><img src="https://img.shields.io/pypi/dm/foundry-dev-tools?label=PyPI%20Downloads&style=flat-square&color=%236272a4"/></a>
43
+ <a href="https://anaconda.org/conda-forge/foundry-dev-tools"><img src="https://img.shields.io/conda/dn/conda-forge/foundry-dev-tools.svg?style=flat-square&label=Conda%20Forge%20Downloads&color=%236272a4" alt="Conda Downloads"/></a>
44
+ <a href="https://github.com/emdgroup/foundry-dev-tools/issues"><img src="https://img.shields.io/github/issues/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
45
+ <a href="https://github.com/emdgroup/foundry-dev-tools/pulls"><img src="https://img.shields.io/github/issues-pr/emdgroup/foundry-dev-tools?style=flat-square&color=%23ff79c6"/></a>
46
+ <a href="http://www.apache.org/licenses/LICENSE-2.0"><img src="https://shields.io/badge/License-Apache%202.0-green.svg?style=flat-square&color=%234c1"/></a>
44
47
  <p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
45
48
 
46
49
  <a href="https://emdgroup.github.io/foundry-dev-tools/installation.html">Installation<a/>
@@ -97,10 +100,16 @@ high level entrypoints to Foundry DevTools:
97
100
 
98
101
  ## Quickstart
99
102
 
103
+ With pip:
100
104
  ```shell
101
105
  pip install foundry-dev-tools
102
106
  ```
103
107
 
108
+ With conda or mamba on the conda-forge channel:
109
+ ```shell
110
+ conda install -c conda-forge foundry-dev-tools
111
+ ```
112
+
104
113
  [Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
105
114
 
106
115
  ## Why did we build this?
@@ -12,7 +12,6 @@ import re
12
12
  from typing import Callable, Dict, Optional
13
13
 
14
14
  import fs
15
- import pandas as pd
16
15
  import pyspark
17
16
 
18
17
  from transforms.api import Input, Output
@@ -121,6 +120,7 @@ class Transform:
121
120
  kwargs["ctx"] = TransformContext()
122
121
 
123
122
  output_df = self(**kwargs)
123
+ import pandas as pd # pylint: disable=import-outside-toplevel
124
124
 
125
125
  if not isinstance(output_df, pd.DataFrame):
126
126
  raise ValueError(
@@ -971,3 +971,43 @@ def test_ls_trailing_slash_empty_folder(random_file, fsspec_write_test_folder):
971
971
  ls_result_no_slash = fs.ls(random_folder, detail=False)
972
972
  ls_result_with_slash = fs.ls(random_folder + "/", detail=False)
973
973
  assert ls_result_no_slash == ls_result_with_slash
974
+
975
+
976
+ @patch(
977
+ "foundry_dev_tools.FoundryRestClient.get_dataset_identity",
978
+ MagicMock(),
979
+ )
980
+ def test_skip_instance_cache():
981
+ fs = FoundryFileSystem(
982
+ dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
983
+ branch="master",
984
+ token="super-secret-token",
985
+ skip_instance_cache=True,
986
+ )
987
+ fs2 = FoundryFileSystem(
988
+ dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
989
+ branch="master",
990
+ token="super-secret-token",
991
+ skip_instance_cache=True,
992
+ )
993
+
994
+ fs3 = FoundryFileSystem(
995
+ dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
996
+ branch="master",
997
+ token="super-secret-token",
998
+ skip_instance_cache=False,
999
+ )
1000
+ fs4 = FoundryFileSystem(
1001
+ dataset="ri.foundry.main.dataset.fake1bb5-be92-4ad9-aa3e-07c161751234",
1002
+ branch="master",
1003
+ token="super-secret-token",
1004
+ skip_instance_cache=False,
1005
+ )
1006
+ fs.x = 123
1007
+ fs2.x = 4567
1008
+ fs3.x = 8901
1009
+ fs4.x = 2345
1010
+ assert fs.x == 123
1011
+ assert fs2.x == 4567
1012
+ assert fs3.x == 2345
1013
+ assert fs4.x == 2345