foundry-dev-tools 1.5.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/PKG-INFO +41 -71
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/README.md +17 -31
- foundry_dev_tools-2.0.0/pyproject.toml +76 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/__about__.py +1 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/__init__.py +20 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/_optional/__init__.py +17 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/_optional/pandas.py +12 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/_optional/polars.py +12 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/_optional/pyarrow.py +15 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/_optional/pyspark.py +18 -0
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/cached_foundry_client.py +120 -120
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/cli/__init__.py +1 -0
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/cli/build.py +85 -95
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/cli/config.py +194 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/cli/git.py +100 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/cli/git_credential_foundry.py +127 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/cli/info.py +303 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/cli/main.py +30 -0
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/cli/s3.py +20 -17
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/cli/stubs.py +37 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/__init__.py +1 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/api_client.py +217 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/build2.py +50 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/catalog.py +575 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/compass.py +1205 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/context_client.py +130 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/data_proxy.py +534 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/foundry_sql_server.py +280 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/foundry_stats.py +61 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/jemma.py +83 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/metadata.py +77 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/multipass.py +560 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/public_ontologies_client.py +45 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/s3_client.py +190 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/clients/schema_inference.py +71 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/config/__init__.py +1 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/config/config.py +213 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/config/config_types.py +31 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/config/context.py +269 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/config/token_provider.py +264 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/errors/__init__.py +1 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/errors/compass.py +210 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/errors/config.py +39 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/errors/dataset.py +68 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/errors/handling.py +186 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/errors/meta.py +98 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/errors/sql.py +23 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/foundry_api_client.py +1472 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/py.typed +1 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/resources/__init__.py +5 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/resources/dataset.py +799 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/resources/folder.py +33 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/resources/resource.py +253 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/__init__.py +1 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/api_types.py +479 -0
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/utils/async_s3.py +8 -11
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/utils/caches/metadata_store.py +30 -14
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/caches/spark_caches.py +286 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/cli.py +12 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/clients.py +17 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/compat.py +107 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/config.py +231 -0
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/utils/converter/foundry_spark.py +61 -67
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/utils/misc.py +44 -15
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/public_api_types.py +16 -0
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/utils/repo.py +34 -66
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/s3.py +60 -0
- foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/spark.py +27 -0
- foundry_dev_tools-1.5.0/.github/ISSUE_TEMPLATE/blank_issue.yml +0 -27
- foundry_dev_tools-1.5.0/.github/ISSUE_TEMPLATE/bug_report.yml +0 -87
- foundry_dev_tools-1.5.0/.github/ISSUE_TEMPLATE/config.yml +0 -1
- foundry_dev_tools-1.5.0/.github/ISSUE_TEMPLATE/feature_request.yml +0 -69
- foundry_dev_tools-1.5.0/.github/pull_request_template.md +0 -10
- foundry_dev_tools-1.5.0/.github/workflows/ci.yml +0 -99
- foundry_dev_tools-1.5.0/.github/workflows/docs.yml +0 -48
- foundry_dev_tools-1.5.0/.gitignore +0 -53
- foundry_dev_tools-1.5.0/.pre-commit-config.yaml +0 -18
- foundry_dev_tools-1.5.0/LICENSE +0 -202
- foundry_dev_tools-1.5.0/docs/Configuration_usage.md +0 -119
- foundry_dev_tools-1.5.0/docs/FoundryFileSystem_usage.md +0 -65
- foundry_dev_tools-1.5.0/docs/FoundryRestClient_usage.md +0 -173
- foundry_dev_tools-1.5.0/docs/SSO_usage.md +0 -35
- foundry_dev_tools-1.5.0/docs/_static/.gitignore +0 -1
- foundry_dev_tools-1.5.0/docs/architecture.md +0 -52
- foundry_dev_tools-1.5.0/docs/changelog.md +0 -245
- foundry_dev_tools-1.5.0/docs/cli.md +0 -93
- foundry_dev_tools-1.5.0/docs/conf.py +0 -319
- foundry_dev_tools-1.5.0/docs/contributors.md +0 -7
- foundry_dev_tools-1.5.0/docs/develop.md +0 -83
- foundry_dev_tools-1.5.0/docs/index.md +0 -45
- foundry_dev_tools-1.5.0/docs/installation.md +0 -73
- foundry_dev_tools-1.5.0/docs/license.md +0 -5
- foundry_dev_tools-1.5.0/docs/pictures/mermaid-diagram-already-cached-dark.svg +0 -1
- foundry_dev_tools-1.5.0/docs/pictures/mermaid-diagram-already-cached-light.svg +0 -1
- foundry_dev_tools-1.5.0/docs/pictures/mermaid-diagram-new-transaction-dark.svg +0 -1
- foundry_dev_tools-1.5.0/docs/pictures/mermaid-diagram-new-transaction-light.svg +0 -1
- foundry_dev_tools-1.5.0/docs/pictures/tpa_config.png +0 -0
- foundry_dev_tools-1.5.0/docs/requirements.txt +0 -7
- foundry_dev_tools-1.5.0/docs/s3.md +0 -108
- foundry_dev_tools-1.5.0/docs/usage_and_examples.md +0 -82
- foundry_dev_tools-1.5.0/pyproject.toml +0 -137
- foundry_dev_tools-1.5.0/setup.cfg +0 -4
- foundry_dev_tools-1.5.0/setup.py +0 -21
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/__about__.py +0 -7
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/__init__.py +0 -14
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/cli/info.py +0 -246
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/cli/main.py +0 -22
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/config.py +0 -338
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/foundry_api_client.py +0 -2637
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/fsspec_impl.py +0 -530
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/utils/__init__.py +0 -0
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -311
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/utils/importer.py +0 -38
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/utils/s3.py +0 -32
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/utils/spark.py +0 -23
- foundry_dev_tools-1.5.0/src/foundry_dev_tools/utils/token_provider.py +0 -143
- foundry_dev_tools-1.5.0/src/foundry_dev_tools.egg-info/PKG-INFO +0 -168
- foundry_dev_tools-1.5.0/src/foundry_dev_tools.egg-info/SOURCES.txt +0 -99
- foundry_dev_tools-1.5.0/src/foundry_dev_tools.egg-info/dependency_links.txt +0 -1
- foundry_dev_tools-1.5.0/src/foundry_dev_tools.egg-info/entry_points.txt +0 -5
- foundry_dev_tools-1.5.0/src/foundry_dev_tools.egg-info/not-zip-safe +0 -1
- foundry_dev_tools-1.5.0/src/foundry_dev_tools.egg-info/requires.txt +0 -35
- foundry_dev_tools-1.5.0/src/foundry_dev_tools.egg-info/top_level.txt +0 -2
- foundry_dev_tools-1.5.0/src/transforms/__init__.py +0 -4
- foundry_dev_tools-1.5.0/src/transforms/api/__init__.py +0 -33
- foundry_dev_tools-1.5.0/src/transforms/api/_configure.py +0 -25
- foundry_dev_tools-1.5.0/src/transforms/api/_dataset.py +0 -333
- foundry_dev_tools-1.5.0/src/transforms/api/_decorators.py +0 -271
- foundry_dev_tools-1.5.0/src/transforms/api/_transform.py +0 -554
- foundry_dev_tools-1.5.0/tests/__init__.py +0 -0
- foundry_dev_tools-1.5.0/tests/conftest.py +0 -219
- foundry_dev_tools-1.5.0/tests/foundry_mock_client.py +0 -611
- foundry_dev_tools-1.5.0/tests/test_cached_foundry_client.py +0 -581
- foundry_dev_tools-1.5.0/tests/test_cached_foundry_client_integration.py +0 -51
- foundry_dev_tools-1.5.0/tests/test_cli_build.py +0 -491
- foundry_dev_tools-1.5.0/tests/test_cli_info.py +0 -10
- foundry_dev_tools-1.5.0/tests/test_config.py +0 -116
- foundry_dev_tools-1.5.0/tests/test_data/binary_dataset/bin +0 -0
- foundry_dev_tools-1.5.0/tests/test_data/iris/iris.csv +0 -151
- foundry_dev_tools-1.5.0/tests/test_foundry_api.py +0 -874
- foundry_dev_tools-1.5.0/tests/test_foundry_local_deprecation.py +0 -149
- foundry_dev_tools-1.5.0/tests/test_foundry_mock.py +0 -342
- foundry_dev_tools-1.5.0/tests/test_foundry_spark_converters.py +0 -566
- foundry_dev_tools-1.5.0/tests/test_foundry_sql_client.py +0 -136
- foundry_dev_tools-1.5.0/tests/test_fsspec_impl.py +0 -1024
- foundry_dev_tools-1.5.0/tests/test_importer.py +0 -11
- foundry_dev_tools-1.5.0/tests/test_multipass_tpa.py +0 -277
- foundry_dev_tools-1.5.0/tests/test_s3.py +0 -145
- foundry_dev_tools-1.5.0/tests/test_spark_caches.py +0 -436
- foundry_dev_tools-1.5.0/tests/test_token_provider.py +0 -116
- foundry_dev_tools-1.5.0/tests/test_transforms.py +0 -700
- foundry_dev_tools-1.5.0/tests/test_transforms_integration.py +0 -223
- foundry_dev_tools-1.5.0/tests/test_utils_misc.py +0 -8
- foundry_dev_tools-1.5.0/tests/test_utils_repo.py +0 -99
- foundry_dev_tools-1.5.0/tests/utils.py +0 -435
- foundry_dev_tools-1.5.0/tox.ini +0 -100
- {foundry_dev_tools-1.5.0 → foundry_dev_tools-2.0.0}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
- {foundry_dev_tools-1.5.0/src/foundry_dev_tools/cli → foundry_dev_tools-2.0.0/src/foundry_dev_tools/utils/converter}/__init__.py +0 -0
|
@@ -1,61 +1,45 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: foundry-dev-tools
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
|
|
5
|
-
Author-
|
|
5
|
+
Author-Email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
|
|
6
6
|
License: Apache-2.0
|
|
7
|
-
Project-URL: Homepage, https://emdgroup.github.io/foundry-dev-tools
|
|
8
|
-
Project-URL: Documentation, https://emdgroup.github.io/foundry-dev-tools
|
|
9
|
-
Project-URL: Source, https://github.com/emdgroup/foundry-dev-tools
|
|
10
|
-
Project-URL: Tracker, https://github.com/emdgroup/foundry-dev-tools/issues
|
|
11
|
-
Project-URL: Changelog, https://emdgroup.github.io/foundry-dev-tools/changelog.html
|
|
12
|
-
Platform: any
|
|
13
7
|
Classifier: Development Status :: 5 - Production/Stable
|
|
14
8
|
Classifier: Programming Language :: Python
|
|
15
9
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
16
10
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
19
11
|
Classifier: Programming Language :: Python :: 3.10
|
|
20
12
|
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
14
|
Classifier: Intended Audience :: Developers
|
|
22
15
|
Classifier: Operating System :: POSIX :: Linux
|
|
23
16
|
Classifier: Operating System :: MacOS
|
|
24
17
|
Classifier: Operating System :: Microsoft :: Windows
|
|
25
18
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
19
|
+
Project-URL: Homepage, https://emdgroup.github.io/foundry-dev-tools
|
|
20
|
+
Project-URL: Documentation, https://emdgroup.github.io/foundry-dev-tools
|
|
21
|
+
Project-URL: Source, https://github.com/emdgroup/foundry-dev-tools
|
|
22
|
+
Project-URL: Tracker, https://github.com/emdgroup/foundry-dev-tools/issues
|
|
23
|
+
Project-URL: Changelog, https://emdgroup.github.io/foundry-dev-tools/changelog.html
|
|
24
|
+
Requires-Python: >=3.10
|
|
31
25
|
Requires-Dist: requests
|
|
32
|
-
Requires-Dist: fs
|
|
33
|
-
Requires-Dist: backoff
|
|
34
26
|
Requires-Dist: palantir-oauth-client
|
|
35
|
-
|
|
36
|
-
Requires-Dist:
|
|
37
|
-
Requires-Dist:
|
|
38
|
-
Requires-Dist:
|
|
39
|
-
Requires-Dist:
|
|
40
|
-
Requires-Dist:
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
Requires-Dist:
|
|
43
|
-
Requires-Dist:
|
|
44
|
-
Requires-Dist:
|
|
45
|
-
Requires-Dist: streamlit; extra == "testing"
|
|
46
|
-
Provides-Extra: integration
|
|
47
|
-
Requires-Dist: dask; extra == "integration"
|
|
48
|
-
Requires-Dist: fastparquet; extra == "integration"
|
|
49
|
-
Provides-Extra: transforms
|
|
50
|
-
Requires-Dist: pyspark>=3.0.0; extra == "transforms"
|
|
51
|
-
Provides-Extra: cli
|
|
52
|
-
Requires-Dist: click; extra == "cli"
|
|
53
|
-
Requires-Dist: inquirer; extra == "cli"
|
|
54
|
-
Requires-Dist: websockets; extra == "cli"
|
|
55
|
-
Requires-Dist: rich; extra == "cli"
|
|
56
|
-
Requires-Dist: packaging; extra == "cli"
|
|
57
|
-
Provides-Extra: s3
|
|
27
|
+
Requires-Dist: platformdirs
|
|
28
|
+
Requires-Dist: tomli; python_version < "3.11"
|
|
29
|
+
Requires-Dist: typing_extensions; python_version < "3.11"
|
|
30
|
+
Requires-Dist: backports-datetime-fromisoformat==2.0.1; python_version < "3.11"
|
|
31
|
+
Requires-Dist: click
|
|
32
|
+
Requires-Dist: inquirer
|
|
33
|
+
Requires-Dist: websockets
|
|
34
|
+
Requires-Dist: rich
|
|
35
|
+
Requires-Dist: packaging
|
|
36
|
+
Requires-Dist: tomli_w
|
|
58
37
|
Requires-Dist: aiobotocore[boto3]; extra == "s3"
|
|
38
|
+
Requires-Dist: foundry-dev-tools-transforms; extra == "full"
|
|
39
|
+
Requires-Dist: foundry-dev-tools[s3]; extra == "full"
|
|
40
|
+
Provides-Extra: s3
|
|
41
|
+
Provides-Extra: full
|
|
42
|
+
Description-Content-Type: text/markdown
|
|
59
43
|
|
|
60
44
|
<div align="center">
|
|
61
45
|
<br/>
|
|
@@ -73,53 +57,39 @@ Requires-Dist: aiobotocore[boto3]; extra == "s3"
|
|
|
73
57
|
|
|
74
58
|
<p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
|
|
75
59
|
|
|
76
|
-
<a href="https://emdgroup.github.io/foundry-dev-tools/
|
|
60
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/getting_started/index.html">Getting Started / Usage<a/>
|
|
77
61
|
•
|
|
78
|
-
<a href="https://emdgroup.github.io/foundry-dev-tools/
|
|
62
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/examples/api.html">Examples<a/>
|
|
79
63
|
•
|
|
80
|
-
<a href="https://emdgroup.github.io/foundry-dev-tools/
|
|
64
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/dev/contribute.html">Development/Contribute<a/>
|
|
81
65
|
|
|
82
66
|
</div>
|
|
83
67
|
|
|
84
68
|
# Foundry DevTools
|
|
85
69
|
|
|
86
70
|
Seamlessly run your Palantir Foundry Repository transforms code and more on your local machine.
|
|
87
|
-
Foundry DevTools is a set of useful libraries to interact with the Foundry APIs.
|
|
88
|
-
|
|
71
|
+
Foundry DevTools is a set of useful libraries to interact with the Foundry APIs.
|
|
72
|
+
It consists of two parts:
|
|
89
73
|
|
|
90
|
-
-
|
|
74
|
+
- The [transforms](https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/) implementation
|
|
91
75
|
|
|
92
76
|
- An implementation of the Foundry `transforms` package that internally uses the `CachedFoundryClient`.
|
|
93
77
|
This allows you to seamlessly run your Palantir Foundry Code Repository transforms code on your local machine.
|
|
94
|
-
Foundry DevTools does not cover all of Foundry's features, more on this [here](https://emdgroup.github.io/foundry-dev-tools/architecture.html#known-limitations-contributions-welcome).
|
|
78
|
+
Foundry DevTools does not cover all of Foundry's features, more on this [here](https://emdgroup.github.io/foundry-dev-tools/dev/architecture.html#known-limitations-contributions-welcome).
|
|
95
79
|
|
|
96
|
-
-
|
|
80
|
+
- API clients
|
|
97
81
|
|
|
98
|
-
|
|
82
|
+
We implemented multiple clients for many foundry APIs like compass, catalog or foundry-sql-server.
|
|
99
83
|
|
|
100
84
|
- For example:
|
|
101
85
|
|
|
102
86
|
```python
|
|
103
|
-
from foundry_dev_tools import
|
|
87
|
+
from foundry_dev_tools import FoundryContext
|
|
104
88
|
|
|
105
|
-
#
|
|
106
|
-
|
|
107
|
-
df = rest_client.query_foundry_sql("SELECT * FROM `/Global/Foundry Training and Resources/Foundry Reference Project/Ontology Project: Aviation/airlines`", branch='master')
|
|
108
|
-
df.shape
|
|
109
|
-
# Out[2]: (17, 10)
|
|
110
|
-
```
|
|
89
|
+
# the context, that contains your credentials and configuration
|
|
90
|
+
ctx = FoundryContext()
|
|
111
91
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
- An implementation of `fsspec` for Foundry. Useful to interact with Foundry from popular data science libraries such as
|
|
115
|
-
`pandas` or `dask`.
|
|
116
|
-
|
|
117
|
-
- For example:
|
|
118
|
-
|
|
119
|
-
```python
|
|
120
|
-
import pandas as pd
|
|
121
|
-
# /Global/Foundry Training and Resources/Foundry Reference Project/Ontology Project: Aviation/airlines
|
|
122
|
-
df = pd.read_parquet("foundry://ri.foundry.main.dataset.5d78f3ae-a588-4fd8-9ba2-66827808c85f")
|
|
92
|
+
df = ctx.foundry_sql_server.query_foundry_sql("SELECT * FROM `/Global/Foundry Training and Resources/Example Data/Aviation Ontology/airlines`", branch='master')
|
|
123
93
|
df.shape
|
|
124
94
|
# Out[2]: (17, 10)
|
|
125
95
|
```
|
|
@@ -138,12 +108,12 @@ With conda or mamba on the conda-forge channel:
|
|
|
138
108
|
conda install -c conda-forge foundry-dev-tools
|
|
139
109
|
```
|
|
140
110
|
|
|
141
|
-
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
|
|
111
|
+
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/getting_started/installation.html) can be found in our documentation.
|
|
142
112
|
|
|
143
113
|
## Why did we build this?
|
|
144
114
|
|
|
145
|
-
- Local development experience in your
|
|
146
|
-
- Access to modern developer tools and workflows such as pylint, black,
|
|
115
|
+
- Local development experience in your favorite IDE (PyCharm, VSCode, ...)
|
|
116
|
+
- Access to modern developer tools and workflows such as ruff, mypy, pylint, black, pre-commit hooks etc.
|
|
147
117
|
- Quicker turnaround time when making changes
|
|
148
118
|
- Debug, change code and run in a matter of seconds instead of minutes
|
|
149
119
|
- No accidental or auto commits
|
|
@@ -151,7 +121,7 @@ conda install -c conda-forge foundry-dev-tools
|
|
|
151
121
|
|
|
152
122
|
# License
|
|
153
123
|
|
|
154
|
-
Copyright (c)
|
|
124
|
+
Copyright (c) 2024 Merck KGaA, Darmstadt, Germany
|
|
155
125
|
|
|
156
126
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
157
127
|
you may not use this file except in compliance with the License.
|
|
@@ -14,53 +14,39 @@
|
|
|
14
14
|
|
|
15
15
|
<p><a href="https://emdgroup.github.io/foundry-dev-tools">Documentation</a></p>
|
|
16
16
|
|
|
17
|
-
<a href="https://emdgroup.github.io/foundry-dev-tools/
|
|
17
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/getting_started/index.html">Getting Started / Usage<a/>
|
|
18
18
|
•
|
|
19
|
-
<a href="https://emdgroup.github.io/foundry-dev-tools/
|
|
19
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/examples/api.html">Examples<a/>
|
|
20
20
|
•
|
|
21
|
-
<a href="https://emdgroup.github.io/foundry-dev-tools/
|
|
21
|
+
<a href="https://emdgroup.github.io/foundry-dev-tools/dev/contribute.html">Development/Contribute<a/>
|
|
22
22
|
|
|
23
23
|
</div>
|
|
24
24
|
|
|
25
25
|
# Foundry DevTools
|
|
26
26
|
|
|
27
27
|
Seamlessly run your Palantir Foundry Repository transforms code and more on your local machine.
|
|
28
|
-
Foundry DevTools is a set of useful libraries to interact with the Foundry APIs.
|
|
29
|
-
|
|
28
|
+
Foundry DevTools is a set of useful libraries to interact with the Foundry APIs.
|
|
29
|
+
It consists of two parts:
|
|
30
30
|
|
|
31
|
-
-
|
|
31
|
+
- The [transforms](https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api/) implementation
|
|
32
32
|
|
|
33
33
|
- An implementation of the Foundry `transforms` package that internally uses the `CachedFoundryClient`.
|
|
34
34
|
This allows you to seamlessly run your Palantir Foundry Code Repository transforms code on your local machine.
|
|
35
|
-
Foundry DevTools does not cover all of Foundry's features, more on this [here](https://emdgroup.github.io/foundry-dev-tools/architecture.html#known-limitations-contributions-welcome).
|
|
35
|
+
Foundry DevTools does not cover all of Foundry's features, more on this [here](https://emdgroup.github.io/foundry-dev-tools/dev/architecture.html#known-limitations-contributions-welcome).
|
|
36
36
|
|
|
37
|
-
-
|
|
37
|
+
- API clients
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
We implemented multiple clients for many foundry APIs like compass, catalog or foundry-sql-server.
|
|
40
40
|
|
|
41
41
|
- For example:
|
|
42
42
|
|
|
43
43
|
```python
|
|
44
|
-
from foundry_dev_tools import
|
|
44
|
+
from foundry_dev_tools import FoundryContext
|
|
45
45
|
|
|
46
|
-
#
|
|
47
|
-
|
|
48
|
-
df = rest_client.query_foundry_sql("SELECT * FROM `/Global/Foundry Training and Resources/Foundry Reference Project/Ontology Project: Aviation/airlines`", branch='master')
|
|
49
|
-
df.shape
|
|
50
|
-
# Out[2]: (17, 10)
|
|
51
|
-
```
|
|
52
|
-
|
|
53
|
-
- [FoundryFileSystem](https://emdgroup.github.io/foundry-dev-tools/FoundryFileSystem_usage.html)
|
|
46
|
+
# the context, that contains your credentials and configuration
|
|
47
|
+
ctx = FoundryContext()
|
|
54
48
|
|
|
55
|
-
|
|
56
|
-
`pandas` or `dask`.
|
|
57
|
-
|
|
58
|
-
- For example:
|
|
59
|
-
|
|
60
|
-
```python
|
|
61
|
-
import pandas as pd
|
|
62
|
-
# /Global/Foundry Training and Resources/Foundry Reference Project/Ontology Project: Aviation/airlines
|
|
63
|
-
df = pd.read_parquet("foundry://ri.foundry.main.dataset.5d78f3ae-a588-4fd8-9ba2-66827808c85f")
|
|
49
|
+
df = ctx.foundry_sql_server.query_foundry_sql("SELECT * FROM `/Global/Foundry Training and Resources/Example Data/Aviation Ontology/airlines`", branch='master')
|
|
64
50
|
df.shape
|
|
65
51
|
# Out[2]: (17, 10)
|
|
66
52
|
```
|
|
@@ -79,12 +65,12 @@ With conda or mamba on the conda-forge channel:
|
|
|
79
65
|
conda install -c conda-forge foundry-dev-tools
|
|
80
66
|
```
|
|
81
67
|
|
|
82
|
-
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/installation.html) can be found in our documentation.
|
|
68
|
+
[Further instructions](https://emdgroup.github.io/foundry-dev-tools/getting_started/installation.html) can be found in our documentation.
|
|
83
69
|
|
|
84
70
|
## Why did we build this?
|
|
85
71
|
|
|
86
|
-
- Local development experience in your
|
|
87
|
-
- Access to modern developer tools and workflows such as pylint, black,
|
|
72
|
+
- Local development experience in your favorite IDE (PyCharm, VSCode, ...)
|
|
73
|
+
- Access to modern developer tools and workflows such as ruff, mypy, pylint, black, pre-commit hooks etc.
|
|
88
74
|
- Quicker turnaround time when making changes
|
|
89
75
|
- Debug, change code and run in a matter of seconds instead of minutes
|
|
90
76
|
- No accidental or auto commits
|
|
@@ -92,7 +78,7 @@ conda install -c conda-forge foundry-dev-tools
|
|
|
92
78
|
|
|
93
79
|
# License
|
|
94
80
|
|
|
95
|
-
Copyright (c)
|
|
81
|
+
Copyright (c) 2024 Merck KGaA, Darmstadt, Germany
|
|
96
82
|
|
|
97
83
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
98
84
|
you may not use this file except in compliance with the License.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = [
|
|
3
|
+
"pdm-backend",
|
|
4
|
+
]
|
|
5
|
+
build-backend = "pdm.backend"
|
|
6
|
+
|
|
7
|
+
[project]
|
|
8
|
+
name = "foundry-dev-tools"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
description = "Seamlessly run your Palantir Foundry Repository transforms code on your local machine."
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Nicolas Renkamp", email = "nicolas.renkamp@merckgroup.com" },
|
|
14
|
+
{ name = "Jonas Wunderlich", email = "jonas.wunderlich@merckgroup.com" },
|
|
15
|
+
]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 5 - Production/Stable",
|
|
18
|
+
"Programming Language :: Python",
|
|
19
|
+
"License :: OSI Approved :: Apache Software License",
|
|
20
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Intended Audience :: Developers",
|
|
25
|
+
"Operating System :: POSIX :: Linux",
|
|
26
|
+
"Operating System :: MacOS",
|
|
27
|
+
"Operating System :: Microsoft :: Windows",
|
|
28
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
29
|
+
]
|
|
30
|
+
dependencies = [
|
|
31
|
+
"requests",
|
|
32
|
+
"palantir-oauth-client",
|
|
33
|
+
"platformdirs",
|
|
34
|
+
"tomli; python_version <\"3.11\"",
|
|
35
|
+
"typing_extensions; python_version < \"3.11\"",
|
|
36
|
+
"backports-datetime-fromisoformat==2.0.1; python_version < \"3.11\"",
|
|
37
|
+
"click",
|
|
38
|
+
"inquirer",
|
|
39
|
+
"websockets",
|
|
40
|
+
"rich",
|
|
41
|
+
"packaging",
|
|
42
|
+
"tomli_w",
|
|
43
|
+
]
|
|
44
|
+
dynamic = []
|
|
45
|
+
version = "2.0.0"
|
|
46
|
+
|
|
47
|
+
[project.license]
|
|
48
|
+
text = "Apache-2.0"
|
|
49
|
+
|
|
50
|
+
[project.scripts]
|
|
51
|
+
fdt = "foundry_dev_tools.cli.main:cli"
|
|
52
|
+
git-credential-foundry = "foundry_dev_tools.cli.git_credential_foundry:_helper"
|
|
53
|
+
|
|
54
|
+
[project.optional-dependencies]
|
|
55
|
+
s3 = [
|
|
56
|
+
"aiobotocore[boto3]",
|
|
57
|
+
]
|
|
58
|
+
full = [
|
|
59
|
+
"foundry-dev-tools-transforms",
|
|
60
|
+
"foundry-dev-tools[s3]",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
[project.urls]
|
|
64
|
+
Homepage = "https://emdgroup.github.io/foundry-dev-tools"
|
|
65
|
+
Documentation = "https://emdgroup.github.io/foundry-dev-tools"
|
|
66
|
+
Source = "https://github.com/emdgroup/foundry-dev-tools"
|
|
67
|
+
Tracker = "https://github.com/emdgroup/foundry-dev-tools/issues"
|
|
68
|
+
Changelog = "https://emdgroup.github.io/foundry-dev-tools/changelog.html"
|
|
69
|
+
|
|
70
|
+
[tool.pdm]
|
|
71
|
+
distribution = true
|
|
72
|
+
|
|
73
|
+
[tool.pdm.version]
|
|
74
|
+
source = "scm"
|
|
75
|
+
write_to = "foundry_dev_tools/__about__.py"
|
|
76
|
+
write_template = "__version__ = '{}'"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = '2.0.0'
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Foundry Clients developed at Merck KGaA, Darmstadt, Germany."""
|
|
2
|
+
|
|
3
|
+
from foundry_dev_tools.__about__ import __version__
|
|
4
|
+
from foundry_dev_tools.cached_foundry_client import CachedFoundryClient
|
|
5
|
+
from foundry_dev_tools.config.config import Config
|
|
6
|
+
from foundry_dev_tools.config.config_types import Host
|
|
7
|
+
from foundry_dev_tools.config.context import FoundryContext
|
|
8
|
+
from foundry_dev_tools.config.token_provider import JWTTokenProvider, OAuthTokenProvider
|
|
9
|
+
from foundry_dev_tools.foundry_api_client import FoundryRestClient
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"__version__",
|
|
13
|
+
"Config",
|
|
14
|
+
"Host",
|
|
15
|
+
"JWTTokenProvider",
|
|
16
|
+
"FoundryContext",
|
|
17
|
+
"OAuthTokenProvider",
|
|
18
|
+
"FoundryRestClient",
|
|
19
|
+
"CachedFoundryClient",
|
|
20
|
+
]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from types import ModuleType
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FakeModule(ModuleType):
|
|
8
|
+
__fake__ = True
|
|
9
|
+
|
|
10
|
+
def __init__(self, name: str):
|
|
11
|
+
self.msg = f"Missing optional dependency '{name}'. Use conda or pip to install {name}"
|
|
12
|
+
super().__init__(name, self.msg)
|
|
13
|
+
|
|
14
|
+
def __getattr__(self, attr: Any) -> Any: # noqa: ANN401
|
|
15
|
+
if isinstance(attr, str) and attr.startswith("__"):
|
|
16
|
+
object.__getattribute__(self, attr)
|
|
17
|
+
raise ImportError(self.msg)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
import pyarrow as pa
|
|
5
|
+
import pyarrow.parquet as pq
|
|
6
|
+
|
|
7
|
+
pa.__fake__ = False
|
|
8
|
+
pq.__fake__ = False
|
|
9
|
+
except ImportError:
|
|
10
|
+
from foundry_dev_tools._optional import FakeModule
|
|
11
|
+
|
|
12
|
+
pa = FakeModule("pyarrow")
|
|
13
|
+
pq = FakeModule("pyarrow")
|
|
14
|
+
|
|
15
|
+
__all__ = ["pa"]
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
try:
|
|
4
|
+
import pyspark
|
|
5
|
+
import pyspark.sql as pyspark_sql
|
|
6
|
+
import pyspark.sql.types as pyspark_sql_types
|
|
7
|
+
|
|
8
|
+
pyspark.__fake__ = False
|
|
9
|
+
pyspark_sql.__fake__ = False
|
|
10
|
+
pyspark_sql_types.__fake__ = False
|
|
11
|
+
except ImportError:
|
|
12
|
+
from foundry_dev_tools._optional import FakeModule
|
|
13
|
+
|
|
14
|
+
pyspark = FakeModule("pyspark")
|
|
15
|
+
pyspark_sql = FakeModule("pyspark")
|
|
16
|
+
pyspark_sql_types = FakeModule("pyspark")
|
|
17
|
+
|
|
18
|
+
__all__ = ["pyspark"]
|