foundry-dev-tools 1.3.5__tar.gz → 1.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.pre-commit-config.yaml +1 -1
  2. {foundry-dev-tools-1.3.5/src/foundry_dev_tools.egg-info → foundry_dev_tools-1.4.1}/PKG-INFO +2 -1
  3. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/Configuration_usage.md +48 -30
  4. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/FoundryRestClient_usage.md +6 -6
  5. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/changelog.md +18 -1
  6. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/installation.md +1 -1
  7. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/pyproject.toml +1 -0
  8. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/build.py +9 -2
  9. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/config.py +1 -1
  10. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/foundry_api_client.py +18 -0
  11. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/repo.py +2 -2
  12. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1/src/foundry_dev_tools.egg-info}/PKG-INFO +2 -1
  13. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/requires.txt +1 -0
  14. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/transforms/api/__init__.py +10 -1
  15. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/transforms/api/_dataset.py +55 -24
  16. foundry_dev_tools-1.4.1/src/transforms/api/_decorators.py +271 -0
  17. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/transforms/api/_transform.py +209 -14
  18. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/conftest.py +11 -0
  19. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_cli_build.py +6 -14
  20. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_foundry_api.py +3 -0
  21. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_transforms.py +90 -8
  22. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_utils_repo.py +39 -12
  23. foundry-dev-tools-1.3.5/src/transforms/api/_decorators.py +0 -135
  24. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/blank_issue.yml +0 -0
  25. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  26. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  27. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  28. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.github/pull_request_template.md +0 -0
  29. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.github/workflows/ci.yml +0 -0
  30. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.github/workflows/docs.yml +0 -0
  31. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/.gitignore +0 -0
  32. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/LICENSE +0 -0
  33. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/README.md +0 -0
  34. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/FoundryFileSystem_usage.md +0 -0
  35. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/SSO_usage.md +0 -0
  36. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/_static/.gitignore +0 -0
  37. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/architecture.md +0 -0
  38. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/cli.md +0 -0
  39. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/conf.py +0 -0
  40. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/contributors.md +0 -0
  41. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/develop.md +0 -0
  42. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/index.md +0 -0
  43. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/license.md +0 -0
  44. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-already-cached-dark.svg +0 -0
  45. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-already-cached-light.svg +0 -0
  46. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-new-transaction-dark.svg +0 -0
  47. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-new-transaction-light.svg +0 -0
  48. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/pictures/tpa_config.png +0 -0
  49. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/requirements.txt +0 -0
  50. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/s3.md +0 -0
  51. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/docs/usage_and_examples.md +0 -0
  52. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/setup.cfg +0 -0
  53. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/setup.py +0 -0
  54. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/__about__.py +0 -0
  55. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/__init__.py +0 -0
  56. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
  57. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/__init__.py +0 -0
  58. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/info.py +0 -0
  59. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/main.py +0 -0
  60. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/s3.py +0 -0
  61. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/fsspec_impl.py +0 -0
  62. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/__init__.py +0 -0
  63. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/async_s3.py +0 -0
  64. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  65. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  66. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  67. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  68. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  69. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/importer.py +0 -0
  70. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/misc.py +0 -0
  71. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/s3.py +0 -0
  72. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/spark.py +0 -0
  73. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/token_provider.py +0 -0
  74. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/SOURCES.txt +0 -0
  75. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/dependency_links.txt +0 -0
  76. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/entry_points.txt +0 -0
  77. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/not-zip-safe +0 -0
  78. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/top_level.txt +0 -0
  79. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/transforms/__init__.py +0 -0
  80. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/src/transforms/api/_configure.py +0 -0
  81. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/__init__.py +0 -0
  82. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/foundry_mock_client.py +0 -0
  83. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_cached_foundry_client.py +0 -0
  84. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_cached_foundry_client_integration.py +0 -0
  85. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_cli_info.py +0 -0
  86. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_config.py +0 -0
  87. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_data/binary_dataset/bin +0 -0
  88. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_data/iris/iris.csv +0 -0
  89. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_foundry_local_deprecation.py +0 -0
  90. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_foundry_mock.py +0 -0
  91. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_foundry_spark_converters.py +0 -0
  92. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_foundry_sql_client.py +0 -0
  93. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_fsspec_impl.py +0 -0
  94. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_importer.py +0 -0
  95. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_multipass_tpa.py +0 -0
  96. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_s3.py +0 -0
  97. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_spark_caches.py +0 -0
  98. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_token_provider.py +0 -0
  99. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_transforms_integration.py +0 -0
  100. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/test_utils_misc.py +0 -0
  101. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tests/utils.py +0 -0
  102. {foundry-dev-tools-1.3.5 → foundry_dev_tools-1.4.1}/tox.ini +0 -0
@@ -6,7 +6,7 @@ repos:
6
6
  args: ['--fix=lf'] # replace 'auto' with 'lf' to enforce Linux/Mac line endings or 'crlf' for Windows
7
7
  - repo: https://github.com/astral-sh/ruff-pre-commit
8
8
  # Ruff version.
9
- rev: 'v0.1.3'
9
+ rev: 'v0.1.15'
10
10
  hooks:
11
11
  - id: ruff
12
12
  args: ["--fix","--exit-non-zero-on-fix","--show-fixes"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.3.5
3
+ Version: 1.4.1
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -44,6 +44,7 @@ Requires-Dist: pytest-asyncio; extra == "testing"
44
44
  Requires-Dist: requests-mock; extra == "testing"
45
45
  Requires-Dist: fsspec; extra == "testing"
46
46
  Requires-Dist: timeflake; extra == "testing"
47
+ Requires-Dist: polars; extra == "testing"
47
48
  Provides-Extra: transforms
48
49
  Requires-Dist: pyspark>=3.0.0; extra == "transforms"
49
50
  Provides-Extra: cli
@@ -1,32 +1,56 @@
1
1
  # Configuration
2
2
 
3
- By default, the code loads the configuration from the '~/.foundry-dev-tools/config' file.
4
- If a project [specific config file](#project-specific-configuration) exists in `your-project-dir/.foundry_dev_tools`, it takes precedence.
5
- If a environment variable like `FOUNDRY_DEV_TOOLS_config_key` exists, `config_key` gets overwritten by this environment variable.
6
- Afterwards, the logic falls back to a default value: ( overwrites > environment variable > project config file > config file > default)
3
+ Configuration options can be supplied to the library through multiple different methods that are processed in the following order:
4
+ 1. Every configuration option is initalized to its default value as given in the [table below](#configuration-options).
5
+ 2. If a file `~/.foundry-dev-tools/config` exists, values defined therein overwrite the corresponding default values.
6
+ See [below](#foundry-dev-tools-config-file) for details on its structure.
7
+ 3. If a project [specific config file](#project-specific-configuration) exists in `your-project-dir/.foundry_dev_tools`, values set in there take precedence.
8
+ 4. If an environment variable like `FOUNDRY_DEV_TOOLS_config_key` exists, `config_key` gets overwritten by this environment variable.
9
+ Note, that environment variable names are always all-caps, e.g. the `foundry_url` config value can be set via `export FOUNDRY_DEV_TOOLS_FOUNDRY_URL=https://foundry.example.com`.
10
+ 5. Configuration options can be [overwritten programmatically](#changing-configuration-programmatically) during runtime.
7
11
 
8
- In the standard setup, the configuration should contain either the Foundry Token (`jwt`) or the Foundry Client
9
- ID (`client_id`) for SSO.
12
+ Thus, the order of precedence for configuration options is: runtime overwrites > environment variable > project config file > config file > default.
13
+
14
+ In the standard setup, the configuration should contain either the Foundry Token (`jwt`) or the Foundry Client ID (`client_id`) for SSO.
10
15
 
11
16
  ## Configuration options
12
17
 
13
- | Name | Description | Values |
14
- |----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|
15
- | foundry_url | Url of the foundry instance. | e.g. https://foundry.example.com |
16
- | jwt | The Bearer Token copied from the Foundry Token Page | eyJhb... |
17
- | client_id | The client_id of the Foundry Third Party application for SSO Authentication. | ... |
18
- | client_secret | The client_secret of the Foundry Third Party application. | ... |
19
- | grant_type | Set this to client_credentials if you use Foundry Third Party application client_credentials flow. Default is authorization_code | authorization_code, client_credentials |
20
- | scopes | Custom scopes for Third Party Application Tokens | "api:read-data offline_access" |
21
- | requests_ca_bundle | Path to custom ca bundle file, useful in corporate networks with SSL inspection | not set |
22
- | transforms_sql_sample_row_limit | Number of rows that are retrieved for each dataset when sql is used | 5000 |
23
- | transforms_sql_dataset_size_threshold | Maximal size of a dataset up to which file gets downloaded. If dataset is larger, sql with limit query will be used to download a partial dataset | 500 |
24
- | transforms_sql_sample_select_random | True or False, if data is randomly sampled when SQL mode is used (query takes more time) | True or False |
25
- | transforms_force_full_dataset_download | if file_download is True, all files of datasets are downloaded | True or False |
26
- | cache_dir | Point cache directory to other directory than default | ~/.foundry-dev-tools/cache |
27
- | transforms_freeze_cache | Uses offline cache as is; no Foundry connection needed | True or False (default: False) |
28
- | transforms_output_folder | When @transform in combination with TransformOutput.filesystem() is used, files are written to this folder. | /projectA/local_files |
29
- | enable_runtime_token_providers | Enables the token providers, Default is True | True or False |
18
+ | Name | Description | Values | Default Values |
19
+ |----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|------------------------------|
20
+ | foundry_url | Url of the foundry instance | e.g. https://foundry.example.com | not set (None) |
21
+ | jwt | The Bearer Token copied from the Foundry Token Page | eyJhb... | not set (None) |
22
+ | client_id | The client_id of the Foundry Third Party application for SSO Authentication | ... | not set (None) |
23
+ | client_secret | The client_secret of the Foundry Third Party application | ... | not set (None) |
24
+ | grant_type | Set this to client_credentials if you use Foundry Third Party application client_credentials flow. Default is authorization_code | authorization_code, client_credentials | authorization_code |
25
+ | scopes | Custom scopes for Third Party Application Tokens | "api:read-data offline_access" | not set (None) |
26
+ | requests_ca_bundle | Path to custom ca bundle file, useful in corporate networks with SSL inspection | not set | not set (None) |
27
+ | transforms_sql_sample_row_limit | Number of rows that are retrieved for each dataset when sql is used | 5000 | 5000 |
28
+ | transforms_sql_dataset_size_threshold | Maximum size of a dataset up to which file gets downloaded. If dataset is larger, sql with limit query will be used to download a partial dataset | 500 | 500 |
29
+ | transforms_sql_sample_select_random | True or False, if data is randomly sampled when SQL mode is used (query takes more time) | True or False | False |
30
+ | transforms_force_full_dataset_download | if file_download is True, all files of datasets are downloaded | True or False | False |
31
+ | cache_dir | Point cache directory to other directory than default | ~/.foundry-dev-tools/cache | `~/.foundry-dev-tools/cache` |
32
+ | transforms_freeze_cache | Uses offline cache as is; no Foundry connection needed | True or False | False |
33
+ | transforms_output_folder | When @transform in combination with `TransformOutput.filesystem()` is used, files are written to this folder. | /projectA/local_files | not set (None) |
34
+ | enable_runtime_token_providers | Enables the token providers | True or False | True |
35
+
36
+ ## `~/.foundry-dev-tools/config` file
37
+ The content of the file `~/.foundry-dev-tools/config` must be compatible with the Python `configparser` library, i.e. it must have a structure similar to what is found in Microsoft Windows INI files.
38
+ See https://docs.python.org/3/library/configparser.html#supported-ini-file-structure for details.
39
+ All configuration options have to reside in a section titled `[default]`.
40
+ Content of all other sections is ignored while parsing the file.
41
+ Thus, an exemplary `~/.foundry-dev-tools/config` file reads
42
+ ```
43
+ [default]
44
+ foundry_url=https://foundry.example.com
45
+ jwt=eyJhb...
46
+ ```
47
+
48
+ ## Project specific configuration
49
+
50
+ You can provide a project-specific configuration via a project-local file `.foundry_dev_tools`.
51
+ This file has to reside in your project's root directory.
52
+ The structure of this file is equivalent to the [`~/.foundry-dev-tools/config`](#foundry-dev-tools-config-file) file.
53
+ Entries of the default config are overwritten with entries from the `.foundry_dev_tools` file as specified [above](#configuration).
30
54
 
31
55
  ## Changing configuration programmatically
32
56
 
@@ -80,13 +104,7 @@ fc2.get_user_info()
80
104
  {"username":"john",...}
81
105
  ```
82
106
 
83
- ## Project specific configuration
84
-
85
- You can provide a configuration per project at the top level of the git repository in file `.foundry_dev_tools`.
86
- The structure of this file is equivalent to the `~/.foundry-dev-tools/config` file. Entries of the default config are
87
- overwritten with entries from the `.foundry_dev_tools` file.
88
-
89
- ## Enable Logging
107
+ # Enable Logging
90
108
 
91
109
  With the following code you can enable the DEBUG logging output of foundry-dev-tools:
92
110
 
@@ -34,13 +34,13 @@ Upload the complete content of a local folder to a dataset in Foundry
34
34
  import os
35
35
  from foundry_dev_tools import FoundryRestClient
36
36
 
37
- upload_folder = "/path/to/folder-to-upload"
38
- target_dataset_path = "/paht/to/test_folder_upload"
37
+ upload_folder = "/local-path/to/folder-to-upload"
38
+ target_dataset_path = "/path/to/foundry-dataset"
39
+
40
+ file_paths = [file for file in Path(upload_folder).rglob("*") if file.is_file() and not file.name.startswith(".")]
41
+ dataset_paths_in_foundry = [str(file_path.relative_to(upload_folder)) for file_path in file_paths]
42
+ path_file_dict = dict(zip(dataset_paths_in_foundry, file_paths))
39
43
 
40
- filenames = os.listdir(upload_folder)
41
- filepaths = list(map(lambda file: os.sep.join([upload_folder, file]), filenames))
42
- dataset_paths_in_foundry = list(map(lambda file: file, filenames))
43
- path_file_dict = dict(zip(dataset_paths_in_foundry, filepaths))
44
44
  rest_client = FoundryRestClient()
45
45
  dataset_rid = rest_client.get_dataset_rid(dataset_path=target_dataset_path)
46
46
  transaction_rid = rest_client.open_transaction(dataset_rid=dataset_rid,
@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog],
6
6
  and this project adheres to [Semantic Versioning].
7
7
 
8
+ ## [1.4.1] - 2024-05-29
9
+
10
+ ## Added
11
+ - Support for git submodules (#56)
12
+
13
+ ## [1.4.0] - 2024-02-05
14
+
15
+ ## Added
16
+
17
+ - support for lightweight transforms (#47)
18
+
19
+ ## Fixed
20
+
21
+ - PySpark link in installation docs (#48)
22
+
8
23
  ## [1.3.5] - 2023-12-05
9
24
 
10
25
  ## Fixed
@@ -189,7 +204,9 @@ and this project adheres to [Semantic Versioning].
189
204
  [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
190
205
  [Semantic Versioning]: https://semver.org/spec/v2.0.0.html
191
206
 
192
- [1.3.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.4...v1.3.5
207
+ [1.4.1]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.4.0...v1.4.1
208
+ [1.4.0]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.5...v1.4.0
209
+ [1.3.5]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.4...v1.3.5
193
210
  [1.3.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.3...v1.3.4
194
211
  [1.3.3]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.2...v1.3.3
195
212
  [1.3.2]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.1...v1.3.2
@@ -65,7 +65,7 @@ pip install 'foundry-dev-tools[transforms]'
65
65
  ```
66
66
 
67
67
  Alternative installation methods, or more information on how to get Spark running,
68
- can be found in the [PySpark Documenation] and the [Spark Documentation].
68
+ can be found in the [PySpark Documentation] and the [Spark Documentation].
69
69
 
70
70
  [PySpark Documentation]: https://spark.apache.org/docs/latest/api/python/getting_started/install.html
71
71
  [Spark Documentation]: https://spark.apache.org/docs/latest/
@@ -54,6 +54,7 @@ testing = [
54
54
  "requests-mock",
55
55
  "fsspec",
56
56
  "timeflake",
57
+ "polars"
57
58
  ]
58
59
  transforms = ["pyspark>=3.0.0"]
59
60
  cli = ["click", "inquirer", "websockets", "rich", "packaging"]
@@ -104,7 +104,13 @@ def create_log_record(log_message: str) -> logging.LogRecord:
104
104
  )
105
105
 
106
106
 
107
- TRANSFORM_DECORATORS = ["transform", "transform_df", "transform_pandas"]
107
+ TRANSFORM_DECORATORS = [
108
+ "lightweight",
109
+ "transform",
110
+ "transform_df",
111
+ "transform_pandas",
112
+ "transform_polars",
113
+ ]
108
114
 
109
115
 
110
116
  def is_transform_file(transform_file: Path) -> bool:
@@ -112,7 +118,7 @@ def is_transform_file(transform_file: Path) -> bool:
112
118
 
113
119
  Conditions are that it must be a file (obviously)
114
120
  the name must end in ".py"
115
- and the file must contain either @transform|@transform_df
121
+ and the file must contain a transform decorator.
116
122
  """
117
123
  if not transform_file.is_file():
118
124
  return False
@@ -126,6 +132,7 @@ def is_transform_file(transform_file: Path) -> bool:
126
132
  if isinstance(node, ast.FunctionDef) and any(
127
133
  decorator.func.id in TRANSFORM_DECORATORS
128
134
  for decorator in node.decorator_list
135
+ if hasattr(decorator, "func")
129
136
  ):
130
137
  return True
131
138
 
@@ -244,7 +244,7 @@ class Config(UserDict):
244
244
  raise KeyError(key)
245
245
  self.data[key] = None
246
246
 
247
- def set(self, key, value): # noqa: A003
247
+ def set(self, key, value):
248
248
  """Deprecated: Stores value in config.
249
249
 
250
250
  Args:
@@ -323,6 +323,24 @@ class FoundryRestClient:
323
323
  _raise_for_status_verbose(response)
324
324
  return response.json()
325
325
 
326
+ def get_branches(self, dataset_rid: str) -> list[str]:
327
+ """Returns a list of branches available a dataset.
328
+
329
+ Args:
330
+ dataset_rid (str): Unique identifier of the dataset
331
+
332
+ Returns:
333
+ list[str]:
334
+ with keys id (name) the dataset branches
335
+
336
+ """
337
+ response = self._request(
338
+ "GET",
339
+ f"{self.catalog}/catalog/datasets/{dataset_rid}/branches2",
340
+ )
341
+ _raise_for_status_verbose(response)
342
+ return response.json()
343
+
326
344
  def get_branch(self, dataset_rid: str, branch: str) -> dict:
327
345
  """Returns branch information.
328
346
 
@@ -113,10 +113,10 @@ def git_toplevel_dir(
113
113
  )
114
114
  if git_dir is None:
115
115
  git_dir = Path.cwd()
116
- if git_dir.joinpath(".git").is_dir():
116
+ if git_dir.joinpath(".git").exists():
117
117
  return git_dir
118
118
  for p in git_dir.resolve().parents:
119
- if p.joinpath(".git").is_dir():
119
+ if p.joinpath(".git").exists():
120
120
  return p
121
121
  return None
122
122
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.3.5
3
+ Version: 1.4.1
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -44,6 +44,7 @@ Requires-Dist: pytest-asyncio; extra == "testing"
44
44
  Requires-Dist: requests-mock; extra == "testing"
45
45
  Requires-Dist: fsspec; extra == "testing"
46
46
  Requires-Dist: timeflake; extra == "testing"
47
+ Requires-Dist: polars; extra == "testing"
47
48
  Provides-Extra: transforms
48
49
  Requires-Dist: pyspark>=3.0.0; extra == "transforms"
49
50
  Provides-Extra: cli
@@ -28,6 +28,7 @@ pytest-asyncio
28
28
  requests-mock
29
29
  fsspec
30
30
  timeflake
31
+ polars
31
32
 
32
33
  [transforms]
33
34
  pyspark>=3.0.0
@@ -7,7 +7,14 @@ https://www.palantir.com/docs/foundry/transforms-python/transforms-python-api-cl
7
7
 
8
8
  from ._configure import configure
9
9
  from ._dataset import Input, Markings, OrgMarkings, Output
10
- from ._decorators import incremental, transform, transform_df, transform_pandas
10
+ from ._decorators import (
11
+ incremental,
12
+ lightweight,
13
+ transform,
14
+ transform_df,
15
+ transform_pandas,
16
+ transform_polars,
17
+ )
11
18
  from ._transform import TransformContext
12
19
 
13
20
  __all__ = (
@@ -18,6 +25,8 @@ __all__ = (
18
25
  "transform_df",
19
26
  "transform_pandas",
20
27
  "transform",
28
+ "lightweight",
29
+ "transform_polars",
21
30
  "TransformContext",
22
31
  "incremental",
23
32
  "configure",
@@ -9,7 +9,7 @@ import inspect
9
9
  import logging
10
10
  import warnings
11
11
  from pathlib import Path
12
- from typing import Any
12
+ from typing import Any, Optional
13
13
 
14
14
  import pyspark
15
15
 
@@ -52,8 +52,8 @@ def _as_list(list_or_single_item: "list[Any] | Any | None") -> "list[Any]":
52
52
  class Input:
53
53
  """Specification of a transform dataset input.
54
54
 
55
- The actual download is initialized when the Input class is constructed.
56
- It is currently not lazy loaded.
55
+ Some API requests may be sent when the Input class is constructed. However, the actual download
56
+ is only initiated when dataframe() or get_local_path_to_dataset() is called.
57
57
 
58
58
  """
59
59
 
@@ -85,24 +85,31 @@ class Input:
85
85
  self.config = foundry_dev_tools.config.Configuration.get_config()
86
86
  self._cached_client = CachedFoundryClient(self.config)
87
87
  self._cache = DiskPersistenceBackedSparkCache(**self.config)
88
+ self._spark_df = None
88
89
  if branch is None:
89
90
  branch = _get_branch(Path(caller_filename))
90
91
 
91
- if (
92
+ if self._is_online:
93
+ (
94
+ self._is_spark_df_retrievable,
95
+ self._dataset_identity,
96
+ self.branch,
97
+ ) = self._online(alias, branch)
98
+ else:
99
+ (
100
+ self._is_spark_df_retrievable,
101
+ self._dataset_identity,
102
+ self.branch,
103
+ ) = self._offline(alias, branch)
104
+
105
+ @property
106
+ def _is_online(self) -> bool:
107
+ return (
92
108
  "transforms_freeze_cache" not in self.config
93
109
  or self.config["transforms_freeze_cache"] is False
94
- ):
95
- self._spark_df, self._dataset_identity, self.branch = self._online(
96
- alias, branch
97
- )
98
- else:
99
- self._spark_df, self._dataset_identity, self.branch = self._offline(
100
- alias, branch
101
- )
110
+ )
102
111
 
103
- def _online(
104
- self, alias, branch
105
- ) -> "tuple[pyspark.sql.DataFrame | None, dict, str]":
112
+ def _online(self, alias, branch) -> "tuple[bool, dict, str]":
106
113
  try:
107
114
  dataset_identity = self._cached_client.api.get_dataset_identity(
108
115
  alias, branch
@@ -122,7 +129,7 @@ class Input:
122
129
  raise DatasetHasNoTransactionsError(alias)
123
130
  if self._dataset_has_schema(dataset_identity, branch):
124
131
  return (
125
- self._retrieve_spark_df(dataset_identity, branch),
132
+ True,
126
133
  dataset_identity,
127
134
  branch,
128
135
  )
@@ -134,16 +141,13 @@ class Input:
134
141
  dataset_identity["dataset_path"],
135
142
  branch,
136
143
  )
137
- self._cached_client._fetch_dataset(dataset_identity, branch)
138
- return None, dataset_identity, branch
144
+ return False, dataset_identity, branch
139
145
 
140
- def _offline(
141
- self, alias: str, branch: str
142
- ) -> "tuple[pyspark.sql.DataFrame | None, dict, str]":
146
+ def _offline(self, alias: str, branch: str) -> "tuple[bool, dict, str]":
143
147
  dataset_identity = self._cache.get_dataset_identity_not_branch_aware(alias)
144
148
  if self._cache.dataset_has_schema(dataset_identity):
145
- return self._cache[dataset_identity], dataset_identity, branch
146
- return None, dataset_identity, branch
149
+ return True, dataset_identity, branch
150
+ return False, dataset_identity, branch
147
151
 
148
152
  def _dataset_has_schema(self, dataset_identity, branch):
149
153
  try:
@@ -228,13 +232,24 @@ class Input:
228
232
  self._cache[dataset_identity] = spark_df
229
233
  return spark_df
230
234
 
231
- def dataframe(self) -> pyspark.sql.DataFrame:
235
+ def dataframe(self) -> Optional[pyspark.sql.DataFrame]:
232
236
  """Get the cached :external+spark:py:class:`~pyspark.sql.DataFrame` of this Input.
233
237
 
238
+ Only available if the input has a schema. The Spark DataFrame will get loaded the first
239
+ time this method is invoked.
240
+
234
241
  Returns:
235
242
  :external+spark:py:class:`~pyspark.sql.DataFrame`: The cached DataFrame of this Input
236
243
 
237
244
  """
245
+ if self._is_spark_df_retrievable and self._spark_df is None:
246
+ if self._is_online:
247
+ self._spark_df = self._retrieve_spark_df(
248
+ self._dataset_identity, self.branch
249
+ )
250
+ else:
251
+ self._spark_df = self._cache[self._dataset_identity]
252
+
238
253
  return self._spark_df
239
254
 
240
255
  def get_dataset_identity(self) -> dict:
@@ -247,6 +262,22 @@ class Input:
247
262
  """
248
263
  return self._dataset_identity
249
264
 
265
+ def get_local_path_to_dataset(self) -> str:
266
+ """Returns path to the dataset's files on disk.
267
+
268
+ Calling this method for the first time may trigger downloading the dataset files.
269
+
270
+ Returns:
271
+ str:
272
+ path to the dataset's files on disk
273
+
274
+ """
275
+ return (
276
+ self._cached_client._fetch_dataset(self._dataset_identity, self.branch)
277
+ if self._is_online
278
+ else self._cache.get_path_to_local_dataset(self._dataset_identity)
279
+ )
280
+
250
281
 
251
282
  def _get_branch(caller_filename: Path) -> str:
252
283
  git_dir = git_toplevel_dir(caller_filename)