foundry-dev-tools 1.4.0__tar.gz → 1.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. {foundry-dev-tools-1.4.0/src/foundry_dev_tools.egg-info → foundry_dev_tools-1.4.1}/PKG-INFO +1 -1
  2. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/Configuration_usage.md +48 -30
  3. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/FoundryRestClient_usage.md +6 -6
  4. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/changelog.md +8 -1
  5. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/foundry_api_client.py +18 -0
  6. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/repo.py +2 -2
  7. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1/src/foundry_dev_tools.egg-info}/PKG-INFO +1 -1
  8. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/conftest.py +11 -0
  9. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_cli_build.py +6 -14
  10. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_foundry_api.py +3 -0
  11. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_utils_repo.py +39 -12
  12. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/blank_issue.yml +0 -0
  13. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  14. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  15. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  16. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.github/pull_request_template.md +0 -0
  17. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.github/workflows/ci.yml +0 -0
  18. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.github/workflows/docs.yml +0 -0
  19. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.gitignore +0 -0
  20. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/.pre-commit-config.yaml +0 -0
  21. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/LICENSE +0 -0
  22. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/README.md +0 -0
  23. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/FoundryFileSystem_usage.md +0 -0
  24. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/SSO_usage.md +0 -0
  25. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/_static/.gitignore +0 -0
  26. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/architecture.md +0 -0
  27. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/cli.md +0 -0
  28. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/conf.py +0 -0
  29. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/contributors.md +0 -0
  30. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/develop.md +0 -0
  31. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/index.md +0 -0
  32. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/installation.md +0 -0
  33. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/license.md +0 -0
  34. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-already-cached-dark.svg +0 -0
  35. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-already-cached-light.svg +0 -0
  36. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-new-transaction-dark.svg +0 -0
  37. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/pictures/mermaid-diagram-new-transaction-light.svg +0 -0
  38. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/pictures/tpa_config.png +0 -0
  39. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/requirements.txt +0 -0
  40. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/s3.md +0 -0
  41. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/docs/usage_and_examples.md +0 -0
  42. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/pyproject.toml +0 -0
  43. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/setup.cfg +0 -0
  44. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/setup.py +0 -0
  45. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/__about__.py +0 -0
  46. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/__init__.py +0 -0
  47. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
  48. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/__init__.py +0 -0
  49. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/build.py +0 -0
  50. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/info.py +0 -0
  51. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/main.py +0 -0
  52. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/cli/s3.py +0 -0
  53. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/config.py +0 -0
  54. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/fsspec_impl.py +0 -0
  55. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/__init__.py +0 -0
  56. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/async_s3.py +0 -0
  57. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  58. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  59. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  60. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  61. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  62. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/importer.py +0 -0
  63. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/misc.py +0 -0
  64. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/s3.py +0 -0
  65. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/spark.py +0 -0
  66. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools/utils/token_provider.py +0 -0
  67. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/SOURCES.txt +0 -0
  68. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/dependency_links.txt +0 -0
  69. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/entry_points.txt +0 -0
  70. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/not-zip-safe +0 -0
  71. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/requires.txt +0 -0
  72. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/foundry_dev_tools.egg-info/top_level.txt +0 -0
  73. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/transforms/__init__.py +0 -0
  74. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/transforms/api/__init__.py +0 -0
  75. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/transforms/api/_configure.py +0 -0
  76. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/transforms/api/_dataset.py +0 -0
  77. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/transforms/api/_decorators.py +0 -0
  78. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/src/transforms/api/_transform.py +0 -0
  79. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/__init__.py +0 -0
  80. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/foundry_mock_client.py +0 -0
  81. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_cached_foundry_client.py +0 -0
  82. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_cached_foundry_client_integration.py +0 -0
  83. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_cli_info.py +0 -0
  84. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_config.py +0 -0
  85. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_data/binary_dataset/bin +0 -0
  86. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_data/iris/iris.csv +0 -0
  87. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_foundry_local_deprecation.py +0 -0
  88. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_foundry_mock.py +0 -0
  89. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_foundry_spark_converters.py +0 -0
  90. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_foundry_sql_client.py +0 -0
  91. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_fsspec_impl.py +0 -0
  92. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_importer.py +0 -0
  93. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_multipass_tpa.py +0 -0
  94. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_s3.py +0 -0
  95. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_spark_caches.py +0 -0
  96. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_token_provider.py +0 -0
  97. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_transforms.py +0 -0
  98. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_transforms_integration.py +0 -0
  99. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/test_utils_misc.py +0 -0
  100. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tests/utils.py +0 -0
  101. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.1}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.4.0
3
+ Version: 1.4.1
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -1,32 +1,56 @@
1
1
  # Configuration
2
2
 
3
- By default, the code loads the configuration from the '~/.foundry-dev-tools/config' file.
4
- If a project [specific config file](#project-specific-configuration) exists in `your-project-dir/.foundry_dev_tools`, it takes precedence.
5
- If a environment variable like `FOUNDRY_DEV_TOOLS_config_key` exists, `config_key` gets overwritten by this environment variable.
6
- Afterwards, the logic falls back to a default value: ( overwrites > environment variable > project config file > config file > default)
3
+ Configuration options can be supplied to the library through multiple different methods that are processed in the following order:
4
+ 1. Every configuration option is initalized to its default value as given in the [table below](#configuration-options).
5
+ 2. If a file `~/.foundry-dev-tools/config` exists, values defined therein overwrite the corresponding default values.
6
+ See [below](#foundry-dev-tools-config-file) for details on its structure.
7
+ 3. If a project [specific config file](#project-specific-configuration) exists in `your-project-dir/.foundry_dev_tools`, values set in there take precedence.
8
+ 4. If an environment variable like `FOUNDRY_DEV_TOOLS_config_key` exists, `config_key` gets overwritten by this environment variable.
9
+ Note, that environment variable names are always all-caps, e.g. the `foundry_url` config value can be set via `export FOUNDRY_DEV_TOOLS_FOUNDRY_URL=https://foundry.example.com`.
10
+ 5. Configuration options can be [overwritten programmatically](#changing-configuration-programmatically) during runtime.
7
11
 
8
- In the standard setup, the configuration should contain either the Foundry Token (`jwt`) or the Foundry Client
9
- ID (`client_id`) for SSO.
12
+ Thus, the order of precedence for configuration options is: runtime overwrites > environment variable > project config file > config file > default.
13
+
14
+ In the standard setup, the configuration should contain either the Foundry Token (`jwt`) or the Foundry Client ID (`client_id`) for SSO.
10
15
 
11
16
  ## Configuration options
12
17
 
13
- | Name | Description | Values |
14
- |----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|
15
- | foundry_url | Url of the foundry instance. | e.g. https://foundry.example.com |
16
- | jwt | The Bearer Token copied from the Foundry Token Page | eyJhb... |
17
- | client_id | The client_id of the Foundry Third Party application for SSO Authentication. | ... |
18
- | client_secret | The client_secret of the Foundry Third Party application. | ... |
19
- | grant_type | Set this to client_credentials if you use Foundry Third Party application client_credentials flow. Default is authorization_code | authorization_code, client_credentials |
20
- | scopes | Custom scopes for Third Party Application Tokens | "api:read-data offline_access" |
21
- | requests_ca_bundle | Path to custom ca bundle file, useful in corporate networks with SSL inspection | not set |
22
- | transforms_sql_sample_row_limit | Number of rows that are retrieved for each dataset when sql is used | 5000 |
23
- | transforms_sql_dataset_size_threshold | Maximal size of a dataset up to which file gets downloaded. If dataset is larger, sql with limit query will be used to download a partial dataset | 500 |
24
- | transforms_sql_sample_select_random | True or False, if data is randomly sampled when SQL mode is used (query takes more time) | True or False |
25
- | transforms_force_full_dataset_download | if file_download is True, all files of datasets are downloaded | True or False |
26
- | cache_dir | Point cache directory to other directory than default | ~/.foundry-dev-tools/cache |
27
- | transforms_freeze_cache | Uses offline cache as is; no Foundry connection needed | True or False (default: False) |
28
- | transforms_output_folder | When @transform in combination with TransformOutput.filesystem() is used, files are written to this folder. | /projectA/local_files |
29
- | enable_runtime_token_providers | Enables the token providers, Default is True | True or False |
18
+ | Name | Description | Values | Default Values |
19
+ |----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|------------------------------|
20
+ | foundry_url | Url of the foundry instance | e.g. https://foundry.example.com | not set (None) |
21
+ | jwt | The Bearer Token copied from the Foundry Token Page | eyJhb... | not set (None) |
22
+ | client_id | The client_id of the Foundry Third Party application for SSO Authentication | ... | not set (None) |
23
+ | client_secret | The client_secret of the Foundry Third Party application | ... | not set (None) |
24
+ | grant_type | Set this to client_credentials if you use Foundry Third Party application client_credentials flow. Default is authorization_code | authorization_code, client_credentials | authorization_code |
25
+ | scopes | Custom scopes for Third Party Application Tokens | "api:read-data offline_access" | not set (None) |
26
+ | requests_ca_bundle | Path to custom ca bundle file, useful in corporate networks with SSL inspection | not set | not set (None) |
27
+ | transforms_sql_sample_row_limit | Number of rows that are retrieved for each dataset when sql is used | 5000 | 5000 |
28
+ | transforms_sql_dataset_size_threshold | Maximum size of a dataset up to which file gets downloaded. If dataset is larger, sql with limit query will be used to download a partial dataset | 500 | 500 |
29
+ | transforms_sql_sample_select_random | True or False, if data is randomly sampled when SQL mode is used (query takes more time) | True or False | False |
30
+ | transforms_force_full_dataset_download | if file_download is True, all files of datasets are downloaded | True or False | False |
31
+ | cache_dir | Point cache directory to other directory than default | ~/.foundry-dev-tools/cache | `~/.foundry-dev-tools/cache` |
32
+ | transforms_freeze_cache | Uses offline cache as is; no Foundry connection needed | True or False | False |
33
+ | transforms_output_folder | When @transform in combination with `TransformOutput.filesystem()` is used, files are written to this folder. | /projectA/local_files | not set (None) |
34
+ | enable_runtime_token_providers | Enables the token providers | True or False | True |
35
+
36
+ ## `~/.foundry-dev-tools/config` file
37
+ The content of the file `~/.foundry-dev-tools/config` must be compatible with the Python `configparser` library, i.e. it must have a structure similar to what is found in Microsoft Windows INI files.
38
+ See https://docs.python.org/3/library/configparser.html#supported-ini-file-structure for details.
39
+ All configuration options have to reside in a section titled `[default]`.
40
+ Content of all other sections is ignored while parsing the file.
41
+ Thus, an exemplary `~/.foundry-dev-tools/config` file reads
42
+ ```
43
+ [default]
44
+ foundry_url=https://foundry.example.com
45
+ jwt=eyJhb...
46
+ ```
47
+
48
+ ## Project specific configuration
49
+
50
+ You can provide a project-specific configuration via a project-local file `.foundry_dev_tools`.
51
+ This file has to reside in your project's root directory.
52
+ The structure of this file is equivalent to the [`~/.foundry-dev-tools/config`](#foundry-dev-tools-config-file) file.
53
+ Entries of the default config are overwritten with entries from the `.foundry_dev_tools` file as specified [above](#configuration).
30
54
 
31
55
  ## Changing configuration programmatically
32
56
 
@@ -80,13 +104,7 @@ fc2.get_user_info()
80
104
  {"username":"john",...}
81
105
  ```
82
106
 
83
- ## Project specific configuration
84
-
85
- You can provide a configuration per project at the top level of the git repository in file `.foundry_dev_tools`.
86
- The structure of this file is equivalent to the `~/.foundry-dev-tools/config` file. Entries of the default config are
87
- overwritten with entries from the `.foundry_dev_tools` file.
88
-
89
- ## Enable Logging
107
+ # Enable Logging
90
108
 
91
109
  With the following code you can enable the DEBUG logging output of foundry-dev-tools:
92
110
 
@@ -34,13 +34,13 @@ Upload the complete content of a local folder to a dataset in Foundry
34
34
  import os
35
35
  from foundry_dev_tools import FoundryRestClient
36
36
 
37
- upload_folder = "/path/to/folder-to-upload"
38
- target_dataset_path = "/paht/to/test_folder_upload"
37
+ upload_folder = "/local-path/to/folder-to-upload"
38
+ target_dataset_path = "/path/to/foundry-dataset"
39
+
40
+ file_paths = [file for file in Path(upload_folder).rglob("*") if file.is_file() and not file.name.startswith(".")]
41
+ dataset_paths_in_foundry = [str(file_path.relative_to(upload_folder)) for file_path in file_paths]
42
+ path_file_dict = dict(zip(dataset_paths_in_foundry, file_paths))
39
43
 
40
- filenames = os.listdir(upload_folder)
41
- filepaths = list(map(lambda file: os.sep.join([upload_folder, file]), filenames))
42
- dataset_paths_in_foundry = list(map(lambda file: file, filenames))
43
- path_file_dict = dict(zip(dataset_paths_in_foundry, filepaths))
44
44
  rest_client = FoundryRestClient()
45
45
  dataset_rid = rest_client.get_dataset_rid(dataset_path=target_dataset_path)
46
46
  transaction_rid = rest_client.open_transaction(dataset_rid=dataset_rid,
@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog],
6
6
  and this project adheres to [Semantic Versioning].
7
7
 
8
+ ## [1.4.1] - 2024-05-29
9
+
10
+ ## Added
11
+ - Support for git submodules (#56)
12
+
8
13
  ## [1.4.0] - 2024-02-05
9
14
 
10
15
  ## Added
@@ -199,7 +204,9 @@ and this project adheres to [Semantic Versioning].
199
204
  [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
200
205
  [Semantic Versioning]: https://semver.org/spec/v2.0.0.html
201
206
 
202
- [1.3.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.4...v1.3.5
207
+ [1.4.1]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.4.0...v1.4.1
208
+ [1.4.0]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.5...v1.4.0
209
+ [1.3.5]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.4...v1.3.5
203
210
  [1.3.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.3...v1.3.4
204
211
  [1.3.3]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.2...v1.3.3
205
212
  [1.3.2]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.1...v1.3.2
@@ -323,6 +323,24 @@ class FoundryRestClient:
323
323
  _raise_for_status_verbose(response)
324
324
  return response.json()
325
325
 
326
+ def get_branches(self, dataset_rid: str) -> list[str]:
327
+ """Returns a list of branches available a dataset.
328
+
329
+ Args:
330
+ dataset_rid (str): Unique identifier of the dataset
331
+
332
+ Returns:
333
+ list[str]:
334
+ with keys id (name) the dataset branches
335
+
336
+ """
337
+ response = self._request(
338
+ "GET",
339
+ f"{self.catalog}/catalog/datasets/{dataset_rid}/branches2",
340
+ )
341
+ _raise_for_status_verbose(response)
342
+ return response.json()
343
+
326
344
  def get_branch(self, dataset_rid: str, branch: str) -> dict:
327
345
  """Returns branch information.
328
346
 
@@ -113,10 +113,10 @@ def git_toplevel_dir(
113
113
  )
114
114
  if git_dir is None:
115
115
  git_dir = Path.cwd()
116
- if git_dir.joinpath(".git").is_dir():
116
+ if git_dir.joinpath(".git").exists():
117
117
  return git_dir
118
118
  for p in git_dir.resolve().parents:
119
- if p.joinpath(".git").is_dir():
119
+ if p.joinpath(".git").exists():
120
120
  return p
121
121
  return None
122
122
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.4.0
3
+ Version: 1.4.1
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -206,3 +206,14 @@ def complex_dataset_fixture(client, spark_session, tmpdir):
206
206
  foundry_schema=FOUNDRY_SCHEMA_COMPLEX_DATASET,
207
207
  )
208
208
  yield ds_rid
209
+
210
+
211
+ @pytest.fixture()
212
+ def git_env():
213
+ return {
214
+ "GIT_CONFIG_NOSYSTEM": "1",
215
+ "GIT_COMMITTER_NAME": "Jon Doe",
216
+ "GIT_COMMITTER_EMAIL": "john@doe",
217
+ "GIT_AUTHOR_NAME": "Jon Doe",
218
+ "GIT_AUTHOR_EMAIL": "john@doe",
219
+ }
@@ -454,15 +454,7 @@ def test_build(a, b, c, d, e, f, g, caplog):
454
454
  assert result.exit_code == 0
455
455
 
456
456
 
457
- def test_get_transform_files(tmpdir: "py.path.LocalPath"):
458
- GIT_ENV = {
459
- "HOME": str(tmpdir),
460
- "GIT_CONFIG_NOSYSTEM": "1",
461
- "GIT_COMMITTER_NAME": "pytest get_transform_files test",
462
- "GIT_COMMITTER_EMAIL": "pytest@get_transform_files.py",
463
- "GIT_AUTHOR_NAME": "pytest get_transform_files test",
464
- "GIT_AUTHOR_EMAIL": "pytest@get_transform_files.py",
465
- } # should use default configs
457
+ def test_get_transform_files(tmpdir: "py.path.LocalPath", git_env: dict):
466
458
  with tmpdir.as_cwd():
467
459
  from foundry_dev_tools.cli.build import (
468
460
  TRANSFORM_DECORATORS,
@@ -470,7 +462,7 @@ def test_get_transform_files(tmpdir: "py.path.LocalPath"):
470
462
  is_transform_file,
471
463
  )
472
464
 
473
- subprocess.check_call(["git", "init"], env=GIT_ENV)
465
+ subprocess.check_call(["git", "init"], env=git_env)
474
466
  t = Path("transforms-python", "examples")
475
467
  t.mkdir(parents=True)
476
468
  tfiles = []
@@ -481,8 +473,8 @@ def test_get_transform_files(tmpdir: "py.path.LocalPath"):
481
473
  f"from transforms.api import {decorator},Output\n\n@transform_df()\ndef test_transform():\n pass"
482
474
  )
483
475
  tfiles.append(transform_file.as_posix()) # git returns with forward slash
484
- subprocess.check_call(["git", "add", "-A"], env=GIT_ENV)
485
- subprocess.check_call(["git", "commit", "-m", "transform commit"], env=GIT_ENV)
476
+ subprocess.check_call(["git", "add", "-A"], env=git_env)
477
+ subprocess.check_call(["git", "commit", "-m", "transform commit"], env=git_env)
486
478
 
487
479
  for f in tfiles:
488
480
  assert is_transform_file(Path(f))
@@ -491,9 +483,9 @@ def test_get_transform_files(tmpdir: "py.path.LocalPath"):
491
483
  assert get_transform_files(Path.cwd()) == tfiles
492
484
  with tmpdir.join("get_transform.txt").open("w+") as gttxt:
493
485
  gttxt.write("something something")
494
- subprocess.check_call(["git", "add", "-A"], env=GIT_ENV)
486
+ subprocess.check_call(["git", "add", "-A"], env=git_env)
495
487
  subprocess.check_call(
496
- ["git", "commit", "-m", "no transform in last commit"], env=GIT_ENV
488
+ ["git", "commit", "-m", "no transform in last commit"], env=git_env
497
489
  )
498
490
  with pytest.raises(UsageError, match="No transform files in the last commit."):
499
491
  get_transform_files(Path.cwd())
@@ -108,6 +108,9 @@ def test_monster_integration_test(client): # noqa: PLR0915, TODO?
108
108
  _ = client.create_branch(ds["rid"], BRANCH)
109
109
  branch_returned = client.get_branch(ds["rid"], BRANCH)
110
110
  assert branch == branch_returned
111
+ branch_list = client.get_branches(ds["rid"])
112
+ assert len(branch_list) == 1
113
+ assert branch_list[0] == BRANCH
111
114
 
112
115
  assert client.get_dataset_identity(ds["rid"], BRANCH) == {
113
116
  "dataset_path": dataset_path,
@@ -14,7 +14,7 @@ if TYPE_CHECKING:
14
14
  @pytest.mark.parametrize(
15
15
  "use_git", [False]
16
16
  ) # only False, as we test use_git=True with test_get_repo
17
- def test_git_toplevel_dir(use_git: bool, tmpdir: "py.path.LocalPath"):
17
+ def test_git_toplevel_dir(use_git: bool, tmpdir: "py.path.LocalPath", git_env: dict):
18
18
  toplevel = Path(tmpdir)
19
19
  subprocess.check_call(["git", "init"], cwd=toplevel)
20
20
  git_dir = git_toplevel_dir(Path(tmpdir.mkdir("subdireectory")), use_git=use_git)
@@ -23,24 +23,51 @@ def test_git_toplevel_dir(use_git: bool, tmpdir: "py.path.LocalPath"):
23
23
  git_dir_cwd = git_toplevel_dir(use_git=use_git)
24
24
  assert git_dir_cwd == toplevel
25
25
 
26
+ # Test if git submodules can be recognized as toplevel_dir
27
+ # First create an empty local repository and initialize it
28
+ subprocess.check_call(["git", "init", "dummy_repo"], cwd=toplevel, env=git_env)
29
+ subprocess.check_call(
30
+ [
31
+ "git",
32
+ "commit",
33
+ "--allow-empty",
34
+ "-m",
35
+ "Initialize",
36
+ ],
37
+ cwd=toplevel.joinpath("dummy_repo"),
38
+ env=git_env,
39
+ )
40
+
41
+ # Add local repository as submodule to toplevel repo
42
+ subprocess.check_call(
43
+ [
44
+ "git",
45
+ "-c",
46
+ "protocol.file.allow=always",
47
+ "submodule",
48
+ "add",
49
+ "./dummy_repo",
50
+ "dummy_submodule",
51
+ ],
52
+ cwd=toplevel,
53
+ )
54
+ toplevel_submodule = toplevel.joinpath("dummy_submodule")
55
+ git_dir_submodule = git_toplevel_dir(
56
+ Path(tmpdir.mkdir("dummy_submodule", "submodule_subdirectory")), use_git=use_git
57
+ )
58
+ assert git_dir_submodule == toplevel_submodule
26
59
 
27
- def test_get_repo(tmpdir: "py.path.LocalPath"):
60
+
61
+ def test_get_repo(tmpdir: "py.path.LocalPath", git_env: dict):
28
62
  repo_rid = f"ri.stemma.main.repository{uuid.uuid4()}"
29
63
  git_dir = Path(tmpdir)
30
64
  subprocess.check_call(["git", "init"], cwd=git_dir)
31
65
  with git_dir.joinpath("gradle.properties").open("w+") as gradle_prop:
32
66
  gradle_prop.write(f"transformsRepoRid = {repo_rid}")
33
- GIT_ENV = {
34
- "HOME": str(git_dir),
35
- "GIT_CONFIG_NOSYSTEM": "1",
36
- "GIT_COMMITTER_NAME": "pytest get_repo test",
37
- "GIT_COMMITTER_EMAIL": "pytest@get_repo.py",
38
- "GIT_AUTHOR_NAME": "pytest get_repo test",
39
- "GIT_AUTHOR_EMAIL": "pytest@get_repo.py",
40
- } # should use default configs
41
- subprocess.check_call(["git", "add", "gradle.properties"], cwd=git_dir, env=GIT_ENV)
67
+
68
+ subprocess.check_call(["git", "add", "gradle.properties"], cwd=git_dir, env=git_env)
42
69
  subprocess.check_call(
43
- ["git", "commit", "-m", "initial commit"], cwd=git_dir, env=GIT_ENV
70
+ ["git", "commit", "-m", "initial commit"], cwd=git_dir, env=git_env
44
71
  )
45
72
  assert get_repo(git_dir)[0] == repo_rid
46
73
  with tmpdir.as_cwd():