foundry-dev-tools 1.4.0__tar.gz → 1.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. {foundry-dev-tools-1.4.0/src/foundry_dev_tools.egg-info → foundry_dev_tools-1.4.2}/PKG-INFO +1 -1
  2. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/Configuration_usage.md +48 -30
  3. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/FoundryRestClient_usage.md +6 -6
  4. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/changelog.md +14 -1
  5. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/foundry_api_client.py +18 -0
  6. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/repo.py +38 -2
  7. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2/src/foundry_dev_tools.egg-info}/PKG-INFO +1 -1
  8. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/transforms/api/_dataset.py +2 -23
  9. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/conftest.py +11 -0
  10. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_cli_build.py +6 -14
  11. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_foundry_api.py +3 -0
  12. foundry_dev_tools-1.4.2/tests/test_utils_repo.py +99 -0
  13. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/utils.py +33 -1
  14. foundry-dev-tools-1.4.0/tests/test_utils_repo.py +0 -49
  15. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.github/ISSUE_TEMPLATE/blank_issue.yml +0 -0
  16. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  17. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  18. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  19. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.github/pull_request_template.md +0 -0
  20. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.github/workflows/ci.yml +0 -0
  21. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.github/workflows/docs.yml +0 -0
  22. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.gitignore +0 -0
  23. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/.pre-commit-config.yaml +0 -0
  24. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/LICENSE +0 -0
  25. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/README.md +0 -0
  26. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/FoundryFileSystem_usage.md +0 -0
  27. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/SSO_usage.md +0 -0
  28. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/_static/.gitignore +0 -0
  29. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/architecture.md +0 -0
  30. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/cli.md +0 -0
  31. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/conf.py +0 -0
  32. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/contributors.md +0 -0
  33. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/develop.md +0 -0
  34. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/index.md +0 -0
  35. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/installation.md +0 -0
  36. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/license.md +0 -0
  37. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/pictures/mermaid-diagram-already-cached-dark.svg +0 -0
  38. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/pictures/mermaid-diagram-already-cached-light.svg +0 -0
  39. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/pictures/mermaid-diagram-new-transaction-dark.svg +0 -0
  40. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/pictures/mermaid-diagram-new-transaction-light.svg +0 -0
  41. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/pictures/tpa_config.png +0 -0
  42. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/requirements.txt +0 -0
  43. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/s3.md +0 -0
  44. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/docs/usage_and_examples.md +0 -0
  45. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/pyproject.toml +0 -0
  46. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/setup.cfg +0 -0
  47. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/setup.py +0 -0
  48. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/__about__.py +0 -0
  49. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/__init__.py +0 -0
  50. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/cached_foundry_client.py +0 -0
  51. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/cli/__init__.py +0 -0
  52. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/cli/build.py +0 -0
  53. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/cli/info.py +0 -0
  54. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/cli/main.py +0 -0
  55. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/cli/s3.py +0 -0
  56. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/config.py +0 -0
  57. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/fsspec_impl.py +0 -0
  58. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/__init__.py +0 -0
  59. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/async_s3.py +0 -0
  60. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/caches/__init__.py +0 -0
  61. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/caches/metadata_store.py +0 -0
  62. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/caches/spark_caches.py +0 -0
  63. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/converter/__init__.py +0 -0
  64. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/converter/foundry_spark.py +0 -0
  65. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/importer.py +0 -0
  66. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/misc.py +0 -0
  67. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/s3.py +0 -0
  68. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/spark.py +0 -0
  69. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools/utils/token_provider.py +0 -0
  70. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools.egg-info/SOURCES.txt +0 -0
  71. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools.egg-info/dependency_links.txt +0 -0
  72. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools.egg-info/entry_points.txt +0 -0
  73. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools.egg-info/not-zip-safe +0 -0
  74. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools.egg-info/requires.txt +0 -0
  75. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/foundry_dev_tools.egg-info/top_level.txt +0 -0
  76. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/transforms/__init__.py +0 -0
  77. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/transforms/api/__init__.py +0 -0
  78. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/transforms/api/_configure.py +0 -0
  79. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/transforms/api/_decorators.py +0 -0
  80. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/src/transforms/api/_transform.py +0 -0
  81. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/__init__.py +0 -0
  82. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/foundry_mock_client.py +0 -0
  83. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_cached_foundry_client.py +0 -0
  84. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_cached_foundry_client_integration.py +0 -0
  85. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_cli_info.py +0 -0
  86. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_config.py +0 -0
  87. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_data/binary_dataset/bin +0 -0
  88. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_data/iris/iris.csv +0 -0
  89. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_foundry_local_deprecation.py +0 -0
  90. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_foundry_mock.py +0 -0
  91. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_foundry_spark_converters.py +0 -0
  92. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_foundry_sql_client.py +0 -0
  93. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_fsspec_impl.py +0 -0
  94. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_importer.py +0 -0
  95. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_multipass_tpa.py +0 -0
  96. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_s3.py +0 -0
  97. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_spark_caches.py +0 -0
  98. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_token_provider.py +0 -0
  99. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_transforms.py +0 -0
  100. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_transforms_integration.py +0 -0
  101. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tests/test_utils_misc.py +0 -0
  102. {foundry-dev-tools-1.4.0 → foundry_dev_tools-1.4.2}/tox.ini +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.4.0
3
+ Version: 1.4.2
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -1,32 +1,56 @@
1
1
  # Configuration
2
2
 
3
- By default, the code loads the configuration from the '~/.foundry-dev-tools/config' file.
4
- If a project [specific config file](#project-specific-configuration) exists in `your-project-dir/.foundry_dev_tools`, it takes precedence.
5
- If a environment variable like `FOUNDRY_DEV_TOOLS_config_key` exists, `config_key` gets overwritten by this environment variable.
6
- Afterwards, the logic falls back to a default value: ( overwrites > environment variable > project config file > config file > default)
3
+ Configuration options can be supplied to the library through multiple different methods that are processed in the following order:
4
+ 1. Every configuration option is initalized to its default value as given in the [table below](#configuration-options).
5
+ 2. If a file `~/.foundry-dev-tools/config` exists, values defined therein overwrite the corresponding default values.
6
+ See [below](#foundry-dev-tools-config-file) for details on its structure.
7
+ 3. If a project [specific config file](#project-specific-configuration) exists in `your-project-dir/.foundry_dev_tools`, values set in there take precedence.
8
+ 4. If an environment variable like `FOUNDRY_DEV_TOOLS_config_key` exists, `config_key` gets overwritten by this environment variable.
9
+ Note, that environment variable names are always all-caps, e.g. the `foundry_url` config value can be set via `export FOUNDRY_DEV_TOOLS_FOUNDRY_URL=https://foundry.example.com`.
10
+ 5. Configuration options can be [overwritten programmatically](#changing-configuration-programmatically) during runtime.
7
11
 
8
- In the standard setup, the configuration should contain either the Foundry Token (`jwt`) or the Foundry Client
9
- ID (`client_id`) for SSO.
12
+ Thus, the order of precedence for configuration options is: runtime overwrites > environment variable > project config file > config file > default.
13
+
14
+ In the standard setup, the configuration should contain either the Foundry Token (`jwt`) or the Foundry Client ID (`client_id`) for SSO.
10
15
 
11
16
  ## Configuration options
12
17
 
13
- | Name | Description | Values |
14
- |----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|
15
- | foundry_url | Url of the foundry instance. | e.g. https://foundry.example.com |
16
- | jwt | The Bearer Token copied from the Foundry Token Page | eyJhb... |
17
- | client_id | The client_id of the Foundry Third Party application for SSO Authentication. | ... |
18
- | client_secret | The client_secret of the Foundry Third Party application. | ... |
19
- | grant_type | Set this to client_credentials if you use Foundry Third Party application client_credentials flow. Default is authorization_code | authorization_code, client_credentials |
20
- | scopes | Custom scopes for Third Party Application Tokens | "api:read-data offline_access" |
21
- | requests_ca_bundle | Path to custom ca bundle file, useful in corporate networks with SSL inspection | not set |
22
- | transforms_sql_sample_row_limit | Number of rows that are retrieved for each dataset when sql is used | 5000 |
23
- | transforms_sql_dataset_size_threshold | Maximal size of a dataset up to which file gets downloaded. If dataset is larger, sql with limit query will be used to download a partial dataset | 500 |
24
- | transforms_sql_sample_select_random | True or False, if data is randomly sampled when SQL mode is used (query takes more time) | True or False |
25
- | transforms_force_full_dataset_download | if file_download is True, all files of datasets are downloaded | True or False |
26
- | cache_dir | Point cache directory to other directory than default | ~/.foundry-dev-tools/cache |
27
- | transforms_freeze_cache | Uses offline cache as is; no Foundry connection needed | True or False (default: False) |
28
- | transforms_output_folder | When @transform in combination with TransformOutput.filesystem() is used, files are written to this folder. | /projectA/local_files |
29
- | enable_runtime_token_providers | Enables the token providers, Default is True | True or False |
18
+ | Name | Description | Values | Default Values |
19
+ |----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|------------------------------|
20
+ | foundry_url | Url of the foundry instance | e.g. https://foundry.example.com | not set (None) |
21
+ | jwt | The Bearer Token copied from the Foundry Token Page | eyJhb... | not set (None) |
22
+ | client_id | The client_id of the Foundry Third Party application for SSO Authentication | ... | not set (None) |
23
+ | client_secret | The client_secret of the Foundry Third Party application | ... | not set (None) |
24
+ | grant_type | Set this to client_credentials if you use Foundry Third Party application client_credentials flow. Default is authorization_code | authorization_code, client_credentials | authorization_code |
25
+ | scopes | Custom scopes for Third Party Application Tokens | "api:read-data offline_access" | not set (None) |
26
+ | requests_ca_bundle | Path to custom ca bundle file, useful in corporate networks with SSL inspection | not set | not set (None) |
27
+ | transforms_sql_sample_row_limit | Number of rows that are retrieved for each dataset when sql is used | 5000 | 5000 |
28
+ | transforms_sql_dataset_size_threshold | Maximum size of a dataset up to which file gets downloaded. If dataset is larger, sql with limit query will be used to download a partial dataset | 500 | 500 |
29
+ | transforms_sql_sample_select_random | True or False, if data is randomly sampled when SQL mode is used (query takes more time) | True or False | False |
30
+ | transforms_force_full_dataset_download | if file_download is True, all files of datasets are downloaded | True or False | False |
31
+ | cache_dir | Point cache directory to other directory than default | ~/.foundry-dev-tools/cache | `~/.foundry-dev-tools/cache` |
32
+ | transforms_freeze_cache | Uses offline cache as is; no Foundry connection needed | True or False | False |
33
+ | transforms_output_folder | When @transform in combination with `TransformOutput.filesystem()` is used, files are written to this folder. | /projectA/local_files | not set (None) |
34
+ | enable_runtime_token_providers | Enables the token providers | True or False | True |
35
+
36
+ ## `~/.foundry-dev-tools/config` file
37
+ The content of the file `~/.foundry-dev-tools/config` must be compatible with the Python `configparser` library, i.e. it must have a structure similar to what is found in Microsoft Windows INI files.
38
+ See https://docs.python.org/3/library/configparser.html#supported-ini-file-structure for details.
39
+ All configuration options have to reside in a section titled `[default]`.
40
+ Content of all other sections is ignored while parsing the file.
41
+ Thus, an exemplary `~/.foundry-dev-tools/config` file reads
42
+ ```
43
+ [default]
44
+ foundry_url=https://foundry.example.com
45
+ jwt=eyJhb...
46
+ ```
47
+
48
+ ## Project specific configuration
49
+
50
+ You can provide a project-specific configuration via a project-local file `.foundry_dev_tools`.
51
+ This file has to reside in your project's root directory.
52
+ The structure of this file is equivalent to the [`~/.foundry-dev-tools/config`](#foundry-dev-tools-config-file) file.
53
+ Entries of the default config are overwritten with entries from the `.foundry_dev_tools` file as specified [above](#configuration).
30
54
 
31
55
  ## Changing configuration programmatically
32
56
 
@@ -80,13 +104,7 @@ fc2.get_user_info()
80
104
  {"username":"john",...}
81
105
  ```
82
106
 
83
- ## Project specific configuration
84
-
85
- You can provide a configuration per project at the top level of the git repository in file `.foundry_dev_tools`.
86
- The structure of this file is equivalent to the `~/.foundry-dev-tools/config` file. Entries of the default config are
87
- overwritten with entries from the `.foundry_dev_tools` file.
88
-
89
- ## Enable Logging
107
+ # Enable Logging
90
108
 
91
109
  With the following code you can enable the DEBUG logging output of foundry-dev-tools:
92
110
 
@@ -34,13 +34,13 @@ Upload the complete content of a local folder to a dataset in Foundry
34
34
  import os
35
35
  from foundry_dev_tools import FoundryRestClient
36
36
 
37
- upload_folder = "/path/to/folder-to-upload"
38
- target_dataset_path = "/paht/to/test_folder_upload"
37
+ upload_folder = "/local-path/to/folder-to-upload"
38
+ target_dataset_path = "/path/to/foundry-dataset"
39
+
40
+ file_paths = [file for file in Path(upload_folder).rglob("*") if file.is_file() and not file.name.startswith(".")]
41
+ dataset_paths_in_foundry = [str(file_path.relative_to(upload_folder)) for file_path in file_paths]
42
+ path_file_dict = dict(zip(dataset_paths_in_foundry, file_paths))
39
43
 
40
- filenames = os.listdir(upload_folder)
41
- filepaths = list(map(lambda file: os.sep.join([upload_folder, file]), filenames))
42
- dataset_paths_in_foundry = list(map(lambda file: file, filenames))
43
- path_file_dict = dict(zip(dataset_paths_in_foundry, filepaths))
44
44
  rest_client = FoundryRestClient()
45
45
  dataset_rid = rest_client.get_dataset_rid(dataset_path=target_dataset_path)
46
46
  transaction_rid = rest_client.open_transaction(dataset_rid=dataset_rid,
@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog],
6
6
  and this project adheres to [Semantic Versioning].
7
7
 
8
+ ## [1.4.2] - 2024-06-03
9
+
10
+ ## Added
11
+ - Support for git submodules in transforms (#57)
12
+
13
+ ## [1.4.1] - 2024-05-29
14
+
15
+ ## Added
16
+ - Support for git submodules (#56)
17
+
8
18
  ## [1.4.0] - 2024-02-05
9
19
 
10
20
  ## Added
@@ -199,7 +209,10 @@ and this project adheres to [Semantic Versioning].
199
209
  [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
200
210
  [Semantic Versioning]: https://semver.org/spec/v2.0.0.html
201
211
 
202
- [1.3.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.4...v1.3.5
212
+ [1.4.2]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.4.1...v1.4.2
213
+ [1.4.1]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.4.0...v1.4.1
214
+ [1.4.0]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.5...v1.4.0
215
+ [1.3.5]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.4...v1.3.5
203
216
  [1.3.4]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.3...v1.3.4
204
217
  [1.3.3]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.2...v1.3.3
205
218
  [1.3.2]: https://github.com/emdgroup/foundry-dev-tools/compare/v1.3.1...v1.3.2
@@ -323,6 +323,24 @@ class FoundryRestClient:
323
323
  _raise_for_status_verbose(response)
324
324
  return response.json()
325
325
 
326
+ def get_branches(self, dataset_rid: str) -> list[str]:
327
+ """Returns a list of branches available a dataset.
328
+
329
+ Args:
330
+ dataset_rid (str): Unique identifier of the dataset
331
+
332
+ Returns:
333
+ list[str]:
334
+ with keys id (name) the dataset branches
335
+
336
+ """
337
+ response = self._request(
338
+ "GET",
339
+ f"{self.catalog}/catalog/datasets/{dataset_rid}/branches2",
340
+ )
341
+ _raise_for_status_verbose(response)
342
+ return response.json()
343
+
326
344
  def get_branch(self, dataset_rid: str, branch: str) -> dict:
327
345
  """Returns branch information.
328
346
 
@@ -113,14 +113,50 @@ def git_toplevel_dir(
113
113
  )
114
114
  if git_dir is None:
115
115
  git_dir = Path.cwd()
116
- if git_dir.joinpath(".git").is_dir():
116
+ if git_dir.joinpath(".git").exists():
117
117
  return git_dir
118
118
  for p in git_dir.resolve().parents:
119
- if p.joinpath(".git").is_dir():
119
+ if p.joinpath(".git").exists():
120
120
  return p
121
121
  return None
122
122
 
123
123
 
124
+ def get_branch(caller_filename: Path) -> str:
125
+ """Get name of current checked out git branch as defined in the git HEAD file.
126
+
127
+ Args:
128
+ caller_filename (Path): Path of file that calls the function
129
+
130
+ Returns:
131
+ str: Name of checked out branch
132
+ """
133
+ git_dir = git_toplevel_dir(caller_filename)
134
+ if not git_dir:
135
+ # fallback for VS Interactive Console
136
+ # or Jupyter lab on Windows
137
+ git_dir = Path.cwd()
138
+
139
+ if git_dir.joinpath(".git").is_file():
140
+ # Infer branch from git submodule
141
+ with git_dir.joinpath(".git").open() as gf:
142
+ rel_submodule_git_dir = gf.read().strip().replace("gitdir: ", "")
143
+ head_file = git_dir.joinpath(rel_submodule_git_dir, "HEAD").resolve()
144
+ else:
145
+ head_file = git_dir.joinpath(".git", "HEAD")
146
+
147
+ if head_file.is_file():
148
+ with head_file.open() as hf:
149
+ ref = hf.read().strip()
150
+
151
+ if ref.startswith("ref: refs/heads/"):
152
+ return ref[16:]
153
+
154
+ return "HEAD" # immitate behaviour of `git rev-parse --abbrev-ref HEAD`
155
+
156
+ warnings.warn("Could not detect git branch of project, falling back to 'master'.")
157
+ return "master"
158
+
159
+
124
160
  def find_project_config_file(
125
161
  project_directory: "Path | None" = None, use_git: bool = False
126
162
  ) -> "Path | None":
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: foundry-dev-tools
3
- Version: 1.4.0
3
+ Version: 1.4.2
4
4
  Summary: Seamlessly run your Palantir Foundry Repository transforms code on your local machine.
5
5
  Author-email: Nicolas Renkamp <nicolas.renkamp@merckgroup.com>, Jonas Wunderlich <jonas.wunderlich@merckgroup.com>
6
6
  License: Apache-2.0
@@ -23,7 +23,7 @@ from foundry_dev_tools.foundry_api_client import (
23
23
  )
24
24
  from foundry_dev_tools.utils.caches.spark_caches import DiskPersistenceBackedSparkCache
25
25
  from foundry_dev_tools.utils.misc import is_dataset_a_view
26
- from foundry_dev_tools.utils.repo import git_toplevel_dir
26
+ from foundry_dev_tools.utils.repo import get_branch
27
27
 
28
28
  LOGGER = logging.getLogger(__name__)
29
29
 
@@ -87,7 +87,7 @@ class Input:
87
87
  self._cache = DiskPersistenceBackedSparkCache(**self.config)
88
88
  self._spark_df = None
89
89
  if branch is None:
90
- branch = _get_branch(Path(caller_filename))
90
+ branch = get_branch(Path(caller_filename))
91
91
 
92
92
  if self._is_online:
93
93
  (
@@ -279,27 +279,6 @@ class Input:
279
279
  )
280
280
 
281
281
 
282
- def _get_branch(caller_filename: Path) -> str:
283
- git_dir = git_toplevel_dir(caller_filename)
284
- if not git_dir:
285
- # fallback for VS Interactive Console
286
- # or Jupyter lab on Windows
287
- git_dir = Path.cwd()
288
-
289
- head_file = git_dir.joinpath(".git", "HEAD")
290
- if head_file.is_file():
291
- with head_file.open() as hf:
292
- ref = hf.read().strip()
293
-
294
- if ref.startswith("ref: refs/heads/"):
295
- return ref[16:]
296
-
297
- return "HEAD" # immitate behaviour of `git rev-parse --abbrev-ref HEAD`
298
-
299
- warnings.warn("Could not detect git branch of project, falling back to 'master'.")
300
- return "master"
301
-
302
-
303
282
  class Output:
304
283
  """Specification of a transform dataset output.
305
284
 
@@ -206,3 +206,14 @@ def complex_dataset_fixture(client, spark_session, tmpdir):
206
206
  foundry_schema=FOUNDRY_SCHEMA_COMPLEX_DATASET,
207
207
  )
208
208
  yield ds_rid
209
+
210
+
211
+ @pytest.fixture()
212
+ def git_env():
213
+ return {
214
+ "GIT_CONFIG_NOSYSTEM": "1",
215
+ "GIT_COMMITTER_NAME": "Jon Doe",
216
+ "GIT_COMMITTER_EMAIL": "john@doe",
217
+ "GIT_AUTHOR_NAME": "Jon Doe",
218
+ "GIT_AUTHOR_EMAIL": "john@doe",
219
+ }
@@ -454,15 +454,7 @@ def test_build(a, b, c, d, e, f, g, caplog):
454
454
  assert result.exit_code == 0
455
455
 
456
456
 
457
- def test_get_transform_files(tmpdir: "py.path.LocalPath"):
458
- GIT_ENV = {
459
- "HOME": str(tmpdir),
460
- "GIT_CONFIG_NOSYSTEM": "1",
461
- "GIT_COMMITTER_NAME": "pytest get_transform_files test",
462
- "GIT_COMMITTER_EMAIL": "pytest@get_transform_files.py",
463
- "GIT_AUTHOR_NAME": "pytest get_transform_files test",
464
- "GIT_AUTHOR_EMAIL": "pytest@get_transform_files.py",
465
- } # should use default configs
457
+ def test_get_transform_files(tmpdir: "py.path.LocalPath", git_env: dict):
466
458
  with tmpdir.as_cwd():
467
459
  from foundry_dev_tools.cli.build import (
468
460
  TRANSFORM_DECORATORS,
@@ -470,7 +462,7 @@ def test_get_transform_files(tmpdir: "py.path.LocalPath"):
470
462
  is_transform_file,
471
463
  )
472
464
 
473
- subprocess.check_call(["git", "init"], env=GIT_ENV)
465
+ subprocess.check_call(["git", "init"], env=git_env)
474
466
  t = Path("transforms-python", "examples")
475
467
  t.mkdir(parents=True)
476
468
  tfiles = []
@@ -481,8 +473,8 @@ def test_get_transform_files(tmpdir: "py.path.LocalPath"):
481
473
  f"from transforms.api import {decorator},Output\n\n@transform_df()\ndef test_transform():\n pass"
482
474
  )
483
475
  tfiles.append(transform_file.as_posix()) # git returns with forward slash
484
- subprocess.check_call(["git", "add", "-A"], env=GIT_ENV)
485
- subprocess.check_call(["git", "commit", "-m", "transform commit"], env=GIT_ENV)
476
+ subprocess.check_call(["git", "add", "-A"], env=git_env)
477
+ subprocess.check_call(["git", "commit", "-m", "transform commit"], env=git_env)
486
478
 
487
479
  for f in tfiles:
488
480
  assert is_transform_file(Path(f))
@@ -491,9 +483,9 @@ def test_get_transform_files(tmpdir: "py.path.LocalPath"):
491
483
  assert get_transform_files(Path.cwd()) == tfiles
492
484
  with tmpdir.join("get_transform.txt").open("w+") as gttxt:
493
485
  gttxt.write("something something")
494
- subprocess.check_call(["git", "add", "-A"], env=GIT_ENV)
486
+ subprocess.check_call(["git", "add", "-A"], env=git_env)
495
487
  subprocess.check_call(
496
- ["git", "commit", "-m", "no transform in last commit"], env=GIT_ENV
488
+ ["git", "commit", "-m", "no transform in last commit"], env=git_env
497
489
  )
498
490
  with pytest.raises(UsageError, match="No transform files in the last commit."):
499
491
  get_transform_files(Path.cwd())
@@ -108,6 +108,9 @@ def test_monster_integration_test(client): # noqa: PLR0915, TODO?
108
108
  _ = client.create_branch(ds["rid"], BRANCH)
109
109
  branch_returned = client.get_branch(ds["rid"], BRANCH)
110
110
  assert branch == branch_returned
111
+ branch_list = client.get_branches(ds["rid"])
112
+ assert len(branch_list) == 1
113
+ assert branch_list[0] == BRANCH
111
114
 
112
115
  assert client.get_dataset_identity(ds["rid"], BRANCH) == {
113
116
  "dataset_path": dataset_path,
@@ -0,0 +1,99 @@
1
+ import os
2
+ import subprocess
3
+ import uuid
4
+ from pathlib import Path
5
+ from typing import TYPE_CHECKING
6
+
7
+ import pytest
8
+
9
+ from foundry_dev_tools.utils.repo import get_branch, get_repo, git_toplevel_dir
10
+ from tests.utils import add_git_submodule
11
+
12
+ if TYPE_CHECKING:
13
+ import py.path
14
+
15
+
16
+ @pytest.mark.parametrize(
17
+ "use_git", [False]
18
+ ) # only False, as we test use_git=True with test_get_repo
19
+ def test_git_toplevel_dir(use_git: bool, tmpdir: "py.path.LocalPath", git_env: dict):
20
+ toplevel = Path(tmpdir)
21
+ subprocess.check_call(["git", "init"], cwd=toplevel)
22
+ git_dir = git_toplevel_dir(Path(tmpdir.mkdir("subdireectory")), use_git=use_git)
23
+ assert git_dir == toplevel
24
+ with tmpdir.mkdir("second_subdirectory").as_cwd():
25
+ git_dir_cwd = git_toplevel_dir(use_git=use_git)
26
+ assert git_dir_cwd == toplevel
27
+
28
+ # Test if git submodules can be recognized as toplevel_dir
29
+ submodule_name = "dummy_submodule"
30
+ add_git_submodule(git_dir=toplevel, submodule_name=submodule_name, git_env=git_env)
31
+ toplevel_submodule = toplevel.joinpath(submodule_name)
32
+ git_dir_submodule = git_toplevel_dir(
33
+ Path(tmpdir.mkdir(submodule_name, "submodule_subdirectory")), use_git=use_git
34
+ )
35
+ assert git_dir_submodule == toplevel_submodule
36
+
37
+
38
+ def test_get_branch(tmpdir: "py.path.LocalPath", git_env: dict):
39
+ toplevel = Path(tmpdir)
40
+
41
+ # Test without git submodule
42
+ parent_repo_name = "parent_repo"
43
+ target_branch_name_parent = "parent_repo_main"
44
+ subprocess.check_call(
45
+ [
46
+ "git",
47
+ "init",
48
+ parent_repo_name,
49
+ f"--initial-branch={target_branch_name_parent}",
50
+ ],
51
+ cwd=toplevel,
52
+ env=git_env,
53
+ )
54
+ branch_name = get_branch(toplevel.joinpath(parent_repo_name))
55
+ assert branch_name == target_branch_name_parent
56
+
57
+ # Test for git submodule
58
+ # Add an empty submodule
59
+ submodule_name = "dummy_submodule"
60
+ add_git_submodule(
61
+ git_dir=toplevel.joinpath(parent_repo_name),
62
+ submodule_name=submodule_name,
63
+ git_env=git_env,
64
+ )
65
+ # Checkout and test new branch from submodule
66
+ target_branch_name_sub = "submodule_repo_dev"
67
+ git_dir_submodule = toplevel.joinpath(parent_repo_name, submodule_name)
68
+ subprocess.check_call(
69
+ [
70
+ "git",
71
+ "checkout",
72
+ "-b",
73
+ target_branch_name_sub,
74
+ ],
75
+ cwd=git_dir_submodule,
76
+ env=git_env,
77
+ )
78
+ assert os.path.isfile(git_dir_submodule.joinpath(".git"))
79
+
80
+ branch_name = get_branch(git_dir_submodule)
81
+ assert branch_name == target_branch_name_sub
82
+
83
+
84
+ def test_get_repo(tmpdir: "py.path.LocalPath", git_env: dict):
85
+ repo_rid = f"ri.stemma.main.repository{uuid.uuid4()}"
86
+ git_dir = Path(tmpdir)
87
+ subprocess.check_call(["git", "init"], cwd=git_dir, env=git_env)
88
+ with git_dir.joinpath("gradle.properties").open("w+") as gradle_prop:
89
+ gradle_prop.write(f"transformsRepoRid = {repo_rid}")
90
+
91
+ subprocess.check_call(["git", "add", "gradle.properties"], cwd=git_dir, env=git_env)
92
+ subprocess.check_call(
93
+ ["git", "commit", "-m", "initial commit"], cwd=git_dir, env=git_env
94
+ )
95
+ assert get_repo(git_dir)[0] == repo_rid
96
+ with tmpdir.as_cwd():
97
+ get_repo_out = get_repo()
98
+ assert get_repo_out[0] == repo_rid
99
+ assert get_repo_out[3] == git_dir
@@ -3,9 +3,10 @@ import glob
3
3
  import os
4
4
  import os.path
5
5
  import random
6
+ import subprocess
6
7
  from decimal import Decimal
7
8
  from pathlib import Path, PurePosixPath
8
- from typing import TYPE_CHECKING
9
+ from typing import TYPE_CHECKING, Dict
9
10
 
10
11
  import numpy as np
11
12
 
@@ -401,3 +402,34 @@ def generate_test_dataset(spark_session, output_folder, n_rows=50000):
401
402
  df = spark_session.createDataFrame(pdf, schema=schema).repartition(10)
402
403
  spark_write_path = f"{output_folder}/spark"
403
404
  df.write.format("parquet").save(path=spark_write_path)
405
+
406
+
407
+ def add_git_submodule(git_dir: Path, submodule_name: str, git_env: Dict) -> None:
408
+ # First create an empty local repository and initialize it
409
+ subprocess.check_call(["git", "init", "dummy_repo"], cwd=git_dir, env=git_env)
410
+ subprocess.check_call(
411
+ [
412
+ "git",
413
+ "commit",
414
+ "--allow-empty",
415
+ "-m",
416
+ "Initialize",
417
+ ],
418
+ cwd=git_dir.joinpath("dummy_repo"),
419
+ env=git_env,
420
+ )
421
+
422
+ # Add local repository as submodule to toplevel repo
423
+ subprocess.check_call(
424
+ [
425
+ "git",
426
+ "-c",
427
+ "protocol.file.allow=always",
428
+ "submodule",
429
+ "add",
430
+ "./dummy_repo",
431
+ submodule_name,
432
+ ],
433
+ cwd=git_dir,
434
+ env=git_env,
435
+ )
@@ -1,49 +0,0 @@
1
- import subprocess
2
- import uuid
3
- from pathlib import Path
4
- from typing import TYPE_CHECKING
5
-
6
- import pytest
7
-
8
- from foundry_dev_tools.utils.repo import get_repo, git_toplevel_dir
9
-
10
- if TYPE_CHECKING:
11
- import py.path
12
-
13
-
14
- @pytest.mark.parametrize(
15
- "use_git", [False]
16
- ) # only False, as we test use_git=True with test_get_repo
17
- def test_git_toplevel_dir(use_git: bool, tmpdir: "py.path.LocalPath"):
18
- toplevel = Path(tmpdir)
19
- subprocess.check_call(["git", "init"], cwd=toplevel)
20
- git_dir = git_toplevel_dir(Path(tmpdir.mkdir("subdireectory")), use_git=use_git)
21
- assert git_dir == toplevel
22
- with tmpdir.mkdir("second_subdirectory").as_cwd():
23
- git_dir_cwd = git_toplevel_dir(use_git=use_git)
24
- assert git_dir_cwd == toplevel
25
-
26
-
27
- def test_get_repo(tmpdir: "py.path.LocalPath"):
28
- repo_rid = f"ri.stemma.main.repository{uuid.uuid4()}"
29
- git_dir = Path(tmpdir)
30
- subprocess.check_call(["git", "init"], cwd=git_dir)
31
- with git_dir.joinpath("gradle.properties").open("w+") as gradle_prop:
32
- gradle_prop.write(f"transformsRepoRid = {repo_rid}")
33
- GIT_ENV = {
34
- "HOME": str(git_dir),
35
- "GIT_CONFIG_NOSYSTEM": "1",
36
- "GIT_COMMITTER_NAME": "pytest get_repo test",
37
- "GIT_COMMITTER_EMAIL": "pytest@get_repo.py",
38
- "GIT_AUTHOR_NAME": "pytest get_repo test",
39
- "GIT_AUTHOR_EMAIL": "pytest@get_repo.py",
40
- } # should use default configs
41
- subprocess.check_call(["git", "add", "gradle.properties"], cwd=git_dir, env=GIT_ENV)
42
- subprocess.check_call(
43
- ["git", "commit", "-m", "initial commit"], cwd=git_dir, env=GIT_ENV
44
- )
45
- assert get_repo(git_dir)[0] == repo_rid
46
- with tmpdir.as_cwd():
47
- get_repo_out = get_repo()
48
- assert get_repo_out[0] == repo_rid
49
- assert get_repo_out[3] == git_dir