bam-masterdata 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/.github/workflows/actions.yml +4 -4
  2. bam_masterdata-0.4.0/AUTHORS +23 -0
  3. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/LICENSE +5 -1
  4. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/PKG-INFO +27 -3
  5. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/README.md +21 -1
  6. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/_version.py +2 -2
  7. bam_masterdata-0.4.0/bam_masterdata/checker/__init__.py +1 -0
  8. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/checker/checker.py +21 -6
  9. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/checker/masterdata_validator.py +10 -4
  10. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/checker/source_loader.py +44 -12
  11. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/cli/cli.py +164 -38
  12. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/cli/entities_to_rdf.py +1 -1
  13. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/cli/fill_masterdata.py +20 -15
  14. bam_masterdata-0.4.0/bam_masterdata/datamodel/vocabulary_types.py +32238 -0
  15. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/excel/excel_to_entities.py +3 -3
  16. bam_masterdata-0.4.0/bam_masterdata/metadata/_maps.py +33 -0
  17. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/metadata/definitions.py +8 -1
  18. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/metadata/entities.py +269 -9
  19. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/openbis/get_entities.py +13 -0
  20. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/utils/__init__.py +1 -0
  21. bam_masterdata-0.4.0/bam_masterdata/utils/paths.py +43 -0
  22. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/utils/utils.py +2 -1
  23. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata.egg-info/PKG-INFO +27 -3
  24. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata.egg-info/SOURCES.txt +4 -0
  25. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/metadata/test_entities.py +8 -2
  26. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/metadata/test_entities_dict.py +44 -44
  27. bam_masterdata-0.4.0/tests/utils/test_paths.py +14 -0
  28. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/utils/test_utils.py +1 -0
  29. bam_masterdata-0.3.0/bam_masterdata/checker/__init__.py +0 -1
  30. bam_masterdata-0.3.0/bam_masterdata/datamodel/vocabulary_types.py +0 -81811
  31. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/.coveragerc +0 -0
  32. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/.github/workflows/coveralls.yml +0 -0
  33. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/.github/workflows/mkdocs-deploy.yml +0 -0
  34. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/.github/workflows/publish.yml +0 -0
  35. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/.gitignore +0 -0
  36. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/.vscode/settings.json +0 -0
  37. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/__init__.py +0 -0
  38. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/checker/validation_rules/all_validation_rules.json +0 -0
  39. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/checker/validation_rules/excel_validation_rules.json +0 -0
  40. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/checker/validation_rules/validation_rules.json +0 -0
  41. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/cli/__init__.py +0 -0
  42. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/cli/entities_to_excel.py +0 -0
  43. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/datamodel/__init__.py +0 -0
  44. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/datamodel/collection_types.py +0 -0
  45. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/datamodel/dataset_types.py +0 -0
  46. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/datamodel/object_types.py +0 -0
  47. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/excel/__init__.py +0 -0
  48. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/logger.py +0 -0
  49. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/metadata/__init__.py +0 -0
  50. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/metadata/entities_dict.py +0 -0
  51. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/openbis/__init__.py +0 -0
  52. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/openbis/login.py +0 -0
  53. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/validation_rules/excel_validation_rules.json +0 -0
  54. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata/validation_rules/validation_rules.json +0 -0
  55. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata.egg-info/dependency_links.txt +0 -0
  56. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata.egg-info/entry_points.txt +0 -0
  57. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata.egg-info/requires.txt +0 -0
  58. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/bam_masterdata.egg-info/top_level.txt +0 -0
  59. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/bammasterdata_blue_transparent.png +0 -0
  60. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/bammasterdata_blue_transparent_text.png +0 -0
  61. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/bammasterdata_blue_transparent_text.svg +0 -0
  62. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/bammasterdata_blue_white.png +0 -0
  63. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/bammasterdata_blue_white_text.png +0 -0
  64. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/datastore-logo-white.png +0 -0
  65. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/datastore-logo.png +0 -0
  66. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/drawio/checker-structure.drawio +0 -0
  67. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/drawio/masterdata-workflow.drawio +0 -0
  68. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/drawio/masterdata-workflow.drawio.png +0 -0
  69. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/fonts/BAMKlavika-Light.ttf +0 -0
  70. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/fonts/BAMKlavika-LightItalic.ttf +0 -0
  71. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/fonts/BAMKlavika-Medium.ttf +0 -0
  72. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/assets/fonts/BAMKlavika-MediumItalic.ttf +0 -0
  73. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/explanations/data-store.md +0 -0
  74. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/explanations/openbis.md +0 -0
  75. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/explanations/pybis.md +0 -0
  76. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/overview.md +0 -0
  77. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/collections.md +0 -0
  78. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/datasets.md +0 -0
  79. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/objects.md +0 -0
  80. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/objects_props.md +0 -0
  81. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/parent_child.md +0 -0
  82. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/projects.md +0 -0
  83. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/searches.md +0 -0
  84. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/setup.md +0 -0
  85. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/howtos/pybis/spaces.md +0 -0
  86. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/index.md +0 -0
  87. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/references/collection.md +0 -0
  88. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/references/controlled-vocabulary.md +0 -0
  89. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/references/dataset.md +0 -0
  90. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/stylesheets/extra.css +0 -0
  91. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/tutorials/create-masterdata.md +0 -0
  92. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/docs/tutorials/pybis.md +0 -0
  93. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/mkdocs.yml +0 -0
  94. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/pyproject.toml +0 -0
  95. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/scripts/install_python_dependencies.sh +0 -0
  96. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/scripts/pkg_size.py +0 -0
  97. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/setup.cfg +0 -0
  98. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/setup.py +0 -0
  99. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/__init__.py +0 -0
  100. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/checker/test_source_loader.py +0 -0
  101. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/cli/test_entities_to_excel.py +0 -0
  102. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/cli/test_entities_to_rdf.py +0 -0
  103. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/conftest.py +0 -0
  104. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/datamodel_test/collection_types.py +0 -0
  105. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/datamodel_test/dataset_types.py +0 -0
  106. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/datamodel_test/object_types.py +0 -0
  107. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example01_codedoesnoteexists.xlsx +0 -0
  108. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example02_codeexists_proptype_datatypechanged.xlsx +0 -0
  109. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example03_codeexists_proptype_contvoccodechanged.xlsx +0 -0
  110. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example04_codeexists_proptype_objectcodechanged.xlsx +0 -0
  111. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example05_codeexists_proptype_warningdefschanged.xlsx +0 -0
  112. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example06_codeexists_proptype_vocobjcodemiss.xlsx +0 -0
  113. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example07_codeexists_objtype_warningdefschanged.xlsx +0 -0
  114. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example08_codeexists_objtype_assignprop_listcodeschanged_add.xlsx +0 -0
  115. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example09_codeexists_objtype_assignprop_listcodeschanged_del.xlsx +0 -0
  116. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example10_codeexists_objtype_assignprop_codedoesnotexists.xlsx +0 -0
  117. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example11_codeexists_objtype_assignprop_listcodeschanged_datatypechanged.xlsx +0 -0
  118. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example12_codeexists_objtype_assignprop_listcodeschanged_warningdefschanged.xlsx +0 -0
  119. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example13_codeexists_objtype_assignprop_coincide2objects.xlsx +0 -0
  120. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example_incoming_excel.xlsx +0 -0
  121. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example_incoming_python/collection_types.py +0 -0
  122. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example_incoming_python/dataset_types.py +0 -0
  123. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/example_incoming_python/object_types.py +0 -0
  124. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/masterdata_example.xlsx +0 -0
  125. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/masterdata_excel_output.json +0 -0
  126. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/checker/masterdata_python_output.json +0 -0
  127. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/utils/example_prop_types_1.py +0 -0
  128. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/data/utils/example_prop_types_2.py +0 -0
  129. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/excel/test_excel_to_entities.py +0 -0
  130. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/metadata/test_definitions.py +0 -0
  131. {bam_masterdata-0.3.0 → bam_masterdata-0.4.0}/tests/test_logger.py +0 -0
@@ -34,10 +34,10 @@ jobs:
34
34
  uv pip install -e '.[dev]'
35
35
  - name: mypy
36
36
  run: |
37
- python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional --exclude dependencies bam_masterdata tests
37
+ python -m mypy --ignore-missing-imports --follow-imports=silent --no-strict-optional bam_masterdata tests
38
38
  - name: Test with pytest
39
39
  run: |
40
- python -m pytest -sv --ignore=bam_masterdata/dependencies tests
40
+ python -m pytest -sv --ignore=tests/cli --ignore=tests/metadata/test_entities_dict.py tests
41
41
  build-and-install:
42
42
  runs-on: ubuntu-latest
43
43
  steps:
@@ -62,11 +62,11 @@ jobs:
62
62
  - uses: actions/checkout@v4
63
63
  - uses: chartboost/ruff-action@v1
64
64
  with:
65
- args: "check . --exclude bam_masterdata/dependencies"
65
+ args: "check ."
66
66
  ruff-formatting:
67
67
  runs-on: ubuntu-latest
68
68
  steps:
69
69
  - uses: actions/checkout@v4
70
70
  - uses: chartboost/ruff-action@v1
71
71
  with:
72
- args: "format . --check --verbose --exclude bam_masterdata/dependencies"
72
+ args: "format . --check --verbose"
@@ -0,0 +1,23 @@
1
+ # This is the list of BAM Masterdata's significant contributors.
2
+ #
3
+ # This does not necessarily list everyone who has contributed code,
4
+ # especially since many employees of one corporation may be contributing to the
5
+ # definition of object types, property types, and vocabularies in the Masterdata
6
+ # datamodel.
7
+ # To see the full list of code contributors, see the revision history in
8
+ # source control.
9
+
10
+ - Angela Ariza de Schellenberger
11
+ - Ingo Bressler
12
+ - Rukeia El-Athman
13
+ - Çağtay Fabry
14
+ - Tobias Fritsch
15
+ - Ralf Herrmann
16
+ - Zoltán Konthur
17
+ - Julius Kruse
18
+ - Pavlina Kruzikova
19
+ - Tarakeshwar Lakshmipathy
20
+ - Julien Lecompagnon
21
+ - Jan Lisec
22
+ - Mathias Röllig
23
+ - Bastian Rühle
@@ -1,4 +1,4 @@
1
- MIT License
1
+ MIT License - MODIFIED
2
2
 
3
3
  Copyright (c) 2025 Bundesanstalt für Materialforschung und -prüfung
4
4
 
@@ -19,3 +19,7 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
19
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
20
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
21
  SOFTWARE.
22
+
23
+ **MODIFICATION**: the datamodel files are excluded from this license agreement and
24
+ we reserve all rights on them for developments. Appropriate recognition has to be
25
+ done if you use parts of this datamodel in your own modelling.
@@ -1,10 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: bam-masterdata
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: The BAM Data Store utility tools and masterdata models wrapped in a Python package with a front-end web interface.
5
5
  Author-email: BAM Data Store <datastore@bam.de>
6
6
  Maintainer-email: "Jose M. Pizarro" <jose.pizarro-blanco@bam.de>
7
- License: MIT License
7
+ License: MIT License - MODIFIED
8
8
 
9
9
  Copyright (c) 2025 Bundesanstalt für Materialforschung und -prüfung
10
10
 
@@ -26,6 +26,9 @@ License: MIT License
26
26
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27
27
  SOFTWARE.
28
28
 
29
+ **MODIFICATION**: the datamodel files are excluded from this license agreement and
30
+ we reserve all rights on them for developments. Appropriate recognition has to be
31
+ done if you use parts of this datamodel in your own modelling.
29
32
  Project-URL: Homepage, https://github.com/BAMresearch/bam-masterdata
30
33
  Project-URL: Bug Tracker, https://github.com/BAMresearch/bam-masterdata/issues
31
34
  Classifier: Natural Language :: English
@@ -39,6 +42,7 @@ Classifier: Programming Language :: Python :: 3.12
39
42
  Requires-Python: >=3.9
40
43
  Description-Content-Type: text/markdown
41
44
  License-File: LICENSE
45
+ License-File: AUTHORS
42
46
  Requires-Dist: pybis~=1.37.1rc4
43
47
  Requires-Dist: openpyxl
44
48
  Requires-Dist: click
@@ -207,11 +211,31 @@ INFO - [14:07:47] Watching paths for changes: 'docs', 'mkdocs.yml'
207
211
  INFO - [14:07:47] Serving on http://127.0.0.1:8000/
208
212
  ```
209
213
 
210
- Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are inmediately reflected when the files are saved (the local web will automatically refresh).
214
+ Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are immediately reflected when the files are saved (the local web will automatically refresh).
211
215
 
212
216
  ## Main contributors
213
217
 
218
+ The main code developers are:
219
+
214
220
  | Name | E-mail |
215
221
  | ------------------- | ------------------------------------------------------------ |
216
222
  | Carlos Madariaga | [carlos.madariaga@bam.de](mailto:carlos.madariaga@bam.de) |
217
223
  | Dr. Jose M. Pizarro | [jose.pizarro-blanco@bam.de](mailto:jose.pizarro-blanco@bam.de) |
224
+
225
+
226
+ The main datamodel developers are:
227
+
228
+ - Angela Ariza de Schellenberger
229
+ - Ingo Bressler
230
+ - Rukeia El-Athman
231
+ - Çağtay Fabry
232
+ - Tobias Fritsch
233
+ - Ralf Herrmann
234
+ - Zoltán Konthur
235
+ - Julius Kruse
236
+ - Pavlina Kruzikova
237
+ - Tarakeshwar Lakshmipathy
238
+ - Julien Lecompagnon
239
+ - Jan Lisec
240
+ - Mathias Röllig
241
+ - Bastian Rühle
@@ -146,11 +146,31 @@ INFO - [14:07:47] Watching paths for changes: 'docs', 'mkdocs.yml'
146
146
  INFO - [14:07:47] Serving on http://127.0.0.1:8000/
147
147
  ```
148
148
 
149
- Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are inmediately reflected when the files are saved (the local web will automatically refresh).
149
+ Simply click on `http://127.0.0.1:8000/`. The changes in the `md` files of the documentation are immediately reflected when the files are saved (the local web will automatically refresh).
150
150
 
151
151
  ## Main contributors
152
152
 
153
+ The main code developers are:
154
+
153
155
  | Name | E-mail |
154
156
  | ------------------- | ------------------------------------------------------------ |
155
157
  | Carlos Madariaga | [carlos.madariaga@bam.de](mailto:carlos.madariaga@bam.de) |
156
158
  | Dr. Jose M. Pizarro | [jose.pizarro-blanco@bam.de](mailto:jose.pizarro-blanco@bam.de) |
159
+
160
+
161
+ The main datamodel developers are:
162
+
163
+ - Angela Ariza de Schellenberger
164
+ - Ingo Bressler
165
+ - Rukeia El-Athman
166
+ - Çağtay Fabry
167
+ - Tobias Fritsch
168
+ - Ralf Herrmann
169
+ - Zoltán Konthur
170
+ - Julius Kruse
171
+ - Pavlina Kruzikova
172
+ - Tarakeshwar Lakshmipathy
173
+ - Julien Lecompagnon
174
+ - Jan Lisec
175
+ - Mathias Röllig
176
+ - Bastian Rühle
@@ -17,5 +17,5 @@ __version__: str
17
17
  __version_tuple__: VERSION_TUPLE
18
18
  version_tuple: VERSION_TUPLE
19
19
 
20
- __version__ = version = '0.3.0'
21
- __version_tuple__ = version_tuple = (0, 3, 0)
20
+ __version__ = version = '0.4.0'
21
+ __version_tuple__ = version_tuple = (0, 4, 0)
@@ -0,0 +1 @@
1
+ from .checker import MasterdataChecker, no_validation_errors
@@ -1,5 +1,3 @@
1
- import os
2
-
3
1
  from bam_masterdata.checker.masterdata_validator import MasterdataValidator
4
2
  from bam_masterdata.checker.source_loader import SourceLoader
5
3
  from bam_masterdata.logger import logger
@@ -47,6 +45,7 @@ class MasterdataChecker:
47
45
  - "validate" -> Validate both the current model and new entities.
48
46
  - "compare" -> Compare new entities against the current model.
49
47
  - "all" -> Run both validation types.
48
+ - "individual" -> Run individual repositories validations.
50
49
 
51
50
  Before running, ensure that required models are loaded based on the mode.
52
51
 
@@ -59,23 +58,23 @@ class MasterdataChecker:
59
58
 
60
59
  # Load required models based on the selected mode
61
60
  if (
62
- mode in ["self", "validate", "compare", "all"]
61
+ mode in ["self", "validate", "compare", "all", "individual"]
63
62
  and self.current_model is None
64
63
  ):
65
64
  self.logger.info("Current model is missing. Loading now...")
66
65
  self.load_current_model()
67
66
 
68
67
  if (
69
- mode in ["incoming", "validate", "compare", "all"]
68
+ mode in ["incoming", "validate", "compare", "all", "individual"]
70
69
  and self.new_entities is None
71
70
  ):
72
71
  raise ValueError(
73
- "New entities must be loaded before validation in 'incoming', 'validate', 'compare', or 'all' modes."
72
+ "New entities must be loaded before validation in 'incoming', 'validate', 'individual', 'compare', or 'all' modes."
74
73
  )
75
74
 
76
75
  # Load the validation rules
77
76
  if (
78
- mode in ["self", "incoming", "validate", "all"]
77
+ mode in ["self", "incoming", "validate", "all", "individual"]
79
78
  and self.validation_rules == {}
80
79
  ):
81
80
  self.validation_rules = load_validation_rules(self.logger)
@@ -84,3 +83,19 @@ class MasterdataChecker:
84
83
  self.new_entities, self.current_model, self.validation_rules
85
84
  )
86
85
  return validator.validate(mode)
86
+
87
+
88
+ def no_validation_errors(validation_results: dict) -> bool:
89
+ """
90
+ Check if there are no validation errors in the results.
91
+
92
+ Args:
93
+ validation_results (dict): The dictionary containing the specific validation results.
94
+
95
+ Returns:
96
+ bool: True if there are no validation errors, False otherwise.
97
+ """
98
+
99
+ if not isinstance(validation_results, dict):
100
+ return False
101
+ return all(no_validation_errors(v) for v in validation_results.values())
@@ -30,6 +30,7 @@ class MasterdataValidator:
30
30
  - "validate": Validate both current and incoming models but do not compare.
31
31
  - "compare": Validate new entities against the current model.
32
32
  - "all": Run both.
33
+ - "individual": Validate new entities and compare them with the current model.
33
34
 
34
35
  Returns:
35
36
  dict: Validation results.
@@ -66,7 +67,7 @@ class MasterdataValidator:
66
67
 
67
68
  if mode == "individual":
68
69
  self.logger.info(
69
- "Validating new DSST entities and comparing them with current model..."
70
+ "Validating new entities and comparing them with current model..."
70
71
  )
71
72
  self.validation_results = {
72
73
  "incoming_model": {},
@@ -426,6 +427,11 @@ class MasterdataValidator:
426
427
  is_terms=True,
427
428
  )
428
429
 
430
+ if not self.validation_results.get("comparisons"):
431
+ logger.info(
432
+ "No critical conflicts found between new entities compared to the current model."
433
+ )
434
+
429
435
  return self.validation_results
430
436
 
431
437
  def _compare_assigned_properties(
@@ -523,7 +529,7 @@ class MasterdataValidator:
523
529
  )
524
530
  other_entity_props = {prop["code"] for prop in other_entity_properties}
525
531
 
526
- if incoming_prop_codes == other_entity_props:
532
+ if (incoming_prop_codes == other_entity_props) and incoming_prop_codes:
527
533
  log_message = (
528
534
  "Entity will not be imported in openBIS. "
529
535
  f"The entity {entity_code} at row {incoming_entity['defs'].get('row_location')} has the same properties defined as {other_entity_code}. "
@@ -568,10 +574,10 @@ class MasterdataValidator:
568
574
  codes.add(property_item["code"])
569
575
  # Recursively check for more nested structures
570
576
  elif isinstance(value, (dict, list)):
571
- codes.extend(self.extract_property_codes(value))
577
+ codes.update(self.extract_property_codes(value))
572
578
 
573
579
  elif isinstance(data, list):
574
580
  for item in data:
575
- codes.extend(self.extract_property_codes(item))
581
+ codes.update(self.extract_property_codes(item))
576
582
 
577
583
  return codes
@@ -64,18 +64,34 @@ class SourceLoader:
64
64
  transformed_data[entity_type] = {}
65
65
 
66
66
  for entity_name, entity_data in entities.items():
67
- transformed_entity = {
68
- "properties": [], # Now placed before "defs"
69
- "defs": { # Metadata moved to the end
70
- "code": entity_data.get("code"),
71
- "description": entity_data.get("description", ""),
72
- "id": format_json_id(entity_name), # PascalCase for entity ID
73
- "row_location": entity_data.get("row_location"),
74
- "validation_script": entity_data.get("validationPlugin")
75
- or None, # Convert "" to None
76
- "iri": entity_data.get("iri") or None, # Convert "" to None
77
- },
78
- }
67
+ if entity_type == "vocabulary_types":
68
+ transformed_entity = {
69
+ "terms": [], # Now placed before "defs"
70
+ "defs": { # Metadata moved to the end
71
+ "code": entity_data.get("code"),
72
+ "description": entity_data.get("description", ""),
73
+ "id": format_json_id(
74
+ entity_name
75
+ ), # PascalCase for entity ID
76
+ "row_location": entity_data.get("row_location"),
77
+ "url_template": entity_data.get("url_template") or None,
78
+ },
79
+ }
80
+ else:
81
+ transformed_entity = {
82
+ "properties": [], # Now placed before "defs"
83
+ "defs": { # Metadata moved to the end
84
+ "code": entity_data.get("code"),
85
+ "description": entity_data.get("description", ""),
86
+ "id": format_json_id(
87
+ entity_name
88
+ ), # PascalCase for entity ID
89
+ "row_location": entity_data.get("row_location"),
90
+ "validation_script": entity_data.get("validationPlugin")
91
+ or None, # Convert "" to None
92
+ "iri": entity_data.get("iri") or None, # Convert "" to None
93
+ },
94
+ }
79
95
 
80
96
  # Handle additional fields specific to dataset_types
81
97
  if entity_type == "dataset_types":
@@ -123,6 +139,22 @@ class SourceLoader:
123
139
  }
124
140
  transformed_entity["properties"].append(transformed_property)
125
141
 
142
+ if "terms" in entity_data:
143
+ for term_name, term_data in entity_data["terms"].items():
144
+ transformed_term = {
145
+ "code": term_data.get("code"),
146
+ "description": term_data.get("description", ""),
147
+ "id": format_json_id(
148
+ term_name
149
+ ), # Now correctly formatted to PascalCase
150
+ "row_location": term_data.get("row_location"),
151
+ "url_template": term_data.get("url_template")
152
+ or None, # Convert "" to None
153
+ "label": term_data.get("label"),
154
+ "official": term_data.get("official"),
155
+ }
156
+ transformed_entity["terms"].append(transformed_term)
157
+
126
158
  transformed_data[entity_type][entity_name] = transformed_entity
127
159
 
128
160
  return transformed_data
@@ -1,6 +1,8 @@
1
+ import glob
2
+ import inspect
1
3
  import json
2
4
  import os
3
- import subprocess
5
+ import shutil
4
6
  import time
5
7
  from pathlib import Path
6
8
 
@@ -9,13 +11,15 @@ from decouple import config as environ
9
11
  from openpyxl import Workbook
10
12
  from rdflib import Graph
11
13
 
12
- from bam_masterdata.checker import MasterdataChecker
14
+ from bam_masterdata.checker import MasterdataChecker, no_validation_errors
13
15
  from bam_masterdata.cli.entities_to_excel import entities_to_excel
14
16
  from bam_masterdata.cli.entities_to_rdf import entities_to_rdf
15
17
  from bam_masterdata.cli.fill_masterdata import MasterdataCodeGenerator
16
18
  from bam_masterdata.logger import logger
17
19
  from bam_masterdata.metadata.entities_dict import EntitiesDict
20
+ from bam_masterdata.openbis.login import ologin
18
21
  from bam_masterdata.utils import (
22
+ DATAMODEL_DIR,
19
23
  delete_and_create_dir,
20
24
  duplicated_property_types,
21
25
  import_module,
@@ -23,27 +27,6 @@ from bam_masterdata.utils import (
23
27
  )
24
28
 
25
29
 
26
- def find_datamodel_dir():
27
- """Search for 'datamodel/' in possible locations and return its absolute path."""
28
- possible_locations = [
29
- # Case: Running from a project with `datamodel/`
30
- Path.cwd() / "datamodel",
31
- # Case: Running inside bam-masterdata
32
- Path.cwd() / "bam_masterdata" / "datamodel",
33
- # Case: Running inside installed package
34
- Path(__file__).parent.parent / "datamodel",
35
- ]
36
-
37
- for path in possible_locations:
38
- if path.exists():
39
- return str(path.resolve())
40
-
41
- raise FileNotFoundError("Could not find a valid 'datamodel/' directory.")
42
-
43
-
44
- DATAMODEL_DIR = find_datamodel_dir()
45
-
46
-
47
30
  @click.group(help="Entry point to run `bam_masterdata` CLI commands.")
48
31
  def cli():
49
32
  pass
@@ -389,7 +372,8 @@ def export_to_rdf(force_delete, python_path, export_dir):
389
372
  "--mode",
390
373
  "mode", # alias
391
374
  type=click.Choice(
392
- ["self", "incoming", "validate", "compare", "all"], case_sensitive=False
375
+ ["self", "incoming", "validate", "compare", "all", "individual"],
376
+ case_sensitive=False,
393
377
  ),
394
378
  default="all",
395
379
  help="""Specify the mode for the checker. Options are:
@@ -397,7 +381,8 @@ def export_to_rdf(force_delete, python_path, export_dir):
397
381
  "incoming" -> Validate only the new entity structure.
398
382
  "validate" -> Validate both the current model and new entities.
399
383
  "compare" -> Compare new entities against the current model.
400
- "all" -> Run all validations and comparison. (Default)""",
384
+ "all" -> Run all validations and comparison. (Default).
385
+ "individual" -> Run individual repositories validations.""",
401
386
  )
402
387
  @click.option(
403
388
  "--datamodel-path",
@@ -420,28 +405,169 @@ def checker(file_path, mode, datamodel_path):
420
405
  validation_results = checker.check(mode=mode)
421
406
 
422
407
  # Check if there are problems with the current model
423
- if mode in ["self", "all", "validate"] and validation_results.get("current_model"):
424
- click.echo(
425
- "There are problems in the current model that need to be solved" #: {validation_results['current_model']}"
426
- )
408
+ if mode in ["self", "all", "validate"] and validation_results.get(
409
+ "current_model", {}
410
+ ):
411
+ for entity, errors in validation_results.get("current_model", {}).items():
412
+ if errors:
413
+ click.echo(
414
+ f"There are problems in the current model for entity {entity} that need to be solved"
415
+ )
427
416
 
428
417
  # Check if there are problems with the incoming model
429
418
  if mode in ["incoming", "all", "validate"] and validation_results.get(
430
- "incoming_model"
419
+ "incoming_model", {}
431
420
  ):
432
- click.echo(
433
- f"There are problems in the incoming model located in {file_path} that need to be solved" #: {validation_results['incoming_model']}"
434
- )
421
+ for entity, errors in validation_results.get("incoming_model", {}).items():
422
+ if errors:
423
+ click.echo(
424
+ f"There are problems in the incoming model in {file_path} for entity {entity} that need to be solved"
425
+ )
435
426
 
436
427
  # Check if there are comparison problems
437
- if mode in ["compare", "all"] and validation_results.get("comparisons"):
438
- click.echo(
439
- f"There are problems when checking the incoming model located in {file_path} against the current data model that need to be solved" #: {validation_results['comparisons']}"
440
- )
428
+ if mode in ["compare", "all"] and validation_results.get("comparisons", {}):
429
+ for entity, errors in validation_results.get("comparisons", {}).items():
430
+ if errors:
431
+ click.echo(
432
+ f"There are problems when checking the incoming model in {file_path} against the current model {datamodel_path} for entity {entity} that need to be solved"
433
+ )
434
+
435
+ # Check if there are individual repository problems
436
+ if (
437
+ mode in ["individual"]
438
+ and validation_results.get("incoming_model", {})
439
+ and validation_results.get("comparisons", {})
440
+ ):
441
+ for entity, errors in validation_results.get("individual", {}).items():
442
+ if errors:
443
+ click.echo(
444
+ f"There are problems in the individual repositories when validating them for entity {entity} that need to be solved"
445
+ )
446
+ for entity, errors in validation_results.get("comparisons", {}).items():
447
+ if errors:
448
+ click.echo(
449
+ f"There are problems in the individual repositories when comparing them with respect to bam-masterdata for entity {entity} that need to be solved"
450
+ )
441
451
 
442
452
  # Check if no problems were found
443
- if all(value == {} for value in validation_results.values()):
453
+ if no_validation_errors(validation_results):
454
+ click.echo("No problems found in the datamodel and incoming model.")
455
+ else:
456
+ raise click.ClickException(
457
+ "Problems found in the datamodel. Please, check the logs."
458
+ )
459
+
460
+
461
+ @cli.command(
462
+ name="push_to_openbis",
463
+ help="Uploads to openBIS the entities contained in the file specified in the tag `--file-path` after passing correctly all the checks from the `checker`.",
464
+ )
465
+ @click.option(
466
+ "--file-path",
467
+ "file_path", # alias
468
+ type=click.Path(exists=True),
469
+ required=True,
470
+ help="""The path to the file or directory containing Python modules or the Excel file to be checked.""",
471
+ )
472
+ @click.option(
473
+ "--datamodel-path",
474
+ "datamodel_path", # alias
475
+ type=click.Path(exists=True, dir_okay=True),
476
+ default=DATAMODEL_DIR,
477
+ help="""Path to the directory containing the Python modules defining the datamodel (defaults to './bam_masterdata/datamodel/').""",
478
+ )
479
+ def push_to_openbis(file_path, datamodel_path):
480
+ # Check if the path is a single .py file OR a directory containing .py files
481
+ if file_path.endswith(".py") or (
482
+ os.path.isdir(file_path) and any(glob.glob(os.path.join(file_path, "*.py")))
483
+ ):
484
+ source_type = "python"
485
+ elif file_path.endswith(".xlsx"):
486
+ source_type = "excel"
487
+ else:
488
+ source_type = None
489
+ logger.warning(f"Unsupported source type for path: {file_path}")
490
+
491
+ # Handle source_type
492
+ if source_type == "python":
493
+ # Copy all .py files to the tmp directory
494
+ if os.path.isdir(file_path):
495
+ tmp_dir = file_path
496
+ else:
497
+ tmp_dir = os.path.dirname(file_path)
498
+ logger.info(f"Copied Python files to {tmp_dir}")
499
+
500
+ elif source_type == "excel":
501
+ tmp_dir = "./bam_masterdata/tmp"
502
+ os.makedirs(tmp_dir, exist_ok=True)
503
+
504
+ generator = MasterdataCodeGenerator(path=file_path, row_cell_info=True)
505
+
506
+ for module_name in ["collection", "dataset", "object", "vocabulary"]:
507
+ output_file = Path(os.path.join(tmp_dir, f"{module_name}_types.py"))
508
+
509
+ # Get the method from `MasterdataCodeGenerator`
510
+ code = getattr(generator, f"generate_{module_name}_types")().rstrip()
511
+
512
+ if code != "":
513
+ output_file.write_text(code + "\n", encoding="utf-8")
514
+ click.echo(f"Generated {module_name} types in {output_file}\n")
515
+ else:
516
+ click.echo(f"Skipping {module_name}_types.py (empty entity data)")
517
+ logger.info(f"Processed Excel file and exported to {tmp_dir}")
518
+
519
+ # Instantiate the checker class and run validation
520
+ checker = MasterdataChecker()
521
+
522
+ # Load current model from datamodel path
523
+ checker.load_current_model(datamodel_dir=datamodel_path)
524
+
525
+ # Load new entities from the specified file path (could be a Python file, directory, or Excel)
526
+ checker.load_new_entities(source=file_path)
527
+
528
+ # Run the checker in the specified mode
529
+ validation_results = checker.check(mode="individual")
530
+
531
+ if not validation_results.get("incoming_model"):
532
+ logger.info("No problems found in the new entities definition.")
533
+
534
+ # If there are no problems, push to openBIS
535
+ if all(
536
+ isinstance(value, dict) and all(not sub_value for sub_value in value.values())
537
+ for value in validation_results.values()
538
+ ):
444
539
  click.echo("No problems found in the datamodel and incoming model.")
540
+ # Push to openBIS
541
+
542
+ url = environ("OPENBIS_URL")
543
+ openbis = ologin(url=url)
544
+ click.echo(f"Using the openBIS instance: {url}\n")
545
+
546
+ # Push each entity type
547
+ for module_path in listdir_py_modules(tmp_dir):
548
+ module = import_module(module_path=module_path)
549
+ for _, obj in inspect.getmembers(module, inspect.isclass):
550
+ if hasattr(obj, "defs") and callable(getattr(obj, "to_openbis")):
551
+ obj_instance = obj()
552
+ obj_instance.to_openbis(openbis=openbis, logger=logger)
553
+
554
+ else:
555
+ logger.error(
556
+ f"The checking of the new entities definition located in {file_path} did not pass. Please, check your entity definitions."
557
+ )
558
+ click.echo(
559
+ f"There are problems in the incoming model located in {file_path} that need to be solved"
560
+ )
561
+ return
562
+
563
+ if source_type == "excel":
564
+ # Clean up the tmp directory
565
+ shutil.rmtree(tmp_dir)
566
+ click.echo(f"Temporary files in {tmp_dir} have been deleted.")
567
+
568
+ logger.info(
569
+ f"Push to openBIS of the new entities located in {file_path} completed."
570
+ )
445
571
 
446
572
 
447
573
  if __name__ == "__main__":
@@ -11,7 +11,7 @@ from rdflib.namespace import DC, OWL, RDF, RDFS
11
11
 
12
12
  from bam_masterdata.utils import import_module
13
13
 
14
- BAM = Namespace("https://bamresearch.github.io/bam-masterdata/")
14
+ BAM = Namespace("https://bamresearch.github.io/bam-masterdata/#")
15
15
  PROV = Namespace("http://www.w3.org/ns/prov#")
16
16
 
17
17