lamindb 1.11a1__tar.gz → 1.11.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. {lamindb-1.11a1 → lamindb-1.11.2}/.gitignore +2 -0
  2. {lamindb-1.11a1 → lamindb-1.11.2}/.gitmodules +0 -3
  3. {lamindb-1.11a1 → lamindb-1.11.2}/.pre-commit-config.yaml +2 -1
  4. lamindb-1.11.2/PKG-INFO +180 -0
  5. lamindb-1.11.2/README.md +126 -0
  6. {lamindb-1.11a1 → lamindb-1.11.2}/docs/api.md +1 -2
  7. {lamindb-1.11a1 → lamindb-1.11.2}/docs/curate.ipynb +88 -13
  8. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/acid.ipynb +1 -1
  9. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/idempotency.ipynb +1 -1
  10. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/keep-artifacts-local.ipynb +2 -2
  11. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/setup.ipynb +1 -1
  12. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/track-run-inputs.ipynb +1 -1
  13. {lamindb-1.11a1 → lamindb-1.11.2}/docs/includes/installation.md +1 -4
  14. {lamindb-1.11a1 → lamindb-1.11.2}/docs/query-search.md +1 -1
  15. {lamindb-1.11a1 → lamindb-1.11.2}/docs/registries.ipynb +4 -23
  16. lamindb-1.11.2/docs/scripts/curate_anndata_uns.py +9 -0
  17. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_attrs.py +2 -9
  18. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_mudata.py +14 -2
  19. lamindb-1.11.2/docs/scripts/define_schema_anndata_uns.py +10 -0
  20. lamindb-1.11.2/docs/scripts/define_schema_df_metadata.py +9 -0
  21. {lamindb-1.11a1 → lamindb-1.11.2}/docs/track.ipynb +2 -2
  22. {lamindb-1.11a1 → lamindb-1.11.2}/docs/transfer.ipynb +1 -1
  23. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/__init__.py +1 -1
  24. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/_finish.py +3 -3
  25. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_context.py +4 -2
  26. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/curators/core.py +13 -5
  27. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/errors.py +6 -0
  28. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/cellxgene/_cellxgene.py +1 -1
  29. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/croissant/__init__.py +20 -4
  30. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/_core.py +8 -1
  31. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/mini_immuno.py +0 -1
  32. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/fixtures/sheets.py +8 -2
  33. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/integrations/_croissant.py +34 -11
  34. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0121_recorduser.py +7 -0
  35. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/__init__.py +1 -0
  36. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_feature_manager.py +78 -18
  37. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/artifact.py +71 -65
  38. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/artifact_set.py +12 -3
  39. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/query_set.py +170 -74
  40. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/record.py +5 -1
  41. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/run.py +2 -27
  42. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/save.py +18 -10
  43. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/sqlrecord.py +47 -33
  44. {lamindb-1.11a1 → lamindb-1.11.2}/noxfile.py +10 -11
  45. {lamindb-1.11a1 → lamindb-1.11.2}/pyproject.toml +14 -19
  46. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_artifact.py +2 -6
  47. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_can_curate.py +8 -8
  48. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_collection.py +1 -1
  49. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_data.py +3 -3
  50. lamindb-1.11.2/tests/core/test_delete.py +18 -0
  51. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_describe_and_df_calls.py +5 -5
  52. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_feature.py +3 -3
  53. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_feature_label_manager.py +26 -26
  54. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_from_values.py +2 -2
  55. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_has_parents.py +2 -2
  56. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_models.py +3 -3
  57. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_queryset.py +56 -3
  58. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_save.py +1 -1
  59. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_schema.py +9 -9
  60. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_search.py +2 -2
  61. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_versioning.py +29 -7
  62. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_cat_managers.py +39 -39
  63. lamindb-1.11.2/tests/curators/test_curate_from_croissant.py +78 -0
  64. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_curators_examples.py +76 -98
  65. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_curators_general.py +16 -16
  66. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_curators_multivalue.py +2 -2
  67. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_cxg_curator.py +5 -5
  68. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_records.py +5 -4
  69. {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/setup_access.py +5 -0
  70. {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/test_permissions.py +61 -2
  71. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_artifact_storage.py +11 -0
  72. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_streaming.py +10 -1
  73. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_transfer.py +8 -4
  74. lamindb-1.11a1/PKG-INFO +0 -144
  75. lamindb-1.11a1/README.md +0 -85
  76. lamindb-1.11a1/tests/core/test_delete.py +0 -9
  77. lamindb-1.11a1/tests/curators/test_curate_from_croissant.py +0 -45
  78. {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  79. {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/config.yml +0 -0
  80. {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/enhancement.yml +0 -0
  81. {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/usage_question.yml +0 -0
  82. {lamindb-1.11a1 → lamindb-1.11.2}/.github/workflows/build.yml +0 -0
  83. {lamindb-1.11a1 → lamindb-1.11.2}/.github/workflows/doc-changes.yml +0 -0
  84. {lamindb-1.11a1 → lamindb-1.11.2}/CONTRIBUTING.md +0 -0
  85. {lamindb-1.11a1 → lamindb-1.11.2}/LICENSE +0 -0
  86. {lamindb-1.11a1 → lamindb-1.11.2}/docs/arrays.ipynb +0 -0
  87. {lamindb-1.11a1 → lamindb-1.11.2}/docs/bio-registries.ipynb +0 -0
  88. {lamindb-1.11a1 → lamindb-1.11.2}/docs/bionty.md +0 -0
  89. {lamindb-1.11a1 → lamindb-1.11.2}/docs/changelog.md +0 -0
  90. {lamindb-1.11a1 → lamindb-1.11.2}/docs/clinicore.md +0 -0
  91. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/curate-any.ipynb +0 -0
  92. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/delete.ipynb +0 -0
  93. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/import-modules.ipynb +0 -0
  94. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/pydantic-pandera.ipynb +0 -0
  95. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/reference-field.ipynb +0 -0
  96. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/search.ipynb +0 -0
  97. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/symbol-mapping.ipynb +0 -0
  98. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/test_notebooks.py +0 -0
  99. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/validate-fields.ipynb +0 -0
  100. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/visibility.ipynb +0 -0
  101. {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq.md +0 -0
  102. {lamindb-1.11a1 → lamindb-1.11.2}/docs/guide.md +0 -0
  103. {lamindb-1.11a1 → lamindb-1.11.2}/docs/index.md +0 -0
  104. {lamindb-1.11a1 → lamindb-1.11.2}/docs/lamindb.md +0 -0
  105. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_anndata_flexible.py +0 -0
  106. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_external_features.py +0 -0
  107. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_flexible.py +0 -0
  108. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_minimal_errors.py +0 -0
  109. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_soma_experiment.py +0 -0
  110. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_spatialdata.py +0 -0
  111. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_mini_immuno_features_labels.py +0 -0
  112. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_mini_immuno_schema_flexible.py +0 -0
  113. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_schema_anndata_ensembl_gene_ids_and_valid_features_in_obs.py +0 -0
  114. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_schema_spatialdata.py +0 -0
  115. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_valid_features.py +0 -0
  116. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/run_track_and_finish.py +0 -0
  117. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/run_track_with_params.py +0 -0
  118. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/run_workflow.py +0 -0
  119. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/save_mini_immuno_datasets.py +0 -0
  120. {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/synced_with_git.py +0 -0
  121. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/add-replace-cache.ipynb +0 -0
  122. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/anndata-accessor.ipynb +0 -0
  123. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/prepare-transfer-local-to-cloud.ipynb +0 -0
  124. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test-files/iris.csv +0 -0
  125. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test-files/iris.data +0 -0
  126. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test-files/new_iris.csv +0 -0
  127. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test_notebooks.py +0 -0
  128. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/transfer-local-to-cloud.ipynb +0 -0
  129. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/upload.ipynb +0 -0
  130. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/vitessce.ipynb +0 -0
  131. {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage.md +0 -0
  132. {lamindb-1.11a1 → lamindb-1.11.2}/docs/test_notebooks.py +0 -0
  133. {lamindb-1.11a1 → lamindb-1.11.2}/docs/wetlab.md +0 -0
  134. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/_tracked.py +0 -0
  135. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/_view.py +0 -0
  136. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/__init__.py +0 -0
  137. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/dtypes.py +0 -0
  138. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/fields.py +0 -0
  139. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/ids.py +0 -0
  140. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/types.py +0 -0
  141. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/uids.py +0 -0
  142. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/users.py +0 -0
  143. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/__init__.py +0 -0
  144. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_compat.py +0 -0
  145. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_mapped_collection.py +0 -0
  146. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_settings.py +0 -0
  147. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_sync_git.py +0 -0
  148. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_track_environment.py +0 -0
  149. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/exceptions.py +0 -0
  150. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/loaders.py +0 -0
  151. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/__init__.py +0 -0
  152. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_anndata_accessor.py +0 -0
  153. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_backed_access.py +0 -0
  154. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_polars_lazy_df.py +0 -0
  155. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_pyarrow_dataset.py +0 -0
  156. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_spatialdata_accessor.py +0 -0
  157. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_tiledbsoma.py +0 -0
  158. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_valid_suffixes.py +0 -0
  159. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_zarr.py +0 -0
  160. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/objects.py +0 -0
  161. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/paths.py +0 -0
  162. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/subsettings/__init__.py +0 -0
  163. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/subsettings/_annotation_settings.py +0 -0
  164. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/subsettings/_creation_settings.py +0 -0
  165. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/types.py +0 -0
  166. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/curators/__init__.py +0 -0
  167. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/curators/_legacy.py +0 -0
  168. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/__init__.py +0 -0
  169. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/cellxgene/__init__.py +0 -0
  170. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/cellxgene/cellxgene_schema_versions.csv +0 -0
  171. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/croissant/mini_immuno.anndata.zarr_metadata.json +0 -0
  172. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/__init__.py +0 -0
  173. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/_fake.py +0 -0
  174. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/_small.py +0 -0
  175. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/fixtures/__init__.py +0 -0
  176. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/schemas/__init__.py +0 -0
  177. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/schemas/_anndata.py +0 -0
  178. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/schemas/_simple.py +0 -0
  179. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/integrations/__init__.py +0 -0
  180. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/integrations/_vitessce.py +0 -0
  181. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0069_squashed.py +0 -0
  182. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0070_lamindbv1_migrate_data.py +0 -0
  183. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0071_lamindbv1_migrate_schema.py +0 -0
  184. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0072_remove_user__branch_code_remove_user_aux_and_more.py +0 -0
  185. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0073_merge_ourprojects.py +0 -0
  186. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0074_lamindbv1_part4.py +0 -0
  187. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0075_lamindbv1_part5.py +0 -0
  188. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0076_lamindbv1_part6.py +0 -0
  189. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0077_lamindbv1_part6b.py +0 -0
  190. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0078_lamindbv1_part6c.py +0 -0
  191. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0079_alter_rundata_value_json_and_more.py +0 -0
  192. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0080_polish_lamindbv1.py +0 -0
  193. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0081_revert_textfield_collection.py +0 -0
  194. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0082_alter_feature_dtype.py +0 -0
  195. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0083_alter_feature_is_type_alter_flextable_is_type_and_more.py +0 -0
  196. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0084_alter_schemafeature_feature_and_more.py +0 -0
  197. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0085_alter_feature_is_type_alter_flextable_is_type_and_more.py +0 -0
  198. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0086_various.py +0 -0
  199. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0087_rename__schemas_m2m_artifact_feature_sets_and_more.py +0 -0
  200. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0088_schema_components.py +0 -0
  201. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0089_subsequent_runs.py +0 -0
  202. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0090_runproject_project_runs.py +0 -0
  203. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0091_alter_featurevalue_options_alter_space_options_and_more.py +0 -0
  204. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0092_alter_artifactfeaturevalue_artifact_and_more.py +0 -0
  205. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0093_alter_schemacomponent_unique_together.py +0 -0
  206. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0094_writeloglock_writelogmigrationstate_and_more.py +0 -0
  207. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0095_remove_rundata_flextable.py +0 -0
  208. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0096_remove_artifact__param_values_and_more.py +0 -0
  209. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0097_remove_schemaparam_param_remove_paramvalue_param_and_more.py +0 -0
  210. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0098_alter_feature_type_alter_project_type_and_more.py +0 -0
  211. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0099_alter_writelog_seqno.py +0 -0
  212. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0100_branch_alter_artifact__branch_code_and_more.py +0 -0
  213. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0101_alter_artifact_hash_alter_feature_name_and_more.py +0 -0
  214. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0102_remove_writelog_branch_code_and_more.py +0 -0
  215. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0103_remove_writelog_migration_state_and_more.py +0 -0
  216. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0104_alter_branch_uid.py +0 -0
  217. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0105_record_unique_name.py +0 -0
  218. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0106_transfer_data_migration.py +0 -0
  219. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0107_add_schema_to_record.py +0 -0
  220. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0108_remove_record_sheet_remove_sheetproject_sheet_and_more.py +0 -0
  221. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0109_record_input_of_runs_alter_record_run_and_more.py +0 -0
  222. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0110_rename_values_artifacts_record_linked_artifacts.py +0 -0
  223. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0111_remove_record__sort_order.py +0 -0
  224. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0112_alter_recordartifact_feature_and_more.py +0 -0
  225. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0113_lower_case_branch_and_space_names.py +0 -0
  226. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0114_alter_run__status_code.py +0 -0
  227. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0115_alter_space_uid.py +0 -0
  228. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0116_remove_artifact_unique_artifact_storage_key_hash_and_more.py +0 -0
  229. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0117_fix_artifact_storage_hash_unique_constraints.py +0 -0
  230. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0118_alter_recordproject_value_projectrecord.py +0 -0
  231. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0119_rename_records_project_linked_in_records.py +0 -0
  232. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0119_squashed.py +0 -0
  233. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0120_add_record_fk_constraint.py +0 -0
  234. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/__init__.py +0 -0
  235. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_describe.py +0 -0
  236. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_django.py +0 -0
  237. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_from_values.py +0 -0
  238. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_is_versioned.py +0 -0
  239. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_label_manager.py +0 -0
  240. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_relations.py +0 -0
  241. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/can_curate.py +0 -0
  242. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/collection.py +0 -0
  243. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/feature.py +0 -0
  244. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/has_parents.py +0 -0
  245. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/project.py +0 -0
  246. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/query_manager.py +0 -0
  247. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/schema.py +0 -0
  248. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/storage.py +0 -0
  249. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/transform.py +0 -0
  250. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/ulabel.py +0 -0
  251. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/py.typed +0 -0
  252. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/__init__.py +0 -0
  253. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/_switch.py +0 -0
  254. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/core/__init__.py +0 -0
  255. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/errors/__init__.py +0 -0
  256. {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/types/__init__.py +0 -0
  257. {lamindb-1.11a1 → lamindb-1.11.2}/tests/conftest.py +0 -0
  258. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/_dataset_fixtures.py +0 -0
  259. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/conftest.py +0 -0
  260. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/basic-r-notebook.Rmd.cleaned.html +0 -0
  261. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/basic-r-notebook.Rmd.html +0 -0
  262. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/duplicate/with-title-initialized-consecutive-finish.ipynb +0 -0
  263. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/no-title.ipynb +0 -0
  264. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/with-title-initialized-consecutive-finish-not-last-cell.ipynb +0 -0
  265. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/with-title-initialized-consecutive-finish.ipynb +0 -0
  266. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate1/script-to-test-versioning.py +0 -0
  267. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate2/script-to-test-versioning.py +0 -0
  268. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate3/script-to-test-versioning.py +0 -0
  269. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate4/script-to-test-versioning.py +0 -0
  270. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate5/script-to-test-versioning.py +0 -0
  271. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/script-to-test-filename-change.py +0 -0
  272. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/script-to-test-versioning.py +0 -0
  273. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_artifact_folders.py +0 -0
  274. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_db.py +0 -0
  275. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_dtype.py +0 -0
  276. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_integrity.py +0 -0
  277. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_load.py +0 -0
  278. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_manager.py +0 -0
  279. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_notebooks.py +0 -0
  280. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_run.py +0 -0
  281. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_settings.py +0 -0
  282. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_sqlrecord.py +0 -0
  283. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_storage.py +0 -0
  284. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_track.py +0 -0
  285. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_tracked.py +0 -0
  286. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_transform.py +0 -0
  287. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_ulabel.py +0 -0
  288. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_view.py +0 -0
  289. {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_visibility.py +0 -0
  290. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/conftest.py +0 -0
  291. {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_dataframe_curators_accounting_example.py +0 -0
  292. {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/conftest.py +0 -0
  293. {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/jwt_utils.py +0 -0
  294. {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/check_lamin_dev.py +0 -0
  295. {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/clean_lamin_dev.py +0 -0
  296. {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/setup_instance.py +0 -0
  297. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/conftest.py +0 -0
  298. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_artifact_zarr.py +0 -0
  299. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_cache.py +0 -0
  300. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_connect_reconnect.py +0 -0
  301. {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_storage_lifecycle.py +0 -0
@@ -66,7 +66,9 @@ run-tests
66
66
  test-django-validation/
67
67
  curate.tiledbsoma
68
68
  small_dataset.tiledbsoma
69
+ nonregistered_storage
69
70
  registered_storage
71
+ tests/core/notebooks/no-uid-renamed.ipynb
70
72
 
71
73
  # General
72
74
  .DS_Store
@@ -10,9 +10,6 @@
10
10
  [submodule "sub/wetlab"]
11
11
  path = sub/wetlab
12
12
  url = https://github.com/laminlabs/wetlab
13
- [submodule "sub/clinicore"]
14
- path = sub/clinicore
15
- url = https://github.com/laminlabs/clinicore
16
13
  [submodule "sub/cellxgene-lamin"]
17
14
  path = sub/cellxgene-lamin
18
15
  url = https://github.com/laminlabs/cellxgene-lamin.git
@@ -68,5 +68,6 @@ repos:
68
68
  tests/writelog/conftest.py|
69
69
  tests/writelog_sqlite/conftest.py|
70
70
  tests/curators/test_curators_examples.py|
71
- tests/core/conftest.py
71
+ tests/core/conftest.py|
72
+ docs/scripts/
72
73
  )
@@ -0,0 +1,180 @@
1
+ Metadata-Version: 2.3
2
+ Name: lamindb
3
+ Version: 1.11.2
4
+ Summary: A data framework for biology.
5
+ Author-email: Lamin Labs <open-source@lamin.ai>
6
+ Requires-Python: >=3.10,<3.14
7
+ Description-Content-Type: text/markdown
8
+ Classifier: Programming Language :: Python :: 3.10
9
+ Classifier: Programming Language :: Python :: 3.11
10
+ Classifier: Programming Language :: Python :: 3.12
11
+ Classifier: Programming Language :: Python :: 3.13
12
+ Requires-Dist: lamin_utils==0.15.0
13
+ Requires-Dist: lamin_cli==1.7.1
14
+ Requires-Dist: lamindb_setup[aws]==1.10.1
15
+ Requires-Dist: bionty>=1.7a1
16
+ Requires-Dist: wetlab>=1.5a1
17
+ Requires-Dist: nbproject==0.11.1
18
+ Requires-Dist: jupytext
19
+ Requires-Dist: nbconvert>=7.2.1
20
+ Requires-Dist: mistune!=3.1.0
21
+ Requires-Dist: pyyaml
22
+ Requires-Dist: pyarrow
23
+ Requires-Dist: pandera>=0.24.0
24
+ Requires-Dist: typing_extensions!=4.6.0
25
+ Requires-Dist: python-dateutil
26
+ Requires-Dist: pandas>=2.0.0
27
+ Requires-Dist: scipy<1.15.0
28
+ Requires-Dist: anndata>=0.8.0,<=0.12.1
29
+ Requires-Dist: fsspec
30
+ Requires-Dist: graphviz
31
+ Requires-Dist: psycopg2-binary
32
+ Requires-Dist: tomlkit ; extra == "dev"
33
+ Requires-Dist: line_profiler ; extra == "dev"
34
+ Requires-Dist: pre-commit ; extra == "dev"
35
+ Requires-Dist: nox ; extra == "dev"
36
+ Requires-Dist: laminci>=0.3 ; extra == "dev"
37
+ Requires-Dist: pytest>=6.0 ; extra == "dev"
38
+ Requires-Dist: coverage ; extra == "dev"
39
+ Requires-Dist: pytest-cov<7.0.0 ; extra == "dev"
40
+ Requires-Dist: mudata ; extra == "dev"
41
+ Requires-Dist: nbproject_test>=0.6.0 ; extra == "dev"
42
+ Requires-Dist: faker-biology ; extra == "dev"
43
+ Requires-Dist: pronto ; extra == "dev"
44
+ Requires-Dist: readfcs>=2.0.1 ; extra == "fcs"
45
+ Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
46
+ Requires-Dist: numcodecs<0.16.0 ; extra == "zarr"
47
+ Requires-Dist: zarr>=2.16.0,<3.0.0a0 ; extra == "zarr"
48
+ Project-URL: Home, https://github.com/laminlabs/lamindb
49
+ Provides-Extra: dev
50
+ Provides-Extra: fcs
51
+ Provides-Extra: gcp
52
+ Provides-Extra: zarr
53
+
54
+ [![Stars](https://img.shields.io/github/stars/laminlabs/lamindb?logo=GitHub)](https://github.com/laminlabs/lamindb)
55
+ [![codecov](https://codecov.io/gh/laminlabs/lamindb/branch/main/graph/badge.svg?token=VKMRJ7OWR3)](https://codecov.io/gh/laminlabs/lamindb)
56
+ [![Docs](https://img.shields.io/badge/docs-humans-yellow)](https://docs.lamin.ai)
57
+ [![DocsLLMs](https://img.shields.io/badge/docs-LLMs-yellow)](https://docs.lamin.ai/summary.md)
58
+ [![pypi](https://img.shields.io/pypi/v/lamindb?color=blue&label=pypi%20package)](https://pypi.org/project/lamindb)
59
+ [![PyPI Downloads](https://img.shields.io/pepy/dt/lamindb?logo=pypi)](https://pepy.tech/project/lamindb)
60
+
61
+ # LaminDB - A data lakehouse for biology
62
+
63
+ LaminDB is an open-source data lakehouse to enable learning at scale in biology.
64
+ It organizes datasets through validation & annotation and provides data lineage, queryability, and reproducibility on top of [FAIR](https://en.wikipedia.org/wiki/FAIR_data) data.
65
+
66
+ <details>
67
+ <summary>Why?</summary>
68
+
69
+ Reproducing analytical results or understanding how a dataset or model was created can be a pain.
70
+ Let alone training models on historical data, LIMS & ELN systems, orthogonal assays, or datasets generated by other teams.
71
+ Even maintaining a mere overview of a project's or team's datasets & analyses is harder than it sounds.
72
+
73
+ Biological datasets are typically managed with versioned storage systems, GUI-focused community or SaaS platforms, structureless data lakes, rigid data warehouses (SQL, monolithic arrays), and data lakehouses for tabular data.
74
+
75
+ LaminDB extends the lakehouse architecture to biological registries & datasets beyond tables (`DataFrame`, `AnnData`, `.zarr`, `.tiledbsoma`, ...) with enough structure to enable queries and enough freedom to keep the pace of R&D high.
76
+ Moreover, it provides context through data lineage -- tracing data and code, scientists and models -- and abstractions for biological domain knowledge and experimental metadata.
77
+
78
+ </details>
79
+
80
+ **Highlights.**
81
+
82
+ - **data lineage:** track inputs & outputs of notebooks, scripts, functions & pipelines with a single line of code
83
+ - **unified infrastructure:** access diverse storage locations (local, S3, GCP, ...), SQL databases (Postgres, SQLite) & ontologies
84
+ - **lakehouse capabilities**: manage, monitor & validate features, labels & dataset schemas; perform distributed queries and batch loading
85
+ - **biological data formats:** validate & annotate formats like `DataFrame`, `AnnData`, `MuData`, ... backed by `parquet`, `zarr`, HDF5, LanceDB, DuckDB, ...
86
+ - **biological entities**: organize experimental metadata & extensible ontologies in registries based on the Django ORM
87
+ - **reproducible & auditable:** auto-version & timestamp execution reports, source code & compute environments, attribute records to users
88
+ - **zero lock-in & scalable:** runs in your infrastructure; is _not_ a client for a rate-limited REST API
89
+ - **extendable:** create custom plug-ins for your own applications based on the Django ecosystem
90
+ - **integrations:** visualization tools like [vitessce](https://docs.lamin.ai/vitessce), workflow managers like [nextflow](https://docs.lamin.ai/nextflow) & [redun](https://docs.lamin.ai/redun), and [other tools](https://docs.lamin.ai/integrations)
91
+ - **production-ready:** used in BigPharma, BioTech, hospitals & top labs
92
+
93
+ LaminDB can be connected to LaminHub to serve as a [LIMS](https://en.wikipedia.org/wiki/Laboratory_information_management_system) for wetlab scientists, closing the drylab-wetlab feedback loop: [lamin.ai](https://lamin.ai)
94
+
95
+ ## Docs
96
+
97
+ Copy [summary.md](https://docs.lamin.ai/summary.md) into an LLM chat and let AI explain or read the [docs](https://docs.lamin.ai).
98
+
99
+ ## Setup
100
+
101
+ <!-- copied from quick-setup-lamindb.md -->
102
+
103
+ Install the `lamindb` Python package:
104
+
105
+ ```shell
106
+ pip install lamindb
107
+ ```
108
+
109
+ Create a LaminDB instance:
110
+
111
+ ```shell
112
+ lamin init --storage ./quickstart-data # or s3://my-bucket, gs://my-bucket
113
+ ```
114
+
115
+ Or if you have write access to an instance, connect to it:
116
+
117
+ ```shell
118
+ lamin connect account/name
119
+ ```
120
+
121
+ ## Quickstart
122
+
123
+ <!-- copied from preface.md -->
124
+
125
+ Track a script or notebook run with source code, inputs, outputs, logs, and environment.
126
+
127
+ <!-- copied from py-quickstart.py -->
128
+
129
+ ```python
130
+ import lamindb as ln
131
+
132
+ ln.track() # track a run
133
+ open("sample.fasta", "w").write(">seq1\nACGT\n")
134
+ ln.Artifact("sample.fasta", key="sample.fasta").save() # create an artifact
135
+ ln.finish() # finish the run
136
+ ```
137
+
138
+ <!-- from here on, slight deviation from preface.md, where all this is treated in the walk through in more depth -->
139
+
140
+ This code snippet creates an artifact, which can store a dataset or model as a file or folder in various formats.
141
+ Running the snippet as a script (`python create-fasta.py`) produces the following data lineage.
142
+
143
+ ```python
144
+ artifact = ln.Artifact.get(key="sample.fasta") # query artifact by key
145
+ artifact.view_lineage()
146
+ ```
147
+
148
+ <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/EkQATsQL5wqC95Wj0005.png" width="250">
149
+
150
+ You'll know how that artifact was created and what it's used for ([interactive visualization](https://lamin.ai/laminlabs/lamindata/artifact/8incOOgjn6F0K1TS)) in addition to capturing basic metadata:
151
+
152
+ ```python
153
+ artifact.describe()
154
+ ```
155
+
156
+ <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/BOTCBgHDAvwglN3U0002.png" width="550">
157
+
158
+ You can organize datasets with validation & annotation of any kind of metadata to then access them via queries & search. Here is a more [comprehensive example](https://lamin.ai/laminlabs/lamindata/artifact/9K1dteZ6Qx0EXK8g).
159
+
160
+ <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/6sofuDVvTANB0f480002.png" width="850">
161
+
162
+ To annotate an artifact with a label, use:
163
+
164
+ ```python
165
+ my_experiment = ln.ULabel(name="My experiment").save() # create a label in the universal label ontology
166
+ artifact.ulabels.add(my_experiment) # annotate the artifact with the label
167
+ ```
168
+
169
+ To query for a set of artifacts, use the `filter()` statement.
170
+
171
+ ```python
172
+ ln.Artifact.filter(ulabels=my_experiment, suffix=".fasta").to_dataframe() # query by suffix and the ulabel we just created
173
+ ln.Artifact.filter(transform__key="create-fasta.py").to_dataframe() # query by the name of the script we just ran
174
+ ```
175
+
176
+ If you have a structured dataset like a `DataFrame`, an `AnnData`, or another array, you can validate the content of the dataset (and parse annotations).
177
+ Here is an example for a dataframe: [docs.lamin.ai/introduction#validate-an-artifact](https://docs.lamin.ai/introduction#validate-an-artifact).
178
+
179
+ With a large body of validated datasets, you can then access data through distributed queries & batch streaming, see here: [docs.lamin.ai/arrays](https://docs.lamin.ai/arrays).
180
+
@@ -0,0 +1,126 @@
1
+ [![Stars](https://img.shields.io/github/stars/laminlabs/lamindb?logo=GitHub)](https://github.com/laminlabs/lamindb)
2
+ [![codecov](https://codecov.io/gh/laminlabs/lamindb/branch/main/graph/badge.svg?token=VKMRJ7OWR3)](https://codecov.io/gh/laminlabs/lamindb)
3
+ [![Docs](https://img.shields.io/badge/docs-humans-yellow)](https://docs.lamin.ai)
4
+ [![DocsLLMs](https://img.shields.io/badge/docs-LLMs-yellow)](https://docs.lamin.ai/summary.md)
5
+ [![pypi](https://img.shields.io/pypi/v/lamindb?color=blue&label=pypi%20package)](https://pypi.org/project/lamindb)
6
+ [![PyPI Downloads](https://img.shields.io/pepy/dt/lamindb?logo=pypi)](https://pepy.tech/project/lamindb)
7
+
8
+ # LaminDB - A data lakehouse for biology
9
+
10
+ LaminDB is an open-source data lakehouse to enable learning at scale in biology.
11
+ It organizes datasets through validation & annotation and provides data lineage, queryability, and reproducibility on top of [FAIR](https://en.wikipedia.org/wiki/FAIR_data) data.
12
+
13
+ <details>
14
+ <summary>Why?</summary>
15
+
16
+ Reproducing analytical results or understanding how a dataset or model was created can be a pain.
17
+ Let alone training models on historical data, LIMS & ELN systems, orthogonal assays, or datasets generated by other teams.
18
+ Even maintaining a mere overview of a project's or team's datasets & analyses is harder than it sounds.
19
+
20
+ Biological datasets are typically managed with versioned storage systems, GUI-focused community or SaaS platforms, structureless data lakes, rigid data warehouses (SQL, monolithic arrays), and data lakehouses for tabular data.
21
+
22
+ LaminDB extends the lakehouse architecture to biological registries & datasets beyond tables (`DataFrame`, `AnnData`, `.zarr`, `.tiledbsoma`, ...) with enough structure to enable queries and enough freedom to keep the pace of R&D high.
23
+ Moreover, it provides context through data lineage -- tracing data and code, scientists and models -- and abstractions for biological domain knowledge and experimental metadata.
24
+
25
+ </details>
26
+
27
+ **Highlights.**
28
+
29
+ - **data lineage:** track inputs & outputs of notebooks, scripts, functions & pipelines with a single line of code
30
+ - **unified infrastructure:** access diverse storage locations (local, S3, GCP, ...), SQL databases (Postgres, SQLite) & ontologies
31
+ - **lakehouse capabilities**: manage, monitor & validate features, labels & dataset schemas; perform distributed queries and batch loading
32
+ - **biological data formats:** validate & annotate formats like `DataFrame`, `AnnData`, `MuData`, ... backed by `parquet`, `zarr`, HDF5, LanceDB, DuckDB, ...
33
+ - **biological entities**: organize experimental metadata & extensible ontologies in registries based on the Django ORM
34
+ - **reproducible & auditable:** auto-version & timestamp execution reports, source code & compute environments, attribute records to users
35
+ - **zero lock-in & scalable:** runs in your infrastructure; is _not_ a client for a rate-limited REST API
36
+ - **extendable:** create custom plug-ins for your own applications based on the Django ecosystem
37
+ - **integrations:** visualization tools like [vitessce](https://docs.lamin.ai/vitessce), workflow managers like [nextflow](https://docs.lamin.ai/nextflow) & [redun](https://docs.lamin.ai/redun), and [other tools](https://docs.lamin.ai/integrations)
38
+ - **production-ready:** used in BigPharma, BioTech, hospitals & top labs
39
+
40
+ LaminDB can be connected to LaminHub to serve as a [LIMS](https://en.wikipedia.org/wiki/Laboratory_information_management_system) for wetlab scientists, closing the drylab-wetlab feedback loop: [lamin.ai](https://lamin.ai)
41
+
42
+ ## Docs
43
+
44
+ Copy [summary.md](https://docs.lamin.ai/summary.md) into an LLM chat and let AI explain or read the [docs](https://docs.lamin.ai).
45
+
46
+ ## Setup
47
+
48
+ <!-- copied from quick-setup-lamindb.md -->
49
+
50
+ Install the `lamindb` Python package:
51
+
52
+ ```shell
53
+ pip install lamindb
54
+ ```
55
+
56
+ Create a LaminDB instance:
57
+
58
+ ```shell
59
+ lamin init --storage ./quickstart-data # or s3://my-bucket, gs://my-bucket
60
+ ```
61
+
62
+ Or if you have write access to an instance, connect to it:
63
+
64
+ ```shell
65
+ lamin connect account/name
66
+ ```
67
+
68
+ ## Quickstart
69
+
70
+ <!-- copied from preface.md -->
71
+
72
+ Track a script or notebook run with source code, inputs, outputs, logs, and environment.
73
+
74
+ <!-- copied from py-quickstart.py -->
75
+
76
+ ```python
77
+ import lamindb as ln
78
+
79
+ ln.track() # track a run
80
+ open("sample.fasta", "w").write(">seq1\nACGT\n")
81
+ ln.Artifact("sample.fasta", key="sample.fasta").save() # create an artifact
82
+ ln.finish() # finish the run
83
+ ```
84
+
85
+ <!-- from here on, slight deviation from preface.md, where all this is treated in the walk through in more depth -->
86
+
87
+ This code snippet creates an artifact, which can store a dataset or model as a file or folder in various formats.
88
+ Running the snippet as a script (`python create-fasta.py`) produces the following data lineage.
89
+
90
+ ```python
91
+ artifact = ln.Artifact.get(key="sample.fasta") # query artifact by key
92
+ artifact.view_lineage()
93
+ ```
94
+
95
+ <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/EkQATsQL5wqC95Wj0005.png" width="250">
96
+
97
+ You'll know how that artifact was created and what it's used for ([interactive visualization](https://lamin.ai/laminlabs/lamindata/artifact/8incOOgjn6F0K1TS)) in addition to capturing basic metadata:
98
+
99
+ ```python
100
+ artifact.describe()
101
+ ```
102
+
103
+ <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/BOTCBgHDAvwglN3U0002.png" width="550">
104
+
105
+ You can organize datasets with validation & annotation of any kind of metadata to then access them via queries & search. Here is a more [comprehensive example](https://lamin.ai/laminlabs/lamindata/artifact/9K1dteZ6Qx0EXK8g).
106
+
107
+ <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/6sofuDVvTANB0f480002.png" width="850">
108
+
109
+ To annotate an artifact with a label, use:
110
+
111
+ ```python
112
+ my_experiment = ln.ULabel(name="My experiment").save() # create a label in the universal label ontology
113
+ artifact.ulabels.add(my_experiment) # annotate the artifact with the label
114
+ ```
115
+
116
+ To query for a set of artifacts, use the `filter()` statement.
117
+
118
+ ```python
119
+ ln.Artifact.filter(ulabels=my_experiment, suffix=".fasta").to_dataframe() # query by suffix and the ulabel we just created
120
+ ln.Artifact.filter(transform__key="create-fasta.py").to_dataframe() # query by the name of the script we just ran
121
+ ```
122
+
123
+ If you have a structured dataset like a `DataFrame`, an `AnnData`, or another array, you can validate the content of the dataset (and parse annotations).
124
+ Here is an example for a dataframe: [docs.lamin.ai/introduction#validate-an-artifact](https://docs.lamin.ai/introduction#validate-an-artifact).
125
+
126
+ With a large body of validated datasets, you can then access data through distributed queries & batch streaming, see here: [docs.lamin.ai/arrays](https://docs.lamin.ai/arrays).
@@ -1,4 +1,4 @@
1
- # API
1
+ # API Reference
2
2
 
3
3
  <meta http-equiv="Refresh" content="0; url=./lamindb.html" />
4
4
 
@@ -18,5 +18,4 @@ lamindb
18
18
 
19
19
  bionty
20
20
  wetlab
21
- clinicore
22
21
  ```
@@ -955,13 +955,88 @@
955
955
  "id": "73",
956
956
  "metadata": {},
957
957
  "source": [
958
- "## MuData"
958
+ "## Unstructured dictionaries"
959
959
  ]
960
960
  },
961
961
  {
962
962
  "cell_type": "markdown",
963
963
  "id": "74",
964
964
  "metadata": {},
965
+ "source": [
966
+ "Most datastructures support unstructured metadata stored as dictionaries:\n",
967
+ "\n",
968
+ "- Pandas DataFrames: `.attrs`\n",
969
+ "- AnnData: `.uns`\n",
970
+ "- MuData: `.uns` and `modality:uns`\n",
971
+ "- SpatialData: `.attrs`\n",
972
+ "\n",
973
+ "Here, we exemplary show how to curate such metadata for AnnData:"
974
+ ]
975
+ },
976
+ {
977
+ "cell_type": "markdown",
978
+ "id": "75",
979
+ "metadata": {},
980
+ "source": [
981
+ "```{eval-rst}\n",
982
+ ".. literalinclude:: scripts/define_schema_anndata_uns.py\n",
983
+ " :language: python\n",
984
+ " :caption: define_schema_anndata_uns.py\n",
985
+ "```"
986
+ ]
987
+ },
988
+ {
989
+ "cell_type": "code",
990
+ "execution_count": null,
991
+ "id": "76",
992
+ "metadata": {
993
+ "tags": [
994
+ "hide-output"
995
+ ]
996
+ },
997
+ "outputs": [],
998
+ "source": [
999
+ "!python scripts/define_schema_anndata_uns.py"
1000
+ ]
1001
+ },
1002
+ {
1003
+ "cell_type": "markdown",
1004
+ "id": "77",
1005
+ "metadata": {},
1006
+ "source": [
1007
+ "```{eval-rst}\n",
1008
+ ".. literalinclude:: scripts/curate_anndata_uns.py\n",
1009
+ " :language: python\n",
1010
+ " :caption: curate_anndata_uns.py\n",
1011
+ "```"
1012
+ ]
1013
+ },
1014
+ {
1015
+ "cell_type": "code",
1016
+ "execution_count": null,
1017
+ "id": "78",
1018
+ "metadata": {
1019
+ "tags": [
1020
+ "hide-output"
1021
+ ]
1022
+ },
1023
+ "outputs": [],
1024
+ "source": [
1025
+ "!python scripts/curate_anndata_uns.py"
1026
+ ]
1027
+ },
1028
+ {
1029
+ "cell_type": "markdown",
1030
+ "id": "79",
1031
+ "metadata": {},
1032
+ "source": [
1033
+ "## MuData"
1034
+ ]
1035
+ },
1036
+ {
1037
+ "cell_type": "markdown",
1038
+ "id": "80",
1039
+ "metadata": {},
965
1040
  "source": [
966
1041
  "```{eval-rst}\n",
967
1042
  ".. literalinclude:: scripts/curate_mudata.py\n",
@@ -973,7 +1048,7 @@
973
1048
  {
974
1049
  "cell_type": "code",
975
1050
  "execution_count": null,
976
- "id": "75",
1051
+ "id": "81",
977
1052
  "metadata": {
978
1053
  "tags": [
979
1054
  "hide-output"
@@ -986,7 +1061,7 @@
986
1061
  },
987
1062
  {
988
1063
  "cell_type": "markdown",
989
- "id": "76",
1064
+ "id": "82",
990
1065
  "metadata": {},
991
1066
  "source": [
992
1067
  "## SpatialData"
@@ -994,7 +1069,7 @@
994
1069
  },
995
1070
  {
996
1071
  "cell_type": "markdown",
997
- "id": "77",
1072
+ "id": "83",
998
1073
  "metadata": {},
999
1074
  "source": [
1000
1075
  "```{eval-rst}\n",
@@ -1007,7 +1082,7 @@
1007
1082
  {
1008
1083
  "cell_type": "code",
1009
1084
  "execution_count": null,
1010
- "id": "78",
1085
+ "id": "84",
1011
1086
  "metadata": {
1012
1087
  "tags": [
1013
1088
  "hide-output"
@@ -1020,7 +1095,7 @@
1020
1095
  },
1021
1096
  {
1022
1097
  "cell_type": "markdown",
1023
- "id": "79",
1098
+ "id": "85",
1024
1099
  "metadata": {},
1025
1100
  "source": [
1026
1101
  "```{eval-rst}\n",
@@ -1033,7 +1108,7 @@
1033
1108
  {
1034
1109
  "cell_type": "code",
1035
1110
  "execution_count": null,
1036
- "id": "80",
1111
+ "id": "86",
1037
1112
  "metadata": {
1038
1113
  "tags": [
1039
1114
  "hide-output"
@@ -1046,7 +1121,7 @@
1046
1121
  },
1047
1122
  {
1048
1123
  "cell_type": "markdown",
1049
- "id": "81",
1124
+ "id": "87",
1050
1125
  "metadata": {},
1051
1126
  "source": [
1052
1127
  "## TiledbsomaExperiment"
@@ -1054,7 +1129,7 @@
1054
1129
  },
1055
1130
  {
1056
1131
  "cell_type": "markdown",
1057
- "id": "82",
1132
+ "id": "88",
1058
1133
  "metadata": {},
1059
1134
  "source": [
1060
1135
  "```{eval-rst}\n",
@@ -1067,7 +1142,7 @@
1067
1142
  {
1068
1143
  "cell_type": "code",
1069
1144
  "execution_count": null,
1070
- "id": "83",
1145
+ "id": "89",
1071
1146
  "metadata": {
1072
1147
  "tags": [
1073
1148
  "hide-output"
@@ -1080,7 +1155,7 @@
1080
1155
  },
1081
1156
  {
1082
1157
  "cell_type": "markdown",
1083
- "id": "84",
1158
+ "id": "90",
1084
1159
  "metadata": {},
1085
1160
  "source": [
1086
1161
  "## Other data structures"
@@ -1088,7 +1163,7 @@
1088
1163
  },
1089
1164
  {
1090
1165
  "cell_type": "markdown",
1091
- "id": "85",
1166
+ "id": "91",
1092
1167
  "metadata": {},
1093
1168
  "source": [
1094
1169
  "If you have other data structures, read: {doc}`/faq/curate-any`."
@@ -1097,7 +1172,7 @@
1097
1172
  {
1098
1173
  "cell_type": "code",
1099
1174
  "execution_count": null,
1100
- "id": "86",
1175
+ "id": "92",
1101
1176
  "metadata": {
1102
1177
  "tags": [
1103
1178
  "hide-cell"
@@ -23,7 +23,7 @@
23
23
  "metadata": {},
24
24
  "outputs": [],
25
25
  "source": [
26
- "# !pip install 'lamindb[jupyter,aws]'\n",
26
+ "# pip install lamindb\n",
27
27
  "from laminci.db import setup_local_test_postgres\n",
28
28
  "\n",
29
29
  "pgurl = setup_local_test_postgres()"
@@ -43,7 +43,7 @@
43
43
  },
44
44
  "outputs": [],
45
45
  "source": [
46
- "# pip install 'lamindb[jupyter]'\n",
46
+ "# pip install lamindb\n",
47
47
  "!lamin init --storage ./test-idempotency"
48
48
  ]
49
49
  },
@@ -47,10 +47,10 @@
47
47
  },
48
48
  "outputs": [],
49
49
  "source": [
50
- "# pip install 'lamindb[jupyter]'\n",
50
+ "# pip install lamindb\n",
51
51
  "import lamindb as ln\n",
52
52
  "\n",
53
- "ln.track(\"l9lFf83aPwRc0000\")"
53
+ "ln.track(\"l9lFf83aPwRc\")"
54
54
  ]
55
55
  },
56
56
  {
@@ -30,7 +30,7 @@
30
30
  "metadata": {},
31
31
  "outputs": [],
32
32
  "source": [
33
- "# pip install 'lamindb[jupyter]'\n",
33
+ "# pip install lamindb\n",
34
34
  "import lamindb as ln\n",
35
35
  "import pytest"
36
36
  ]
@@ -26,7 +26,7 @@
26
26
  },
27
27
  "outputs": [],
28
28
  "source": [
29
- "# !pip install 'lamindb[jupyter]'\n",
29
+ "# pip install lamindb\n",
30
30
  "!lamin init --storage test-run-inputs"
31
31
  ]
32
32
  },
@@ -7,15 +7,12 @@ pip install lamindb
7
7
  You can configure the installation using `extras`, e.g.,
8
8
 
9
9
  ```shell
10
- pip install 'lamindb[jupyter,bionty]'
10
+ pip install 'lamindb[gcp]'
11
11
  ```
12
12
 
13
13
  Supported `extras` are:
14
14
 
15
15
  ```yaml
16
- # commonly used
17
- jupyter # parse Jupyter notebook metadata
18
- bionty # basic biological ontologies
19
16
  # cloud backends (AWS is assumed)
20
17
  gcp # Google Cloud (gcfs, etc.)
21
18
  # biological artifact formats
@@ -1,4 +1,4 @@
1
- # Query & search
1
+ # Query, search & stream
2
2
 
3
3
  ```{toctree}
4
4
  :maxdepth: 1