lamindb 1.11a1__tar.gz → 1.11.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lamindb-1.11a1 → lamindb-1.11.2}/.gitignore +2 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/.gitmodules +0 -3
- {lamindb-1.11a1 → lamindb-1.11.2}/.pre-commit-config.yaml +2 -1
- lamindb-1.11.2/PKG-INFO +180 -0
- lamindb-1.11.2/README.md +126 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/api.md +1 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/curate.ipynb +88 -13
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/acid.ipynb +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/idempotency.ipynb +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/keep-artifacts-local.ipynb +2 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/setup.ipynb +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/track-run-inputs.ipynb +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/includes/installation.md +1 -4
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/query-search.md +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/registries.ipynb +4 -23
- lamindb-1.11.2/docs/scripts/curate_anndata_uns.py +9 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_attrs.py +2 -9
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_mudata.py +14 -2
- lamindb-1.11.2/docs/scripts/define_schema_anndata_uns.py +10 -0
- lamindb-1.11.2/docs/scripts/define_schema_df_metadata.py +9 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/track.ipynb +2 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/transfer.ipynb +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/__init__.py +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/_finish.py +3 -3
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_context.py +4 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/curators/core.py +13 -5
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/errors.py +6 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/cellxgene/_cellxgene.py +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/croissant/__init__.py +20 -4
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/_core.py +8 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/mini_immuno.py +0 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/fixtures/sheets.py +8 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/integrations/_croissant.py +34 -11
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0121_recorduser.py +7 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/__init__.py +1 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_feature_manager.py +78 -18
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/artifact.py +71 -65
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/artifact_set.py +12 -3
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/query_set.py +170 -74
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/record.py +5 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/run.py +2 -27
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/save.py +18 -10
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/sqlrecord.py +47 -33
- {lamindb-1.11a1 → lamindb-1.11.2}/noxfile.py +10 -11
- {lamindb-1.11a1 → lamindb-1.11.2}/pyproject.toml +14 -19
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_artifact.py +2 -6
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_can_curate.py +8 -8
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_collection.py +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_data.py +3 -3
- lamindb-1.11.2/tests/core/test_delete.py +18 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_describe_and_df_calls.py +5 -5
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_feature.py +3 -3
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_feature_label_manager.py +26 -26
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_from_values.py +2 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_has_parents.py +2 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_models.py +3 -3
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_queryset.py +56 -3
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_save.py +1 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_schema.py +9 -9
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_search.py +2 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_versioning.py +29 -7
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_cat_managers.py +39 -39
- lamindb-1.11.2/tests/curators/test_curate_from_croissant.py +78 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_curators_examples.py +76 -98
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_curators_general.py +16 -16
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_curators_multivalue.py +2 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_cxg_curator.py +5 -5
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_records.py +5 -4
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/setup_access.py +5 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/test_permissions.py +61 -2
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_artifact_storage.py +11 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_streaming.py +10 -1
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_transfer.py +8 -4
- lamindb-1.11a1/PKG-INFO +0 -144
- lamindb-1.11a1/README.md +0 -85
- lamindb-1.11a1/tests/core/test_delete.py +0 -9
- lamindb-1.11a1/tests/curators/test_curate_from_croissant.py +0 -45
- {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/enhancement.yml +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/.github/ISSUE_TEMPLATE/usage_question.yml +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/.github/workflows/build.yml +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/.github/workflows/doc-changes.yml +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/CONTRIBUTING.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/LICENSE +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/arrays.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/bio-registries.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/bionty.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/changelog.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/clinicore.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/curate-any.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/delete.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/import-modules.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/pydantic-pandera.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/reference-field.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/search.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/symbol-mapping.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/test_notebooks.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/validate-fields.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq/visibility.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/faq.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/guide.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/index.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/lamindb.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_anndata_flexible.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_external_features.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_flexible.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_dataframe_minimal_errors.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_soma_experiment.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/curate_spatialdata.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_mini_immuno_features_labels.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_mini_immuno_schema_flexible.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_schema_anndata_ensembl_gene_ids_and_valid_features_in_obs.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_schema_spatialdata.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/define_valid_features.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/run_track_and_finish.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/run_track_with_params.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/run_workflow.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/save_mini_immuno_datasets.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/scripts/synced_with_git.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/add-replace-cache.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/anndata-accessor.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/prepare-transfer-local-to-cloud.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test-files/iris.csv +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test-files/iris.data +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test-files/new_iris.csv +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/test_notebooks.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/transfer-local-to-cloud.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/upload.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage/vitessce.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/storage.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/test_notebooks.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/docs/wetlab.md +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/_tracked.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/_view.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/dtypes.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/fields.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/ids.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/types.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/uids.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/base/users.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_compat.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_mapped_collection.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_settings.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_sync_git.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/_track_environment.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/exceptions.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/loaders.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_anndata_accessor.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_backed_access.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_polars_lazy_df.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_pyarrow_dataset.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_spatialdata_accessor.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_tiledbsoma.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_valid_suffixes.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/_zarr.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/objects.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/storage/paths.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/subsettings/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/subsettings/_annotation_settings.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/subsettings/_creation_settings.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/core/types.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/curators/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/curators/_legacy.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/cellxgene/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/cellxgene/cellxgene_schema_versions.csv +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/croissant/mini_immuno.anndata.zarr_metadata.json +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/_fake.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/datasets/_small.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/fixtures/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/schemas/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/schemas/_anndata.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/examples/schemas/_simple.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/integrations/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/integrations/_vitessce.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0069_squashed.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0070_lamindbv1_migrate_data.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0071_lamindbv1_migrate_schema.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0072_remove_user__branch_code_remove_user_aux_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0073_merge_ourprojects.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0074_lamindbv1_part4.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0075_lamindbv1_part5.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0076_lamindbv1_part6.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0077_lamindbv1_part6b.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0078_lamindbv1_part6c.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0079_alter_rundata_value_json_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0080_polish_lamindbv1.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0081_revert_textfield_collection.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0082_alter_feature_dtype.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0083_alter_feature_is_type_alter_flextable_is_type_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0084_alter_schemafeature_feature_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0085_alter_feature_is_type_alter_flextable_is_type_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0086_various.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0087_rename__schemas_m2m_artifact_feature_sets_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0088_schema_components.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0089_subsequent_runs.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0090_runproject_project_runs.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0091_alter_featurevalue_options_alter_space_options_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0092_alter_artifactfeaturevalue_artifact_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0093_alter_schemacomponent_unique_together.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0094_writeloglock_writelogmigrationstate_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0095_remove_rundata_flextable.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0096_remove_artifact__param_values_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0097_remove_schemaparam_param_remove_paramvalue_param_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0098_alter_feature_type_alter_project_type_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0099_alter_writelog_seqno.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0100_branch_alter_artifact__branch_code_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0101_alter_artifact_hash_alter_feature_name_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0102_remove_writelog_branch_code_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0103_remove_writelog_migration_state_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0104_alter_branch_uid.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0105_record_unique_name.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0106_transfer_data_migration.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0107_add_schema_to_record.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0108_remove_record_sheet_remove_sheetproject_sheet_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0109_record_input_of_runs_alter_record_run_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0110_rename_values_artifacts_record_linked_artifacts.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0111_remove_record__sort_order.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0112_alter_recordartifact_feature_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0113_lower_case_branch_and_space_names.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0114_alter_run__status_code.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0115_alter_space_uid.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0116_remove_artifact_unique_artifact_storage_key_hash_and_more.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0117_fix_artifact_storage_hash_unique_constraints.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0118_alter_recordproject_value_projectrecord.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0119_rename_records_project_linked_in_records.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0119_squashed.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/0120_add_record_fk_constraint.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/migrations/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_describe.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_django.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_from_values.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_is_versioned.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_label_manager.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/_relations.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/can_curate.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/collection.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/feature.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/has_parents.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/project.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/query_manager.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/schema.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/storage.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/transform.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/models/ulabel.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/py.typed +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/_switch.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/core/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/errors/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/lamindb/setup/types/__init__.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/conftest.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/_dataset_fixtures.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/conftest.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/basic-r-notebook.Rmd.cleaned.html +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/basic-r-notebook.Rmd.html +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/duplicate/with-title-initialized-consecutive-finish.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/no-title.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/with-title-initialized-consecutive-finish-not-last-cell.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/notebooks/with-title-initialized-consecutive-finish.ipynb +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate1/script-to-test-versioning.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate2/script-to-test-versioning.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate3/script-to-test-versioning.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate4/script-to-test-versioning.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/duplicate5/script-to-test-versioning.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/script-to-test-filename-change.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/scripts/script-to-test-versioning.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_artifact_folders.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_db.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_dtype.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_integrity.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_load.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_manager.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_notebooks.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_run.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_settings.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_sqlrecord.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_storage.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_track.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_tracked.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_transform.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_ulabel.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_view.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/core/test_visibility.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/conftest.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/curators/test_dataframe_curators_accounting_example.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/conftest.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/jwt_utils.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/check_lamin_dev.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/clean_lamin_dev.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/permissions/scripts/setup_instance.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/conftest.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_artifact_zarr.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_cache.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_connect_reconnect.py +0 -0
- {lamindb-1.11a1 → lamindb-1.11.2}/tests/storage/test_storage_lifecycle.py +0 -0
@@ -10,9 +10,6 @@
|
|
10
10
|
[submodule "sub/wetlab"]
|
11
11
|
path = sub/wetlab
|
12
12
|
url = https://github.com/laminlabs/wetlab
|
13
|
-
[submodule "sub/clinicore"]
|
14
|
-
path = sub/clinicore
|
15
|
-
url = https://github.com/laminlabs/clinicore
|
16
13
|
[submodule "sub/cellxgene-lamin"]
|
17
14
|
path = sub/cellxgene-lamin
|
18
15
|
url = https://github.com/laminlabs/cellxgene-lamin.git
|
lamindb-1.11.2/PKG-INFO
ADDED
@@ -0,0 +1,180 @@
|
|
1
|
+
Metadata-Version: 2.3
|
2
|
+
Name: lamindb
|
3
|
+
Version: 1.11.2
|
4
|
+
Summary: A data framework for biology.
|
5
|
+
Author-email: Lamin Labs <open-source@lamin.ai>
|
6
|
+
Requires-Python: >=3.10,<3.14
|
7
|
+
Description-Content-Type: text/markdown
|
8
|
+
Classifier: Programming Language :: Python :: 3.10
|
9
|
+
Classifier: Programming Language :: Python :: 3.11
|
10
|
+
Classifier: Programming Language :: Python :: 3.12
|
11
|
+
Classifier: Programming Language :: Python :: 3.13
|
12
|
+
Requires-Dist: lamin_utils==0.15.0
|
13
|
+
Requires-Dist: lamin_cli==1.7.1
|
14
|
+
Requires-Dist: lamindb_setup[aws]==1.10.1
|
15
|
+
Requires-Dist: bionty>=1.7a1
|
16
|
+
Requires-Dist: wetlab>=1.5a1
|
17
|
+
Requires-Dist: nbproject==0.11.1
|
18
|
+
Requires-Dist: jupytext
|
19
|
+
Requires-Dist: nbconvert>=7.2.1
|
20
|
+
Requires-Dist: mistune!=3.1.0
|
21
|
+
Requires-Dist: pyyaml
|
22
|
+
Requires-Dist: pyarrow
|
23
|
+
Requires-Dist: pandera>=0.24.0
|
24
|
+
Requires-Dist: typing_extensions!=4.6.0
|
25
|
+
Requires-Dist: python-dateutil
|
26
|
+
Requires-Dist: pandas>=2.0.0
|
27
|
+
Requires-Dist: scipy<1.15.0
|
28
|
+
Requires-Dist: anndata>=0.8.0,<=0.12.1
|
29
|
+
Requires-Dist: fsspec
|
30
|
+
Requires-Dist: graphviz
|
31
|
+
Requires-Dist: psycopg2-binary
|
32
|
+
Requires-Dist: tomlkit ; extra == "dev"
|
33
|
+
Requires-Dist: line_profiler ; extra == "dev"
|
34
|
+
Requires-Dist: pre-commit ; extra == "dev"
|
35
|
+
Requires-Dist: nox ; extra == "dev"
|
36
|
+
Requires-Dist: laminci>=0.3 ; extra == "dev"
|
37
|
+
Requires-Dist: pytest>=6.0 ; extra == "dev"
|
38
|
+
Requires-Dist: coverage ; extra == "dev"
|
39
|
+
Requires-Dist: pytest-cov<7.0.0 ; extra == "dev"
|
40
|
+
Requires-Dist: mudata ; extra == "dev"
|
41
|
+
Requires-Dist: nbproject_test>=0.6.0 ; extra == "dev"
|
42
|
+
Requires-Dist: faker-biology ; extra == "dev"
|
43
|
+
Requires-Dist: pronto ; extra == "dev"
|
44
|
+
Requires-Dist: readfcs>=2.0.1 ; extra == "fcs"
|
45
|
+
Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
|
46
|
+
Requires-Dist: numcodecs<0.16.0 ; extra == "zarr"
|
47
|
+
Requires-Dist: zarr>=2.16.0,<3.0.0a0 ; extra == "zarr"
|
48
|
+
Project-URL: Home, https://github.com/laminlabs/lamindb
|
49
|
+
Provides-Extra: dev
|
50
|
+
Provides-Extra: fcs
|
51
|
+
Provides-Extra: gcp
|
52
|
+
Provides-Extra: zarr
|
53
|
+
|
54
|
+
[](https://github.com/laminlabs/lamindb)
|
55
|
+
[](https://codecov.io/gh/laminlabs/lamindb)
|
56
|
+
[](https://docs.lamin.ai)
|
57
|
+
[](https://docs.lamin.ai/summary.md)
|
58
|
+
[](https://pypi.org/project/lamindb)
|
59
|
+
[](https://pepy.tech/project/lamindb)
|
60
|
+
|
61
|
+
# LaminDB - A data lakehouse for biology
|
62
|
+
|
63
|
+
LaminDB is an open-source data lakehouse to enable learning at scale in biology.
|
64
|
+
It organizes datasets through validation & annotation and provides data lineage, queryability, and reproducibility on top of [FAIR](https://en.wikipedia.org/wiki/FAIR_data) data.
|
65
|
+
|
66
|
+
<details>
|
67
|
+
<summary>Why?</summary>
|
68
|
+
|
69
|
+
Reproducing analytical results or understanding how a dataset or model was created can be a pain.
|
70
|
+
Let alone training models on historical data, LIMS & ELN systems, orthogonal assays, or datasets generated by other teams.
|
71
|
+
Even maintaining a mere overview of a project's or team's datasets & analyses is harder than it sounds.
|
72
|
+
|
73
|
+
Biological datasets are typically managed with versioned storage systems, GUI-focused community or SaaS platforms, structureless data lakes, rigid data warehouses (SQL, monolithic arrays), and data lakehouses for tabular data.
|
74
|
+
|
75
|
+
LaminDB extends the lakehouse architecture to biological registries & datasets beyond tables (`DataFrame`, `AnnData`, `.zarr`, `.tiledbsoma`, ...) with enough structure to enable queries and enough freedom to keep the pace of R&D high.
|
76
|
+
Moreover, it provides context through data lineage -- tracing data and code, scientists and models -- and abstractions for biological domain knowledge and experimental metadata.
|
77
|
+
|
78
|
+
</details>
|
79
|
+
|
80
|
+
**Highlights.**
|
81
|
+
|
82
|
+
- **data lineage:** track inputs & outputs of notebooks, scripts, functions & pipelines with a single line of code
|
83
|
+
- **unified infrastructure:** access diverse storage locations (local, S3, GCP, ...), SQL databases (Postgres, SQLite) & ontologies
|
84
|
+
- **lakehouse capabilities**: manage, monitor & validate features, labels & dataset schemas; perform distributed queries and batch loading
|
85
|
+
- **biological data formats:** validate & annotate formats like `DataFrame`, `AnnData`, `MuData`, ... backed by `parquet`, `zarr`, HDF5, LanceDB, DuckDB, ...
|
86
|
+
- **biological entities**: organize experimental metadata & extensible ontologies in registries based on the Django ORM
|
87
|
+
- **reproducible & auditable:** auto-version & timestamp execution reports, source code & compute environments, attribute records to users
|
88
|
+
- **zero lock-in & scalable:** runs in your infrastructure; is _not_ a client for a rate-limited REST API
|
89
|
+
- **extendable:** create custom plug-ins for your own applications based on the Django ecosystem
|
90
|
+
- **integrations:** visualization tools like [vitessce](https://docs.lamin.ai/vitessce), workflow managers like [nextflow](https://docs.lamin.ai/nextflow) & [redun](https://docs.lamin.ai/redun), and [other tools](https://docs.lamin.ai/integrations)
|
91
|
+
- **production-ready:** used in BigPharma, BioTech, hospitals & top labs
|
92
|
+
|
93
|
+
LaminDB can be connected to LaminHub to serve as a [LIMS](https://en.wikipedia.org/wiki/Laboratory_information_management_system) for wetlab scientists, closing the drylab-wetlab feedback loop: [lamin.ai](https://lamin.ai)
|
94
|
+
|
95
|
+
## Docs
|
96
|
+
|
97
|
+
Copy [summary.md](https://docs.lamin.ai/summary.md) into an LLM chat and let AI explain or read the [docs](https://docs.lamin.ai).
|
98
|
+
|
99
|
+
## Setup
|
100
|
+
|
101
|
+
<!-- copied from quick-setup-lamindb.md -->
|
102
|
+
|
103
|
+
Install the `lamindb` Python package:
|
104
|
+
|
105
|
+
```shell
|
106
|
+
pip install lamindb
|
107
|
+
```
|
108
|
+
|
109
|
+
Create a LaminDB instance:
|
110
|
+
|
111
|
+
```shell
|
112
|
+
lamin init --storage ./quickstart-data # or s3://my-bucket, gs://my-bucket
|
113
|
+
```
|
114
|
+
|
115
|
+
Or if you have write access to an instance, connect to it:
|
116
|
+
|
117
|
+
```shell
|
118
|
+
lamin connect account/name
|
119
|
+
```
|
120
|
+
|
121
|
+
## Quickstart
|
122
|
+
|
123
|
+
<!-- copied from preface.md -->
|
124
|
+
|
125
|
+
Track a script or notebook run with source code, inputs, outputs, logs, and environment.
|
126
|
+
|
127
|
+
<!-- copied from py-quickstart.py -->
|
128
|
+
|
129
|
+
```python
|
130
|
+
import lamindb as ln
|
131
|
+
|
132
|
+
ln.track() # track a run
|
133
|
+
open("sample.fasta", "w").write(">seq1\nACGT\n")
|
134
|
+
ln.Artifact("sample.fasta", key="sample.fasta").save() # create an artifact
|
135
|
+
ln.finish() # finish the run
|
136
|
+
```
|
137
|
+
|
138
|
+
<!-- from here on, slight deviation from preface.md, where all this is treated in the walk through in more depth -->
|
139
|
+
|
140
|
+
This code snippet creates an artifact, which can store a dataset or model as a file or folder in various formats.
|
141
|
+
Running the snippet as a script (`python create-fasta.py`) produces the following data lineage.
|
142
|
+
|
143
|
+
```python
|
144
|
+
artifact = ln.Artifact.get(key="sample.fasta") # query artifact by key
|
145
|
+
artifact.view_lineage()
|
146
|
+
```
|
147
|
+
|
148
|
+
<img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/EkQATsQL5wqC95Wj0005.png" width="250">
|
149
|
+
|
150
|
+
You'll know how that artifact was created and what it's used for ([interactive visualization](https://lamin.ai/laminlabs/lamindata/artifact/8incOOgjn6F0K1TS)) in addition to capturing basic metadata:
|
151
|
+
|
152
|
+
```python
|
153
|
+
artifact.describe()
|
154
|
+
```
|
155
|
+
|
156
|
+
<img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/BOTCBgHDAvwglN3U0002.png" width="550">
|
157
|
+
|
158
|
+
You can organize datasets with validation & annotation of any kind of metadata to then access them via queries & search. Here is a more [comprehensive example](https://lamin.ai/laminlabs/lamindata/artifact/9K1dteZ6Qx0EXK8g).
|
159
|
+
|
160
|
+
<img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/6sofuDVvTANB0f480002.png" width="850">
|
161
|
+
|
162
|
+
To annotate an artifact with a label, use:
|
163
|
+
|
164
|
+
```python
|
165
|
+
my_experiment = ln.ULabel(name="My experiment").save() # create a label in the universal label ontology
|
166
|
+
artifact.ulabels.add(my_experiment) # annotate the artifact with the label
|
167
|
+
```
|
168
|
+
|
169
|
+
To query for a set of artifacts, use the `filter()` statement.
|
170
|
+
|
171
|
+
```python
|
172
|
+
ln.Artifact.filter(ulabels=my_experiment, suffix=".fasta").to_dataframe() # query by suffix and the ulabel we just created
|
173
|
+
ln.Artifact.filter(transform__key="create-fasta.py").to_dataframe() # query by the name of the script we just ran
|
174
|
+
```
|
175
|
+
|
176
|
+
If you have a structured dataset like a `DataFrame`, an `AnnData`, or another array, you can validate the content of the dataset (and parse annotations).
|
177
|
+
Here is an example for a dataframe: [docs.lamin.ai/introduction#validate-an-artifact](https://docs.lamin.ai/introduction#validate-an-artifact).
|
178
|
+
|
179
|
+
With a large body of validated datasets, you can then access data through distributed queries & batch streaming, see here: [docs.lamin.ai/arrays](https://docs.lamin.ai/arrays).
|
180
|
+
|
lamindb-1.11.2/README.md
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
[](https://github.com/laminlabs/lamindb)
|
2
|
+
[](https://codecov.io/gh/laminlabs/lamindb)
|
3
|
+
[](https://docs.lamin.ai)
|
4
|
+
[](https://docs.lamin.ai/summary.md)
|
5
|
+
[](https://pypi.org/project/lamindb)
|
6
|
+
[](https://pepy.tech/project/lamindb)
|
7
|
+
|
8
|
+
# LaminDB - A data lakehouse for biology
|
9
|
+
|
10
|
+
LaminDB is an open-source data lakehouse to enable learning at scale in biology.
|
11
|
+
It organizes datasets through validation & annotation and provides data lineage, queryability, and reproducibility on top of [FAIR](https://en.wikipedia.org/wiki/FAIR_data) data.
|
12
|
+
|
13
|
+
<details>
|
14
|
+
<summary>Why?</summary>
|
15
|
+
|
16
|
+
Reproducing analytical results or understanding how a dataset or model was created can be a pain.
|
17
|
+
Let alone training models on historical data, LIMS & ELN systems, orthogonal assays, or datasets generated by other teams.
|
18
|
+
Even maintaining a mere overview of a project's or team's datasets & analyses is harder than it sounds.
|
19
|
+
|
20
|
+
Biological datasets are typically managed with versioned storage systems, GUI-focused community or SaaS platforms, structureless data lakes, rigid data warehouses (SQL, monolithic arrays), and data lakehouses for tabular data.
|
21
|
+
|
22
|
+
LaminDB extends the lakehouse architecture to biological registries & datasets beyond tables (`DataFrame`, `AnnData`, `.zarr`, `.tiledbsoma`, ...) with enough structure to enable queries and enough freedom to keep the pace of R&D high.
|
23
|
+
Moreover, it provides context through data lineage -- tracing data and code, scientists and models -- and abstractions for biological domain knowledge and experimental metadata.
|
24
|
+
|
25
|
+
</details>
|
26
|
+
|
27
|
+
**Highlights.**
|
28
|
+
|
29
|
+
- **data lineage:** track inputs & outputs of notebooks, scripts, functions & pipelines with a single line of code
|
30
|
+
- **unified infrastructure:** access diverse storage locations (local, S3, GCP, ...), SQL databases (Postgres, SQLite) & ontologies
|
31
|
+
- **lakehouse capabilities**: manage, monitor & validate features, labels & dataset schemas; perform distributed queries and batch loading
|
32
|
+
- **biological data formats:** validate & annotate formats like `DataFrame`, `AnnData`, `MuData`, ... backed by `parquet`, `zarr`, HDF5, LanceDB, DuckDB, ...
|
33
|
+
- **biological entities**: organize experimental metadata & extensible ontologies in registries based on the Django ORM
|
34
|
+
- **reproducible & auditable:** auto-version & timestamp execution reports, source code & compute environments, attribute records to users
|
35
|
+
- **zero lock-in & scalable:** runs in your infrastructure; is _not_ a client for a rate-limited REST API
|
36
|
+
- **extendable:** create custom plug-ins for your own applications based on the Django ecosystem
|
37
|
+
- **integrations:** visualization tools like [vitessce](https://docs.lamin.ai/vitessce), workflow managers like [nextflow](https://docs.lamin.ai/nextflow) & [redun](https://docs.lamin.ai/redun), and [other tools](https://docs.lamin.ai/integrations)
|
38
|
+
- **production-ready:** used in BigPharma, BioTech, hospitals & top labs
|
39
|
+
|
40
|
+
LaminDB can be connected to LaminHub to serve as a [LIMS](https://en.wikipedia.org/wiki/Laboratory_information_management_system) for wetlab scientists, closing the drylab-wetlab feedback loop: [lamin.ai](https://lamin.ai)
|
41
|
+
|
42
|
+
## Docs
|
43
|
+
|
44
|
+
Copy [summary.md](https://docs.lamin.ai/summary.md) into an LLM chat and let AI explain or read the [docs](https://docs.lamin.ai).
|
45
|
+
|
46
|
+
## Setup
|
47
|
+
|
48
|
+
<!-- copied from quick-setup-lamindb.md -->
|
49
|
+
|
50
|
+
Install the `lamindb` Python package:
|
51
|
+
|
52
|
+
```shell
|
53
|
+
pip install lamindb
|
54
|
+
```
|
55
|
+
|
56
|
+
Create a LaminDB instance:
|
57
|
+
|
58
|
+
```shell
|
59
|
+
lamin init --storage ./quickstart-data # or s3://my-bucket, gs://my-bucket
|
60
|
+
```
|
61
|
+
|
62
|
+
Or if you have write access to an instance, connect to it:
|
63
|
+
|
64
|
+
```shell
|
65
|
+
lamin connect account/name
|
66
|
+
```
|
67
|
+
|
68
|
+
## Quickstart
|
69
|
+
|
70
|
+
<!-- copied from preface.md -->
|
71
|
+
|
72
|
+
Track a script or notebook run with source code, inputs, outputs, logs, and environment.
|
73
|
+
|
74
|
+
<!-- copied from py-quickstart.py -->
|
75
|
+
|
76
|
+
```python
|
77
|
+
import lamindb as ln
|
78
|
+
|
79
|
+
ln.track() # track a run
|
80
|
+
open("sample.fasta", "w").write(">seq1\nACGT\n")
|
81
|
+
ln.Artifact("sample.fasta", key="sample.fasta").save() # create an artifact
|
82
|
+
ln.finish() # finish the run
|
83
|
+
```
|
84
|
+
|
85
|
+
<!-- from here on, slight deviation from preface.md, where all this is treated in the walk through in more depth -->
|
86
|
+
|
87
|
+
This code snippet creates an artifact, which can store a dataset or model as a file or folder in various formats.
|
88
|
+
Running the snippet as a script (`python create-fasta.py`) produces the following data lineage.
|
89
|
+
|
90
|
+
```python
|
91
|
+
artifact = ln.Artifact.get(key="sample.fasta") # query artifact by key
|
92
|
+
artifact.view_lineage()
|
93
|
+
```
|
94
|
+
|
95
|
+
<img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/EkQATsQL5wqC95Wj0005.png" width="250">
|
96
|
+
|
97
|
+
You'll know how that artifact was created and what it's used for ([interactive visualization](https://lamin.ai/laminlabs/lamindata/artifact/8incOOgjn6F0K1TS)) in addition to capturing basic metadata:
|
98
|
+
|
99
|
+
```python
|
100
|
+
artifact.describe()
|
101
|
+
```
|
102
|
+
|
103
|
+
<img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/BOTCBgHDAvwglN3U0002.png" width="550">
|
104
|
+
|
105
|
+
You can organize datasets with validation & annotation of any kind of metadata to then access them via queries & search. Here is a more [comprehensive example](https://lamin.ai/laminlabs/lamindata/artifact/9K1dteZ6Qx0EXK8g).
|
106
|
+
|
107
|
+
<img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/6sofuDVvTANB0f480002.png" width="850">
|
108
|
+
|
109
|
+
To annotate an artifact with a label, use:
|
110
|
+
|
111
|
+
```python
|
112
|
+
my_experiment = ln.ULabel(name="My experiment").save() # create a label in the universal label ontology
|
113
|
+
artifact.ulabels.add(my_experiment) # annotate the artifact with the label
|
114
|
+
```
|
115
|
+
|
116
|
+
To query for a set of artifacts, use the `filter()` statement.
|
117
|
+
|
118
|
+
```python
|
119
|
+
ln.Artifact.filter(ulabels=my_experiment, suffix=".fasta").to_dataframe() # query by suffix and the ulabel we just created
|
120
|
+
ln.Artifact.filter(transform__key="create-fasta.py").to_dataframe() # query by the name of the script we just ran
|
121
|
+
```
|
122
|
+
|
123
|
+
If you have a structured dataset like a `DataFrame`, an `AnnData`, or another array, you can validate the content of the dataset (and parse annotations).
|
124
|
+
Here is an example for a dataframe: [docs.lamin.ai/introduction#validate-an-artifact](https://docs.lamin.ai/introduction#validate-an-artifact).
|
125
|
+
|
126
|
+
With a large body of validated datasets, you can then access data through distributed queries & batch streaming, see here: [docs.lamin.ai/arrays](https://docs.lamin.ai/arrays).
|
@@ -955,13 +955,88 @@
|
|
955
955
|
"id": "73",
|
956
956
|
"metadata": {},
|
957
957
|
"source": [
|
958
|
-
"##
|
958
|
+
"## Unstructured dictionaries"
|
959
959
|
]
|
960
960
|
},
|
961
961
|
{
|
962
962
|
"cell_type": "markdown",
|
963
963
|
"id": "74",
|
964
964
|
"metadata": {},
|
965
|
+
"source": [
|
966
|
+
"Most datastructures support unstructured metadata stored as dictionaries:\n",
|
967
|
+
"\n",
|
968
|
+
"- Pandas DataFrames: `.attrs`\n",
|
969
|
+
"- AnnData: `.uns`\n",
|
970
|
+
"- MuData: `.uns` and `modality:uns`\n",
|
971
|
+
"- SpatialData: `.attrs`\n",
|
972
|
+
"\n",
|
973
|
+
"Here, we exemplary show how to curate such metadata for AnnData:"
|
974
|
+
]
|
975
|
+
},
|
976
|
+
{
|
977
|
+
"cell_type": "markdown",
|
978
|
+
"id": "75",
|
979
|
+
"metadata": {},
|
980
|
+
"source": [
|
981
|
+
"```{eval-rst}\n",
|
982
|
+
".. literalinclude:: scripts/define_schema_anndata_uns.py\n",
|
983
|
+
" :language: python\n",
|
984
|
+
" :caption: define_schema_anndata_uns.py\n",
|
985
|
+
"```"
|
986
|
+
]
|
987
|
+
},
|
988
|
+
{
|
989
|
+
"cell_type": "code",
|
990
|
+
"execution_count": null,
|
991
|
+
"id": "76",
|
992
|
+
"metadata": {
|
993
|
+
"tags": [
|
994
|
+
"hide-output"
|
995
|
+
]
|
996
|
+
},
|
997
|
+
"outputs": [],
|
998
|
+
"source": [
|
999
|
+
"!python scripts/define_schema_anndata_uns.py"
|
1000
|
+
]
|
1001
|
+
},
|
1002
|
+
{
|
1003
|
+
"cell_type": "markdown",
|
1004
|
+
"id": "77",
|
1005
|
+
"metadata": {},
|
1006
|
+
"source": [
|
1007
|
+
"```{eval-rst}\n",
|
1008
|
+
".. literalinclude:: scripts/curate_anndata_uns.py\n",
|
1009
|
+
" :language: python\n",
|
1010
|
+
" :caption: curate_anndata_uns.py\n",
|
1011
|
+
"```"
|
1012
|
+
]
|
1013
|
+
},
|
1014
|
+
{
|
1015
|
+
"cell_type": "code",
|
1016
|
+
"execution_count": null,
|
1017
|
+
"id": "78",
|
1018
|
+
"metadata": {
|
1019
|
+
"tags": [
|
1020
|
+
"hide-output"
|
1021
|
+
]
|
1022
|
+
},
|
1023
|
+
"outputs": [],
|
1024
|
+
"source": [
|
1025
|
+
"!python scripts/curate_anndata_uns.py"
|
1026
|
+
]
|
1027
|
+
},
|
1028
|
+
{
|
1029
|
+
"cell_type": "markdown",
|
1030
|
+
"id": "79",
|
1031
|
+
"metadata": {},
|
1032
|
+
"source": [
|
1033
|
+
"## MuData"
|
1034
|
+
]
|
1035
|
+
},
|
1036
|
+
{
|
1037
|
+
"cell_type": "markdown",
|
1038
|
+
"id": "80",
|
1039
|
+
"metadata": {},
|
965
1040
|
"source": [
|
966
1041
|
"```{eval-rst}\n",
|
967
1042
|
".. literalinclude:: scripts/curate_mudata.py\n",
|
@@ -973,7 +1048,7 @@
|
|
973
1048
|
{
|
974
1049
|
"cell_type": "code",
|
975
1050
|
"execution_count": null,
|
976
|
-
"id": "
|
1051
|
+
"id": "81",
|
977
1052
|
"metadata": {
|
978
1053
|
"tags": [
|
979
1054
|
"hide-output"
|
@@ -986,7 +1061,7 @@
|
|
986
1061
|
},
|
987
1062
|
{
|
988
1063
|
"cell_type": "markdown",
|
989
|
-
"id": "
|
1064
|
+
"id": "82",
|
990
1065
|
"metadata": {},
|
991
1066
|
"source": [
|
992
1067
|
"## SpatialData"
|
@@ -994,7 +1069,7 @@
|
|
994
1069
|
},
|
995
1070
|
{
|
996
1071
|
"cell_type": "markdown",
|
997
|
-
"id": "
|
1072
|
+
"id": "83",
|
998
1073
|
"metadata": {},
|
999
1074
|
"source": [
|
1000
1075
|
"```{eval-rst}\n",
|
@@ -1007,7 +1082,7 @@
|
|
1007
1082
|
{
|
1008
1083
|
"cell_type": "code",
|
1009
1084
|
"execution_count": null,
|
1010
|
-
"id": "
|
1085
|
+
"id": "84",
|
1011
1086
|
"metadata": {
|
1012
1087
|
"tags": [
|
1013
1088
|
"hide-output"
|
@@ -1020,7 +1095,7 @@
|
|
1020
1095
|
},
|
1021
1096
|
{
|
1022
1097
|
"cell_type": "markdown",
|
1023
|
-
"id": "
|
1098
|
+
"id": "85",
|
1024
1099
|
"metadata": {},
|
1025
1100
|
"source": [
|
1026
1101
|
"```{eval-rst}\n",
|
@@ -1033,7 +1108,7 @@
|
|
1033
1108
|
{
|
1034
1109
|
"cell_type": "code",
|
1035
1110
|
"execution_count": null,
|
1036
|
-
"id": "
|
1111
|
+
"id": "86",
|
1037
1112
|
"metadata": {
|
1038
1113
|
"tags": [
|
1039
1114
|
"hide-output"
|
@@ -1046,7 +1121,7 @@
|
|
1046
1121
|
},
|
1047
1122
|
{
|
1048
1123
|
"cell_type": "markdown",
|
1049
|
-
"id": "
|
1124
|
+
"id": "87",
|
1050
1125
|
"metadata": {},
|
1051
1126
|
"source": [
|
1052
1127
|
"## TiledbsomaExperiment"
|
@@ -1054,7 +1129,7 @@
|
|
1054
1129
|
},
|
1055
1130
|
{
|
1056
1131
|
"cell_type": "markdown",
|
1057
|
-
"id": "
|
1132
|
+
"id": "88",
|
1058
1133
|
"metadata": {},
|
1059
1134
|
"source": [
|
1060
1135
|
"```{eval-rst}\n",
|
@@ -1067,7 +1142,7 @@
|
|
1067
1142
|
{
|
1068
1143
|
"cell_type": "code",
|
1069
1144
|
"execution_count": null,
|
1070
|
-
"id": "
|
1145
|
+
"id": "89",
|
1071
1146
|
"metadata": {
|
1072
1147
|
"tags": [
|
1073
1148
|
"hide-output"
|
@@ -1080,7 +1155,7 @@
|
|
1080
1155
|
},
|
1081
1156
|
{
|
1082
1157
|
"cell_type": "markdown",
|
1083
|
-
"id": "
|
1158
|
+
"id": "90",
|
1084
1159
|
"metadata": {},
|
1085
1160
|
"source": [
|
1086
1161
|
"## Other data structures"
|
@@ -1088,7 +1163,7 @@
|
|
1088
1163
|
},
|
1089
1164
|
{
|
1090
1165
|
"cell_type": "markdown",
|
1091
|
-
"id": "
|
1166
|
+
"id": "91",
|
1092
1167
|
"metadata": {},
|
1093
1168
|
"source": [
|
1094
1169
|
"If you have other data structures, read: {doc}`/faq/curate-any`."
|
@@ -1097,7 +1172,7 @@
|
|
1097
1172
|
{
|
1098
1173
|
"cell_type": "code",
|
1099
1174
|
"execution_count": null,
|
1100
|
-
"id": "
|
1175
|
+
"id": "92",
|
1101
1176
|
"metadata": {
|
1102
1177
|
"tags": [
|
1103
1178
|
"hide-cell"
|
@@ -7,15 +7,12 @@ pip install lamindb
|
|
7
7
|
You can configure the installation using `extras`, e.g.,
|
8
8
|
|
9
9
|
```shell
|
10
|
-
pip install 'lamindb[
|
10
|
+
pip install 'lamindb[gcp]'
|
11
11
|
```
|
12
12
|
|
13
13
|
Supported `extras` are:
|
14
14
|
|
15
15
|
```yaml
|
16
|
-
# commonly used
|
17
|
-
jupyter # parse Jupyter notebook metadata
|
18
|
-
bionty # basic biological ontologies
|
19
16
|
# cloud backends (AWS is assumed)
|
20
17
|
gcp # Google Cloud (gcfs, etc.)
|
21
18
|
# biological artifact formats
|