lamindb 0.59.5__tar.gz → 0.60.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lamindb-0.59.5 → lamindb-0.60.0}/PKG-INFO +4 -4
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/changelog.md +9 -7
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/tutorial.ipynb +1 -1
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/__init__.py +14 -11
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_dataset.py +24 -15
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_file.py +11 -79
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_view.py +4 -4
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/_data.py +100 -69
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/_label_manager.py +2 -2
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/storage/__init__.py +0 -2
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/storage/file.py +1 -61
- {lamindb-0.59.5 → lamindb-0.60.0}/pyproject.toml +3 -3
- lamindb-0.60.0/sub/lamin-cli/.gitignore +208 -0
- lamindb-0.60.0/sub/lamin-cli/.pre-commit-config.yaml +64 -0
- lamindb-0.60.0/sub/lamin-cli/README.md +3 -0
- lamindb-0.60.0/sub/lamin-cli/lamin_cli/__init__.py +2 -0
- lamindb-0.60.0/sub/lamin-cli/lamin_cli/__main__.py +266 -0
- lamindb-0.60.0/sub/lamin-cli/lamin_cli/_notebook.py +226 -0
- lamindb-0.60.0/sub/lamin-cli/pyproject.toml +20 -0
- lamindb-0.60.0/sub/lamin-cli/tests/conftest.py +19 -0
- lamindb-0.60.0/sub/lamin-cli/tests/notebooks/no-title.ipynb +60 -0
- lamindb-0.60.0/sub/lamin-cli/tests/notebooks/not-initialized.ipynb +39 -0
- lamindb-0.60.0/sub/lamin-cli/tests/notebooks/with-title-and-initialized-consecutive.ipynb +208 -0
- lamindb-0.60.0/sub/lamin-cli/tests/notebooks/with-title-and-initialized-non-consecutive.ipynb +153 -0
- lamindb-0.60.0/sub/lamin-cli/tests/test_cli.py +34 -0
- lamindb-0.60.0/sub/lamin-cli/tests/test_notebooks.py +118 -0
- lamindb-0.60.0/sub/lamindb-setup/.github/workflows/build.yml +239 -0
- lamindb-0.60.0/sub/lamindb-setup/.github/workflows/latest-changes.jinja2 +2 -0
- lamindb-0.60.0/sub/lamindb-setup/.github/workflows/latest-changes.yml +25 -0
- lamindb-0.60.0/sub/lamindb-setup/.gitignore +110 -0
- lamindb-0.60.0/sub/lamindb-setup/.pre-commit-config.yaml +64 -0
- lamindb-0.60.0/sub/lamindb-setup/LICENSE +201 -0
- lamindb-0.60.0/sub/lamindb-setup/README.md +6 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/changelog.md +490 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/index.md +12 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/notebooks.md +19 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-only/test-cache-management.ipynb +305 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-only/test-empty-init.ipynb +129 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-only/test-import-schema.ipynb +133 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-only/test-insufficient-user-info.ipynb +288 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-only/test-load-lock.ipynb +273 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-only/test-sqlite-sync.ipynb +314 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-only/test_notebooks2.py +7 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-staging/01-init-instance.ipynb +372 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-staging/02-load-instance.ipynb +209 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-staging/03-set-storage.ipynb +271 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-staging/04-test-bionty.ipynb +167 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-staging/test-multi-session.ipynb +159 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/prod-staging/test_notebooks.py +7 -0
- lamindb-0.60.0/sub/lamindb-setup/docs/reference.md +5 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/__init__.py +81 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_cache.py +33 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_check_instance_setup.py +49 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_close.py +32 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_delete.py +75 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_django.py +38 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_docstrings.py +11 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_export.py +76 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_info.py +11 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_init_instance.py +274 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_init_vault.py +47 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_load_instance.py +300 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_migrate.py +64 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_register_instance.py +27 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_schema.py +24 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_set.py +66 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_settings.py +77 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_setup_user.py +100 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/_silence_loggers.py +32 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/__init__.py +26 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_deprecated.py +60 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_docs.py +12 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_hub_client.py +139 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_hub_core.py +348 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_hub_crud.py +193 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_hub_utils.py +180 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_settings_instance.py +283 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_settings_load.py +76 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_settings_save.py +75 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_settings_storage.py +175 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_settings_store.py +75 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_settings_user.py +46 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/_setup_bionty_sources.py +85 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/cloud_sqlite_locker.py +234 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/django.py +142 -0
- lamindb-0.60.0/sub/lamindb-setup/lamindb_setup/dev/upath.py +358 -0
- lamindb-0.60.0/sub/lamindb-setup/noxfile.py +110 -0
- lamindb-0.60.0/sub/lamindb-setup/pyproject.toml +52 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/hub-local/conftest.py +50 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/hub-local/test_all.py +205 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-only/conftest.py +20 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-only/test_django.py +13 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-only/test_switch_and_fallback_env.py +47 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-staging/test_delete_instance.py +12 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-staging/test_init_instance.py +186 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-staging/test_load_instance.py +125 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-staging/test_login.py +17 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-staging/test_migrate.py +21 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/prod-staging/test_set_storage.py +8 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/test_load_persistent_instance.py +25 -0
- lamindb-0.60.0/sub/lamindb-setup/tests/test_vault.py +240 -0
- lamindb-0.60.0/sub/lnschema-core/.github/workflows/build.yml +34 -0
- lamindb-0.60.0/sub/lnschema-core/.github/workflows/latest-changes.jinja2 +2 -0
- lamindb-0.60.0/sub/lnschema-core/.github/workflows/latest-changes.yml +25 -0
- lamindb-0.60.0/sub/lnschema-core/.gitignore +117 -0
- lamindb-0.60.0/sub/lnschema-core/.pre-commit-config.yaml +66 -0
- lamindb-0.60.0/sub/lnschema-core/CHANGELOG.md +277 -0
- lamindb-0.60.0/sub/lnschema-core/LICENSE +201 -0
- lamindb-0.60.0/sub/lnschema-core/README.md +5 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/__init__.py +28 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/ids.py +102 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0001_initial.py +220 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0001_initial_squashed_0023.py +446 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0002_alter_user_name.py +17 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0003_alter_storage_region_alter_transform_short_name.py +22 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0004_rename_folder_tag_alter_project_folders.py +62 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0005_alter_run_inputs_delete_runinput.py +24 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0006_feature_dataset.py +65 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0007_feature_synonyms_featureset_field_and_more.py +33 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0008_file_hash_type_transform_parents.py +22 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0009_remove_featureset_files_feature_unit_and_more.py +98 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0010_dataset_categories_file_categories.py +71 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0011_label_remove_tag_created_by_remove_tag_parents_and_more.py +125 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0012_remove_label_ref_id_remove_label_ref_orm_and_more.py +135 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0013_remove_feature_labels_orm_and_more.py +77 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0014_rename_ref_field_featureset_registry.py +17 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0015_file_initial_version_file_version.py +23 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0016_dataset_input_of_dataset_run_dataset_transform_and_more.py +91 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0017_dataset_initial_version_dataset_version.py +23 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0018_rename_datasetlabel_datasetulabel_and_more.py +65 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0019_dataset_reference_dataset_reference_type_and_more.py +37 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0020_run_report_transform_latest_report_and_more.py +38 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0021_dataset_storage_alter_dataset_file.py +23 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0022_migrate_to_integer_pks.py +297 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0023_export_legacy_data.py +49 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0024_import_legacy_data.py +80 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0025_remove_user_email.py +16 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0026_dataset_visibility_file_visibility.py +22 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/0027_file_key_is_virtual.py +18 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/migrations/__init__.py +0 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/mocks.py +10 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/models.py +2276 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/types.py +47 -0
- lamindb-0.60.0/sub/lnschema-core/lnschema_core/users.py +20 -0
- lamindb-0.60.0/sub/lnschema-core/noxfile.py +16 -0
- lamindb-0.60.0/sub/lnschema-core/pyproject.toml +45 -0
- lamindb-0.60.0/sub/lnschema-core/tests/test_integrity.py +13 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_dataset.py +1 -1
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_file.py +7 -3
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_label_manager.py +3 -1
- {lamindb-0.59.5 → lamindb-0.60.0}/.github/workflows/build.yml +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/.github/workflows/latest-changes.jinja2 +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/.github/workflows/latest-changes.yml +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/.gitignore +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/.gitmodules +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/.pre-commit-config.yaml +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/LICENSE +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/README.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/bio-registries.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/bionty.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/data.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/acid.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/delete.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/idempotency.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/import-schema.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/notebooks.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/reference-field.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/setup.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/test_notebooks.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/track-run-inputs.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/validator.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq/visibility.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/faq.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/features-lamindb.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/features-laminhub.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/guide.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/index.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/installation.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/introduction.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/lamin-utils.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/lamindb.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/lnschema-bionty.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/meta.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/nbproject.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/query-search.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/readfcs.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/reference.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/schemas.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/setup.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/signup-login.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage/add-replace-stage.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage/anndata-accessor.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage/test-files/iris.csv +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage/test-files/iris.data +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage/test-files/new_iris.csv +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage/test_notebooks.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage/upload.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/storage.md +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/test_notebooks.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/transfer.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/tutorial2.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/docs/validate.ipynb +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_delete.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_feature.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_feature_set.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_filter.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_from_values.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_parents.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_query_manager.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_query_set.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_registry.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_run.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_save.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_storage.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_transform.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_ulabel.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_utils.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/_validate.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/__init__.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/_feature_manager.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/_priors.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/_run_context.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/_settings.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/datasets/__init__.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/datasets/_core.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/datasets/_fake.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/exceptions.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/fields.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/hashing.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/storage/_anndata_sizes.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/storage/_backed_access.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/storage/_zarr.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/storage/object.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/types.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/dev/versioning.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/setup/__init__.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/lamindb/setup/dev/__init__.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/noxfile.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/conftest.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_cache.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_data.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_db.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_delete.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_feature.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_feature_manager.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_feature_set.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_from_values.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_hashing.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_manager.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_parents.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_queryset.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_registry.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_run.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_run_context.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_save.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_settings.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_storage.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_transfer.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_transform.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_ulabel.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_validate.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_versioning.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_view.py +0 -0
- {lamindb-0.59.5 → lamindb-0.60.0}/tests/test_visibility.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: lamindb
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.60.0
|
4
4
|
Summary: A data framework for biology.
|
5
5
|
Author-email: Lamin Labs <open-source@lamin.ai>
|
6
6
|
Requires-Python: >=3.8
|
@@ -8,8 +8,8 @@ Description-Content-Type: text/markdown
|
|
8
8
|
Classifier: Programming Language :: Python :: 3.8
|
9
9
|
Classifier: Programming Language :: Python :: 3.9
|
10
10
|
Classifier: Programming Language :: Python :: 3.10
|
11
|
-
Requires-Dist: lnschema_core==0.54.
|
12
|
-
Requires-Dist: lamindb_setup==0.
|
11
|
+
Requires-Dist: lnschema_core==0.54.1
|
12
|
+
Requires-Dist: lamindb_setup==0.58.0
|
13
13
|
Requires-Dist: lamin_utils==0.11.7
|
14
14
|
Requires-Dist: lamin_cli==0.1.2
|
15
15
|
Requires-Dist: rapidfuzz
|
@@ -27,7 +27,7 @@ Requires-Dist: boto3==1.28.17 ; extra == "aws"
|
|
27
27
|
Requires-Dist: aiobotocore==2.5.4 ; extra == "aws"
|
28
28
|
Requires-Dist: fsspec[s3]==2023.9.0 ; extra == "aws"
|
29
29
|
Requires-Dist: s3fs>=2023.1.0 ; extra == "aws"
|
30
|
-
Requires-Dist: lnschema_bionty==0.34.
|
30
|
+
Requires-Dist: lnschema_bionty==0.34.2 ; extra == "bionty"
|
31
31
|
Requires-Dist: pandas<2 ; extra == "dev"
|
32
32
|
Requires-Dist: pre-commit ; extra == "dev"
|
33
33
|
Requires-Dist: nox ; extra == "dev"
|
@@ -1,18 +1,20 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
## 0.59
|
4
|
-
|
5
|
-
### 0.59.5 {small}`2023-11-09`
|
6
3
|
|
7
|
-
- ⬆️ Upgrade lamindb-setup [PR1284](https://github.com/laminlabs/lamindb/pull/1284) [@fredericenard](https://github.com/fredericenard)
|
8
|
-
- ♻️ Perfect pre-population of registries for conde22 [PR1283](https://github.com/laminlabs/lamindb/pull/1283) [@falexwolf](https://github.com/falexwolf)
|
9
4
|
```{eval-rst}
|
10
5
|
.. role:: small
|
11
6
|
```
|
12
7
|
|
13
|
-
|
8
|
+
## 0.60
|
9
|
+
|
10
|
+
### 0.60.0 {small}`2023-11-09`
|
14
11
|
|
15
|
-
-
|
12
|
+
- 🚸 Enable to pass metadata to dataset [PR1289](https://github.com/laminlabs/lamindb/pull/1289) [@falexwolf](https://github.com/falexwolf)
|
13
|
+
- 🚚 Move `view_tree` to `UPath` [PR1288](https://github.com/laminlabs/lamindb/pull/1288) [@falexwolf](https://github.com/falexwolf)
|
14
|
+
- ♻️ Move `UPath` to root-level API [PR1286](https://github.com/laminlabs/lamindb/pull/1286) [@falexwolf](https://github.com/falexwolf)
|
15
|
+
- 🚸 Enable to add labels without a feature [PR1285](https://github.com/laminlabs/lamindb/pull/1285) [@sunnyosun](https://github.com/sunnyosun)
|
16
|
+
|
17
|
+
## 0.59
|
16
18
|
|
17
19
|
### 0.59.3 {small}`2023-11-04`
|
18
20
|
|
@@ -1,11 +1,6 @@
|
|
1
1
|
"""A data framework for biology.
|
2
2
|
|
3
|
-
|
4
|
-
HDF5, TileDB or DuckDB.
|
5
|
-
|
6
|
-
LaminDB helps you manage these data with registries for metadata.
|
7
|
-
|
8
|
-
The two most important are:
|
3
|
+
LaminDB helps you manage data with registries for metadata:
|
9
4
|
|
10
5
|
.. autosummary::
|
11
6
|
:toctree: .
|
@@ -13,7 +8,7 @@ The two most important are:
|
|
13
8
|
File
|
14
9
|
Dataset
|
15
10
|
|
16
|
-
|
11
|
+
Registries to track provenance:
|
17
12
|
|
18
13
|
.. autosummary::
|
19
14
|
:toctree: .
|
@@ -23,7 +18,7 @@ Four registries track provenance of data batches:
|
|
23
18
|
User
|
24
19
|
Storage
|
25
20
|
|
26
|
-
|
21
|
+
Registries to validate & contextualize:
|
27
22
|
|
28
23
|
.. autosummary::
|
29
24
|
:toctree: .
|
@@ -33,7 +28,14 @@ Four registries validate & contextualize data batches:
|
|
33
28
|
FeatureSet
|
34
29
|
Modality
|
35
30
|
|
36
|
-
|
31
|
+
You can also access data directly via paths:
|
32
|
+
|
33
|
+
.. autosummary::
|
34
|
+
:toctree: .
|
35
|
+
|
36
|
+
UPath
|
37
|
+
|
38
|
+
Functions:
|
37
39
|
|
38
40
|
.. autosummary::
|
39
41
|
:toctree: .
|
@@ -42,7 +44,7 @@ Functional tools:
|
|
42
44
|
view
|
43
45
|
save
|
44
46
|
|
45
|
-
|
47
|
+
Modules & settings:
|
46
48
|
|
47
49
|
.. autosummary::
|
48
50
|
:toctree: .
|
@@ -53,7 +55,7 @@ Static classes & modules:
|
|
53
55
|
|
54
56
|
"""
|
55
57
|
|
56
|
-
__version__ = "0.
|
58
|
+
__version__ = "0.60.0" # denote a release candidate for 0.1.0 with 0.1rc1
|
57
59
|
|
58
60
|
import os as _os
|
59
61
|
|
@@ -64,6 +66,7 @@ from lamin_utils import py_version_warning as _py_version_warning
|
|
64
66
|
from lamindb_setup import _check_instance_setup
|
65
67
|
from lamindb_setup._check_instance_setup import _INSTANCE_NOT_SETUP_WARNING
|
66
68
|
from lamindb_setup._init_instance import reload_schema_modules as _reload_schema_modules
|
69
|
+
from lamindb_setup.dev.upath import UPath
|
67
70
|
|
68
71
|
_py_version_warning("3.8", "3.11")
|
69
72
|
|
@@ -45,6 +45,7 @@ def __init__(
|
|
45
45
|
data: Union[pd.DataFrame, ad.AnnData, File, Iterable[File]] = (
|
46
46
|
kwargs.pop("data") if len(args) == 0 else args[0]
|
47
47
|
)
|
48
|
+
meta: Optional[str] = kwargs.pop("meta") if "meta" in kwargs else None
|
48
49
|
name: Optional[str] = kwargs.pop("name") if "name" in kwargs else None
|
49
50
|
description: Optional[str] = (
|
50
51
|
kwargs.pop("description") if "description" in kwargs else None
|
@@ -92,9 +93,26 @@ def __init__(
|
|
92
93
|
)
|
93
94
|
|
94
95
|
run = get_run(run)
|
96
|
+
data_init_complete = False
|
95
97
|
# there are exactly 3 ways of creating a Dataset object right now
|
96
98
|
# using exactly one file or using more than one file
|
97
99
|
# init file
|
100
|
+
# init from directory or bucket
|
101
|
+
if isinstance(data, (str, Path, UPath)):
|
102
|
+
file = None
|
103
|
+
files = None
|
104
|
+
upath = UPath(data)
|
105
|
+
if not upath.is_dir:
|
106
|
+
raise ValueError(f"Can only pass buckets or directories, not {data}")
|
107
|
+
upath_str = upath.as_posix().rstrip("/")
|
108
|
+
region = get_storage_region(upath_str)
|
109
|
+
storage_settings = StorageSettings(upath_str, region)
|
110
|
+
storage = register_storage(storage_settings)
|
111
|
+
hash = None
|
112
|
+
data_init_complete = True
|
113
|
+
# now handle the metadata
|
114
|
+
if isinstance(meta, (pd.DataFrame, ad.AnnData, File)):
|
115
|
+
data = meta
|
98
116
|
if isinstance(data, (pd.DataFrame, ad.AnnData, File)):
|
99
117
|
files = None
|
100
118
|
storage = None
|
@@ -126,28 +144,19 @@ def __init__(
|
|
126
144
|
file.description = f"See dataset {provisional_uid}" # type: ignore
|
127
145
|
file._feature_sets = feature_sets
|
128
146
|
storage = None
|
129
|
-
|
130
|
-
|
131
|
-
file = None
|
132
|
-
files = None
|
133
|
-
upath = UPath(data)
|
134
|
-
if not upath.is_dir:
|
135
|
-
raise ValueError(f"Can only pass buckets or directories, not {data}")
|
136
|
-
upath_str = upath.as_posix().rstrip("/")
|
137
|
-
region = get_storage_region(upath_str)
|
138
|
-
storage_settings = StorageSettings(upath_str, region)
|
139
|
-
storage = register_storage(storage_settings)
|
140
|
-
hash = None
|
141
|
-
# init files
|
142
|
-
else:
|
147
|
+
data_init_complete = True
|
148
|
+
if not data_init_complete:
|
143
149
|
file = None
|
144
150
|
storage = None
|
145
151
|
if hasattr(data, "__getitem__"):
|
146
152
|
assert isinstance(data[0], File) # type: ignore
|
147
153
|
files = data
|
148
154
|
hash, feature_sets = from_files(files) # type: ignore
|
155
|
+
data_init_complete = True
|
149
156
|
else:
|
150
|
-
raise ValueError(
|
157
|
+
raise ValueError(
|
158
|
+
"Only DataFrame, AnnData, folder or list of File is allowed."
|
159
|
+
)
|
151
160
|
# we ignore datasets in trash containing the same hash
|
152
161
|
existing_dataset = Dataset.filter(hash=hash).one_or_none()
|
153
162
|
if existing_dataset is not None:
|
@@ -1,7 +1,5 @@
|
|
1
|
-
from collections import defaultdict
|
2
|
-
from itertools import islice
|
3
1
|
from pathlib import Path, PurePath, PurePosixPath
|
4
|
-
from typing import Any, List, Optional,
|
2
|
+
from typing import Any, List, Optional, Tuple, Union
|
5
3
|
|
6
4
|
import anndata as ad
|
7
5
|
import fsspec
|
@@ -14,7 +12,7 @@ from lamindb_setup._init_instance import register_storage
|
|
14
12
|
from lamindb_setup.dev import StorageSettings
|
15
13
|
from lamindb_setup.dev._docs import doc_args
|
16
14
|
from lamindb_setup.dev._hub_utils import get_storage_region
|
17
|
-
from lamindb_setup.dev.upath import create_path
|
15
|
+
from lamindb_setup.dev.upath import create_path, extract_suffix_from_path
|
18
16
|
from lnschema_core import Feature, FeatureSet, File, Modality, Run, Storage
|
19
17
|
from lnschema_core.types import AnnDataLike, DataLike, FieldAttr, PathLike
|
20
18
|
|
@@ -36,7 +34,6 @@ from lamindb.dev.storage.file import (
|
|
36
34
|
ProgressCallback,
|
37
35
|
auto_storage_key_from_file,
|
38
36
|
auto_storage_key_from_id_suffix,
|
39
|
-
extract_suffix_from_path,
|
40
37
|
filepath_from_file,
|
41
38
|
)
|
42
39
|
from lamindb.dev.versioning import get_ids_from_old_version, init_uid
|
@@ -928,7 +925,6 @@ def path(self) -> Union[Path, UPath]:
|
|
928
925
|
return filepath_from_file(self)
|
929
926
|
|
930
927
|
|
931
|
-
# adapted from: https://stackoverflow.com/questions/9727673
|
932
928
|
@classmethod # type: ignore
|
933
929
|
@doc_args(File.view_tree.__doc__)
|
934
930
|
def view_tree(
|
@@ -940,82 +936,18 @@ def view_tree(
|
|
940
936
|
length_limit: int = 1000,
|
941
937
|
) -> None:
|
942
938
|
"""{}"""
|
943
|
-
|
944
|
-
|
945
|
-
tee = "├── "
|
946
|
-
last = "└── "
|
947
|
-
max_files_per_dir_per_type = 7
|
948
|
-
|
949
|
-
if path is None:
|
950
|
-
dir_path = settings.storage
|
951
|
-
else:
|
952
|
-
dir_path = create_path(path) # returns Path for local
|
953
|
-
n_files = 0
|
954
|
-
n_directories = 0
|
955
|
-
|
956
|
-
# by default only including registered files
|
957
|
-
# need a flag and a proper implementation
|
958
|
-
registered_paths: Set[Any] = set()
|
959
|
-
registered_dirs: Set[Any] = set()
|
939
|
+
logger.warning("Deprecated: Please use UPath.view_tree()")
|
940
|
+
include_paths = None
|
960
941
|
if path is None:
|
961
|
-
|
942
|
+
path = settings.storage
|
943
|
+
include_paths = {
|
962
944
|
file.path for file in cls.filter(storage_id=setup_settings.storage.id).all()
|
963
945
|
}
|
964
|
-
|
965
|
-
|
966
|
-
|
967
|
-
|
968
|
-
|
969
|
-
if not level:
|
970
|
-
return # 0, stop iterating
|
971
|
-
stripped_dir_path = dir_path.as_posix().rstrip("/")
|
972
|
-
# do not iterate through zarr directories
|
973
|
-
if stripped_dir_path.endswith((".zarr", ".zrad")):
|
974
|
-
return
|
975
|
-
# this is needed so that the passed folder is not listed
|
976
|
-
contents = [
|
977
|
-
i
|
978
|
-
for i in dir_path.iterdir()
|
979
|
-
if i.as_posix().rstrip("/") != stripped_dir_path
|
980
|
-
]
|
981
|
-
if limit_to_directories:
|
982
|
-
contents = [d for d in contents if d.is_dir()]
|
983
|
-
pointers = [tee] * (len(contents) - 1) + [last]
|
984
|
-
n_files_per_dir_per_type = defaultdict(lambda: 0) # type: ignore
|
985
|
-
for pointer, path in zip(pointers, contents):
|
986
|
-
if path.is_dir():
|
987
|
-
if registered_dirs and path not in registered_dirs:
|
988
|
-
continue
|
989
|
-
yield prefix + pointer + path.name
|
990
|
-
n_directories += 1
|
991
|
-
n_files_per_dir_per_type = defaultdict(lambda: 0)
|
992
|
-
extension = branch if pointer == tee else space
|
993
|
-
yield from inner(path, prefix=prefix + extension, level=level - 1)
|
994
|
-
elif not limit_to_directories:
|
995
|
-
if registered_paths and path not in registered_paths:
|
996
|
-
continue
|
997
|
-
suffix = extract_suffix_from_path(path)
|
998
|
-
suffixes.add(suffix)
|
999
|
-
n_files_per_dir_per_type[suffix] += 1
|
1000
|
-
n_files += 1
|
1001
|
-
if n_files_per_dir_per_type[suffix] == max_files_per_dir_per_type:
|
1002
|
-
yield prefix + "..."
|
1003
|
-
elif n_files_per_dir_per_type[suffix] > max_files_per_dir_per_type:
|
1004
|
-
continue
|
1005
|
-
else:
|
1006
|
-
yield prefix + pointer + path.name
|
1007
|
-
|
1008
|
-
folder_tree = ""
|
1009
|
-
iterator = inner(dir_path, level=level)
|
1010
|
-
for line in islice(iterator, length_limit):
|
1011
|
-
folder_tree += f"\n{line}"
|
1012
|
-
if next(iterator, None):
|
1013
|
-
folder_tree += f"... length_limit, {length_limit}, reached, counted:"
|
1014
|
-
directory_info = "directory" if n_directories == 1 else "directories"
|
1015
|
-
display_suffixes = ", ".join([f"{suffix!r}" for suffix in suffixes])
|
1016
|
-
logger.print(
|
1017
|
-
f"{dir_path.name} ({n_directories} sub-{directory_info} & {n_files} files with"
|
1018
|
-
f" suffixes {display_suffixes}): {folder_tree}"
|
946
|
+
UPath(path).view_tree(
|
947
|
+
level=level,
|
948
|
+
limit_to_directories=limit_to_directories,
|
949
|
+
length_limit=length_limit,
|
950
|
+
include_paths=include_paths,
|
1019
951
|
)
|
1020
952
|
|
1021
953
|
|
@@ -11,15 +11,15 @@ from lnschema_core import Registry
|
|
11
11
|
|
12
12
|
def view(
|
13
13
|
n: int = 7, schema: Optional[str] = None, registries: Optional[List[str]] = None
|
14
|
-
):
|
15
|
-
"""View
|
14
|
+
) -> None:
|
15
|
+
"""View latest metadata state.
|
16
16
|
|
17
17
|
Args:
|
18
|
-
n: Display the last `n` rows of a
|
18
|
+
n: Display the last `n` rows of a registry.
|
19
19
|
schema: Schema module to view. Default's to
|
20
20
|
`None` and displays all schema modules.
|
21
21
|
registries: List of Registry names. Defaults to
|
22
|
-
`None` and lists all
|
22
|
+
`None` and lists all registries.
|
23
23
|
|
24
24
|
Examples:
|
25
25
|
>>> ln.view()
|
@@ -19,7 +19,10 @@ from lnschema_core.types import StrField
|
|
19
19
|
|
20
20
|
from lamindb.dev._settings import settings
|
21
21
|
|
22
|
-
from .._feature_set import
|
22
|
+
from .._feature_set import (
|
23
|
+
dict_related_model_to_related_name,
|
24
|
+
dict_schema_name_to_model_name,
|
25
|
+
)
|
23
26
|
from .._parents import view_flow
|
24
27
|
from .._query_set import QuerySet
|
25
28
|
from ._feature_manager import (
|
@@ -202,19 +205,29 @@ def get_labels(
|
|
202
205
|
def add_labels(
|
203
206
|
self,
|
204
207
|
records: Union[Registry, List[Registry], QuerySet, Iterable],
|
205
|
-
feature: Feature,
|
208
|
+
feature: Optional[Feature] = None,
|
206
209
|
*,
|
207
210
|
field: Optional[StrField] = None,
|
208
211
|
) -> None:
|
209
212
|
"""{}"""
|
213
|
+
if self._state.adding:
|
214
|
+
raise ValueError("Please save the file/dataset before adding a label!")
|
215
|
+
|
210
216
|
if isinstance(records, (QuerySet, QuerySet.__base__)): # need to have both
|
211
217
|
records = records.list()
|
212
218
|
if isinstance(records, (str, Registry)):
|
213
219
|
records = [records]
|
214
220
|
if not isinstance(records, List): # avoids warning for pd Series
|
215
221
|
records = list(records)
|
222
|
+
# create records from values
|
216
223
|
if isinstance(records[0], str): # type: ignore
|
217
224
|
records_validated = []
|
225
|
+
# feature is needed if we want to create records from values
|
226
|
+
if feature is None:
|
227
|
+
raise ValueError(
|
228
|
+
"Please pass a feature, e.g., via: label = ln.ULabel(name='my_label',"
|
229
|
+
" feature=ln.Feature(name='my_feature'))"
|
230
|
+
)
|
218
231
|
if feature.registries is not None:
|
219
232
|
orm_dict = dict_schema_name_to_model_name(File)
|
220
233
|
for reg in feature.registries.split("|"):
|
@@ -231,80 +244,98 @@ def add_labels(
|
|
231
244
|
)
|
232
245
|
records = records_validated
|
233
246
|
|
234
|
-
if self._state.adding:
|
235
|
-
raise ValueError("Please save the file/dataset before adding a label!")
|
236
247
|
for record in records:
|
237
248
|
if record._state.adding:
|
238
249
|
raise ValidationError(
|
239
250
|
f"{record} not validated. If it looks correct: record.save()"
|
240
251
|
)
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
252
|
+
|
253
|
+
if feature is None:
|
254
|
+
d = dict_related_model_to_related_name(self.__class__)
|
255
|
+
# strategy: group records by registry to reduce number of transactions
|
256
|
+
records_by_related_name: Dict = {}
|
257
|
+
for record in records:
|
258
|
+
related_name = d.get(record.__class__.__get_name_with_schema__())
|
259
|
+
if related_name is None:
|
260
|
+
raise ValueError(f"Can't add labels to {record.__class__} record!")
|
261
|
+
if related_name not in records_by_related_name:
|
262
|
+
records_by_related_name[related_name] = []
|
263
|
+
records_by_related_name[related_name].append(record)
|
264
|
+
for related_name, records in records_by_related_name.items():
|
265
|
+
print(related_name, records)
|
266
|
+
getattr(self, related_name).add(*records)
|
267
|
+
else:
|
268
|
+
validate_feature(feature, records) # type:ignore
|
269
|
+
records_by_registry = defaultdict(list)
|
270
|
+
for record in records:
|
271
|
+
records_by_registry[record.__class__.__get_name_with_schema__()].append(
|
272
|
+
record
|
273
|
+
)
|
274
|
+
for registry_name, records in records_by_registry.items():
|
275
|
+
getattr(self, self.features._accessor_by_orm[registry_name]).add(
|
276
|
+
*records, through_defaults={"feature_id": feature.id}
|
277
|
+
)
|
278
|
+
feature_set_links = get_feature_set_links(self)
|
279
|
+
feature_set_ids = [link.feature_set_id for link in feature_set_links.all()]
|
280
|
+
# get all linked features of type Feature
|
281
|
+
feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
|
282
|
+
linked_features_by_slot = {
|
283
|
+
feature_set_links.filter(feature_set_id=feature_set.id)
|
284
|
+
.one()
|
285
|
+
.slot: feature_set.features.all()
|
286
|
+
for feature_set in feature_sets
|
287
|
+
if "core.Feature" == feature_set.registry
|
288
|
+
}
|
289
|
+
for registry_name, records in records_by_registry.items():
|
290
|
+
msg = ""
|
291
|
+
if feature.registries is None or registry_name not in feature.registries:
|
292
|
+
if len(msg) > 0:
|
293
|
+
msg += ", "
|
294
|
+
msg += f"linked feature '{feature.name}' to registry '{registry_name}'"
|
295
|
+
if feature.registries is None:
|
296
|
+
feature.registries = registry_name
|
297
|
+
elif registry_name not in feature.registries:
|
298
|
+
feature.registries += f"|{registry_name}"
|
299
|
+
feature.save()
|
263
300
|
if len(msg) > 0:
|
264
|
-
msg
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
feature
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
else:
|
284
|
-
features_list = []
|
285
|
-
features_list.append(feature)
|
286
|
-
feature_set = FeatureSet(features_list, modality=priors.modalities.meta)
|
287
|
-
feature_set.save()
|
288
|
-
if "external" in linked_features_by_slot:
|
289
|
-
old_feature_set_link = feature_set_links.filter(slot="external").one()
|
290
|
-
old_feature_set_link.delete()
|
291
|
-
remaining_links = self.feature_sets.through.objects.filter(
|
292
|
-
feature_set_id=feature_set.id
|
293
|
-
).all()
|
294
|
-
if len(remaining_links) == 0:
|
295
|
-
old_feature_set = FeatureSet.filter(
|
296
|
-
id=old_feature_set_link.feature_set_id
|
301
|
+
logger.save(msg)
|
302
|
+
# check whether we have to update the feature set that manages labels
|
303
|
+
# (Feature) to account for a new feature
|
304
|
+
found_feature = False
|
305
|
+
for _, linked_features in linked_features_by_slot.items():
|
306
|
+
if feature in linked_features:
|
307
|
+
found_feature = True
|
308
|
+
if not found_feature:
|
309
|
+
if "external" in linked_features_by_slot:
|
310
|
+
feature_set = self.features._feature_set_by_slot["external"]
|
311
|
+
features_list = feature_set.features.list()
|
312
|
+
else:
|
313
|
+
features_list = []
|
314
|
+
features_list.append(feature)
|
315
|
+
feature_set = FeatureSet(features_list, modality=priors.modalities.meta)
|
316
|
+
feature_set.save()
|
317
|
+
if "external" in linked_features_by_slot:
|
318
|
+
old_feature_set_link = feature_set_links.filter(
|
319
|
+
slot="external"
|
297
320
|
).one()
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
)
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
321
|
+
old_feature_set_link.delete()
|
322
|
+
remaining_links = self.feature_sets.through.objects.filter(
|
323
|
+
feature_set_id=feature_set.id
|
324
|
+
).all()
|
325
|
+
if len(remaining_links) == 0:
|
326
|
+
old_feature_set = FeatureSet.filter(
|
327
|
+
id=old_feature_set_link.feature_set_id
|
328
|
+
).one()
|
329
|
+
logger.info(
|
330
|
+
"no file links to it anymore, deleting feature set"
|
331
|
+
f" {old_feature_set}"
|
332
|
+
)
|
333
|
+
old_feature_set.delete()
|
334
|
+
self.features.add_feature_set(feature_set, slot="external")
|
335
|
+
logger.save(
|
336
|
+
f"linked new feature '{feature.name}' together with new feature set"
|
337
|
+
f" {feature_set}"
|
338
|
+
)
|
308
339
|
|
309
340
|
|
310
341
|
def _track_run_input(
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from typing import Dict, List, Union
|
1
|
+
from typing import Dict, List, Optional, Union
|
2
2
|
|
3
3
|
import numpy as np
|
4
4
|
from lamin_utils import colors, logger
|
@@ -116,7 +116,7 @@ class LabelManager:
|
|
116
116
|
def add(
|
117
117
|
self,
|
118
118
|
records: Union[Registry, List[Registry], QuerySet],
|
119
|
-
feature: Feature,
|
119
|
+
feature: Optional[Feature] = None,
|
120
120
|
) -> None:
|
121
121
|
"""Add one or several labels and associate them with a feature.
|
122
122
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import os
|
2
2
|
import shutil
|
3
3
|
from pathlib import Path
|
4
|
-
from typing import Literal,
|
4
|
+
from typing import Literal, Union
|
5
5
|
|
6
6
|
import anndata as ad
|
7
7
|
import fsspec
|
@@ -28,66 +28,6 @@ except ImportError:
|
|
28
28
|
AUTO_KEY_PREFIX = ".lamindb/"
|
29
29
|
|
30
30
|
|
31
|
-
# also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
|
32
|
-
# ".gz" is not listed here as it typically occurs with another suffix
|
33
|
-
KNOWN_SUFFIXES = {
|
34
|
-
#
|
35
|
-
# without readers
|
36
|
-
#
|
37
|
-
".fasta",
|
38
|
-
".fastq",
|
39
|
-
".jpg",
|
40
|
-
".mtx",
|
41
|
-
".obo",
|
42
|
-
".pdf",
|
43
|
-
".png",
|
44
|
-
".tar",
|
45
|
-
".tiff",
|
46
|
-
".txt",
|
47
|
-
".tsv",
|
48
|
-
".zip",
|
49
|
-
".xml",
|
50
|
-
#
|
51
|
-
# with readers (see below)
|
52
|
-
#
|
53
|
-
".h5ad",
|
54
|
-
".parquet",
|
55
|
-
".csv",
|
56
|
-
".fcs",
|
57
|
-
".xslx",
|
58
|
-
".zarr",
|
59
|
-
".zrad",
|
60
|
-
}
|
61
|
-
|
62
|
-
|
63
|
-
def extract_suffix_from_path(
|
64
|
-
path: Union[UPath, Path], arg_name: Optional[str] = None
|
65
|
-
) -> str:
|
66
|
-
if len(path.suffixes) <= 1:
|
67
|
-
return path.suffix
|
68
|
-
else:
|
69
|
-
arg_name = "file" if arg_name is None else arg_name # for the warning
|
70
|
-
msg = f"{arg_name} has more than one suffix (path.suffixes), "
|
71
|
-
# first check the 2nd-to-last suffix because it might be followed by .gz
|
72
|
-
# or another compression-related suffix
|
73
|
-
# Alex thought about adding logic along the lines of path.suffixes[-1]
|
74
|
-
# in COMPRESSION_SUFFIXES to detect something like .random.gz and then
|
75
|
-
# add ".random.gz" but concluded it's too dangerous it's safer to just
|
76
|
-
# use ".gz" in such a case
|
77
|
-
if path.suffixes[-2] in KNOWN_SUFFIXES:
|
78
|
-
suffix = "".join(path.suffixes[-2:])
|
79
|
-
msg += f"inferring: '{suffix}'"
|
80
|
-
else:
|
81
|
-
suffix = path.suffixes[-1] # this is equivalent to path.suffix!!!
|
82
|
-
msg += (
|
83
|
-
f"using only last suffix: '{suffix}' - if you want your file format to"
|
84
|
-
" be recognized, make an issue:"
|
85
|
-
" https://github.com/laminlabs/lamindb/issues/new"
|
86
|
-
)
|
87
|
-
logger.warning(msg)
|
88
|
-
return suffix
|
89
|
-
|
90
|
-
|
91
31
|
# add type annotations back asap when re-organizing the module
|
92
32
|
def auto_storage_key_from_file(file: File):
|
93
33
|
if file.key is None or file.key_is_virtual:
|