lamindb 0.66.1__py3-none-any.whl → 0.67.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +15 -15
- lamindb/_artifact.py +4 -3
- lamindb/_collection.py +5 -1
- lamindb/_from_values.py +17 -17
- lamindb/_query_manager.py +3 -6
- lamindb/_query_set.py +17 -1
- lamindb/_registry.py +3 -3
- lamindb/_run.py +23 -0
- lamindb/_transform.py +26 -1
- lamindb/_validate.py +6 -6
- lamindb/dev/_label_manager.py +1 -1
- lamindb/dev/_run_context.py +2 -2
- lamindb/dev/datasets/_core.py +9 -9
- lamindb/dev/storage/file.py +33 -1
- {lamindb-0.66.1.dist-info → lamindb-0.67.1.dist-info}/METADATA +6 -6
- {lamindb-0.66.1.dist-info → lamindb-0.67.1.dist-info}/RECORD +18 -19
- lamindb/_delete.py +0 -65
- {lamindb-0.66.1.dist-info → lamindb-0.67.1.dist-info}/LICENSE +0 -0
- {lamindb-0.66.1.dist-info → lamindb-0.67.1.dist-info}/WHEEL +0 -0
lamindb/__init__.py
CHANGED
@@ -54,7 +54,7 @@ Modules & settings:
|
|
54
54
|
|
55
55
|
"""
|
56
56
|
|
57
|
-
__version__ = "0.
|
57
|
+
__version__ = "0.67.1" # denote a release candidate for 0.1.0 with 0.1rc1
|
58
58
|
|
59
59
|
import os as _os
|
60
60
|
|
@@ -102,20 +102,20 @@ if _INSTANCE_SETUP:
|
|
102
102
|
User,
|
103
103
|
)
|
104
104
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
105
|
+
from . import (
|
106
|
+
_artifact,
|
107
|
+
_collection,
|
108
|
+
_feature,
|
109
|
+
_feature_set,
|
110
|
+
_parents,
|
111
|
+
_registry,
|
112
|
+
_run,
|
113
|
+
_storage,
|
114
|
+
_transform,
|
115
|
+
_ulabel,
|
116
|
+
_validate,
|
117
|
+
dev,
|
118
|
+
)
|
119
119
|
from ._save import save
|
120
120
|
from ._view import view
|
121
121
|
from .dev._run_context import run_context
|
lamindb/_artifact.py
CHANGED
@@ -911,13 +911,13 @@ def delete(
|
|
911
911
|
self, permanent: Optional[bool] = None, storage: Optional[bool] = None
|
912
912
|
) -> None:
|
913
913
|
# by default, we only move artifacts into the trash
|
914
|
-
if self.visibility >
|
914
|
+
if self.visibility > -1 and permanent is not True:
|
915
915
|
if storage is not None:
|
916
916
|
logger.warning("moving artifact to trash, storage arg is ignored")
|
917
917
|
# move to trash
|
918
|
-
self.visibility =
|
918
|
+
self.visibility = -1
|
919
919
|
self.save()
|
920
|
-
logger.warning("moved artifact to trash")
|
920
|
+
logger.warning("moved artifact to trash (visibility = -1)")
|
921
921
|
return
|
922
922
|
|
923
923
|
# if the artifact is already in the trash
|
@@ -929,6 +929,7 @@ def delete(
|
|
929
929
|
)
|
930
930
|
delete_record = response == "y"
|
931
931
|
else:
|
932
|
+
# this second option doesn't feel very intuitive
|
932
933
|
delete_record = permanent
|
933
934
|
|
934
935
|
if delete_record:
|
lamindb/_collection.py
CHANGED
@@ -424,8 +424,12 @@ def save(self, *args, **kwargs) -> None:
|
|
424
424
|
self.artifact.save()
|
425
425
|
# we don't need to save feature sets again
|
426
426
|
save_feature_sets(self)
|
427
|
+
state_was_adding = self._state.adding
|
427
428
|
super(Collection, self).save()
|
428
|
-
|
429
|
+
# we don't allow updating the collection of artifacts
|
430
|
+
# if users want to update the set of artifacts, they
|
431
|
+
# have to create a new collection
|
432
|
+
if state_was_adding and hasattr(self, "_artifacts"):
|
429
433
|
if self._artifacts is not None and len(self._artifacts) > 0:
|
430
434
|
links = [
|
431
435
|
CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
|
lamindb/_from_values.py
CHANGED
@@ -14,7 +14,7 @@ def get_or_create_records(
|
|
14
14
|
iterable: ListLike,
|
15
15
|
field: StrField,
|
16
16
|
*,
|
17
|
-
|
17
|
+
from_public: bool = False,
|
18
18
|
**kwargs,
|
19
19
|
) -> List[Registry]:
|
20
20
|
"""Get or create records from iterables."""
|
@@ -32,14 +32,14 @@ def get_or_create_records(
|
|
32
32
|
|
33
33
|
# new records to be created based on new values
|
34
34
|
if len(nonexist_values) > 0:
|
35
|
-
if
|
36
|
-
records_bionty, unmapped_values =
|
35
|
+
if from_public:
|
36
|
+
records_bionty, unmapped_values = create_records_from_public(
|
37
37
|
iterable_idx=nonexist_values, field=field, msg=msg, **kwargs
|
38
38
|
)
|
39
39
|
if len(records_bionty) > 0:
|
40
40
|
msg = ""
|
41
41
|
for record in records_bionty:
|
42
|
-
record.
|
42
|
+
record._from_public = True
|
43
43
|
records += records_bionty
|
44
44
|
else:
|
45
45
|
unmapped_values = nonexist_values
|
@@ -81,8 +81,8 @@ def get_existing_records(
|
|
81
81
|
model = field.field.model
|
82
82
|
condition: Dict = {} if len(kwargs) == 0 else kwargs.copy()
|
83
83
|
# existing records matching is agnostic to the bionty source
|
84
|
-
if "
|
85
|
-
condition.pop("
|
84
|
+
if "public_source" in condition:
|
85
|
+
condition.pop("public_source")
|
86
86
|
|
87
87
|
if _has_organism_field(model):
|
88
88
|
from lnschema_bionty._bionty import create_or_get_organism_record
|
@@ -160,7 +160,7 @@ def get_existing_records(
|
|
160
160
|
return records, nonexist_values, msg
|
161
161
|
|
162
162
|
|
163
|
-
def
|
163
|
+
def create_records_from_public(
|
164
164
|
iterable_idx: pd.Index,
|
165
165
|
field: StrField,
|
166
166
|
msg: str = "",
|
@@ -169,24 +169,24 @@ def create_records_from_bionty(
|
|
169
169
|
model = field.field.model
|
170
170
|
records: List = []
|
171
171
|
# populate additional fields from bionty
|
172
|
-
from lnschema_bionty._bionty import
|
172
|
+
from lnschema_bionty._bionty import get_public_source_record
|
173
173
|
|
174
174
|
# create the corresponding bionty object from model
|
175
175
|
try:
|
176
|
-
|
177
|
-
organism=kwargs.get("organism"),
|
176
|
+
public_ontology = model.public(
|
177
|
+
organism=kwargs.get("organism"), public_source=kwargs.get("public_source")
|
178
178
|
)
|
179
179
|
except Exception:
|
180
180
|
# for custom records that are not created from bionty sources
|
181
181
|
return records, iterable_idx
|
182
|
-
# add
|
183
|
-
kwargs.update({"
|
182
|
+
# add public_source record to the kwargs
|
183
|
+
kwargs.update({"public_source": get_public_source_record(public_ontology)})
|
184
184
|
|
185
185
|
# filter the columns in bionty df based on fields
|
186
|
-
bionty_df = _filter_bionty_df_columns(model=model,
|
186
|
+
bionty_df = _filter_bionty_df_columns(model=model, public_ontology=public_ontology)
|
187
187
|
|
188
188
|
# standardize in the bionty reference
|
189
|
-
result =
|
189
|
+
result = public_ontology.inspect(iterable_idx, field=field.field.name, mute=True)
|
190
190
|
syn_mapper = result.synonyms_mapper
|
191
191
|
|
192
192
|
msg_syn: str = ""
|
@@ -254,13 +254,13 @@ def _print_values(names: List, n: int = 20) -> str:
|
|
254
254
|
return print_values
|
255
255
|
|
256
256
|
|
257
|
-
def _filter_bionty_df_columns(model: Registry,
|
257
|
+
def _filter_bionty_df_columns(model: Registry, public_ontology: Any) -> pd.DataFrame:
|
258
258
|
bionty_df = pd.DataFrame()
|
259
|
-
if
|
259
|
+
if public_ontology is not None:
|
260
260
|
model_field_names = {i.name for i in model._meta.fields}
|
261
261
|
# parents needs to be added here as relationships aren't in fields
|
262
262
|
model_field_names.add("parents")
|
263
|
-
bionty_df =
|
263
|
+
bionty_df = public_ontology.df().reset_index()
|
264
264
|
if model.__name__ == "Gene":
|
265
265
|
# groupby ensembl_gene_id and concat ncbi_gene_ids
|
266
266
|
groupby_id_col = (
|
lamindb/_query_manager.py
CHANGED
@@ -46,14 +46,11 @@ class QueryManager(models.Manager):
|
|
46
46
|
|
47
47
|
Examples:
|
48
48
|
>>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))
|
49
|
-
>>> labels = ln.ULabel.filter(name__icontains
|
49
|
+
>>> labels = ln.ULabel.filter(name__icontains="label").all()
|
50
50
|
>>> ln.ULabel(name="ULabel1").save()
|
51
51
|
>>> label = ln.ULabel.filter(name="ULabel1").one()
|
52
52
|
>>> label.parents.set(labels)
|
53
53
|
>>> label.parents.list()
|
54
|
-
[ULabel(id=sFMcPepC, name=ULabel1, updated_at=2023-07-19 19:45:17, created_by_id=DzTjkKse), # noqa
|
55
|
-
ULabel(id=2SscQvsM, name=ULabel2, updated_at=2023-07-19 19:45:17, created_by_id=DzTjkKse), # noqa
|
56
|
-
ULabel(id=lecV87vi, name=ULabel3, updated_at=2023-07-19 19:45:17, created_by_id=DzTjkKse)] # noqa
|
57
54
|
>>> label.parents.list("name")
|
58
55
|
['ULabel1', 'ULabel2', 'ULabel3']
|
59
56
|
"""
|
@@ -76,7 +73,7 @@ class QueryManager(models.Manager):
|
|
76
73
|
For `**kwargs`, see :meth:`lamindb.dev.QuerySet.df`.
|
77
74
|
"""
|
78
75
|
self._track_run_input_manager()
|
79
|
-
return self.
|
76
|
+
return self._all_base_class()
|
80
77
|
|
81
78
|
@doc_args(Registry.search.__doc__)
|
82
79
|
def search(self, string: str, **kwargs):
|
@@ -115,5 +112,5 @@ models.Manager.__getitem__ = QueryManager.__getitem__
|
|
115
112
|
models.Manager._track_run_input_manager = QueryManager._track_run_input_manager
|
116
113
|
# the two lines below would be easy if we could actually inherit; like this,
|
117
114
|
# they're suboptimal
|
118
|
-
models.Manager.
|
115
|
+
models.Manager._all_base_class = models.Manager.all
|
119
116
|
models.Manager.all = QueryManager.all
|
lamindb/_query_set.py
CHANGED
@@ -4,7 +4,14 @@ from typing import Dict, Iterable, List, NamedTuple, Optional, Union
|
|
4
4
|
import pandas as pd
|
5
5
|
from django.db import models
|
6
6
|
from lamindb_setup.dev._docs import doc_args
|
7
|
-
from lnschema_core.models import
|
7
|
+
from lnschema_core.models import (
|
8
|
+
Artifact,
|
9
|
+
CanValidate,
|
10
|
+
Collection,
|
11
|
+
IsTree,
|
12
|
+
Registry,
|
13
|
+
Transform,
|
14
|
+
)
|
8
15
|
from lnschema_core.types import ListLike, StrField
|
9
16
|
|
10
17
|
|
@@ -147,6 +154,13 @@ class QuerySet(models.QuerySet, CanValidate, IsTree):
|
|
147
154
|
df.rename(columns={values_expression: expression}, inplace=True)
|
148
155
|
return df
|
149
156
|
|
157
|
+
def delete(self, *args, **kwargs):
|
158
|
+
if self.model in {Artifact, Collection, Transform}:
|
159
|
+
for record in self:
|
160
|
+
record.delete(*args, **kwargs)
|
161
|
+
else:
|
162
|
+
self._delete_base_class(*args, **kwargs)
|
163
|
+
|
150
164
|
def list(self, field: Optional[str] = None) -> List[Registry]:
|
151
165
|
"""Populate a list with the results.
|
152
166
|
|
@@ -282,3 +296,5 @@ models.QuerySet.lookup = QuerySet.lookup
|
|
282
296
|
models.QuerySet.validate = QuerySet.validate
|
283
297
|
models.QuerySet.inspect = QuerySet.inspect
|
284
298
|
models.QuerySet.standardize = QuerySet.standardize
|
299
|
+
models.QuerySet._delete_base_class = models.QuerySet.delete
|
300
|
+
models.QuerySet.delete = QuerySet.delete
|
lamindb/_registry.py
CHANGED
@@ -129,12 +129,12 @@ def from_values(
|
|
129
129
|
cls, values: ListLike, field: Optional[StrField] = None, **kwargs
|
130
130
|
) -> List["Registry"]:
|
131
131
|
"""{}."""
|
132
|
-
|
132
|
+
from_public = True if cls.__module__.startswith("lnschema_bionty.") else False
|
133
133
|
field_str = get_default_str_field(cls, field=field)
|
134
134
|
return get_or_create_records(
|
135
135
|
iterable=values,
|
136
136
|
field=getattr(cls, field_str),
|
137
|
-
|
137
|
+
from_public=from_public,
|
138
138
|
**kwargs,
|
139
139
|
)
|
140
140
|
|
@@ -412,7 +412,7 @@ def update_fk_to_default_db(
|
|
412
412
|
def transfer_fk_to_default_db_bulk(records: Union[List, QuerySet]):
|
413
413
|
for fk in [
|
414
414
|
"organism",
|
415
|
-
"
|
415
|
+
"public_source",
|
416
416
|
"initial_version",
|
417
417
|
"latest_report", # Transform
|
418
418
|
"source_code", # Transform
|
lamindb/_run.py
CHANGED
@@ -30,4 +30,27 @@ def __init__(run: Run, *args, **kwargs):
|
|
30
30
|
)
|
31
31
|
|
32
32
|
|
33
|
+
def delete_run_artifacts(run: Run) -> None:
|
34
|
+
environment = None
|
35
|
+
if run.environment is not None:
|
36
|
+
environment = run.environment
|
37
|
+
run.environment = None
|
38
|
+
report = None
|
39
|
+
if run.report is not None:
|
40
|
+
report = run.report
|
41
|
+
run.report = None
|
42
|
+
if environment is not None or report is not None:
|
43
|
+
run.save()
|
44
|
+
if environment is not None:
|
45
|
+
environment.delete(permanent=True)
|
46
|
+
if report is not None:
|
47
|
+
report.delete(permanent=True)
|
48
|
+
|
49
|
+
|
50
|
+
def delete(self) -> None:
|
51
|
+
delete_run_artifacts(self)
|
52
|
+
super(Run, self).delete()
|
53
|
+
|
54
|
+
|
33
55
|
Run.__init__ = __init__
|
56
|
+
Run.delete = delete
|
lamindb/_transform.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
from typing import TYPE_CHECKING, Optional
|
2
2
|
|
3
|
-
from lnschema_core.models import TRANSFORM_TYPE_DEFAULT, Transform
|
3
|
+
from lnschema_core.models import TRANSFORM_TYPE_DEFAULT, Artifact, Run, Transform
|
4
4
|
|
5
|
+
from ._run import delete_run_artifacts
|
5
6
|
from .dev.versioning import get_uid_from_old_version, init_uid
|
6
7
|
|
7
8
|
if TYPE_CHECKING:
|
@@ -63,4 +64,28 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
63
64
|
)
|
64
65
|
|
65
66
|
|
67
|
+
def delete(self) -> None:
|
68
|
+
# set latest_report to None, it's tracked through the latest run
|
69
|
+
latest_report = None
|
70
|
+
if self.latest_report is not None:
|
71
|
+
latest_report = self.latest_report
|
72
|
+
self.latest_report = None
|
73
|
+
source_code = None
|
74
|
+
if self.source_code is not None:
|
75
|
+
source_code = self.source_code
|
76
|
+
self.source_code = None
|
77
|
+
if latest_report is not None or source_code is not None:
|
78
|
+
self.save()
|
79
|
+
if source_code is not None:
|
80
|
+
source_code.delete(permanent=True)
|
81
|
+
# query all runs and delete their artifacts
|
82
|
+
runs = Run.filter(transform=self)
|
83
|
+
for run in runs:
|
84
|
+
delete_run_artifacts(run)
|
85
|
+
# at this point, all artifacts have been taken care of
|
86
|
+
# we can now leverage CASCADE delete
|
87
|
+
super(Transform, self).delete()
|
88
|
+
|
89
|
+
|
66
90
|
Transform.__init__ = __init__
|
91
|
+
Transform.delete = delete
|
lamindb/_validate.py
CHANGED
@@ -84,7 +84,7 @@ def _inspect(
|
|
84
84
|
|
85
85
|
if len(nonval) > 0 and orm.__get_schema_name__() == "bionty":
|
86
86
|
try:
|
87
|
-
bionty_result = orm.
|
87
|
+
bionty_result = orm.public(organism=kwargs.get("organism")).inspect(
|
88
88
|
values=nonval, field=field, mute=True, **kwargs
|
89
89
|
)
|
90
90
|
bionty_validated = bionty_result.validated
|
@@ -186,7 +186,7 @@ def standardize(
|
|
186
186
|
return_mapper: bool = False,
|
187
187
|
case_sensitive: bool = False,
|
188
188
|
mute: bool = False,
|
189
|
-
|
189
|
+
public_aware: bool = True,
|
190
190
|
keep: Literal["first", "last", False] = "first",
|
191
191
|
synonyms_field: str = "synonyms",
|
192
192
|
**kwargs,
|
@@ -200,7 +200,7 @@ def standardize(
|
|
200
200
|
return_mapper=return_mapper,
|
201
201
|
case_sensitive=case_sensitive,
|
202
202
|
mute=mute,
|
203
|
-
|
203
|
+
public_aware=public_aware,
|
204
204
|
keep=keep,
|
205
205
|
synonyms_field=synonyms_field,
|
206
206
|
**kwargs,
|
@@ -248,7 +248,7 @@ def _standardize(
|
|
248
248
|
return_mapper: bool = False,
|
249
249
|
case_sensitive: bool = False,
|
250
250
|
mute: bool = False,
|
251
|
-
|
251
|
+
public_aware: bool = True,
|
252
252
|
keep: Literal["first", "last", False] = "first",
|
253
253
|
synonyms_field: str = "synonyms",
|
254
254
|
**kwargs,
|
@@ -308,7 +308,7 @@ def _standardize(
|
|
308
308
|
return result
|
309
309
|
|
310
310
|
# map synonyms in Bionty
|
311
|
-
if orm.__get_schema_name__() == "bionty" and
|
311
|
+
if orm.__get_schema_name__() == "bionty" and public_aware:
|
312
312
|
mapper = {}
|
313
313
|
if return_mapper:
|
314
314
|
mapper = std_names_db
|
@@ -321,7 +321,7 @@ def _standardize(
|
|
321
321
|
return _return(result=std_names_db, mapper=mapper)
|
322
322
|
|
323
323
|
nonval = np.array(std_names_db)[~val_res]
|
324
|
-
std_names_bt_mapper = orm.
|
324
|
+
std_names_bt_mapper = orm.public(organism=organism).standardize(
|
325
325
|
nonval, return_mapper=True, mute=True, **_kwargs
|
326
326
|
)
|
327
327
|
|
lamindb/dev/_label_manager.py
CHANGED
lamindb/dev/_run_context.py
CHANGED
@@ -27,9 +27,9 @@ msg_path_failed = (
|
|
27
27
|
|
28
28
|
msg_manual_init = (
|
29
29
|
"\n(1) save your notebook!"
|
30
|
-
"\n(2)
|
30
|
+
"\n(2) add stem_uid & version to your notebook file by running the CLI:\n"
|
31
31
|
"lamin track {notebook_path}"
|
32
|
-
"\n(3) reload
|
32
|
+
"\n(3) reload your notebook from disk"
|
33
33
|
)
|
34
34
|
|
35
35
|
|
lamindb/dev/datasets/_core.py
CHANGED
@@ -38,10 +38,10 @@ def file_fcs_alpert19(populate_registries: bool = False) -> Path: # pragma: no
|
|
38
38
|
verbosity = ln.settings.verbosity
|
39
39
|
ln.settings.verbosity = "error"
|
40
40
|
adata = readfcs.read(filepath)
|
41
|
-
std = lb.CellMarker.
|
41
|
+
std = lb.CellMarker.public().standardize(adata.var.index)
|
42
42
|
ln.save(
|
43
43
|
lb.CellMarker.from_values(
|
44
|
-
lb.CellMarker.
|
44
|
+
lb.CellMarker.public().inspect(std, "name").validated, "name"
|
45
45
|
)
|
46
46
|
)
|
47
47
|
ln.Feature(
|
@@ -86,7 +86,7 @@ def file_tsv_rnaseq_nfcore_salmon_merged_gene_counts(
|
|
86
86
|
name="assay", type="category", registries=[lb.ExperimentalFactor]
|
87
87
|
).save()
|
88
88
|
ln.Feature(name="organism", type="category", registries=[lb.Organism]).save()
|
89
|
-
lb.ExperimentalFactor.
|
89
|
+
lb.ExperimentalFactor.from_public(ontology_id="EFO:0008896").save()
|
90
90
|
ln.settings.verbosity = verbosity
|
91
91
|
|
92
92
|
return Path(filepath)
|
@@ -191,20 +191,20 @@ def anndata_mouse_sc_lymph_node(
|
|
191
191
|
auto_save_parents = lb.settings.auto_save_parents
|
192
192
|
lb.settings.auto_save_parents = False
|
193
193
|
# strain
|
194
|
-
lb.ExperimentalFactor.
|
194
|
+
lb.ExperimentalFactor.from_public(ontology_id="EFO:0004472").save()
|
195
195
|
# developmental stage
|
196
|
-
lb.ExperimentalFactor.
|
196
|
+
lb.ExperimentalFactor.from_public(ontology_id="EFO:0001272").save()
|
197
197
|
# tissue
|
198
|
-
lb.Tissue.
|
198
|
+
lb.Tissue.from_public(ontology_id="UBERON:0001542").save()
|
199
199
|
# cell types
|
200
200
|
ln.save(lb.CellType.from_values(["CL:0000115", "CL:0000738"], "ontology_id"))
|
201
201
|
# assays
|
202
202
|
ln.Feature(
|
203
203
|
name="assay", type="category", registries=[lb.ExperimentalFactor]
|
204
204
|
).save()
|
205
|
-
lb.ExperimentalFactor.
|
205
|
+
lb.ExperimentalFactor.from_public(ontology_id="EFO:0008913").save()
|
206
206
|
# genes
|
207
|
-
validated = lb.Gene.
|
207
|
+
validated = lb.Gene.public(organism="mouse").validate(
|
208
208
|
adata.var.index, field="ensembl_gene_id"
|
209
209
|
)
|
210
210
|
ln.save(
|
@@ -335,7 +335,7 @@ def anndata_human_immune_cells(
|
|
335
335
|
ln.Feature(name="tissue", type="category", registries=[lb.Tissue]).save()
|
336
336
|
ln.Feature(name="organism", type="category", registries=[lb.Organism]).save()
|
337
337
|
ln.Feature(name="donor", type="category", registries=[ln.ULabel]).save()
|
338
|
-
lb.ExperimentalFactor.
|
338
|
+
lb.ExperimentalFactor.from_public(ontology_id="EFO:0008913").save()
|
339
339
|
ln.save([ln.ULabel(name=name) for name in adata.obs.donor.unique()])
|
340
340
|
ln.settings.verbosity = verbosity
|
341
341
|
lb.settings.auto_save_parents = auto_save_parents
|
lamindb/dev/storage/file.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
+
import builtins
|
2
|
+
import re
|
1
3
|
import shutil
|
2
4
|
from pathlib import Path
|
3
|
-
from typing import Union
|
5
|
+
from typing import Optional, Union
|
4
6
|
|
5
7
|
import anndata as ad
|
6
8
|
import pandas as pd
|
@@ -24,6 +26,7 @@ except ImportError:
|
|
24
26
|
|
25
27
|
|
26
28
|
AUTO_KEY_PREFIX = ".lamindb/"
|
29
|
+
is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
|
27
30
|
|
28
31
|
|
29
32
|
# add type annotations back asap when re-organizing the module
|
@@ -143,6 +146,33 @@ def read_tsv(path: Union[str, Path, UPath]) -> pd.DataFrame:
|
|
143
146
|
return pd.read_csv(path_sanitized, sep="\t")
|
144
147
|
|
145
148
|
|
149
|
+
def load_html(path: Union[str, Path, UPath]):
|
150
|
+
if is_run_from_ipython:
|
151
|
+
with open(path, encoding="utf-8") as f:
|
152
|
+
html_content = f.read()
|
153
|
+
# Extract the body content using regular expressions
|
154
|
+
body_content = re.findall(
|
155
|
+
r"<body(?:.*?)>(?:.*?)</body>", html_content, re.DOTALL
|
156
|
+
)
|
157
|
+
# Remove any empty body tags
|
158
|
+
if body_content:
|
159
|
+
body_content = body_content[0]
|
160
|
+
body_content = body_content.strip() # type: ignore
|
161
|
+
from IPython.display import HTML, display
|
162
|
+
|
163
|
+
display(HTML(data=body_content))
|
164
|
+
else:
|
165
|
+
return path
|
166
|
+
|
167
|
+
|
168
|
+
def load_json(path: Union[str, Path, UPath]):
|
169
|
+
import json
|
170
|
+
|
171
|
+
with open(path) as f:
|
172
|
+
data = json.load(f)
|
173
|
+
return data
|
174
|
+
|
175
|
+
|
146
176
|
def load_to_memory(filepath: Union[str, Path, UPath], stream: bool = False, **kwargs):
|
147
177
|
"""Load a file into memory.
|
148
178
|
|
@@ -170,6 +200,8 @@ def load_to_memory(filepath: Union[str, Path, UPath], stream: bool = False, **kw
|
|
170
200
|
".fcs": read_fcs,
|
171
201
|
".zarr": read_adata_zarr,
|
172
202
|
".zrad": read_adata_zarr,
|
203
|
+
".html": load_html,
|
204
|
+
".json": load_json,
|
173
205
|
}
|
174
206
|
|
175
207
|
reader = READER_FUNCS.get(filepath.suffix)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: lamindb
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.67.1
|
4
4
|
Summary: A data framework for biology.
|
5
5
|
Author-email: Lamin Labs <open-source@lamin.ai>
|
6
6
|
Requires-Python: >=3.8
|
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
|
|
9
9
|
Classifier: Programming Language :: Python :: 3.9
|
10
10
|
Classifier: Programming Language :: Python :: 3.10
|
11
11
|
Classifier: Programming Language :: Python :: 3.11
|
12
|
-
Requires-Dist: lnschema_core==0.61.
|
13
|
-
Requires-Dist: lamindb_setup==0.64.
|
12
|
+
Requires-Dist: lnschema_core==0.61.1
|
13
|
+
Requires-Dist: lamindb_setup==0.64.2
|
14
14
|
Requires-Dist: lamin_utils==0.13.0
|
15
|
-
Requires-Dist: lamin_cli==0.6.
|
15
|
+
Requires-Dist: lamin_cli==0.6.1
|
16
16
|
Requires-Dist: rapidfuzz
|
17
17
|
Requires-Dist: pyarrow
|
18
18
|
Requires-Dist: typing_extensions!=4.6.0
|
@@ -28,7 +28,7 @@ Requires-Dist: boto3==1.28.17 ; extra == "aws"
|
|
28
28
|
Requires-Dist: aiobotocore==2.5.4 ; extra == "aws"
|
29
29
|
Requires-Dist: fsspec[s3]==2023.9.0 ; extra == "aws"
|
30
30
|
Requires-Dist: s3fs>=2023.9.0 ; extra == "aws"
|
31
|
-
Requires-Dist: lnschema_bionty==0.
|
31
|
+
Requires-Dist: lnschema_bionty==0.39.0 ; extra == "bionty"
|
32
32
|
Requires-Dist: pandas<2 ; extra == "dev"
|
33
33
|
Requires-Dist: pre-commit ; extra == "dev"
|
34
34
|
Requires-Dist: nox ; extra == "dev"
|
@@ -42,7 +42,7 @@ Requires-Dist: lamin_vault ; extra == "dev"
|
|
42
42
|
Requires-Dist: django-schema-graph ; extra == "erdiagram"
|
43
43
|
Requires-Dist: readfcs>=1.1.7 ; extra == "fcs"
|
44
44
|
Requires-Dist: fsspec[gs]==2023.9.0 ; extra == "gcp"
|
45
|
-
Requires-Dist: nbproject==0.
|
45
|
+
Requires-Dist: nbproject==0.10.0 ; extra == "jupyter"
|
46
46
|
Requires-Dist: nbstripout ; extra == "jupyter"
|
47
47
|
Requires-Dist: nbconvert ; extra == "jupyter"
|
48
48
|
Requires-Dist: zarr>=2.16.0 ; extra == "zarr"
|
@@ -1,29 +1,28 @@
|
|
1
|
-
lamindb/__init__.py,sha256=
|
2
|
-
lamindb/_artifact.py,sha256=
|
3
|
-
lamindb/_collection.py,sha256=
|
4
|
-
lamindb/_delete.py,sha256=jO6kcIoxY6EFgqiVF2vlbXaCaqlI25AvBo7nre3JXkQ,1968
|
1
|
+
lamindb/__init__.py,sha256=hMge9XlaVuCCh6oLRPJWy1lAE4u58fiIrWHxORYbIv0,2530
|
2
|
+
lamindb/_artifact.py,sha256=4GIo987_3AJgQRFDC7LwokXBYFKPwmnOUeGcFxG8zjI,38110
|
3
|
+
lamindb/_collection.py,sha256=Y-GU-MOvlLmqlcBhfGVojDbnZFD7V_FZEbrsF37rvqI,17850
|
5
4
|
lamindb/_feature.py,sha256=tEcqFoEj5yp4LSJfMGyiVvxDUuLoZaik6lo05ZKcCtE,6036
|
6
5
|
lamindb/_feature_set.py,sha256=KYgdmMdXb21pfpir1J1O21in3nJvUeznECOB38qfTvk,8654
|
7
6
|
lamindb/_filter.py,sha256=YwWqviJ34kHTMJ8NYlrEw-vsrXkKrVIPsEZSBVvMcrI,1163
|
8
|
-
lamindb/_from_values.py,sha256=
|
7
|
+
lamindb/_from_values.py,sha256=WMyD2FiEgxgpSYpj8fFChv7Mdz8Y--2a8RoEZjLFR30,11885
|
9
8
|
lamindb/_parents.py,sha256=UVvCOR9CUkEWc2_qr3iHJUM_j7s88L5H6F3Xz5Ni5gY,14240
|
10
|
-
lamindb/_query_manager.py,sha256=
|
11
|
-
lamindb/_query_set.py,sha256=
|
12
|
-
lamindb/_registry.py,sha256=
|
13
|
-
lamindb/_run.py,sha256=
|
9
|
+
lamindb/_query_manager.py,sha256=OGBFv-l16t2g0TqBIUvMjfEZhu0a5AT0LsFBHVEOyv8,4001
|
10
|
+
lamindb/_query_set.py,sha256=LOHuiGok0FRrG_CBOHVkIsVTDdw406eJalsNwI0MNng,10904
|
11
|
+
lamindb/_registry.py,sha256=56ax7_mrUbmQveGCkHypvQrGlrw5LMF3yFZPqj8_gkk,17535
|
12
|
+
lamindb/_run.py,sha256=O5TcVPZqgdUvk4nBmI_HXzvWOG_Zr3B_HMDaw228w-4,1719
|
14
13
|
lamindb/_save.py,sha256=UlRHJGUiHGOXv90wmawZVsOqhJIqk8f1wj8MW3Rlq_c,10535
|
15
14
|
lamindb/_storage.py,sha256=mz2Cy0CTaeJGA03A1FPQmmH0Vt2ib_KlXklaLqtN1mU,394
|
16
|
-
lamindb/_transform.py,sha256=
|
15
|
+
lamindb/_transform.py,sha256=O5fJ6X-sHicoGj3aHTikYjd3Nsru0mstlI6gRMgRamQ,3360
|
17
16
|
lamindb/_ulabel.py,sha256=HALoy6HerRnehR-u8zPH-qmiFQHWxeAwkZ31jxjrfgI,1893
|
18
17
|
lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
|
19
|
-
lamindb/_validate.py,sha256=
|
18
|
+
lamindb/_validate.py,sha256=KFzU3aLEd5RcimT61FWDlX7_uojFyEL0BaVZVzmoyow,14400
|
20
19
|
lamindb/_view.py,sha256=yFMu4vnt0YqvN1q11boAkwigxCH1gdliDUSbzh3IuDw,2175
|
21
20
|
lamindb/dev/__init__.py,sha256=LLqivujL8c-oKWC15SJepAYyrTlLNvql5Vdwunc0qvE,1174
|
22
21
|
lamindb/dev/_data.py,sha256=YPZ664qGKMl34LbZCMCEFIxQ-E81iAt_b3lvMiTe-oc,17066
|
23
22
|
lamindb/dev/_feature_manager.py,sha256=jn8x_JbrtLFelmaFh4noOXqGSCfqVuVX0quoa7gTJtM,9366
|
24
|
-
lamindb/dev/_label_manager.py,sha256=
|
23
|
+
lamindb/dev/_label_manager.py,sha256=eViJrKRnVqbO4zUPZk0yW4IP224sbTcGFL-UHSE1otg,8750
|
25
24
|
lamindb/dev/_mapped_collection.py,sha256=UhlDRuXOySQNqNabcJQ3dXmSz-zPwYxk2VZzdLikc4Y,13754
|
26
|
-
lamindb/dev/_run_context.py,sha256=
|
25
|
+
lamindb/dev/_run_context.py,sha256=3p-2GdxNREKvUW86gW79MtDHLZ0IxuruYJJ_7MbfZwI,22973
|
27
26
|
lamindb/dev/_settings.py,sha256=nixk8lVijCbq_fRlUpkX5gvO9AdgUFjbXzFThAJhGBA,3824
|
28
27
|
lamindb/dev/_track_environment.py,sha256=QjHWbyl2u8J4hbJG8Q_ToFaZIgS-H15Ej6syJgk-dvY,662
|
29
28
|
lamindb/dev/_view_tree.py,sha256=K-C1BsOiEupwgkhyrsGxLFxHU45SAkiKsQbeOV9PbaY,3421
|
@@ -33,17 +32,17 @@ lamindb/dev/hashing.py,sha256=wmioFBv_5G-Gjnm-QU_UtofpB0wBjeEkycof_V9F454,1745
|
|
33
32
|
lamindb/dev/types.py,sha256=dzzoSaxjtJ1zJ_C9LZJcydiULf9XfZji1KI2DvO6S8g,224
|
34
33
|
lamindb/dev/versioning.py,sha256=NlwOH7Vf4bad5iC1l9erkuE7MBcj1vQd-AfgAZRpNew,2499
|
35
34
|
lamindb/dev/datasets/__init__.py,sha256=zRP98oqUAaXhqWyKMiH0s_ImVIuNeziQQ2kQ_t0f-DI,1353
|
36
|
-
lamindb/dev/datasets/_core.py,sha256=
|
35
|
+
lamindb/dev/datasets/_core.py,sha256=pdJn25vlX9NQ0AaPo2E9LYGHQqH02rQnjHgZ8U-5Z6o,18882
|
37
36
|
lamindb/dev/datasets/_fake.py,sha256=S8mNho-oSh1M9x9oOSsUBLLHmBAegsOLlFk6LnF81EA,942
|
38
37
|
lamindb/dev/storage/__init__.py,sha256=P6LYolhSu4QNtvp4hEnVpLpW1GueX2BTewbUE8A12ak,368
|
39
38
|
lamindb/dev/storage/_anndata_sizes.py,sha256=0XVzA6AQeVGPaGPrhGusKyxFgFjeo3qSN29hxb8D5E8,993
|
40
39
|
lamindb/dev/storage/_backed_access.py,sha256=NFMS--MZHjh1obyNjUG_wIe9NGpYhv9G08sr8PslafI,22794
|
41
40
|
lamindb/dev/storage/_zarr.py,sha256=FEwAkOhAp_1PcgBCJF2q-OxZsr2iVFQQ88svuAJ_AKU,2815
|
42
|
-
lamindb/dev/storage/file.py,sha256=
|
41
|
+
lamindb/dev/storage/file.py,sha256=4PVxVzPGzoSizODRPW1cejbyVG_7ur5qNKWYHa_BjUw,6654
|
43
42
|
lamindb/dev/storage/object.py,sha256=KGuOwwYuN2yCJxTXn9v0LanC0fjKwy_62P-WksHcf40,1140
|
44
43
|
lamindb/setup/__init__.py,sha256=WaWKO-2XT67S65lSbS80hUojL-Mr_Wms9UxH6U54TsY,289
|
45
44
|
lamindb/setup/dev/__init__.py,sha256=tBty426VGF2PGqqt2XuNU-WgvOrbOp1aZBDowjLuzgA,242
|
46
|
-
lamindb-0.
|
47
|
-
lamindb-0.
|
48
|
-
lamindb-0.
|
49
|
-
lamindb-0.
|
45
|
+
lamindb-0.67.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
46
|
+
lamindb-0.67.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
|
47
|
+
lamindb-0.67.1.dist-info/METADATA,sha256=qwqkIHZP36qqzxe9xe3rwEM924X5eYlyLEgihVjuPgU,3077
|
48
|
+
lamindb-0.67.1.dist-info/RECORD,,
|
lamindb/_delete.py
DELETED
@@ -1,65 +0,0 @@
|
|
1
|
-
from typing import List, Union, overload
|
2
|
-
|
3
|
-
from lamin_utils import colors, logger
|
4
|
-
from lnschema_core import Registry
|
5
|
-
|
6
|
-
|
7
|
-
@overload
|
8
|
-
def delete(
|
9
|
-
record: Registry,
|
10
|
-
) -> None:
|
11
|
-
... # pragma: no cover
|
12
|
-
|
13
|
-
|
14
|
-
@overload
|
15
|
-
def delete(
|
16
|
-
records: List[Registry],
|
17
|
-
) -> None: # type: ignore
|
18
|
-
... # pragma: no cover
|
19
|
-
|
20
|
-
|
21
|
-
def delete( # type: ignore
|
22
|
-
records: Union[Registry, List[Registry]],
|
23
|
-
) -> None:
|
24
|
-
"""Delete metadata records & files.
|
25
|
-
|
26
|
-
Args:
|
27
|
-
records: `Union[Registry, List[Registry]]` One or multiple records.
|
28
|
-
|
29
|
-
Returns:
|
30
|
-
`None`
|
31
|
-
|
32
|
-
Examples:
|
33
|
-
|
34
|
-
Delete by record:
|
35
|
-
|
36
|
-
>>> experiment = ln.filter(Experiment, id=experiment_id).one()
|
37
|
-
>>> ln.delete(experiment)
|
38
|
-
|
39
|
-
Delete files (delete the metadata record and the file in storage):
|
40
|
-
|
41
|
-
>>> file = ln.filter(File, id=artifact_id).one()
|
42
|
-
>>> ln.delete(file)
|
43
|
-
>>> # deleting the record occurs automatically
|
44
|
-
>>> # you will be asked whether to delete the file in storage
|
45
|
-
>>> # for more control, use:
|
46
|
-
>>> artifact.delete(storage=True)
|
47
|
-
|
48
|
-
Bulk delete via QuerySet:
|
49
|
-
|
50
|
-
>>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))
|
51
|
-
>>> queryset = ln.ULabel.filter(name__icontains = "label")
|
52
|
-
>>> queryset.list()
|
53
|
-
[ULabel(id=o3FY3c5n, name=ULabel2, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9), # noqa
|
54
|
-
ULabel(id=Qi3c4utq, name=ULabel3, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9), # noqa
|
55
|
-
ULabel(id=CcFPLmpq, name=ULabel1, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9)] # noqa
|
56
|
-
>>> queryset.delete()
|
57
|
-
"""
|
58
|
-
logger.warning("for efficient bulk delete, use `queryset.delete` instead")
|
59
|
-
if isinstance(records, list):
|
60
|
-
records = records
|
61
|
-
elif isinstance(records, Registry):
|
62
|
-
records = [records]
|
63
|
-
for record in records:
|
64
|
-
record.delete()
|
65
|
-
logger.success(f"deleted {colors.yellow(f'{record}')}")
|
File without changes
|
File without changes
|