lamin_cli 1.11.0__py2.py3-none-any.whl → 1.12.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamin_cli/__init__.py +27 -27
- lamin_cli/__main__.py +826 -625
- lamin_cli/_annotate.py +47 -47
- lamin_cli/_cache.py +49 -41
- lamin_cli/_context.py +76 -76
- lamin_cli/_delete.py +85 -44
- lamin_cli/_io.py +147 -144
- lamin_cli/_load.py +203 -203
- lamin_cli/_migration.py +50 -50
- lamin_cli/_save.py +350 -325
- lamin_cli/_settings.py +154 -96
- lamin_cli/clone/_clone_verification.py +56 -56
- lamin_cli/clone/create_sqlite_clone_and_import_db.py +53 -51
- lamin_cli/compute/modal.py +174 -175
- lamin_cli/urls.py +10 -8
- {lamin_cli-1.11.0.dist-info → lamin_cli-1.12.1.dist-info}/METADATA +3 -2
- lamin_cli-1.12.1.dist-info/RECORD +22 -0
- {lamin_cli-1.11.0.dist-info → lamin_cli-1.12.1.dist-info}/WHEEL +1 -1
- {lamin_cli-1.11.0.dist-info → lamin_cli-1.12.1.dist-info/licenses}/LICENSE +201 -201
- lamin_cli-1.11.0.dist-info/RECORD +0 -22
- {lamin_cli-1.11.0.dist-info → lamin_cli-1.12.1.dist-info}/entry_points.txt +0 -0
lamin_cli/_settings.py
CHANGED
|
@@ -1,96 +1,154 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import os
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
if os.environ.get("NO_RICH"):
|
|
7
|
-
import click as click
|
|
8
|
-
else:
|
|
9
|
-
import rich_click as click
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
@click.group(invoke_without_command=True)
|
|
13
|
-
@click.pass_context
|
|
14
|
-
def settings(ctx):
|
|
15
|
-
"""Manage settings.
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
lamin settings
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
lamin settings
|
|
41
|
-
|
|
42
|
-
lamin settings
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
@
|
|
75
|
-
@click.argument(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
if os.environ.get("NO_RICH"):
|
|
7
|
+
import click as click
|
|
8
|
+
else:
|
|
9
|
+
import rich_click as click
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@click.group(invoke_without_command=True)
|
|
13
|
+
@click.pass_context
|
|
14
|
+
def settings(ctx):
|
|
15
|
+
"""Manage development & cache directories, branch, and space settings.
|
|
16
|
+
|
|
17
|
+
Get or set a setting by name:
|
|
18
|
+
|
|
19
|
+
- `dev-dir` → development directory {attr}`~lamindb.setup.core.SetupSettings.dev_dir`
|
|
20
|
+
- `cache-dir` → cache directory {attr}`~lamindb.setup.core.SetupSettings.cache_dir`
|
|
21
|
+
- `branch` → branch {attr}`~lamindb.setup.core.SetupSettings.branch`
|
|
22
|
+
- `space` → space {attr}`~lamindb.setup.core.SetupSettings.space`
|
|
23
|
+
|
|
24
|
+
Display via [lamin info](https://docs.lamin.ai/cli#info)
|
|
25
|
+
|
|
26
|
+
Examples:
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
# dev-dir
|
|
30
|
+
lamin settings dev-dir get
|
|
31
|
+
lamin settings dev-dir set . # set to current directory
|
|
32
|
+
lamin settings dev-dir set ~/my-project
|
|
33
|
+
lamin settings dev-dir unset
|
|
34
|
+
# cache-dir
|
|
35
|
+
lamin settings cache-dir get
|
|
36
|
+
lamin settings cache-dir set /path/to/cache
|
|
37
|
+
lamin settings cache-dir clear
|
|
38
|
+
# branch
|
|
39
|
+
lamin settings branch get
|
|
40
|
+
lamin settings branch set main
|
|
41
|
+
# space
|
|
42
|
+
lamin settings space get
|
|
43
|
+
lamin settings space set all
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
→ Python/R alternative: {attr}`~lamindb.setup.core.SetupSettings.dev_dir`, {attr}`~lamindb.setup.core.SetupSettings.cache_dir`, {attr}`~lamindb.setup.core.SetupSettings.branch`, and {attr}`~lamindb.setup.core.SetupSettings.space`
|
|
47
|
+
"""
|
|
48
|
+
if ctx.invoked_subcommand is None:
|
|
49
|
+
from lamindb_setup import settings as settings_
|
|
50
|
+
|
|
51
|
+
click.echo("Configure: see `lamin settings --help`")
|
|
52
|
+
click.echo(settings_)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# -----------------------------------------------------------------------------
|
|
56
|
+
# dev-dir group (pattern: lamin settings dev-dir get/set)
|
|
57
|
+
# -----------------------------------------------------------------------------
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@click.group("dev-dir")
|
|
61
|
+
def dev_dir_group():
|
|
62
|
+
"""Get or set the development directory."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dev_dir_group.command("get")
|
|
66
|
+
def dev_dir_get():
|
|
67
|
+
"""Show the current development directory."""
|
|
68
|
+
from lamindb_setup import settings as settings_
|
|
69
|
+
|
|
70
|
+
value = settings_.dev_dir
|
|
71
|
+
click.echo(value if value is not None else "None")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dev_dir_group.command("set")
|
|
75
|
+
@click.argument("value", type=str)
|
|
76
|
+
def dev_dir_set(value: str):
|
|
77
|
+
"""Set the development directory."""
|
|
78
|
+
from lamindb_setup import settings as settings_
|
|
79
|
+
|
|
80
|
+
if value.lower() == "none":
|
|
81
|
+
value = None # type: ignore[assignment]
|
|
82
|
+
settings_.dev_dir = value
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dev_dir_group.command("unset")
|
|
86
|
+
def dev_dir_unset():
|
|
87
|
+
"""Unset the development directory."""
|
|
88
|
+
from lamindb_setup import settings as settings_
|
|
89
|
+
|
|
90
|
+
settings_.dev_dir = None
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
settings.add_command(dev_dir_group)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# -----------------------------------------------------------------------------
|
|
97
|
+
# Legacy get/set (hidden, backward compatibility)
|
|
98
|
+
# -----------------------------------------------------------------------------
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@settings.command("set", hidden=True)
|
|
102
|
+
@click.argument(
|
|
103
|
+
"setting",
|
|
104
|
+
type=click.Choice(
|
|
105
|
+
["auto-connect", "private-django-api", "dev-dir"], case_sensitive=False
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
@click.argument("value") # No explicit type - let Click handle it
|
|
109
|
+
def set_legacy(setting: str, value: str):
|
|
110
|
+
"""Set a setting (legacy). Use lamin settings <name> set <value> instead."""
|
|
111
|
+
from lamindb_setup import settings as settings_
|
|
112
|
+
|
|
113
|
+
if setting == "auto-connect":
|
|
114
|
+
settings_.auto_connect = click.BOOL(value)
|
|
115
|
+
if setting == "private-django-api":
|
|
116
|
+
settings_.private_django_api = click.BOOL(value)
|
|
117
|
+
if setting == "dev-dir":
|
|
118
|
+
if value.lower() == "none":
|
|
119
|
+
value = None # type: ignore[assignment]
|
|
120
|
+
settings_.dev_dir = value
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
@settings.command("get", hidden=True)
|
|
124
|
+
@click.argument(
|
|
125
|
+
"setting",
|
|
126
|
+
type=click.Choice(
|
|
127
|
+
["auto-connect", "private-django-api", "space", "branch", "dev-dir"],
|
|
128
|
+
case_sensitive=False,
|
|
129
|
+
),
|
|
130
|
+
)
|
|
131
|
+
def get_legacy(setting: str):
|
|
132
|
+
"""Get a setting (legacy). Use lamin settings <name> get instead."""
|
|
133
|
+
from lamindb_setup import settings as settings_
|
|
134
|
+
|
|
135
|
+
if setting == "branch":
|
|
136
|
+
_, value = settings_._read_branch_idlike_name()
|
|
137
|
+
elif setting == "space":
|
|
138
|
+
_, value = settings_._read_space_idlike_name()
|
|
139
|
+
elif setting == "dev-dir":
|
|
140
|
+
value = settings_.dev_dir
|
|
141
|
+
if value is None:
|
|
142
|
+
value = "None"
|
|
143
|
+
else:
|
|
144
|
+
value = getattr(settings_, setting.replace("-", "_"))
|
|
145
|
+
click.echo(value)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# -----------------------------------------------------------------------------
|
|
149
|
+
# cache-dir (already uses lamin settings cache-dir get/set/clear)
|
|
150
|
+
# -----------------------------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
from lamin_cli._cache import cache
|
|
153
|
+
|
|
154
|
+
settings.add_command(cache, "cache-dir")
|
|
@@ -1,56 +1,56 @@
|
|
|
1
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
2
|
-
|
|
3
|
-
from django.db import OperationalError, ProgrammingError
|
|
4
|
-
from lamin_utils import logger
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def _count_instance_records() -> dict[str, int]:
|
|
8
|
-
"""Count all records across SQLRecord registries in parallel.
|
|
9
|
-
|
|
10
|
-
Returns:
|
|
11
|
-
Dictionary mapping table names (format: "app_label.ModelName") to their record counts.
|
|
12
|
-
|
|
13
|
-
Example:
|
|
14
|
-
>>> counts = _count_all_records()
|
|
15
|
-
>>> counts
|
|
16
|
-
{'lamindb.Artifact': 1523, 'lamindb.Collection': 42, 'bionty.Gene': 60000}
|
|
17
|
-
"""
|
|
18
|
-
# Import here to ensure that models are loaded
|
|
19
|
-
from django.apps import apps
|
|
20
|
-
from lamindb.models import SQLRecord
|
|
21
|
-
|
|
22
|
-
def _count_model(model):
|
|
23
|
-
"""Count records for a single model."""
|
|
24
|
-
table_name = f"{model._meta.app_label}.{model.__name__}"
|
|
25
|
-
try:
|
|
26
|
-
return (table_name, model.objects.count())
|
|
27
|
-
except (OperationalError, ProgrammingError) as e:
|
|
28
|
-
logger.warning(f"Could not count {table_name}: {e}")
|
|
29
|
-
return (table_name, 0)
|
|
30
|
-
|
|
31
|
-
models = [m for m in apps.get_models() if issubclass(m, SQLRecord)]
|
|
32
|
-
|
|
33
|
-
with ThreadPoolExecutor(max_workers=10) as executor:
|
|
34
|
-
results = executor.map(_count_model, models)
|
|
35
|
-
|
|
36
|
-
return dict(results)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def _compare_record_counts(
|
|
40
|
-
original: dict[str, int], clone: dict[str, int]
|
|
41
|
-
) -> dict[str, tuple[int, int]]:
|
|
42
|
-
"""Compare record counts and return mismatches."""
|
|
43
|
-
mismatches = {}
|
|
44
|
-
|
|
45
|
-
all_tables = set(original.keys()) | set(clone.keys())
|
|
46
|
-
|
|
47
|
-
for table in all_tables:
|
|
48
|
-
orig_count = original.get(table, 0)
|
|
49
|
-
clone_count = clone.get(table, 0)
|
|
50
|
-
|
|
51
|
-
# we allow a difference of 1 because of tracking
|
|
52
|
-
# new records during the cloning process
|
|
53
|
-
if abs(clone_count - orig_count) > 1:
|
|
54
|
-
mismatches[table] = (orig_count, clone_count)
|
|
55
|
-
|
|
56
|
-
return mismatches
|
|
1
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
2
|
+
|
|
3
|
+
from django.db import OperationalError, ProgrammingError
|
|
4
|
+
from lamin_utils import logger
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _count_instance_records() -> dict[str, int]:
|
|
8
|
+
"""Count all records across SQLRecord registries in parallel.
|
|
9
|
+
|
|
10
|
+
Returns:
|
|
11
|
+
Dictionary mapping table names (format: "app_label.ModelName") to their record counts.
|
|
12
|
+
|
|
13
|
+
Example:
|
|
14
|
+
>>> counts = _count_all_records()
|
|
15
|
+
>>> counts
|
|
16
|
+
{'lamindb.Artifact': 1523, 'lamindb.Collection': 42, 'bionty.Gene': 60000}
|
|
17
|
+
"""
|
|
18
|
+
# Import here to ensure that models are loaded
|
|
19
|
+
from django.apps import apps
|
|
20
|
+
from lamindb.models import SQLRecord
|
|
21
|
+
|
|
22
|
+
def _count_model(model):
|
|
23
|
+
"""Count records for a single model."""
|
|
24
|
+
table_name = f"{model._meta.app_label}.{model.__name__}"
|
|
25
|
+
try:
|
|
26
|
+
return (table_name, model.objects.count())
|
|
27
|
+
except (OperationalError, ProgrammingError) as e:
|
|
28
|
+
logger.warning(f"Could not count {table_name}: {e}")
|
|
29
|
+
return (table_name, 0)
|
|
30
|
+
|
|
31
|
+
models = [m for m in apps.get_models() if issubclass(m, SQLRecord)]
|
|
32
|
+
|
|
33
|
+
with ThreadPoolExecutor(max_workers=10) as executor:
|
|
34
|
+
results = executor.map(_count_model, models)
|
|
35
|
+
|
|
36
|
+
return dict(results)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _compare_record_counts(
|
|
40
|
+
original: dict[str, int], clone: dict[str, int]
|
|
41
|
+
) -> dict[str, tuple[int, int]]:
|
|
42
|
+
"""Compare record counts and return mismatches."""
|
|
43
|
+
mismatches = {}
|
|
44
|
+
|
|
45
|
+
all_tables = set(original.keys()) | set(clone.keys())
|
|
46
|
+
|
|
47
|
+
for table in all_tables:
|
|
48
|
+
orig_count = original.get(table, 0)
|
|
49
|
+
clone_count = clone.get(table, 0)
|
|
50
|
+
|
|
51
|
+
# we allow a difference of 1 because of tracking
|
|
52
|
+
# new records during the cloning process
|
|
53
|
+
if abs(clone_count - orig_count) > 1:
|
|
54
|
+
mismatches[table] = (orig_count, clone_count)
|
|
55
|
+
|
|
56
|
+
return mismatches
|
|
@@ -1,51 +1,53 @@
|
|
|
1
|
-
import argparse
|
|
2
|
-
import json
|
|
3
|
-
import sys
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
import lamindb_setup as ln_setup
|
|
7
|
-
|
|
8
|
-
if __name__ == "__main__":
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
parser.
|
|
14
|
-
parser.add_argument("--
|
|
15
|
-
parser.add_argument("--
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
connections
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import lamindb_setup as ln_setup
|
|
7
|
+
|
|
8
|
+
if __name__ == "__main__":
|
|
9
|
+
from lamindb_setup.io import import_db
|
|
10
|
+
|
|
11
|
+
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
12
|
+
|
|
13
|
+
parser = argparse.ArgumentParser()
|
|
14
|
+
parser.add_argument("--instance-name", required=True)
|
|
15
|
+
parser.add_argument("--export-dir", required=True)
|
|
16
|
+
parser.add_argument("--modules", required=True)
|
|
17
|
+
parser.add_argument("--original-counts", required=True)
|
|
18
|
+
args = parser.parse_args()
|
|
19
|
+
|
|
20
|
+
instance_name = args.instance_name
|
|
21
|
+
export_dir = args.export_dir
|
|
22
|
+
modules_without_lamindb = {m for m in args.modules.split(",") if m}
|
|
23
|
+
modules_complete = modules_without_lamindb.copy()
|
|
24
|
+
modules_complete.add("lamindb")
|
|
25
|
+
|
|
26
|
+
ln_setup.init(
|
|
27
|
+
storage=f"{instance_name}-clone", modules=f"{','.join(modules_without_lamindb)}"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
import_db(
|
|
31
|
+
module_names=list(modules_complete), input_dir=export_dir, if_exists="replace"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
from django.db import connection
|
|
35
|
+
|
|
36
|
+
with connection.cursor() as cursor:
|
|
37
|
+
cursor.execute("PRAGMA wal_checkpoint(FULL)")
|
|
38
|
+
|
|
39
|
+
from lamin_cli.clone._clone_verification import (
|
|
40
|
+
_compare_record_counts,
|
|
41
|
+
_count_instance_records,
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
clone_counts = _count_instance_records()
|
|
45
|
+
original_counts = json.loads(args.original_counts)
|
|
46
|
+
mismatches = _compare_record_counts(original_counts, clone_counts)
|
|
47
|
+
if mismatches:
|
|
48
|
+
print(json.dumps(mismatches), file=sys.stderr)
|
|
49
|
+
sys.exit(1)
|
|
50
|
+
|
|
51
|
+
from django.db import connections
|
|
52
|
+
|
|
53
|
+
connections.close_all()
|