lamindb_setup 1.15.0__py3-none-any.whl → 1.15.2__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Migration management
 
 """
 
-__version__ = "1.15.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.15.2"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 import warnings

lamindb_setup/_connect_instance.py

@@ -12,7 +12,6 @@ from lamin_utils import logger
 from ._check_setup import (
     _check_instance_setup,
     _get_current_instance_settings,
-    find_module_candidates,
 )
 from ._disconnect import disconnect
 from ._init_instance import load_from_isettings

lamindb_setup/_set_managed_storage.py

@@ -41,7 +41,7 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             "use a tuple of (local_root, host) instead"
         )
 
-    # here the storage is registered in the hub
+    # here the storage location is registered in the hub
     # hub_record_status="hub-record-created" if a new record is created
     # "hub-record-retrieved" if the storage is in the hub already
     ssettings, hub_record_status = init_storage(
@@ -65,5 +65,13 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             delete_storage_record(ssettings)
         raise e
 
-    settings.instance._storage = ssettings
-    settings.storage._set_fs_kwargs(**fs_kwargs)
+    if ssettings._instance_id != settings.instance._id:
+        logger.warning(
+            f"registered storage location {root} as read-only for this instance (it's written by instance with uid: {ssettings.instance_uid})"
+        )
+        logger.warning(
+            f"did *not* switch default storage location, it's still: {settings.storage.root_as_str}"
+        )
+    else:
+        settings.instance._storage = ssettings
+        settings.storage._set_fs_kwargs(**fs_kwargs)

lamindb_setup/core/__init__.py

@@ -23,7 +23,12 @@ Storage
 """
 
 from . import django, upath
-from ._clone import connect_local_sqlite, init_local_sqlite
+from ._clone import (
+    connect_local_sqlite,
+    connect_remote_sqlite,
+    init_local_sqlite,
+    upload_sqlite_clone,
+)
 from ._deprecated import deprecated  # documented in lamindb.base
 from ._docs import doc_args  # documented in lamindb.base
 from ._settings import SetupSettings

lamindb_setup/core/_aws_options.py

@@ -20,6 +20,7 @@ lamin_env = os.getenv("LAMIN_ENV")
 if lamin_env is None or lamin_env == "prod":
     HOSTED_BUCKETS = tuple([f"s3://lamin-{region}" for region in HOSTED_REGIONS])
 else:
+    logger.warning("loaded LAMIN_ENV: staging")
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 

lamindb_setup/core/_clone.py

@@ -5,14 +5,19 @@
 
    init_local_sqlite
    connect_local_sqlite
+   upload_sqlite_clone
 """
 
+import gzip
 import os
+import shutil
+from pathlib import Path
 
 from lamindb_setup.core._settings_instance import InstanceSettings
 from lamindb_setup.core._settings_load import load_instance_settings
 from lamindb_setup.core._settings_store import instance_settings_file
 from lamindb_setup.core.django import reset_django
+from lamindb_setup.core.upath import create_path
 
 
 def init_local_sqlite(
@@ -53,28 +58,31 @@ def init_local_sqlite(
         if copy_suffix is not None
         else ln_setup.settings.instance.name
     )
-    isettings = InstanceSettings(
-        id=ln_setup.settings.instance._id,
-        owner=ln_setup.settings.instance.owner,  # type: ignore
-        name=name,
-        storage=ln_setup.settings.storage,
-        db=None,
-        modules=",".join(ln_setup.settings.instance.modules),
-        is_on_hub=False,
+    isettings = ln_setup._connect_instance._connect_instance(
+        owner=ln_setup.settings.instance.owner, name=name
     )
-
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
     isettings._persist(write_to_disk=True)
 
     if not isettings._sqlite_file_local.exists():
         # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
-        # Without this reset, the if not settings.configured check in setup_django() would skip reconfiguration,
+        # Without this reset, the `if not settings.configured`` check in `setup_django()` would skip reconfiguration,
         # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
         reset_django()
         isettings._init_db()
 
 
-def connect_local_sqlite(instance: str) -> None:
-    """Load a SQLite instance of which a remote hub Postgres instance exists.
+def connect_local_sqlite(
+    instance: str,
+) -> None:
+    """Load a locally stored SQLite instance of which a remote hub Postgres instance exists.
 
     This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
     The clone must first be created via `init_local_sqlite()`.
@@ -86,8 +94,81 @@ def connect_local_sqlite(instance: str) -> None:
     settings_file = instance_settings_file(name=name, owner=owner)
 
     if not settings_file.exists():
-        raise ValueError("SQLite clone not found. Run init_local_sqlite() first.")
+        raise ValueError(
+            "SQLite clone not found."
+            " Run `init_local_sqlite()` to create a local copy or connect to a remote copy using `connect_remote_sqlite`."
+        )
 
     isettings = load_instance_settings(settings_file)
     isettings._persist(write_to_disk=False)
-    isettings._load_db()
+
+    # Using `setup_django` instead of `_load_db` to not ping AWS RDS
+    from lamindb_setup._check_setup import disable_auto_connect
+
+    from .django import setup_django
+
+    disable_auto_connect(setup_django)(isettings)
+
+
+def connect_remote_sqlite(instance: str, *, copy_suffix: str | None = None) -> None:
+    """Load an existing SQLite copy of a hub instance.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+        copy_suffix: Optional suffix of the local clone.
+    """
+    import lamindb_setup as ln_setup
+
+    owner, name = instance.split("/")
+
+    # Step 1: Create the settings file
+    isettings = ln_setup._connect_instance._connect_instance(owner=owner, name=name)
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    isettings._db_permissions = "read"
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
+    isettings._persist(write_to_disk=True)
+
+    connect_local_sqlite(instance=instance + (copy_suffix or ""))
+
+
+def upload_sqlite_clone(
+    local_sqlite_path: Path | str | None = None, compress: bool = True
+) -> None:
+    """Uploads the SQLite clone to the default storage.
+
+    Args:
+        local_sqlite_path: Path to the SQLite file.
+            Defaults to the local storage path if not specified.
+        compress: Whether to compress the database with gzip before uploading.
+    """
+    import lamindb_setup as ln_setup
+
+    if local_sqlite_path is None:
+        local_sqlite_path = ln_setup.settings.instance._sqlite_file_local
+    else:
+        local_sqlite_path = Path(local_sqlite_path)
+
+    if not local_sqlite_path.exists():
+        raise FileNotFoundError(f"Database not found at {local_sqlite_path}")
+
+    cloud_db_path = ln_setup.settings.instance._sqlite_file
+
+    if compress:
+        temp_gz_path = local_sqlite_path.with_suffix(".db.gz")
+        with (
+            open(local_sqlite_path, "rb") as f_in,
+            gzip.open(temp_gz_path, "wb") as f_out,
+        ):
+            shutil.copyfileobj(f_in, f_out)
+        cloud_destination = create_path(f"{cloud_db_path}.gz")
+        cloud_destination.upload_from(temp_gz_path, print_progress=True)
+        temp_gz_path.unlink()
+    else:
+        cloud_destination = create_path(cloud_db_path)
+        cloud_destination.upload_from(local_sqlite_path, print_progress=True)

lamindb_setup/core/_hub_client.py

@@ -96,9 +96,12 @@ def connect_hub(
         transports.append(
             RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
         )
+    # this overwrites transports of existing httpx clients
+    # if proxies are set, the default transports that were created on clients init
+    # will be used, irrespective of these re-settings
     client.auth._http_client._transport = transports[0]
     client.postgrest.session._transport = transports[1]
     # POST is not retryable by default, but for our functions it should be safe to retry
@@ -116,7 +119,7 @@ def connect_hub(
                 "POST",
             ],
         ),
-        transport=httpx.HTTPTransport(verify=True, http2=True),
+        transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
     )
     return client
 
@@ -246,9 +249,13 @@ def httpx_client():
         else:
             transport = RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
-            client = httpx.Client(transport=transport)
+            # first we create a client to build the proxy map from the env variables
+            # if proxies are set, the default transports will be used
+            # otherwise the RetryTransport object that we assign below
+            client = httpx.Client(trust_env=True)
+            client._transport = transport
         yield client
     finally:
         if client is not None:

lamindb_setup/core/_settings.py

@@ -320,8 +320,7 @@ class SetupSettings:
     def paths(self) -> type[SetupPaths]:
         """Convert cloud paths to lamindb local paths.
 
-        Use `settings.paths.cloud_to_local_no_update`
-        or `settings.paths.cloud_to_local`.
+        Use `settings.paths.cloud_to_local_no_update` or `settings.paths.cloud_to_local`.
         """
         return SetupPaths
 

lamindb_setup/core/_settings_instance.py

@@ -54,8 +54,7 @@ def is_local_db_url(db_url: str) -> bool:
 
 
 def check_is_instance_remote(root: UPathStr, db: str | None) -> bool:
-    # returns True for cloud SQLite
-    # and remote postgres
+    # returns True for cloud SQLite and remote postgres
     root_str = str(root)
     if not root_str.startswith("create-s3") and get_storage_type(root_str) == "local":
         return False
@@ -83,7 +82,8 @@ class InstanceSettings:
         schema_id: UUID | None = None,
         fine_grained_access: bool = False,
         db_permissions: str | None = None,
-        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite
+        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite,
+        _is_clone: bool = False,
     ):
         from ._hub_utils import validate_db_arg
 
@@ -109,6 +109,7 @@ class InstanceSettings:
         self._db_permissions = db_permissions
         # if None then settings.user is used
         self._locker_user = _locker_user
+        self._is_clone = _is_clone
 
     def __repr__(self):
         """Rich string representation."""
@@ -434,7 +435,7 @@ class InstanceSettings:
 
     def _update_cloud_sqlite_file(self, unlock_cloud_sqlite: bool = True) -> None:
         """Upload the local sqlite file to the cloud file."""
-        if self._is_cloud_sqlite:
+        if self._is_cloud_sqlite and not self._is_clone:
             sqlite_file = self._sqlite_file
             logger.warning(
                 f"updating{' & unlocking' if unlock_cloud_sqlite else ''} cloud SQLite "
@@ -602,6 +603,14 @@ class InstanceSettings:
         disable_auto_connect(setup_django)(self, init=True)
 
     def _load_db(self) -> tuple[bool, str]:
+        """Load the database connection.
+
+        For cloud SQLite instances, downloads the database file from cloud storage.
+        For all instances, initializes Django ORM with the database connection.
+
+        Returns:
+            Tuple of (success: bool, error_message: str). Returns (True, "") on success.
+        """
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -615,8 +624,8 @@ class InstanceSettings:
                 return False, f"SQLite file {self._sqlite_file} does not exist"
         # we need the local sqlite to setup django
         self._update_local_sqlite_file()
-        # setting up django also performs a check for migrations & prints them
-        # as warnings
+
+        # setting up django also performs a check for migrations & prints them as warnings
         # this should fail, e.g., if the db is not reachable
         from lamindb_setup._check_setup import disable_auto_connect
 

lamindb_setup/core/_settings_load.py

@@ -69,8 +69,7 @@ def load_or_create_user_settings(api_key: str | None = None) -> UserSettings:
     """Return current user settings.
 
     Args:
-        api_key: if provided and there is no current user,
-            perform login and return the user settings.
+        api_key: if provided and there is no current user, perform login and return the user settings.
     """
     current_user_settings = current_user_settings_file()
     if not current_user_settings.exists():
@@ -125,6 +124,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         schema_id=None if store.schema_id in {None, "null"} else UUID(store.schema_id),
         fine_grained_access=store.fine_grained_access,
         db_permissions=_null_to_value(store.db_permissions),
+        _is_clone=store.is_clone,
     )
 
 

lamindb_setup/core/_settings_save.py

@@ -63,6 +63,7 @@ def save_settings(
                         "schema_id",
                         "fine_grained_access",
                         "db_permissions",
+                        "is_clone",
                     }:
                         settings_key = f"_{store_key.rstrip('_')}"
                     else:

lamindb_setup/core/_settings_storage.py

@@ -12,17 +12,14 @@ from lamin_utils import logger
 from lamindb_setup.errors import StorageAlreadyManaged
 
 from ._aws_options import (
-    HOSTED_REGIONS,
     LAMIN_ENDPOINTS,
     get_aws_options_manager,
 )
-from ._aws_storage import find_closest_aws_region
 from ._deprecated import deprecated
 from .hashing import hash_and_encode_as_b62
 from .upath import (
     LocalPathClasses,
     UPath,
-    _split_path_query,
     create_path,
     get_storage_region,
 )
@@ -58,12 +55,40 @@ def get_storage_type(root_as_str: str) -> StorageType:
     return convert.get(protocol, protocol)  # type: ignore
 
 
+def sanitize_root_user_input(root: UPathStr) -> UPath:
+    """Format a root path string."""
+    root_upath = root if isinstance(root, UPath) else UPath(root)
+    root_upath = root_upath.expanduser()
+    if isinstance(root_upath, LocalPathClasses):  # local paths
+        try:
+            (root_upath / ".lamindb").mkdir(parents=True, exist_ok=True)
+            root_upath = root_upath.resolve()
+        except Exception:
+            logger.warning(f"unable to create .lamindb/ folder in {root_upath}")
+    return root_upath
+
+
+def convert_sanitized_root_path_to_str(root_upath: UPath) -> str:
+    # embed endpoint_url into path string for storing and displaying
+    if root_upath.protocol == "s3":
+        endpoint_url = root_upath.storage_options.get("endpoint_url", None)
+        # LAMIN_ENDPOINTS include None
+        if endpoint_url not in LAMIN_ENDPOINTS:
+            return f"s3://{root_upath.path.rstrip('/')}?endpoint_url={endpoint_url}"
+    return root_upath.as_posix().rstrip("/")
+
+
+def convert_root_path_to_str(root: UPathStr) -> str:
+    """Format a root path string."""
+    sanitized_root_upath = sanitize_root_user_input(root)
+    return convert_sanitized_root_path_to_str(sanitized_root_upath)
+
+
 def mark_storage_root(
     root: UPathStr, uid: str, instance_id: UUID, instance_slug: str
 ) -> Literal["__marked__"] | str:
     # we need a file in folder-like storage locations on S3 to avoid
-    # permission errors from leveraging s3fs on an empty hosted storage location
-    # (path.fs.find raises a PermissionError)
+    # permission errors from leveraging s3fs on an empty hosted storage location (path.fs.find raises a PermissionError)
     # we also need it in case a storage location is ambiguous because a server / local environment
     # doesn't have a globally unique identifier, then we screen for this file to map the
     # path on a storage location in the registry
@@ -214,15 +239,7 @@ class StorageSettings:
     ):
         self._uid = uid
         self._uuid_ = uuid
-        self._root_init = UPath(root).expanduser()
-        if isinstance(self._root_init, LocalPathClasses):  # local paths
-            try:
-                (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
-                self._root_init = self._root_init.resolve()
-            except Exception:
-                logger.warning(
-                    f"unable to create .lamindb/ folder in {self._root_init}"
-                )
+        self._root_init: UPath = sanitize_root_user_input(root)
         self._root = None
         self._instance_id = instance_id
         # we don't yet infer region here to make init fast
@@ -337,13 +354,7 @@ class StorageSettings:
     @property
     def root_as_str(self) -> str:
         """Formatted root string."""
-        # embed endpoint_url into path string for storing and displaying
-        if self._root_init.protocol == "s3":
-            endpoint_url = self._root_init.storage_options.get("endpoint_url", None)
-            # LAMIN_ENDPOINTS include None
-            if endpoint_url not in LAMIN_ENDPOINTS:
-                return f"s3://{self._root_init.path.rstrip('/')}?endpoint_url={endpoint_url}"
-        return self._root_init.as_posix().rstrip("/")
+        return convert_sanitized_root_path_to_str(self._root_init)
 
     @property
     def cache_dir(

lamindb_setup/core/_settings_store.py

@@ -67,8 +67,8 @@ class InstanceSettingsStore(BaseSettings):
     owner: str
     name: str
     storage_root: str
-    storage_region: str | None  # take old type annotations here because pydantic
-    db: str | None  # doesn't like new types on 3.9 even with future annotations
+    storage_region: str | None
+    db: str | None
     schema_str: str | None
     schema_id: str | None = None
     fine_grained_access: bool = False
@@ -76,6 +76,7 @@ class InstanceSettingsStore(BaseSettings):
     id: str
     git_repo: str | None
     keep_artifacts_local: bool | None
+    is_clone: bool = False
     model_config = SettingsConfigDict(env_prefix="lamindb_instance_", env_file=".env")
 
 

lamindb_setup/core/upath.py

@@ -908,12 +908,9 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str | None, str | None]:
     elif "blob_id" in stat:
         hash = b16_to_b64(stat["blob_id"])
         hash_type = "sha1"
-    # s3
-    # StorageClass is checked to be sure that it is indeed s3
-    # because http also has ETag
     elif "ETag" in stat:
         etag = stat["ETag"]
-        if "mimetype" in stat:
+        if "mimetype" in stat or ("url" in stat and stat["url"].startswith("http")):
             # http
             hash = hash_string(etag.strip('"'))
             hash_type = "md5-etag"

lamindb_setup/io.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+import io
 import json
 import warnings
+from concurrent.futures import ProcessPoolExecutor, as_completed
 from importlib import import_module
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+import numpy as np
 import pandas as pd
 from django.db import models, transaction
 from rich.progress import Progress
@@ -43,21 +46,101 @@ def _get_registries(module_name: str) -> list[str]:
     ]
 
 
-def _export_registry_to_parquet(registry: type[models.Model], directory: Path) -> None:
-    """Export a single registry table to parquet."""
+def _export_full_table(
+    registry_info: tuple[str, str, str | None],
+    directory: Path,
+    chunk_size: int,
+) -> list[tuple[str, Path]] | str:
+    """Export a registry table to parquet.
+
+    For PostgreSQL, uses COPY TO which streams the table directly to CSV format,
+    bypassing query planner overhead and row-by-row conversion (10-50x faster than SELECT).
+
+    For SQLite with large tables, reads in chunks to avoid memory issues when tables exceed available RAM.
+
+    Args:
+        registry_info: Tuple of (module_name, model_name, field_name) where field_name
+            is None for regular tables or the field name for M2M link tables.
+        directory: Output directory for parquet files.
+        chunk_size: Maximum rows per chunk for SQLite large tables.
+
+    Returns:
+        String identifier for single-file exports, or list of (table_name, chunk_path) tuples for chunked exports that need merging.
+    """
+    from django.db import connection
+
     import lamindb_setup as ln_setup
 
+    module_name, model_name, field_name = registry_info
+    schema_module = import_module(module_name)
+    registry = getattr(schema_module, model_name)
+
+    if field_name:
+        registry = getattr(registry, field_name).through
+
     table_name = registry._meta.db_table
-    with warnings.catch_warnings():
-        warnings.filterwarnings("ignore", message="Skipped unsupported reflection")
-        df = pd.read_sql_table(table_name, ln_setup.settings.instance.db)
-    df.to_parquet(directory / f"{table_name}.parquet", compression=None)
+
+    try:
+        if ln_setup.settings.instance.dialect == "postgresql":
+            buffer = io.StringIO()
+            with connection.cursor() as cursor:
+                cursor.copy_expert(
+                    f'COPY "{table_name}" TO STDOUT WITH (FORMAT CSV, HEADER TRUE)',
+                    buffer,
+                )
+            buffer.seek(0)
+            df = pd.read_csv(buffer)
+            df.to_parquet(directory / f"{table_name}.parquet", compression=None)
+            return (
+                f"{module_name}.{model_name}.{field_name}"
+                if field_name
+                else f"{module_name}.{model_name}"
+            )
+        else:
+            with warnings.catch_warnings():
+                warnings.filterwarnings(
+                    "ignore", message="Skipped unsupported reflection"
+                )
+                row_count = pd.read_sql(
+                    f"SELECT COUNT(*) as count FROM {table_name}",
+                    ln_setup.settings.instance.db,
+                ).iloc[0]["count"]
+
+                if row_count > chunk_size:
+                    chunk_files = []
+                    num_chunks = (row_count + chunk_size - 1) // chunk_size
+                    for chunk_id in range(num_chunks):
+                        offset = chunk_id * chunk_size
+                        df = pd.read_sql(
+                            f"SELECT * FROM {table_name} LIMIT {chunk_size} OFFSET {offset}",
+                            ln_setup.settings.instance.db,
+                        )
+                        chunk_file = (
+                            directory / f"{table_name}_chunk_{chunk_id}.parquet"
+                        )
+                        df.to_parquet(chunk_file, compression=None)
+                        chunk_files.append((table_name, chunk_file))
+                    return chunk_files
+                else:
+                    df = pd.read_sql_table(table_name, ln_setup.settings.instance.db)
+                    df.to_parquet(directory / f"{table_name}.parquet", compression=None)
+                    return (
+                        f"{module_name}.{model_name}.{field_name}"
+                        if field_name
+                        else f"{module_name}.{model_name}"
+                    )
+    except (ValueError, pd.errors.DatabaseError):
+        raise ValueError(
+            f"Table '{table_name}' was not found. The instance might need to be migrated."
+        ) from None
 
 
 def export_db(
     module_names: Sequence[str] | None = None,
     *,
     output_dir: str | Path = "./lamindb_export/",
+    max_workers: int = 8,
+    chunk_size: int = 500_000,
 ) -> None:
     """Export registry tables and many-to-many link tables to parquet files.
 
@@ -67,26 +150,64 @@ def export_db(
         module_names: Module names to export (e.g., ["lamindb", "bionty", "wetlab"]).
             Defaults to "lamindb" if not provided.
         output_dir: Directory path for exported parquet files.
+        max_workers: Number of parallel processes.
+        chunk_size: Number of rows per chunk for large tables.
     """
     directory = Path(output_dir)
     directory.mkdir(parents=True, exist_ok=True)
 
     module_names = module_names or ["lamindb"]
     modules = {name: _get_registries(name) for name in module_names}
-    total_models = sum(len(models) for models in modules.values())
+
+    tasks = []
+    for module_name, model_names in modules.items():
+        schema_module = import_module(module_name)
+        for model_name in model_names:
+            registry = getattr(schema_module, model_name)
+            tasks.append((module_name, model_name, None))
+            for field in registry._meta.many_to_many:
+                tasks.append((module_name, model_name, field.name))
+
+    chunk_files_by_table: dict[str, list[Path]] = {}
 
     with Progress() as progress:
-        task = progress.add_task("Exporting", total=total_models)
-        for module_name, model_names in modules.items():
-            schema_module = import_module(module_name)
-            for model_name in model_names:
-                progress.update(task, description=f"[cyan]{module_name}.{model_name}")
-                registry = getattr(schema_module, model_name)
-                _export_registry_to_parquet(registry, directory)
-                for field in registry._meta.many_to_many:
-                    link_orm = getattr(registry, field.name).through
-                    _export_registry_to_parquet(link_orm, directory)
-                progress.advance(task)
+        task_id = progress.add_task("Exporting", total=len(tasks))
+
+        import multiprocessing
+
+        mp_context = multiprocessing.get_context("spawn")
+
+        with ProcessPoolExecutor(
+            max_workers=max_workers, mp_context=mp_context
+        ) as executor:
+            futures = {
+                executor.submit(_export_full_table, task, directory, chunk_size): task
+                for task in tasks
+            }
+
+            for future in as_completed(futures):
+                result = future.result()
+                if isinstance(result, list):
+                    for table_name, chunk_file in result:
+                        chunk_files_by_table.setdefault(table_name, []).append(
+                            chunk_file
+                        )
+                progress.advance(task_id)
+
+    for table_name, chunk_files in chunk_files_by_table.items():
+        merged_df = pd.concat([pd.read_parquet(f) for f in sorted(chunk_files)])
+        merged_df.to_parquet(directory / f"{table_name}.parquet", compression=None)
+        for chunk_file in chunk_files:
+            chunk_file.unlink()
+
+
+def _serialize_value(val):
+    """Convert value to JSON string if it's a dict, list, or numpy array, otherwise return as-is."""
+    if isinstance(val, (dict, list, np.ndarray)):
+        return json.dumps(
+            val, default=lambda o: o.tolist() if isinstance(o, np.ndarray) else None
+        )
+    return val
 
 
 def _import_registry(
@@ -96,8 +217,14 @@ def _import_registry(
 ) -> None:
     """Import a single registry table from parquet.
 
-    Uses raw SQL export instead of django to later circumvent FK constraints.
+    For PostgreSQL, uses COPY FROM which bypasses SQL parsing and writes directly to
+    table pages (20-50x faster than multi-row INSERTs).
+
+    For SQLite, uses multi-row INSERTs with dynamic chunking to stay under the 999
+    variable limit (2-5x faster than single-row INSERTs).
     """
+    from django.db import connection
+
     table_name = registry._meta.db_table
     parquet_file = directory / f"{table_name}.parquet"
 
@@ -113,13 +240,60 @@ def _import_registry(
 
     for col in df.columns:
         if df[col].dtype == "object":
-            df[col] = df[col].apply(
-                lambda x: json.dumps(x) if isinstance(x, (dict, list)) else x
-            )
+            mask = df[col].apply(lambda x: isinstance(x, (dict, list, np.ndarray)))
+            if mask.any():
+                df.loc[mask, col] = df.loc[mask, col].map(_serialize_value)
+
+    if if_exists == "append":
+        # Fill NULL values in NOT NULL columns to handle schema mismatches between postgres source and SQLite target
+        # This allows importing data where fields were nullable
+        for field in registry._meta.fields:
+            if field.column in df.columns and not field.null:
+                df[field.column] = df[field.column].fillna("")
+
+    if df.empty:
+        return
 
-    from django.db import connection
+    if if_exists == "append":
+        # Clear existing data before import
+        # When appending we would run into duplicate errors because of existing values like branches etc
+        with connection.cursor() as cursor:
+            cursor.execute(f'DELETE FROM "{table_name}"')
+
+    if connection.vendor == "postgresql":
+        columns = df.columns.tolist()
+        column_names = ", ".join(f'"{col}"' for col in columns)
+
+        buffer = io.StringIO()
+        df.to_csv(buffer, index=False, header=False, sep="\t", na_rep="\\N")
+        buffer.seek(0)
 
-    df.to_sql(table_name, connection.connection, if_exists=if_exists, index=False)
+        with connection.cursor() as cursor:
+            if if_exists == "replace":
+                cursor.execute(f'DELETE FROM "{table_name}"')
+            elif if_exists == "fail":
+                cursor.execute(f'SELECT COUNT(*) FROM "{table_name}"')
+                if cursor.fetchone()[0] > 0:
+                    raise ValueError(f"Table {table_name} already contains data")
+
+            cursor.copy_expert(
+                f"COPY \"{table_name}\" ({column_names}) FROM STDIN WITH (FORMAT CSV, DELIMITER E'\\t', NULL '\\N')",
+                buffer,
+            )
+    else:
+        num_cols = len(df.columns)
+        max_vars = 900  # SQLite has a limit of 999 variables per statement
+        chunksize = max(1, max_vars // num_cols)
+
+        # Always use append mode since we set up the tables from a fresh instance
+        df.to_sql(
+            table_name,
+            connection.connection,
+            if_exists=if_exists,
+            index=False,
+            method="multi",
+            chunksize=chunksize,
+        )
 
 
 def import_db(
@@ -137,6 +311,9 @@ def import_db(
         input_dir: Directory containing parquet files to import.
         module_names: Module names to import (e.g., ["lamindb", "bionty", "wetlab"]).
         if_exists: How to behave if table exists: 'fail', 'replace', or 'append'.
+            If set to 'replace', existing data is deleted and new data is imported. PKs and indices are not guaranteed to be preserved which can lead to write errors.
+            If set to 'append', new data is added to existing data without clearing the table. PKs and indices are preserved but database size will greatly increase.
+            If set to 'fail', raises an error if the table contains any data.
     """
     from django.db import connection
 
@@ -157,38 +334,47 @@ def import_db(
     modules = {name: _get_registries(name) for name in module_names}
     total_models = sum(len(models) for models in modules.values())
 
-    # Disable FK constraints to allow insertion in arbitrary order
-    if ln_setup.settings.instance.dialect == "sqlite":
+    is_sqlite = ln_setup.settings.instance.dialect == "sqlite"
+
+    try:
         with connection.cursor() as cursor:
             if ln_setup.settings.instance.dialect == "postgresql":
                 cursor.execute("SET session_replication_role = 'replica'")
-            elif ln_setup.settings.instance.dialect == "sqlite":
+            elif is_sqlite:
                 cursor.execute("PRAGMA foreign_keys = OFF")
-
-    with transaction.atomic():
-        if ln_setup.settings.instance.dialect == "postgresql":
-            with connection.cursor() as cursor:
-                cursor.execute("SET CONSTRAINTS ALL DEFERRED")
-
-        with Progress() as progress:
-            task = progress.add_task("Importing", total=total_models)
-            for module_name, model_names in modules.items():
-                schema_module = import_module(module_name)
-                for model_name in model_names:
-                    progress.update(
-                        task, description=f"[cyan]{module_name}.{model_name}"
-                    )
-                    registry = getattr(schema_module, model_name)
-                    _import_registry(registry, directory, if_exists=if_exists)
-                    for field in registry._meta.many_to_many:
-                        link_orm = getattr(registry, field.name).through
-                        _import_registry(link_orm, directory, if_exists=if_exists)
-                    progress.advance(task)
-
-    # Re-enable FK constraints again
-    if ln_setup.settings.instance.dialect == "sqlite":
+                # Disables fsync - OS buffers writes to disk, 10-50x faster but can corrupt DB on crash
+                cursor.execute("PRAGMA synchronous = OFF")
+                # Keeps rollback journal in RAM - 2-5x faster but cannot rollback on crash
+                cursor.execute("PRAGMA journal_mode = MEMORY")
+                # 64MB page cache for better performance on large imports
+                cursor.execute("PRAGMA cache_size = -64000")
+
+        with transaction.atomic():
+            if ln_setup.settings.instance.dialect == "postgresql":
+                with connection.cursor() as cursor:
+                    cursor.execute("SET CONSTRAINTS ALL DEFERRED")
+
+            with Progress() as progress:
+                task = progress.add_task("Importing", total=total_models)
+                for module_name, model_names in modules.items():
+                    schema_module = import_module(module_name)
+                    for model_name in model_names:
+                        progress.update(
+                            task, description=f"[cyan]{module_name}.{model_name}"
+                        )
+                        registry = getattr(schema_module, model_name)
+                        _import_registry(registry, directory, if_exists=if_exists)
+                        for field in registry._meta.many_to_many:
+                            link_orm = getattr(registry, field.name).through
+                            _import_registry(link_orm, directory, if_exists=if_exists)
+                        progress.advance(task)
+    finally:
         with connection.cursor() as cursor:
             if ln_setup.settings.instance.dialect == "postgresql":
                 cursor.execute("SET session_replication_role = 'origin'")
-            elif ln_setup.settings.instance.dialect == "sqlite":
+            elif is_sqlite:
+                cursor.execute("PRAGMA synchronous = FULL")
+                cursor.execute("PRAGMA journal_mode = DELETE")
                 cursor.execute("PRAGMA foreign_keys = ON")
+                # Reclaim space from DELETEs
+                cursor.execute("VACUUM")

{lamindb_setup-1.15.0.dist-info → lamindb_setup-1.15.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.15.0
+Version: 1.15.2
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.15.0.dist-info → lamindb_setup-1.15.2.dist-info}/RECORD

@@ -1,8 +1,8 @@
-lamindb_setup/__init__.py,sha256=jFfiLt0yIvOHoum_g8DmraECRdU-EOd-iSmAKTZyG3c,3215
+lamindb_setup/__init__.py,sha256=T0CRWMhViQuKtrYM_foGZnApH3-uh0ta8YyS7rd9Fj0,3215
 lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=ToKMxsUq8dQBQh8baOrNVlSb1iC8h4zTg5dV8wMu0W4,6760
-lamindb_setup/_connect_instance.py,sha256=Yp24MMAnypKE14fMzOYCE8oqW0EJTiRHrBuugubfkBM,17816
+lamindb_setup/_connect_instance.py,sha256=3dsaZ7LGzJYtOWDbi2RkiVJIJqdxy43suNjQ-6C96_U,17788
 lamindb_setup/_delete.py,sha256=KS3r-xGFuDmAbzPUy-9JR-YnPShYdaHjDRQrAmXQ0qM,5863
 lamindb_setup/_disconnect.py,sha256=FT8EpCm5XXDdhDH7QtAnkO3KPatq2HqT9VXGNjgJDbk,1232
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
@@ -12,30 +12,30 @@ lamindb_setup/_migrate.py,sha256=SN8uphuQX-8XShH5odLyzV8-eyXATDxB5hWoxwxmgBU,112
 lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=af1Es7qFKGPRdNmk48384HiB2r-cDTdBPu0wB9qrga4,15526
-lamindb_setup/_set_managed_storage.py,sha256=y5YQASsWNYVWUYeLgh3N2YBETYP7mBtbpxe3X_Vgb5I,2699
+lamindb_setup/_set_managed_storage.py,sha256=xQe5DXCRiQ5VseAjVC2Bki0wB0n0tSTchvVKSx9I6eo,3094
 lamindb_setup/_setup_user.py,sha256=ojq7UP2Aia8GTCr6m8fylFx9VSuvGu0HmvIJ8RzymE0,6108
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/errors.py,sha256=lccF3X3M2mcbHVG_0HxfuJRFFpUE-42paccIxFOfefQ,1958
-lamindb_setup/io.py,sha256=7wU3g2AH1h18wQerltA7p8VP0n47ZOsy32XlRcBbBAc,7321
+lamindb_setup/io.py,sha256=jLJ3WbWWDctXmzl6XTES6_CutX3TsvG1BSW-Jd5UITo,15081
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=fuQxZJnrGYe7a_Ju9n1RqO-HhkOAr1l1xjpAg9dmBu8,605
-lamindb_setup/core/__init__.py,sha256=gMqbkPeD-T6o_KykonvNiUJJQYz5SCk2yGgFNWFUBNc,603
-lamindb_setup/core/_aws_options.py,sha256=6SAs3dhKxNadQqt7Ce7uHSus0DsU_LcNZoCQWvhgBWA,9626
+lamindb_setup/core/__init__.py,sha256=adZtacDwG2C0tgx-ypp9yOAqw9qaR-IRWkgLurKpXVE,668
+lamindb_setup/core/_aws_options.py,sha256=9kQ5BB-cuJQrlJRGNqMRe1m48dP67xMbefOJP2c9OQw,9674
 lamindb_setup/core/_aws_storage.py,sha256=QEtV-riQrwfivcwqHnXBbkJ-9YyNEXL4fLoCmOHZ1BI,2003
-lamindb_setup/core/_clone.py,sha256=2NlXV04yykqg_k7z59C_kD1F1Hi4H-55H-JtNjhenQ0,3691
+lamindb_setup/core/_clone.py,sha256=Vikc0LckB7fWTS06ei5enErWqe1mx6XvP7Ikkj-fzdg,6493
 lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=J0x43at0zb0yWP-RoT2lyqaHV66ewUP3OiYVYQCjxe8,9974
+lamindb_setup/core/_hub_client.py,sha256=vem145S5ppRPcWob7iclGhos8k-BfwJi9AI-l5PteDs,10481
 lamindb_setup/core/_hub_core.py,sha256=GAQK5XkHROIuqA-H8sOQZVlxvN4QIH_cmHY0TENnq2U,29090
 lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=bAwvJhin7hLNliE_yaM-CMC3d5YD6-kY3raGpnse91Y,15458
-lamindb_setup/core/_settings_instance.py,sha256=7VXd1W88fgqEnAfzFQKUlDnTr3pmA_e8aIag7FqPrJI,23899
-lamindb_setup/core/_settings_load.py,sha256=j20cy3J56ZBHLDfB2A8oKjekNetMNsy0_W3eWD36pWI,5161
-lamindb_setup/core/_settings_save.py,sha256=jh412jXIAbIYvnSoW9riBFePRAa4vmPm-ScYD0smlnw,3292
-lamindb_setup/core/_settings_storage.py,sha256=yFz0w7yB7_3UWdrndVHJ_BZyV4ONj2lPRaQn_cksFQw,15150
-lamindb_setup/core/_settings_store.py,sha256=ykJeBA9IODK4G_jrfBE9pb0c1xkfePkARPpb306DT08,2687
+lamindb_setup/core/_settings.py,sha256=QbTrSkkdx0u685NJ4neNtWzhdHoaGMKcIvrfFnctTQ4,15450
+lamindb_setup/core/_settings_instance.py,sha256=eDkueLK5JZOGFhZRbGa-OffS9iBFlxMp47vF_MfmCYI,24301
+lamindb_setup/core/_settings_load.py,sha256=NQDOln8e3qyGphk8ucU7mm3HVkCv4QV4rDZro3TIwfo,5183
+lamindb_setup/core/_settings_save.py,sha256=96mWdYLyfvbnG_ok_vK4x7jm-rtqcWCD1OHEt2QSAms,3328
+lamindb_setup/core/_settings_storage.py,sha256=mO5WVlybloyizKILGpbJf_Fe5yp32D-fYERURcncQ3o,15505
+lamindb_setup/core/_settings_store.py,sha256=auZssUBb6qE5oSqdGiHhqI2B46qSpegX89VwObPQksk,2601
 lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
@@ -43,8 +43,8 @@ lamindb_setup/core/django.py,sha256=2HwhtfUEX4peSkczc0VSfA-CpfCGL4vNgkPe9Pwu5kw,
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=bi3k8AYeiGB_NtVTO9e9gHsfs2AFB4fXiVHcbNpnlpI,35780
-lamindb_setup-1.15.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.15.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.15.0.dist-info/METADATA,sha256=aVm2bsYK6bWwBDEcp1IJR3kp8xrRnJb5CYK_KT09HCA,1798
-lamindb_setup-1.15.0.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=DgdCBCTv1HRx94kOUw3DGVcsSJrY68S9aVwOT3ep1D8,35730
+lamindb_setup-1.15.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.15.2.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.15.2.dist-info/METADATA,sha256=vWKrDEJ5PzX3wM-qkSJwTEPHWNXsw0zROmRNjNMpibw,1798
+lamindb_setup-1.15.2.dist-info/RECORD,,