lamindb_setup 1.15.1__py3-none-any.whl → 1.16.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Migration management
 
 """
 
-__version__ = "1.15.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.16.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 import warnings

lamindb_setup/_connect_instance.py

@@ -12,7 +12,6 @@ from lamin_utils import logger
 from ._check_setup import (
     _check_instance_setup,
     _get_current_instance_settings,
-    find_module_candidates,
 )
 from ._disconnect import disconnect
 from ._init_instance import load_from_isettings

lamindb_setup/_schema_metadata.py

@@ -172,7 +172,8 @@ class _ModelHandler:
         self.table_name = model._meta.db_table
         self.included_modules = included_modules
         self.fields = self._get_fields_metadata(self.model)
-        self.is_link_table = issubclass(model, IsLink)
+        self.is_auto_created = bool(model._meta.auto_created)
+        self.is_link_table = issubclass(model, IsLink) or self.is_auto_created
         self.name_field = model._name_field if hasattr(model, "_name_field") else None
         self.ontology_id_field = (
             model._ontology_id_field if hasattr(model, "_ontology_id_field") else None
@@ -183,6 +184,7 @@ class _ModelHandler:
             "fields": self.fields.copy(),
             "class_name": self.class_name,
             "table_name": self.table_name,
+            "is_auto_created": self.is_auto_created,
             "is_link_table": self.is_link_table,
             "name_field": self.name_field,
             "ontology_id_field": self.ontology_id_field,
@@ -249,13 +251,13 @@ class _ModelHandler:
         return related_fields
 
     def _get_field_metadata(self, model, field: Field):
-        from lamindb.models import IsLink
+        from lamindb.models import IsLink, Registry
 
         internal_type = field.get_internal_type()
         model_name = field.model._meta.model_name
         relation_type = self._get_relation_type(model, field)
 
-        schema_name = field.model.__get_module_name__()
+        schema_name = Registry.__get_module_name__(field.model)
 
         if field.related_model is None:
             related_model_name = None
@@ -265,7 +267,7 @@ class _ModelHandler:
             max_length = field.max_length
         else:
             related_model_name = field.related_model._meta.model_name
-            related_schema_name = field.related_model.__get_module_name__()
+            related_schema_name = Registry.__get_module_name__(field.related_model)
             related_field_name = field.remote_field.name
             is_editable = False
             max_length = None
@@ -418,14 +420,10 @@ class _SchemaHandler:
         all_models = {module_name: {} for module_name in self.included_modules}
 
         # Iterate through all registered Django models
-        for model in apps.get_models():
+        for model in apps.get_models(include_auto_created=True):
             # Check if model meets the criteria
-            if (
-                model.__class__ is Registry
-                and model is not SQLRecord
-                and not model._meta.abstract
-            ):
-                module_name = model.__get_module_name__()
+            if model is not SQLRecord and not model._meta.abstract:
+                module_name = Registry.__get_module_name__(model)
                 # Only include if module is in our included list
                 if module_name in self.included_modules:
                     model_name = model._meta.model_name

lamindb_setup/_set_managed_storage.py

@@ -41,7 +41,7 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             "use a tuple of (local_root, host) instead"
         )
 
-    # here the storage is registered in the hub
+    # here the storage location is registered in the hub
     # hub_record_status="hub-record-created" if a new record is created
     # "hub-record-retrieved" if the storage is in the hub already
     ssettings, hub_record_status = init_storage(
@@ -65,5 +65,13 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             delete_storage_record(ssettings)
         raise e
 
-    settings.instance._storage = ssettings
-    settings.storage._set_fs_kwargs(**fs_kwargs)
+    if ssettings._instance_id != settings.instance._id:
+        logger.warning(
+            f"registered storage location {root} as read-only for this instance (it's written by instance with uid: {ssettings.instance_uid})"
+        )
+        logger.warning(
+            f"did *not* switch default storage location, it's still: {settings.storage.root_as_str}"
+        )
+    else:
+        settings.instance._storage = ssettings
+        settings.storage._set_fs_kwargs(**fs_kwargs)

lamindb_setup/_setup_user.py

@@ -43,6 +43,14 @@ def load_user(email: str | None = None, handle: str | None = None) -> UserSettin
     return user_settings
 
 
+def current_user_uid() -> str:
+    current_user_settings = current_user_settings_file()
+    if current_user_settings.exists():
+        return load_user_settings(current_user_settings).uid
+
+    return "00000000"  # anonymous
+
+
 def login(
     user: str | None = None, *, api_key: str | None = None, **kwargs
 ) -> UserSettings:
@@ -90,6 +98,9 @@ def login(
             "the legacy API key is deprecated and will likely be removed in a future version"
         )
 
+    # do this here because load_user overwrites current_user_settings_file
+    previous_user_uid = current_user_uid()
+
     if api_key is None:
         if "@" in user:  # type: ignore
             email, handle = user, None
@@ -144,8 +155,15 @@ def login(
     user_settings.api_key = api_key
     save_user_settings(user_settings)
 
-    if settings._instance_exists and _check_instance_setup():
-        register_user(user_settings)
+    if settings._instance_exists:
+        if (
+            isettings := settings.instance
+        ).is_on_hub and previous_user_uid != user_settings.uid:
+            logger.important_hint(
+                f"consider re-connecting to update permissions: lamin connect {isettings.slug}"
+            )
+        if _check_instance_setup():
+            register_user(user_settings)
 
     settings._user_settings = None
     # aws s3 credentials are scoped to the user

lamindb_setup/core/__init__.py

@@ -23,7 +23,12 @@ Storage
 """
 
 from . import django, upath
-from ._clone import connect_local_sqlite, init_local_sqlite
+from ._clone import (
+    connect_local_sqlite,
+    connect_remote_sqlite,
+    init_local_sqlite,
+    upload_sqlite_clone,
+)
 from ._deprecated import deprecated  # documented in lamindb.base
 from ._docs import doc_args  # documented in lamindb.base
 from ._settings import SetupSettings

lamindb_setup/core/_clone.py

@@ -5,14 +5,19 @@
 
    init_local_sqlite
    connect_local_sqlite
+   connect_remote_sqlite
+   upload_sqlite_clone
 """
 
+import gzip
 import os
+import shutil
+from pathlib import Path
 
-from lamindb_setup.core._settings_instance import InstanceSettings
 from lamindb_setup.core._settings_load import load_instance_settings
 from lamindb_setup.core._settings_store import instance_settings_file
 from lamindb_setup.core.django import reset_django
+from lamindb_setup.core.upath import create_path
 
 
 def init_local_sqlite(
@@ -53,28 +58,31 @@ def init_local_sqlite(
         if copy_suffix is not None
         else ln_setup.settings.instance.name
     )
-    isettings = InstanceSettings(
-        id=ln_setup.settings.instance._id,
-        owner=ln_setup.settings.instance.owner,  # type: ignore
-        name=name,
-        storage=ln_setup.settings.storage,
-        db=None,
-        modules=",".join(ln_setup.settings.instance.modules),
-        is_on_hub=False,
+    isettings = ln_setup._connect_instance._connect_instance(
+        owner=ln_setup.settings.instance.owner, name=name
     )
-
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
     isettings._persist(write_to_disk=True)
 
     if not isettings._sqlite_file_local.exists():
         # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
-        # Without this reset, the if not settings.configured check in setup_django() would skip reconfiguration,
+        # Without this reset, the `if not settings.configured`` check in `setup_django()` would skip reconfiguration,
         # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
         reset_django()
         isettings._init_db()
 
 
-def connect_local_sqlite(instance: str) -> None:
-    """Load a SQLite instance of which a remote hub Postgres instance exists.
+def connect_local_sqlite(
+    instance: str,
+) -> None:
+    """Load a locally stored SQLite instance of which a remote hub Postgres instance exists.
 
     This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
     The clone must first be created via `init_local_sqlite()`.
@@ -86,8 +94,81 @@ def connect_local_sqlite(instance: str) -> None:
     settings_file = instance_settings_file(name=name, owner=owner)
 
     if not settings_file.exists():
-        raise ValueError("SQLite clone not found. Run init_local_sqlite() first.")
+        raise ValueError(
+            "SQLite clone not found."
+            " Run `init_local_sqlite()` to create a local copy or connect to a remote copy using `connect_remote_sqlite`."
+        )
 
     isettings = load_instance_settings(settings_file)
     isettings._persist(write_to_disk=False)
-    isettings._load_db()
+
+    # Using `setup_django` instead of `_load_db` to not ping AWS RDS
+    from lamindb_setup._check_setup import disable_auto_connect
+
+    from .django import setup_django
+
+    disable_auto_connect(setup_django)(isettings)
+
+
+def connect_remote_sqlite(instance: str, *, copy_suffix: str | None = None) -> None:
+    """Load an existing SQLite copy of a hub instance.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+        copy_suffix: Optional suffix of the local clone.
+    """
+    import lamindb_setup as ln_setup
+
+    owner, name = instance.split("/")
+
+    # Step 1: Create the settings file
+    isettings = ln_setup._connect_instance._connect_instance(owner=owner, name=name)
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    isettings._db_permissions = "read"
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
+    isettings._persist(write_to_disk=True)
+
+    connect_local_sqlite(instance=instance + (copy_suffix or ""))
+
+
+def upload_sqlite_clone(
+    local_sqlite_path: Path | str | None = None, compress: bool = True
+) -> None:
+    """Uploads the SQLite clone to the default storage.
+
+    Args:
+        local_sqlite_path: Path to the SQLite file.
+            Defaults to the local storage path if not specified.
+        compress: Whether to compress the database with gzip before uploading.
+    """
+    import lamindb_setup as ln_setup
+
+    if local_sqlite_path is None:
+        local_sqlite_path = ln_setup.settings.instance._sqlite_file_local
+    else:
+        local_sqlite_path = Path(local_sqlite_path)
+
+    if not local_sqlite_path.exists():
+        raise FileNotFoundError(f"Database not found at {local_sqlite_path}")
+
+    cloud_db_path = ln_setup.settings.instance._sqlite_file
+
+    if compress:
+        temp_gz_path = local_sqlite_path.with_suffix(".db.gz")
+        with (
+            open(local_sqlite_path, "rb") as f_in,
+            gzip.open(temp_gz_path, "wb") as f_out,
+        ):
+            shutil.copyfileobj(f_in, f_out)
+        cloud_destination = create_path(f"{cloud_db_path}.gz")
+        cloud_destination.upload_from(temp_gz_path, print_progress=True)
+        temp_gz_path.unlink()
+    else:
+        cloud_destination = create_path(cloud_db_path)
+        cloud_destination.upload_from(local_sqlite_path, print_progress=True)

lamindb_setup/core/_hub_client.py

@@ -96,9 +96,12 @@ def connect_hub(
         transports.append(
             RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
         )
+    # this overwrites transports of existing httpx clients
+    # if proxies are set, the default transports that were created on clients init
+    # will be used, irrespective of these re-settings
     client.auth._http_client._transport = transports[0]
     client.postgrest.session._transport = transports[1]
     # POST is not retryable by default, but for our functions it should be safe to retry
@@ -116,7 +119,7 @@ def connect_hub(
                 "POST",
             ],
         ),
-        transport=httpx.HTTPTransport(verify=True, http2=True),
+        transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
     )
     return client
 
@@ -246,9 +249,13 @@ def httpx_client():
         else:
             transport = RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
-            client = httpx.Client(transport=transport)
+            # first we create a client to build the proxy map from the env variables
+            # if proxies are set, the default transports will be used
+            # otherwise the RetryTransport object that we assign below
+            client = httpx.Client(trust_env=True)
+            client._transport = transport
         yield client
     finally:
         if client is not None:

lamindb_setup/core/_private_django_api.py

@@ -37,7 +37,6 @@ def private_django_api(reverse=False):
         "MultipleObjectsReturned",
         "add_to_class",
         "adelete",
-        "refresh_from_db",
         "asave",
         "clean",
         "clean_fields",

lamindb_setup/core/_settings.py

@@ -320,8 +320,7 @@ class SetupSettings:
     def paths(self) -> type[SetupPaths]:
         """Convert cloud paths to lamindb local paths.
 
-        Use `settings.paths.cloud_to_local_no_update`
-        or `settings.paths.cloud_to_local`.
+        Use `settings.paths.cloud_to_local_no_update` or `settings.paths.cloud_to_local`.
         """
         return SetupPaths
 

lamindb_setup/core/_settings_instance.py

@@ -54,8 +54,7 @@ def is_local_db_url(db_url: str) -> bool:
 
 
 def check_is_instance_remote(root: UPathStr, db: str | None) -> bool:
-    # returns True for cloud SQLite
-    # and remote postgres
+    # returns True for cloud SQLite and remote postgres
     root_str = str(root)
     if not root_str.startswith("create-s3") and get_storage_type(root_str) == "local":
         return False
@@ -83,7 +82,8 @@ class InstanceSettings:
         schema_id: UUID | None = None,
         fine_grained_access: bool = False,
         db_permissions: str | None = None,
-        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite
+        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite,
+        _is_clone: bool = False,
     ):
         from ._hub_utils import validate_db_arg
 
@@ -109,6 +109,7 @@ class InstanceSettings:
         self._db_permissions = db_permissions
         # if None then settings.user is used
         self._locker_user = _locker_user
+        self._is_clone = _is_clone
 
     def __repr__(self):
         """Rich string representation."""
@@ -434,7 +435,7 @@ class InstanceSettings:
 
     def _update_cloud_sqlite_file(self, unlock_cloud_sqlite: bool = True) -> None:
         """Upload the local sqlite file to the cloud file."""
-        if self._is_cloud_sqlite:
+        if self._is_cloud_sqlite and not self._is_clone:
             sqlite_file = self._sqlite_file
             logger.warning(
                 f"updating{' & unlocking' if unlock_cloud_sqlite else ''} cloud SQLite "
@@ -602,6 +603,14 @@ class InstanceSettings:
         disable_auto_connect(setup_django)(self, init=True)
 
     def _load_db(self) -> tuple[bool, str]:
+        """Load the database connection.
+
+        For cloud SQLite instances, downloads the database file from cloud storage.
+        For all instances, initializes Django ORM with the database connection.
+
+        Returns:
+            Tuple of (success: bool, error_message: str). Returns (True, "") on success.
+        """
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -615,8 +624,8 @@ class InstanceSettings:
                 return False, f"SQLite file {self._sqlite_file} does not exist"
         # we need the local sqlite to setup django
         self._update_local_sqlite_file()
-        # setting up django also performs a check for migrations & prints them
-        # as warnings
+
+        # setting up django also performs a check for migrations & prints them as warnings
         # this should fail, e.g., if the db is not reachable
         from lamindb_setup._check_setup import disable_auto_connect
 

lamindb_setup/core/_settings_load.py

@@ -69,8 +69,7 @@ def load_or_create_user_settings(api_key: str | None = None) -> UserSettings:
     """Return current user settings.
 
     Args:
-        api_key: if provided and there is no current user,
-            perform login and return the user settings.
+        api_key: if provided and there is no current user, perform login and return the user settings.
     """
     current_user_settings = current_user_settings_file()
     if not current_user_settings.exists():
@@ -125,6 +124,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         schema_id=None if store.schema_id in {None, "null"} else UUID(store.schema_id),
         fine_grained_access=store.fine_grained_access,
         db_permissions=_null_to_value(store.db_permissions),
+        _is_clone=store.is_clone,
     )
 
 

lamindb_setup/core/_settings_save.py

@@ -63,6 +63,7 @@ def save_settings(
                         "schema_id",
                         "fine_grained_access",
                         "db_permissions",
+                        "is_clone",
                     }:
                         settings_key = f"_{store_key.rstrip('_')}"
                     else:

lamindb_setup/core/_settings_storage.py

@@ -12,17 +12,14 @@ from lamin_utils import logger
 from lamindb_setup.errors import StorageAlreadyManaged
 
 from ._aws_options import (
-    HOSTED_REGIONS,
     LAMIN_ENDPOINTS,
     get_aws_options_manager,
 )
-from ._aws_storage import find_closest_aws_region
 from ._deprecated import deprecated
 from .hashing import hash_and_encode_as_b62
 from .upath import (
     LocalPathClasses,
     UPath,
-    _split_path_query,
     create_path,
     get_storage_region,
 )
@@ -58,12 +55,40 @@ def get_storage_type(root_as_str: str) -> StorageType:
     return convert.get(protocol, protocol)  # type: ignore
 
 
+def sanitize_root_user_input(root: UPathStr) -> UPath:
+    """Format a root path string."""
+    root_upath = root if isinstance(root, UPath) else UPath(root)
+    root_upath = root_upath.expanduser()
+    if isinstance(root_upath, LocalPathClasses):  # local paths
+        try:
+            (root_upath / ".lamindb").mkdir(parents=True, exist_ok=True)
+            root_upath = root_upath.resolve()
+        except Exception:
+            logger.warning(f"unable to create .lamindb/ folder in {root_upath}")
+    return root_upath
+
+
+def convert_sanitized_root_path_to_str(root_upath: UPath) -> str:
+    # embed endpoint_url into path string for storing and displaying
+    if root_upath.protocol == "s3":
+        endpoint_url = root_upath.storage_options.get("endpoint_url", None)
+        # LAMIN_ENDPOINTS include None
+        if endpoint_url not in LAMIN_ENDPOINTS:
+            return f"s3://{root_upath.path.rstrip('/')}?endpoint_url={endpoint_url}"
+    return root_upath.as_posix().rstrip("/")
+
+
+def convert_root_path_to_str(root: UPathStr) -> str:
+    """Format a root path string."""
+    sanitized_root_upath = sanitize_root_user_input(root)
+    return convert_sanitized_root_path_to_str(sanitized_root_upath)
+
+
 def mark_storage_root(
     root: UPathStr, uid: str, instance_id: UUID, instance_slug: str
 ) -> Literal["__marked__"] | str:
     # we need a file in folder-like storage locations on S3 to avoid
-    # permission errors from leveraging s3fs on an empty hosted storage location
-    # (path.fs.find raises a PermissionError)
+    # permission errors from leveraging s3fs on an empty hosted storage location (path.fs.find raises a PermissionError)
     # we also need it in case a storage location is ambiguous because a server / local environment
     # doesn't have a globally unique identifier, then we screen for this file to map the
     # path on a storage location in the registry
@@ -97,6 +122,7 @@ def init_storage(
     access_token: str | None = None,
     region: str | None = None,
     space_uuid: UUID | None = None,
+    skip_mark_storage_root: bool = False,
 ) -> tuple[
     StorageSettings,
     Literal["hub-record-not-created", "hub-record-retrieved", "hub-record-created"],
@@ -156,7 +182,8 @@ def init_storage(
         space_id=space_uuid,
     )
     # we check the write access here if the storage record has not been retrieved from the hub
-    if hub_record_status != "hub-record-retrieved":
+    # Sergei: should it in fact still go through if hub_record_status == "hub-record-not-created"?
+    if hub_record_status != "hub-record-retrieved" and not skip_mark_storage_root:
         try:
             # (federated) credentials for AWS access are provisioned under-the-hood
             # discussion: https://laminlabs.slack.com/archives/C04FPE8V01W/p1719260587167489
@@ -214,15 +241,7 @@ class StorageSettings:
     ):
         self._uid = uid
         self._uuid_ = uuid
-        self._root_init = UPath(root).expanduser()
-        if isinstance(self._root_init, LocalPathClasses):  # local paths
-            try:
-                (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
-                self._root_init = self._root_init.resolve()
-            except Exception:
-                logger.warning(
-                    f"unable to create .lamindb/ folder in {self._root_init}"
-                )
+        self._root_init: UPath = sanitize_root_user_input(root)
         self._root = None
         self._instance_id = instance_id
         # we don't yet infer region here to make init fast
@@ -337,13 +356,7 @@ class StorageSettings:
     @property
     def root_as_str(self) -> str:
         """Formatted root string."""
-        # embed endpoint_url into path string for storing and displaying
-        if self._root_init.protocol == "s3":
-            endpoint_url = self._root_init.storage_options.get("endpoint_url", None)
-            # LAMIN_ENDPOINTS include None
-            if endpoint_url not in LAMIN_ENDPOINTS:
-                return f"s3://{self._root_init.path.rstrip('/')}?endpoint_url={endpoint_url}"
-        return self._root_init.as_posix().rstrip("/")
+        return convert_sanitized_root_path_to_str(self._root_init)
 
     @property
     def cache_dir(

lamindb_setup/core/_settings_store.py

@@ -67,8 +67,8 @@ class InstanceSettingsStore(BaseSettings):
     owner: str
     name: str
     storage_root: str
-    storage_region: str | None  # take old type annotations here because pydantic
-    db: str | None  # doesn't like new types on 3.9 even with future annotations
+    storage_region: str | None
+    db: str | None
     schema_str: str | None
     schema_id: str | None = None
     fine_grained_access: bool = False
@@ -76,6 +76,7 @@ class InstanceSettingsStore(BaseSettings):
     id: str
     git_repo: str | None
     keep_artifacts_local: bool | None
+    is_clone: bool = False
     model_config = SettingsConfigDict(env_prefix="lamindb_instance_", env_file=".env")
 
 

lamindb_setup/core/django.py

@@ -238,6 +238,8 @@ def setup_django(
         if view_schema:
             installed_apps = installed_apps[::-1]  # to fix how apps appear
             installed_apps += ["schema_graph", "django.contrib.staticfiles"]
+        if isettings.dialect == "postgresql":
+            installed_apps.insert(0, "pgtrigger")
 
         kwargs = dict(
             INSTALLED_APPS=installed_apps,

lamindb_setup/core/upath.py

@@ -93,10 +93,12 @@ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
         else:
             return suffix
 
-    if len(path.suffixes) <= 1:
+    suffixes = path.suffixes
+
+    if len(suffixes) <= 1:
         return process_digits(path.suffix)
 
-    total_suffix = "".join(path.suffixes)
+    total_suffix = "".join(suffixes)
     if total_suffix in VALID_SIMPLE_SUFFIXES:
         return total_suffix
     elif total_suffix.endswith(tuple(VALID_COMPOSITE_SUFFIXES)):
@@ -115,14 +117,24 @@ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
         # in COMPRESSION_SUFFIXES to detect something like .random.gz and then
         # add ".random.gz" but concluded it's too dangerous it's safer to just
         # use ".gz" in such a case
-        if path.suffixes[-2] in VALID_SIMPLE_SUFFIXES:
-            suffix = "".join(path.suffixes[-2:])
-            msg += f"inferring: '{suffix}'"
+        if suffixes[-2] in VALID_SIMPLE_SUFFIXES:
+            suffix = "".join(suffixes[-2:])
+            # if the suffix preceding the compression suffixes is a valid suffix,
+            # we account for it; otherwise we don't.
+            # i.e. we should have .h5ad.tar.gz or .csv.tar.gz, not just .tar.gz
+            if (
+                suffix == ".tar.gz"
+                and len(suffixes) > 2
+                and (suffix_3 := suffixes[-3]) in VALID_SIMPLE_SUFFIXES
+            ):
+                suffix = suffix_3 + suffix
             # do not print a warning for things like .tar.gz, .fastq.gz
-            if path.suffixes[-1] == ".gz":
+            if suffixes[-1] == ".gz":
                 print_hint = False
+            else:
+                msg += f"inferring: '{suffix}'"
         else:
-            suffix = path.suffixes[-1]  # this is equivalent to path.suffix
+            suffix = suffixes[-1]  # this is equivalent to path.suffix
             msg += (
                 f"using only last suffix: '{suffix}' - if you want your composite"
                 " suffix to be recognized add it to"
@@ -908,12 +920,9 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str | None, str | None]:
     elif "blob_id" in stat:
         hash = b16_to_b64(stat["blob_id"])
         hash_type = "sha1"
-    # s3
-    # StorageClass is checked to be sure that it is indeed s3
-    # because http also has ETag
     elif "ETag" in stat:
         etag = stat["ETag"]
-        if "mimetype" in stat:
+        if "mimetype" in stat or ("url" in stat and stat["url"].startswith("http")):
             # http
             hash = hash_string(etag.strip('"'))
             hash_type = "md5-etag"

lamindb_setup/io.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import io
 import json
 import warnings
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from importlib import import_module
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -14,24 +14,13 @@ from django.db import models, transaction
 from rich.progress import Progress
 
 if TYPE_CHECKING:
-    from collections.abc import Sequence
+    from collections.abc import Iterable, Sequence
     from typing import Literal
 
 
 def _get_registries(module_name: str) -> list[str]:
     """Get registry class names from a module."""
     schema_module = import_module(module_name)
-    exclude = {"SQLRecord", "BaseSQLRecord"}
-
-    if module_name == "lamindb":
-        module_filter = lambda cls, name: cls.__module__.startswith(
-            f"{module_name}.models."
-        ) and name in dir(schema_module)
-    else:
-        module_filter = (
-            lambda cls, name: cls.__module__ == f"{module_name}.models"
-            and name in dir(schema_module)
-        )
 
     return [
         name
@@ -40,8 +29,8 @@ def _get_registries(module_name: str) -> list[str]:
             name[0].isupper()
             and isinstance(cls := getattr(schema_module.models, name, None), type)
             and issubclass(cls, models.Model)
-            and module_filter(cls, name)
-            and name not in exclude
+            # Table names starting with `None_` are abstract base classes or Django mixins
+            and not cls._meta.db_table.startswith("None_")  # type: ignore
         )
     ]
 
@@ -59,7 +48,7 @@ def _export_full_table(
     For SQLite with large tables, reads in chunks to avoid memory issues when tables exceed available RAM.
 
     Args:
-        registry_info: Tuple of (module_name, model_name, field_name) where field_name
+        registry_info: Tuple of (module_name, model_name, field_name) where `field_name`
             is None for regular tables or the field name for M2M link tables.
         directory: Output directory for parquet files.
         chunk_size: Maximum rows per chunk for SQLite large tables.
@@ -73,7 +62,7 @@ def _export_full_table(
 
     module_name, model_name, field_name = registry_info
     schema_module = import_module(module_name)
-    registry = getattr(schema_module, model_name)
+    registry = getattr(schema_module.models, model_name)
 
     if field_name:
         registry = getattr(registry, field_name).through
@@ -84,12 +73,19 @@ def _export_full_table(
         if ln_setup.settings.instance.dialect == "postgresql":
             buffer = io.StringIO()
             with connection.cursor() as cursor:
+                cursor.execute("SET statement_timeout = 0")
                 cursor.copy_expert(
                     f'COPY "{table_name}" TO STDOUT WITH (FORMAT CSV, HEADER TRUE)',
                     buffer,
                 )
             buffer.seek(0)
-            df = pd.read_csv(buffer)
+            # Prevent pandas from converting empty strings to float NaN (which PyArrow rejects)
+            df = pd.read_csv(buffer, keep_default_na=False)
+            # Convert object columns to string to handle mixed types from data corruption,
+            # schema migrations, or manual SQL inserts. PyArrow rejects mixed-type objects.
+            df = df.astype(
+                {col: str for col in df.columns if df[col].dtype == "object"}
+            )
             df.to_parquet(directory / f"{table_name}.parquet", compression=None)
             return (
                 f"{module_name}.{model_name}.{field_name}"
@@ -118,11 +114,21 @@ def _export_full_table(
                         chunk_file = (
                             directory / f"{table_name}_chunk_{chunk_id}.parquet"
                         )
+                        df = df.astype(
+                            {
+                                col: str
+                                for col in df.columns
+                                if df[col].dtype == "object"
+                            }
+                        )
                         df.to_parquet(chunk_file, compression=None)
                         chunk_files.append((table_name, chunk_file))
                     return chunk_files
                 else:
                     df = pd.read_sql_table(table_name, ln_setup.settings.instance.db)
+                    df = df.astype(
+                        {col: str for col in df.columns if df[col].dtype == "object"}
+                    )
                     df.to_parquet(directory / f"{table_name}.parquet", compression=None)
                     return (
                         f"{module_name}.{model_name}.{field_name}"
@@ -163,7 +169,7 @@ def export_db(
     for module_name, model_names in modules.items():
         schema_module = import_module(module_name)
         for model_name in model_names:
-            registry = getattr(schema_module, model_name)
+            registry = getattr(schema_module.models, model_name)
             tasks.append((module_name, model_name, None))
             for field in registry._meta.many_to_many:
                 tasks.append((module_name, model_name, field.name))
@@ -172,7 +178,9 @@ def export_db(
 
     with Progress() as progress:
         task_id = progress.add_task("Exporting", total=len(tasks))
-        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+
+        # This must be a ThreadPoolExecutor and not a ProcessPoolExecutor to inherit JWTs
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
             futures = {
                 executor.submit(_export_full_table, task, directory, chunk_size): task
                 for task in tasks
@@ -222,7 +230,6 @@ def _import_registry(
     parquet_file = directory / f"{table_name}.parquet"
 
     if not parquet_file.exists():
-        print(f"Skipped {table_name} (file not found)")
         return
 
     df = pd.read_parquet(parquet_file)
@@ -237,9 +244,47 @@ def _import_registry(
             if mask.any():
                 df.loc[mask, col] = df.loc[mask, col].map(_serialize_value)
 
+    for field in registry._meta.fields:
+        # Convert PostgreSQL boolean string literals ('t'/'f') to Python booleans for SQLite compatibility
+        if field.get_internal_type() == "BooleanField" and field.column in df.columns:
+            df[field.column] = df[field.column].map(
+                {"t": True, "f": False, True: True, False: False, None: None}
+            )
+
+        # PostgreSQL CSV export writes NULL as empty string; convert back to None for nullable fields
+        if field.null and field.column in df.columns:
+            df[field.column] = df[field.column].replace("", None)
+
+        # Convert numeric fields from strings to proper types for SQLite
+        if (
+            field.get_internal_type()
+            in (
+                "IntegerField",
+                "BigIntegerField",
+                "PositiveIntegerField",
+                "FloatField",
+                "DecimalField",
+            )
+            and field.column in df.columns
+        ):
+            df[field.column] = pd.to_numeric(df[field.column], errors="coerce")
+
+    if if_exists == "append":
+        # Fill NULL values in NOT NULL columns to handle schema mismatches between postgres source and SQLite target
+        # This allows importing data where fields were nullable
+        for field in registry._meta.fields:
+            if field.column in df.columns and not field.null:
+                df[field.column] = df[field.column].fillna("").infer_objects(copy=False)
+
     if df.empty:
         return
 
+    if if_exists == "append":
+        # Clear existing data before import
+        # When appending we would run into duplicate errors because of existing values like branches etc
+        with connection.cursor() as cursor:
+            cursor.execute(f'DELETE FROM "{table_name}"')
+
     if connection.vendor == "postgresql":
         columns = df.columns.tolist()
         column_names = ", ".join(f'"{col}"' for col in columns)
@@ -265,6 +310,7 @@ def _import_registry(
         max_vars = 900  # SQLite has a limit of 999 variables per statement
         chunksize = max(1, max_vars // num_cols)
 
+        # Always use append mode since we set up the tables from a fresh instance
         df.to_sql(
             table_name,
             connection.connection,
@@ -276,7 +322,7 @@ def _import_registry(
 
 
 def import_db(
-    module_names: Sequence[str] | None = None,
+    module_names: Iterable[str] | None = None,
     *,
     input_dir: str | Path = "./lamindb_export/",
     if_exists: Literal["fail", "replace", "append"] = "replace",
@@ -290,6 +336,9 @@ def import_db(
         input_dir: Directory containing parquet files to import.
         module_names: Module names to import (e.g., ["lamindb", "bionty", "wetlab"]).
         if_exists: How to behave if table exists: 'fail', 'replace', or 'append'.
+            If set to 'replace', existing data is deleted and new data is imported. PKs and indices are not guaranteed to be preserved which can lead to write errors.
+            If set to 'append', new data is added to existing data without clearing the table. PKs and indices are preserved but database size will greatly increase.
+            If set to 'fail', raises an error if the table contains any data.
     """
     from django.db import connection
 
@@ -338,7 +387,7 @@ def import_db(
                         progress.update(
                             task, description=f"[cyan]{module_name}.{model_name}"
                         )
-                        registry = getattr(schema_module, model_name)
+                        registry = getattr(schema_module.models, model_name)
                         _import_registry(registry, directory, if_exists=if_exists)
                         for field in registry._meta.many_to_many:
                             link_orm = getattr(registry, field.name).through
@@ -352,3 +401,5 @@ def import_db(
                 cursor.execute("PRAGMA synchronous = FULL")
                 cursor.execute("PRAGMA journal_mode = DELETE")
                 cursor.execute("PRAGMA foreign_keys = ON")
+                # Reclaim space from DELETEs
+                cursor.execute("VACUUM")

{lamindb_setup-1.15.1.dist-info → lamindb_setup-1.16.0.dist-info}/METADATA

@@ -1,14 +1,14 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.15.1
+Version: 1.16.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: lamin_utils>=0.3.3
 Requires-Dist: django>=5.2,<5.3
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
+Requires-Dist: django-pgtrigger
 Requires-Dist: pydantic-settings
 Requires-Dist: platformdirs<5.0.0
 Requires-Dist: httpx_retries<1.0.0

{lamindb_setup-1.15.1.dist-info → lamindb_setup-1.16.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-lamindb_setup/__init__.py,sha256=3Oaw6Mj_4lWEvxbDmE5Y6Q-hyP20r-8yeSKOpJXvjso,3215
+lamindb_setup/__init__.py,sha256=zAgDC8o3cyH4eXcM60mbulFPECs5XZbqFRxLfGpgpc8,3215
 lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=ToKMxsUq8dQBQh8baOrNVlSb1iC8h4zTg5dV8wMu0W4,6760
-lamindb_setup/_connect_instance.py,sha256=Yp24MMAnypKE14fMzOYCE8oqW0EJTiRHrBuugubfkBM,17816
+lamindb_setup/_connect_instance.py,sha256=3dsaZ7LGzJYtOWDbi2RkiVJIJqdxy43suNjQ-6C96_U,17788
 lamindb_setup/_delete.py,sha256=KS3r-xGFuDmAbzPUy-9JR-YnPShYdaHjDRQrAmXQ0qM,5863
 lamindb_setup/_disconnect.py,sha256=FT8EpCm5XXDdhDH7QtAnkO3KPatq2HqT9VXGNjgJDbk,1232
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
@@ -11,40 +11,40 @@ lamindb_setup/_init_instance.py,sha256=zNXmZPUHYda1CfLGtsvo4gNhHprK9QVPfffUIfBlT
 lamindb_setup/_migrate.py,sha256=SN8uphuQX-8XShH5odLyzV8-eyXATDxB5hWoxwxmgBU,11264
 lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
-lamindb_setup/_schema_metadata.py,sha256=af1Es7qFKGPRdNmk48384HiB2r-cDTdBPu0wB9qrga4,15526
-lamindb_setup/_set_managed_storage.py,sha256=y5YQASsWNYVWUYeLgh3N2YBETYP7mBtbpxe3X_Vgb5I,2699
-lamindb_setup/_setup_user.py,sha256=ojq7UP2Aia8GTCr6m8fylFx9VSuvGu0HmvIJ8RzymE0,6108
+lamindb_setup/_schema_metadata.py,sha256=Whs-e4ZMnA1niZ2l5Eu8il-33IxI4Hr5ylGEgPxx8wk,15628
+lamindb_setup/_set_managed_storage.py,sha256=xQe5DXCRiQ5VseAjVC2Bki0wB0n0tSTchvVKSx9I6eo,3094
+lamindb_setup/_setup_user.py,sha256=cjQ-Md-FkP04PnBxocbHW6wCsZsNtD2T2NB52vAOnHI,6730
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/errors.py,sha256=lccF3X3M2mcbHVG_0HxfuJRFFpUE-42paccIxFOfefQ,1958
-lamindb_setup/io.py,sha256=9LstFkIaki_m_oE7hFSPN8j1eXLjfEyarf13v2wcGso,13702
+lamindb_setup/io.py,sha256=9s4Itt4rrHzsUATY79r4nhGp9zVAm-9uBhiQgg60l6U,16708
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=fuQxZJnrGYe7a_Ju9n1RqO-HhkOAr1l1xjpAg9dmBu8,605
-lamindb_setup/core/__init__.py,sha256=gMqbkPeD-T6o_KykonvNiUJJQYz5SCk2yGgFNWFUBNc,603
+lamindb_setup/core/__init__.py,sha256=adZtacDwG2C0tgx-ypp9yOAqw9qaR-IRWkgLurKpXVE,668
 lamindb_setup/core/_aws_options.py,sha256=9kQ5BB-cuJQrlJRGNqMRe1m48dP67xMbefOJP2c9OQw,9674
 lamindb_setup/core/_aws_storage.py,sha256=QEtV-riQrwfivcwqHnXBbkJ-9YyNEXL4fLoCmOHZ1BI,2003
-lamindb_setup/core/_clone.py,sha256=2NlXV04yykqg_k7z59C_kD1F1Hi4H-55H-JtNjhenQ0,3691
+lamindb_setup/core/_clone.py,sha256=oLTMItGxRQB1THSDP-RG2eH1qPTVcuZ7_-eElttJ518,6451
 lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=J0x43at0zb0yWP-RoT2lyqaHV66ewUP3OiYVYQCjxe8,9974
+lamindb_setup/core/_hub_client.py,sha256=vem145S5ppRPcWob7iclGhos8k-BfwJi9AI-l5PteDs,10481
 lamindb_setup/core/_hub_core.py,sha256=GAQK5XkHROIuqA-H8sOQZVlxvN4QIH_cmHY0TENnq2U,29090
 lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
-lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=bAwvJhin7hLNliE_yaM-CMC3d5YD6-kY3raGpnse91Y,15458
-lamindb_setup/core/_settings_instance.py,sha256=7VXd1W88fgqEnAfzFQKUlDnTr3pmA_e8aIag7FqPrJI,23899
-lamindb_setup/core/_settings_load.py,sha256=j20cy3J56ZBHLDfB2A8oKjekNetMNsy0_W3eWD36pWI,5161
-lamindb_setup/core/_settings_save.py,sha256=jh412jXIAbIYvnSoW9riBFePRAa4vmPm-ScYD0smlnw,3292
-lamindb_setup/core/_settings_storage.py,sha256=yFz0w7yB7_3UWdrndVHJ_BZyV4ONj2lPRaQn_cksFQw,15150
-lamindb_setup/core/_settings_store.py,sha256=ykJeBA9IODK4G_jrfBE9pb0c1xkfePkARPpb306DT08,2687
+lamindb_setup/core/_private_django_api.py,sha256=Z9uGL4CK0OX58rc8R_qarg9rIBp1DgjsjfP9Vj2vJHI,2629
+lamindb_setup/core/_settings.py,sha256=QbTrSkkdx0u685NJ4neNtWzhdHoaGMKcIvrfFnctTQ4,15450
+lamindb_setup/core/_settings_instance.py,sha256=eDkueLK5JZOGFhZRbGa-OffS9iBFlxMp47vF_MfmCYI,24301
+lamindb_setup/core/_settings_load.py,sha256=NQDOln8e3qyGphk8ucU7mm3HVkCv4QV4rDZro3TIwfo,5183
+lamindb_setup/core/_settings_save.py,sha256=96mWdYLyfvbnG_ok_vK4x7jm-rtqcWCD1OHEt2QSAms,3328
+lamindb_setup/core/_settings_storage.py,sha256=22EBagIr5qOZr9pqVkJsTcQtgE14en-Wh0y9rgF4FEQ,15677
+lamindb_setup/core/_settings_store.py,sha256=auZssUBb6qE5oSqdGiHhqI2B46qSpegX89VwObPQksk,2601
 lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
-lamindb_setup/core/django.py,sha256=2HwhtfUEX4peSkczc0VSfA-CpfCGL4vNgkPe9Pwu5kw,12429
+lamindb_setup/core/django.py,sha256=aBdIN07ZCD8PGT04sjF1rruwlppx0cEobBnjk2Due70,12525
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=bi3k8AYeiGB_NtVTO9e9gHsfs2AFB4fXiVHcbNpnlpI,35780
-lamindb_setup-1.15.1.dist-info/licenses/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.15.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-lamindb_setup-1.15.1.dist-info/METADATA,sha256=z9yk-pHFnYB7gv59trxqyhbbnpxzRC2XPuhMoTuujDE,1820
-lamindb_setup-1.15.1.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=_xs6CgqQezOe6h8oQURjpOl1WT_1ctROzH3yzesVceE,36188
+lamindb_setup-1.16.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.16.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.16.0.dist-info/METADATA,sha256=mkCWug_dpZRqU6Y0ajukECuIC8kezhh4bTYAIuDLAao,1830
+lamindb_setup-1.16.0.dist-info/RECORD,,

{lamindb_setup-1.15.1.dist-info → lamindb_setup-1.16.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.12.0
+Generator: flit 3.10.1
 Root-Is-Purelib: true
 Tag: py3-none-any