lamindb_setup 1.15.1__tar.gz → 1.15.2__tar.gz

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/PKG-INFO

@@ -1,11 +1,10 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.15.1
+Version: 1.15.2
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: lamin_utils>=0.3.3
 Requires-Dist: django>=5.2,<5.3
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Migration management
 
 """
 
-__version__ = "1.15.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.15.2"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 import warnings

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/_connect_instance.py

@@ -12,7 +12,6 @@ from lamin_utils import logger
 from ._check_setup import (
     _check_instance_setup,
     _get_current_instance_settings,
-    find_module_candidates,
 )
 from ._disconnect import disconnect
 from ._init_instance import load_from_isettings

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/_set_managed_storage.py

@@ -41,7 +41,7 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             "use a tuple of (local_root, host) instead"
         )
 
-    # here the storage is registered in the hub
+    # here the storage location is registered in the hub
     # hub_record_status="hub-record-created" if a new record is created
     # "hub-record-retrieved" if the storage is in the hub already
     ssettings, hub_record_status = init_storage(
@@ -65,5 +65,13 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             delete_storage_record(ssettings)
         raise e
 
-    settings.instance._storage = ssettings
-    settings.storage._set_fs_kwargs(**fs_kwargs)
+    if ssettings._instance_id != settings.instance._id:
+        logger.warning(
+            f"registered storage location {root} as read-only for this instance (it's written by instance with uid: {ssettings.instance_uid})"
+        )
+        logger.warning(
+            f"did *not* switch default storage location, it's still: {settings.storage.root_as_str}"
+        )
+    else:
+        settings.instance._storage = ssettings
+        settings.storage._set_fs_kwargs(**fs_kwargs)

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/__init__.py

@@ -23,7 +23,12 @@ Storage
 """
 
 from . import django, upath
-from ._clone import connect_local_sqlite, init_local_sqlite
+from ._clone import (
+    connect_local_sqlite,
+    connect_remote_sqlite,
+    init_local_sqlite,
+    upload_sqlite_clone,
+)
 from ._deprecated import deprecated  # documented in lamindb.base
 from ._docs import doc_args  # documented in lamindb.base
 from ._settings import SetupSettings

lamindb_setup-1.15.2/lamindb_setup/core/_clone.py

@@ -0,0 +1,174 @@
+"""Utilities to copy, clone and load Postgres instances as local SQLite databases.
+
+.. autosummary::
+   :toctree:
+
+   init_local_sqlite
+   connect_local_sqlite
+   upload_sqlite_clone
+"""
+
+import gzip
+import os
+import shutil
+from pathlib import Path
+
+from lamindb_setup.core._settings_instance import InstanceSettings
+from lamindb_setup.core._settings_load import load_instance_settings
+from lamindb_setup.core._settings_store import instance_settings_file
+from lamindb_setup.core.django import reset_django
+from lamindb_setup.core.upath import create_path
+
+
+def init_local_sqlite(
+    instance: str | None = None, copy_suffix: str | None = None
+) -> None:
+    """Initialize SQLite copy of an existing Postgres instance.
+
+    Creates a SQLite database with the same schema as the source Postgres instance.
+    The copy shares the same storage location as the original instance.
+
+    The copy is intended for read-only access to instance data without requiring a Postgres connection.
+    Data synchronization to complete the clone happens via a separate Lambda function.
+
+    Note that essential user, branch and storage tables are missing.
+    Therefore, it is not possible to store Artifacts without having replayed these records first.
+
+    Args:
+        instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
+            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
+            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+        copy_suffix: Optional suffix to append to the local clone name.
+    """
+    import lamindb_setup as ln_setup
+
+    if instance is None:  # pragma: no cover
+        instance = os.environ.get("LAMIN_CURRENT_INSTANCE")
+
+    if instance is None:
+        raise ValueError(
+            "No instance identifier provided and LAMIN_CURRENT_INSTANCE is not set"
+        )
+
+    if ln_setup.settings.instance is None:  # pragma: no cover
+        ln_setup.connect(instance)
+
+    name = (
+        f"{ln_setup.settings.instance.name}{copy_suffix}"
+        if copy_suffix is not None
+        else ln_setup.settings.instance.name
+    )
+    isettings = ln_setup._connect_instance._connect_instance(
+        owner=ln_setup.settings.instance.owner, name=name
+    )
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
+    isettings._persist(write_to_disk=True)
+
+    if not isettings._sqlite_file_local.exists():
+        # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
+        # Without this reset, the `if not settings.configured`` check in `setup_django()` would skip reconfiguration,
+        # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
+        reset_django()
+        isettings._init_db()
+
+
+def connect_local_sqlite(
+    instance: str,
+) -> None:
+    """Load a locally stored SQLite instance of which a remote hub Postgres instance exists.
+
+    This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
+    The clone must first be created via `init_local_sqlite()`.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+    """
+    owner, name = instance.split("/")
+    settings_file = instance_settings_file(name=name, owner=owner)
+
+    if not settings_file.exists():
+        raise ValueError(
+            "SQLite clone not found."
+            " Run `init_local_sqlite()` to create a local copy or connect to a remote copy using `connect_remote_sqlite`."
+        )
+
+    isettings = load_instance_settings(settings_file)
+    isettings._persist(write_to_disk=False)
+
+    # Using `setup_django` instead of `_load_db` to not ping AWS RDS
+    from lamindb_setup._check_setup import disable_auto_connect
+
+    from .django import setup_django
+
+    disable_auto_connect(setup_django)(isettings)
+
+
+def connect_remote_sqlite(instance: str, *, copy_suffix: str | None = None) -> None:
+    """Load an existing SQLite copy of a hub instance.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+        copy_suffix: Optional suffix of the local clone.
+    """
+    import lamindb_setup as ln_setup
+
+    owner, name = instance.split("/")
+
+    # Step 1: Create the settings file
+    isettings = ln_setup._connect_instance._connect_instance(owner=owner, name=name)
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    isettings._db_permissions = "read"
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
+    isettings._persist(write_to_disk=True)
+
+    connect_local_sqlite(instance=instance + (copy_suffix or ""))
+
+
+def upload_sqlite_clone(
+    local_sqlite_path: Path | str | None = None, compress: bool = True
+) -> None:
+    """Uploads the SQLite clone to the default storage.
+
+    Args:
+        local_sqlite_path: Path to the SQLite file.
+            Defaults to the local storage path if not specified.
+        compress: Whether to compress the database with gzip before uploading.
+    """
+    import lamindb_setup as ln_setup
+
+    if local_sqlite_path is None:
+        local_sqlite_path = ln_setup.settings.instance._sqlite_file_local
+    else:
+        local_sqlite_path = Path(local_sqlite_path)
+
+    if not local_sqlite_path.exists():
+        raise FileNotFoundError(f"Database not found at {local_sqlite_path}")
+
+    cloud_db_path = ln_setup.settings.instance._sqlite_file
+
+    if compress:
+        temp_gz_path = local_sqlite_path.with_suffix(".db.gz")
+        with (
+            open(local_sqlite_path, "rb") as f_in,
+            gzip.open(temp_gz_path, "wb") as f_out,
+        ):
+            shutil.copyfileobj(f_in, f_out)
+        cloud_destination = create_path(f"{cloud_db_path}.gz")
+        cloud_destination.upload_from(temp_gz_path, print_progress=True)
+        temp_gz_path.unlink()
+    else:
+        cloud_destination = create_path(cloud_db_path)
+        cloud_destination.upload_from(local_sqlite_path, print_progress=True)

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/_hub_client.py

@@ -96,9 +96,12 @@ def connect_hub(
         transports.append(
             RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
         )
+    # this overwrites transports of existing httpx clients
+    # if proxies are set, the default transports that were created on clients init
+    # will be used, irrespective of these re-settings
     client.auth._http_client._transport = transports[0]
     client.postgrest.session._transport = transports[1]
     # POST is not retryable by default, but for our functions it should be safe to retry
@@ -116,7 +119,7 @@ def connect_hub(
                 "POST",
             ],
         ),
-        transport=httpx.HTTPTransport(verify=True, http2=True),
+        transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
     )
     return client
 
@@ -246,9 +249,13 @@ def httpx_client():
         else:
             transport = RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
-            client = httpx.Client(transport=transport)
+            # first we create a client to build the proxy map from the env variables
+            # if proxies are set, the default transports will be used
+            # otherwise the RetryTransport object that we assign below
+            client = httpx.Client(trust_env=True)
+            client._transport = transport
         yield client
     finally:
         if client is not None:

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings.py

@@ -320,8 +320,7 @@ class SetupSettings:
     def paths(self) -> type[SetupPaths]:
         """Convert cloud paths to lamindb local paths.
 
-        Use `settings.paths.cloud_to_local_no_update`
-        or `settings.paths.cloud_to_local`.
+        Use `settings.paths.cloud_to_local_no_update` or `settings.paths.cloud_to_local`.
         """
         return SetupPaths
 

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_instance.py

@@ -54,8 +54,7 @@ def is_local_db_url(db_url: str) -> bool:
 
 
 def check_is_instance_remote(root: UPathStr, db: str | None) -> bool:
-    # returns True for cloud SQLite
-    # and remote postgres
+    # returns True for cloud SQLite and remote postgres
     root_str = str(root)
     if not root_str.startswith("create-s3") and get_storage_type(root_str) == "local":
         return False
@@ -83,7 +82,8 @@ class InstanceSettings:
         schema_id: UUID | None = None,
         fine_grained_access: bool = False,
         db_permissions: str | None = None,
-        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite
+        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite,
+        _is_clone: bool = False,
     ):
         from ._hub_utils import validate_db_arg
 
@@ -109,6 +109,7 @@ class InstanceSettings:
         self._db_permissions = db_permissions
         # if None then settings.user is used
         self._locker_user = _locker_user
+        self._is_clone = _is_clone
 
     def __repr__(self):
         """Rich string representation."""
@@ -434,7 +435,7 @@ class InstanceSettings:
 
     def _update_cloud_sqlite_file(self, unlock_cloud_sqlite: bool = True) -> None:
         """Upload the local sqlite file to the cloud file."""
-        if self._is_cloud_sqlite:
+        if self._is_cloud_sqlite and not self._is_clone:
             sqlite_file = self._sqlite_file
             logger.warning(
                 f"updating{' & unlocking' if unlock_cloud_sqlite else ''} cloud SQLite "
@@ -602,6 +603,14 @@ class InstanceSettings:
         disable_auto_connect(setup_django)(self, init=True)
 
     def _load_db(self) -> tuple[bool, str]:
+        """Load the database connection.
+
+        For cloud SQLite instances, downloads the database file from cloud storage.
+        For all instances, initializes Django ORM with the database connection.
+
+        Returns:
+            Tuple of (success: bool, error_message: str). Returns (True, "") on success.
+        """
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -615,8 +624,8 @@ class InstanceSettings:
                 return False, f"SQLite file {self._sqlite_file} does not exist"
         # we need the local sqlite to setup django
         self._update_local_sqlite_file()
-        # setting up django also performs a check for migrations & prints them
-        # as warnings
+
+        # setting up django also performs a check for migrations & prints them as warnings
         # this should fail, e.g., if the db is not reachable
         from lamindb_setup._check_setup import disable_auto_connect
 

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_load.py

@@ -69,8 +69,7 @@ def load_or_create_user_settings(api_key: str | None = None) -> UserSettings:
     """Return current user settings.
 
     Args:
-        api_key: if provided and there is no current user,
-            perform login and return the user settings.
+        api_key: if provided and there is no current user, perform login and return the user settings.
     """
     current_user_settings = current_user_settings_file()
     if not current_user_settings.exists():
@@ -125,6 +124,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         schema_id=None if store.schema_id in {None, "null"} else UUID(store.schema_id),
         fine_grained_access=store.fine_grained_access,
         db_permissions=_null_to_value(store.db_permissions),
+        _is_clone=store.is_clone,
     )
 
 

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_save.py

@@ -63,6 +63,7 @@ def save_settings(
                         "schema_id",
                         "fine_grained_access",
                         "db_permissions",
+                        "is_clone",
                     }:
                         settings_key = f"_{store_key.rstrip('_')}"
                     else:

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_storage.py

@@ -12,17 +12,14 @@ from lamin_utils import logger
 from lamindb_setup.errors import StorageAlreadyManaged
 
 from ._aws_options import (
-    HOSTED_REGIONS,
     LAMIN_ENDPOINTS,
     get_aws_options_manager,
 )
-from ._aws_storage import find_closest_aws_region
 from ._deprecated import deprecated
 from .hashing import hash_and_encode_as_b62
 from .upath import (
     LocalPathClasses,
     UPath,
-    _split_path_query,
     create_path,
     get_storage_region,
 )
@@ -58,12 +55,40 @@ def get_storage_type(root_as_str: str) -> StorageType:
     return convert.get(protocol, protocol)  # type: ignore
 
 
+def sanitize_root_user_input(root: UPathStr) -> UPath:
+    """Format a root path string."""
+    root_upath = root if isinstance(root, UPath) else UPath(root)
+    root_upath = root_upath.expanduser()
+    if isinstance(root_upath, LocalPathClasses):  # local paths
+        try:
+            (root_upath / ".lamindb").mkdir(parents=True, exist_ok=True)
+            root_upath = root_upath.resolve()
+        except Exception:
+            logger.warning(f"unable to create .lamindb/ folder in {root_upath}")
+    return root_upath
+
+
+def convert_sanitized_root_path_to_str(root_upath: UPath) -> str:
+    # embed endpoint_url into path string for storing and displaying
+    if root_upath.protocol == "s3":
+        endpoint_url = root_upath.storage_options.get("endpoint_url", None)
+        # LAMIN_ENDPOINTS include None
+        if endpoint_url not in LAMIN_ENDPOINTS:
+            return f"s3://{root_upath.path.rstrip('/')}?endpoint_url={endpoint_url}"
+    return root_upath.as_posix().rstrip("/")
+
+
+def convert_root_path_to_str(root: UPathStr) -> str:
+    """Format a root path string."""
+    sanitized_root_upath = sanitize_root_user_input(root)
+    return convert_sanitized_root_path_to_str(sanitized_root_upath)
+
+
 def mark_storage_root(
     root: UPathStr, uid: str, instance_id: UUID, instance_slug: str
 ) -> Literal["__marked__"] | str:
     # we need a file in folder-like storage locations on S3 to avoid
-    # permission errors from leveraging s3fs on an empty hosted storage location
-    # (path.fs.find raises a PermissionError)
+    # permission errors from leveraging s3fs on an empty hosted storage location (path.fs.find raises a PermissionError)
     # we also need it in case a storage location is ambiguous because a server / local environment
     # doesn't have a globally unique identifier, then we screen for this file to map the
     # path on a storage location in the registry
@@ -214,15 +239,7 @@ class StorageSettings:
     ):
         self._uid = uid
         self._uuid_ = uuid
-        self._root_init = UPath(root).expanduser()
-        if isinstance(self._root_init, LocalPathClasses):  # local paths
-            try:
-                (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
-                self._root_init = self._root_init.resolve()
-            except Exception:
-                logger.warning(
-                    f"unable to create .lamindb/ folder in {self._root_init}"
-                )
+        self._root_init: UPath = sanitize_root_user_input(root)
         self._root = None
         self._instance_id = instance_id
         # we don't yet infer region here to make init fast
@@ -337,13 +354,7 @@ class StorageSettings:
     @property
     def root_as_str(self) -> str:
         """Formatted root string."""
-        # embed endpoint_url into path string for storing and displaying
-        if self._root_init.protocol == "s3":
-            endpoint_url = self._root_init.storage_options.get("endpoint_url", None)
-            # LAMIN_ENDPOINTS include None
-            if endpoint_url not in LAMIN_ENDPOINTS:
-                return f"s3://{self._root_init.path.rstrip('/')}?endpoint_url={endpoint_url}"
-        return self._root_init.as_posix().rstrip("/")
+        return convert_sanitized_root_path_to_str(self._root_init)
 
     @property
     def cache_dir(

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_store.py

@@ -67,8 +67,8 @@ class InstanceSettingsStore(BaseSettings):
     owner: str
     name: str
     storage_root: str
-    storage_region: str | None  # take old type annotations here because pydantic
-    db: str | None  # doesn't like new types on 3.9 even with future annotations
+    storage_region: str | None
+    db: str | None
     schema_str: str | None
     schema_id: str | None = None
     fine_grained_access: bool = False
@@ -76,6 +76,7 @@ class InstanceSettingsStore(BaseSettings):
     id: str
     git_repo: str | None
     keep_artifacts_local: bool | None
+    is_clone: bool = False
     model_config = SettingsConfigDict(env_prefix="lamindb_instance_", env_file=".env")
 
 

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/core/upath.py

@@ -908,12 +908,9 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str | None, str | None]:
     elif "blob_id" in stat:
         hash = b16_to_b64(stat["blob_id"])
         hash_type = "sha1"
-    # s3
-    # StorageClass is checked to be sure that it is indeed s3
-    # because http also has ETag
     elif "ETag" in stat:
         etag = stat["ETag"]
-        if "mimetype" in stat:
+        if "mimetype" in stat or ("url" in stat and stat["url"].startswith("http")):
             # http
             hash = hash_string(etag.strip('"'))
             hash_type = "md5-etag"

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/lamindb_setup/io.py

@@ -172,7 +172,14 @@ def export_db(
 
     with Progress() as progress:
         task_id = progress.add_task("Exporting", total=len(tasks))
-        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+
+        import multiprocessing
+
+        mp_context = multiprocessing.get_context("spawn")
+
+        with ProcessPoolExecutor(
+            max_workers=max_workers, mp_context=mp_context
+        ) as executor:
             futures = {
                 executor.submit(_export_full_table, task, directory, chunk_size): task
                 for task in tasks
@@ -237,9 +244,22 @@ def _import_registry(
             if mask.any():
                 df.loc[mask, col] = df.loc[mask, col].map(_serialize_value)
 
+    if if_exists == "append":
+        # Fill NULL values in NOT NULL columns to handle schema mismatches between postgres source and SQLite target
+        # This allows importing data where fields were nullable
+        for field in registry._meta.fields:
+            if field.column in df.columns and not field.null:
+                df[field.column] = df[field.column].fillna("")
+
     if df.empty:
         return
 
+    if if_exists == "append":
+        # Clear existing data before import
+        # When appending we would run into duplicate errors because of existing values like branches etc
+        with connection.cursor() as cursor:
+            cursor.execute(f'DELETE FROM "{table_name}"')
+
     if connection.vendor == "postgresql":
         columns = df.columns.tolist()
         column_names = ", ".join(f'"{col}"' for col in columns)
@@ -265,6 +285,7 @@ def _import_registry(
         max_vars = 900  # SQLite has a limit of 999 variables per statement
         chunksize = max(1, max_vars // num_cols)
 
+        # Always use append mode since we set up the tables from a fresh instance
         df.to_sql(
             table_name,
             connection.connection,
@@ -290,6 +311,9 @@ def import_db(
         input_dir: Directory containing parquet files to import.
         module_names: Module names to import (e.g., ["lamindb", "bionty", "wetlab"]).
         if_exists: How to behave if table exists: 'fail', 'replace', or 'append'.
+            If set to 'replace', existing data is deleted and new data is imported. PKs and indices are not guaranteed to be preserved which can lead to write errors.
+            If set to 'append', new data is added to existing data without clearing the table. PKs and indices are preserved but database size will greatly increase.
+            If set to 'fail', raises an error if the table contains any data.
     """
     from django.db import connection
 
@@ -352,3 +376,5 @@ def import_db(
                 cursor.execute("PRAGMA synchronous = FULL")
                 cursor.execute("PRAGMA journal_mode = DELETE")
                 cursor.execute("PRAGMA foreign_keys = ON")
+                # Reclaim space from DELETEs
+                cursor.execute("VACUUM")

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/tests/hub-cloud/test_clone_instance.py

@@ -1,5 +1,8 @@
 import shutil
+from pathlib import Path
 from subprocess import DEVNULL, run
+from unittest.mock import MagicMock, Mock, patch
+from uuid import uuid4
 
 import lamindb_setup as ln_setup
 import pandas as pd
@@ -85,3 +88,50 @@ def test_connect_local_sqlite(local_postgres_instance):
 
     with pytest.raises(ValueError, match="SQLite clone not found"):
         ln_setup.core.connect_local_sqlite(f"{original_owner}/nonexistent")
+
+
+def test_connect_remote_sqlite(tmp_path):
+    mock_instance_id = uuid4()
+    mock_storage_root = "s3://my-bucket/data"
+
+    with (
+        patch(
+            "lamindb_setup._connect_instance._connect_instance"
+        ) as mock_connect_instance,
+        patch("lamindb_setup.settings") as mock_settings,
+        patch("lamindb_setup.core._clone.InstanceSettings"),
+        patch("lamindb_setup.core._clone.create_path") as mock_create_path,
+        patch("lamindb_setup.core._clone.connect_local_sqlite"),
+    ):
+        mock_instance = Mock()
+        mock_instance._id = mock_instance_id
+        mock_instance.owner = "testowner"
+        mock_instance.name = "testname"
+        mock_instance.modules = ["module1", "module2"]
+        mock_instance.storage.root = mock_storage_root
+        mock_instance.storage.root_as_str = mock_storage_root
+
+        mock_settings.instance = mock_instance
+        mock_settings.storage = Mock()
+        mock_settings.cache_dir = Path(tmp_path)
+
+        mock_connect_instance.return_value = mock_instance
+
+        def fake_download(target_path):
+            target_path.parent.mkdir(parents=True, exist_ok=True)
+            target_path.write_bytes(b"fake db content")
+
+        def mock_create_path_fn(path):
+            mock_file = MagicMock()
+            if path.endswith(".gz"):
+                mock_file.exists.return_value = False
+            else:
+                mock_file.exists.return_value = True
+                mock_file.download_to.side_effect = fake_download
+            return mock_file
+
+        mock_create_path.side_effect = mock_create_path_fn
+
+        from lamindb_setup.core._clone import connect_remote_sqlite
+
+        connect_remote_sqlite("testowner/testname", copy_suffix="-copy")

{lamindb_setup-1.15.1 → lamindb_setup-1.15.2}/tests/storage/test_db_import_export.py

@@ -213,3 +213,13 @@ def test_import_db_from_parquet(simple_instance: Callable, tmp_path):
     linked_gene = imported_artifact.genes.first()
     assert linked_gene.id == 999
     assert linked_gene.symbol == "TESTGENE"
+
+    # Verify PRIMARY KEY constraint is preserved for "append" mode that we used here
+    from django.db import connection
+
+    with connection.cursor() as cursor:
+        cursor.execute(
+            "SELECT sql FROM sqlite_master WHERE type='table' AND name='lamindb_artifact'"
+        )
+        create_stmt = cursor.fetchone()[0]
+        assert "PRIMARY KEY" in create_stmt

lamindb_setup-1.15.2/tests/storage/test_httpx_client.py

@@ -0,0 +1,21 @@
+import os
+
+from lamindb_setup.core._hub_client import httpx_client
+
+
+def test_proxy_from_env():
+    with httpx_client() as client:
+        assert client._mounts == {}
+
+    os.environ["HTTP_PROXY"] = "http://localhost:8080"
+    os.environ["HTTPS_PROXY"] = "http://localhost:8080"
+
+    with httpx_client() as client:
+        patterns = {p.scheme for p in client._mounts}
+    assert patterns == {"http", "https"}
+
+    del os.environ["HTTP_PROXY"]
+    del os.environ["HTTPS_PROXY"]
+
+    with httpx_client() as client:
+        assert client._mounts == {}

lamindb_setup-1.15.1/lamindb_setup/core/_clone.py

@@ -1,93 +0,0 @@
-"""Utilities to copy, clone and load Postgres instances as local SQLite databases.
-
-.. autosummary::
-   :toctree:
-
-   init_local_sqlite
-   connect_local_sqlite
-"""
-
-import os
-
-from lamindb_setup.core._settings_instance import InstanceSettings
-from lamindb_setup.core._settings_load import load_instance_settings
-from lamindb_setup.core._settings_store import instance_settings_file
-from lamindb_setup.core.django import reset_django
-
-
-def init_local_sqlite(
-    instance: str | None = None, copy_suffix: str | None = None
-) -> None:
-    """Initialize SQLite copy of an existing Postgres instance.
-
-    Creates a SQLite database with the same schema as the source Postgres instance.
-    The copy shares the same storage location as the original instance.
-
-    The copy is intended for read-only access to instance data without requiring a Postgres connection.
-    Data synchronization to complete the clone happens via a separate Lambda function.
-
-    Note that essential user, branch and storage tables are missing.
-    Therefore, it is not possible to store Artifacts without having replayed these records first.
-
-    Args:
-        instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
-            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
-            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
-        copy_suffix: Optional suffix to append to the local clone name.
-    """
-    import lamindb_setup as ln_setup
-
-    if instance is None:  # pragma: no cover
-        instance = os.environ.get("LAMIN_CURRENT_INSTANCE")
-
-    if instance is None:
-        raise ValueError(
-            "No instance identifier provided and LAMIN_CURRENT_INSTANCE is not set"
-        )
-
-    if ln_setup.settings.instance is None:  # pragma: no cover
-        ln_setup.connect(instance)
-
-    name = (
-        f"{ln_setup.settings.instance.name}{copy_suffix}"
-        if copy_suffix is not None
-        else ln_setup.settings.instance.name
-    )
-    isettings = InstanceSettings(
-        id=ln_setup.settings.instance._id,
-        owner=ln_setup.settings.instance.owner,  # type: ignore
-        name=name,
-        storage=ln_setup.settings.storage,
-        db=None,
-        modules=",".join(ln_setup.settings.instance.modules),
-        is_on_hub=False,
-    )
-
-    isettings._persist(write_to_disk=True)
-
-    if not isettings._sqlite_file_local.exists():
-        # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
-        # Without this reset, the if not settings.configured check in setup_django() would skip reconfiguration,
-        # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
-        reset_django()
-        isettings._init_db()
-
-
-def connect_local_sqlite(instance: str) -> None:
-    """Load a SQLite instance of which a remote hub Postgres instance exists.
-
-    This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
-    The clone must first be created via `init_local_sqlite()`.
-
-    Args:
-        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
-    """
-    owner, name = instance.split("/")
-    settings_file = instance_settings_file(name=name, owner=owner)
-
-    if not settings_file.exists():
-        raise ValueError("SQLite clone not found. Run init_local_sqlite() first.")
-
-    isettings = load_instance_settings(settings_file)
-    isettings._persist(write_to_disk=False)
-    isettings._load_db()