lamindb_setup 1.15.0__tar.gz → 1.15.2__tar.gz

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/.gitignore

@@ -114,4 +114,4 @@ _docs_tmp*
 storage_uid.txt
 test.ipynb
 test2.ipynb
-lamindb_export
+*_export

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.15.0
+Version: 1.15.2
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Migration management
 
 """
 
-__version__ = "1.15.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.15.2"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 import warnings

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/_connect_instance.py

@@ -12,7 +12,6 @@ from lamin_utils import logger
 from ._check_setup import (
     _check_instance_setup,
     _get_current_instance_settings,
-    find_module_candidates,
 )
 from ._disconnect import disconnect
 from ._init_instance import load_from_isettings

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/_set_managed_storage.py

@@ -41,7 +41,7 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             "use a tuple of (local_root, host) instead"
         )
 
-    # here the storage is registered in the hub
+    # here the storage location is registered in the hub
     # hub_record_status="hub-record-created" if a new record is created
     # "hub-record-retrieved" if the storage is in the hub already
     ssettings, hub_record_status = init_storage(
@@ -65,5 +65,13 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
             delete_storage_record(ssettings)
         raise e
 
-    settings.instance._storage = ssettings
-    settings.storage._set_fs_kwargs(**fs_kwargs)
+    if ssettings._instance_id != settings.instance._id:
+        logger.warning(
+            f"registered storage location {root} as read-only for this instance (it's written by instance with uid: {ssettings.instance_uid})"
+        )
+        logger.warning(
+            f"did *not* switch default storage location, it's still: {settings.storage.root_as_str}"
+        )
+    else:
+        settings.instance._storage = ssettings
+        settings.storage._set_fs_kwargs(**fs_kwargs)

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/__init__.py

@@ -23,7 +23,12 @@ Storage
 """
 
 from . import django, upath
-from ._clone import connect_local_sqlite, init_local_sqlite
+from ._clone import (
+    connect_local_sqlite,
+    connect_remote_sqlite,
+    init_local_sqlite,
+    upload_sqlite_clone,
+)
 from ._deprecated import deprecated  # documented in lamindb.base
 from ._docs import doc_args  # documented in lamindb.base
 from ._settings import SetupSettings

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_aws_options.py

@@ -20,6 +20,7 @@ lamin_env = os.getenv("LAMIN_ENV")
 if lamin_env is None or lamin_env == "prod":
     HOSTED_BUCKETS = tuple([f"s3://lamin-{region}" for region in HOSTED_REGIONS])
 else:
+    logger.warning("loaded LAMIN_ENV: staging")
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 

lamindb_setup-1.15.2/lamindb_setup/core/_clone.py

@@ -0,0 +1,174 @@
+"""Utilities to copy, clone and load Postgres instances as local SQLite databases.
+
+.. autosummary::
+   :toctree:
+
+   init_local_sqlite
+   connect_local_sqlite
+   upload_sqlite_clone
+"""
+
+import gzip
+import os
+import shutil
+from pathlib import Path
+
+from lamindb_setup.core._settings_instance import InstanceSettings
+from lamindb_setup.core._settings_load import load_instance_settings
+from lamindb_setup.core._settings_store import instance_settings_file
+from lamindb_setup.core.django import reset_django
+from lamindb_setup.core.upath import create_path
+
+
+def init_local_sqlite(
+    instance: str | None = None, copy_suffix: str | None = None
+) -> None:
+    """Initialize SQLite copy of an existing Postgres instance.
+
+    Creates a SQLite database with the same schema as the source Postgres instance.
+    The copy shares the same storage location as the original instance.
+
+    The copy is intended for read-only access to instance data without requiring a Postgres connection.
+    Data synchronization to complete the clone happens via a separate Lambda function.
+
+    Note that essential user, branch and storage tables are missing.
+    Therefore, it is not possible to store Artifacts without having replayed these records first.
+
+    Args:
+        instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
+            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
+            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+        copy_suffix: Optional suffix to append to the local clone name.
+    """
+    import lamindb_setup as ln_setup
+
+    if instance is None:  # pragma: no cover
+        instance = os.environ.get("LAMIN_CURRENT_INSTANCE")
+
+    if instance is None:
+        raise ValueError(
+            "No instance identifier provided and LAMIN_CURRENT_INSTANCE is not set"
+        )
+
+    if ln_setup.settings.instance is None:  # pragma: no cover
+        ln_setup.connect(instance)
+
+    name = (
+        f"{ln_setup.settings.instance.name}{copy_suffix}"
+        if copy_suffix is not None
+        else ln_setup.settings.instance.name
+    )
+    isettings = ln_setup._connect_instance._connect_instance(
+        owner=ln_setup.settings.instance.owner, name=name
+    )
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
+    isettings._persist(write_to_disk=True)
+
+    if not isettings._sqlite_file_local.exists():
+        # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
+        # Without this reset, the `if not settings.configured`` check in `setup_django()` would skip reconfiguration,
+        # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
+        reset_django()
+        isettings._init_db()
+
+
+def connect_local_sqlite(
+    instance: str,
+) -> None:
+    """Load a locally stored SQLite instance of which a remote hub Postgres instance exists.
+
+    This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
+    The clone must first be created via `init_local_sqlite()`.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+    """
+    owner, name = instance.split("/")
+    settings_file = instance_settings_file(name=name, owner=owner)
+
+    if not settings_file.exists():
+        raise ValueError(
+            "SQLite clone not found."
+            " Run `init_local_sqlite()` to create a local copy or connect to a remote copy using `connect_remote_sqlite`."
+        )
+
+    isettings = load_instance_settings(settings_file)
+    isettings._persist(write_to_disk=False)
+
+    # Using `setup_django` instead of `_load_db` to not ping AWS RDS
+    from lamindb_setup._check_setup import disable_auto_connect
+
+    from .django import setup_django
+
+    disable_auto_connect(setup_django)(isettings)
+
+
+def connect_remote_sqlite(instance: str, *, copy_suffix: str | None = None) -> None:
+    """Load an existing SQLite copy of a hub instance.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+        copy_suffix: Optional suffix of the local clone.
+    """
+    import lamindb_setup as ln_setup
+
+    owner, name = instance.split("/")
+
+    # Step 1: Create the settings file
+    isettings = ln_setup._connect_instance._connect_instance(owner=owner, name=name)
+    isettings._db = None
+    isettings._is_on_hub = False
+    isettings._fine_grained_access = False
+    isettings._db_permissions = "read"
+    name = (
+        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
+    )
+    isettings._name = name
+    isettings._is_clone = True
+    isettings._persist(write_to_disk=True)
+
+    connect_local_sqlite(instance=instance + (copy_suffix or ""))
+
+
+def upload_sqlite_clone(
+    local_sqlite_path: Path | str | None = None, compress: bool = True
+) -> None:
+    """Uploads the SQLite clone to the default storage.
+
+    Args:
+        local_sqlite_path: Path to the SQLite file.
+            Defaults to the local storage path if not specified.
+        compress: Whether to compress the database with gzip before uploading.
+    """
+    import lamindb_setup as ln_setup
+
+    if local_sqlite_path is None:
+        local_sqlite_path = ln_setup.settings.instance._sqlite_file_local
+    else:
+        local_sqlite_path = Path(local_sqlite_path)
+
+    if not local_sqlite_path.exists():
+        raise FileNotFoundError(f"Database not found at {local_sqlite_path}")
+
+    cloud_db_path = ln_setup.settings.instance._sqlite_file
+
+    if compress:
+        temp_gz_path = local_sqlite_path.with_suffix(".db.gz")
+        with (
+            open(local_sqlite_path, "rb") as f_in,
+            gzip.open(temp_gz_path, "wb") as f_out,
+        ):
+            shutil.copyfileobj(f_in, f_out)
+        cloud_destination = create_path(f"{cloud_db_path}.gz")
+        cloud_destination.upload_from(temp_gz_path, print_progress=True)
+        temp_gz_path.unlink()
+    else:
+        cloud_destination = create_path(cloud_db_path)
+        cloud_destination.upload_from(local_sqlite_path, print_progress=True)

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_hub_client.py

@@ -96,9 +96,12 @@ def connect_hub(
         transports.append(
             RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
         )
+    # this overwrites transports of existing httpx clients
+    # if proxies are set, the default transports that were created on clients init
+    # will be used, irrespective of these re-settings
     client.auth._http_client._transport = transports[0]
     client.postgrest.session._transport = transports[1]
     # POST is not retryable by default, but for our functions it should be safe to retry
@@ -116,7 +119,7 @@ def connect_hub(
                 "POST",
             ],
         ),
-        transport=httpx.HTTPTransport(verify=True, http2=True),
+        transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
     )
     return client
 
@@ -246,9 +249,13 @@ def httpx_client():
         else:
             transport = RetryTransport(
                 retry=LogRetry(total=2, backoff_factor=0.2),
-                transport=httpx.HTTPTransport(verify=True, http2=True),
+                transport=httpx.HTTPTransport(verify=True, http2=True, trust_env=True),
             )
-            client = httpx.Client(transport=transport)
+            # first we create a client to build the proxy map from the env variables
+            # if proxies are set, the default transports will be used
+            # otherwise the RetryTransport object that we assign below
+            client = httpx.Client(trust_env=True)
+            client._transport = transport
         yield client
     finally:
         if client is not None:

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings.py

@@ -320,8 +320,7 @@ class SetupSettings:
     def paths(self) -> type[SetupPaths]:
         """Convert cloud paths to lamindb local paths.
 
-        Use `settings.paths.cloud_to_local_no_update`
-        or `settings.paths.cloud_to_local`.
+        Use `settings.paths.cloud_to_local_no_update` or `settings.paths.cloud_to_local`.
         """
         return SetupPaths
 

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_instance.py

@@ -54,8 +54,7 @@ def is_local_db_url(db_url: str) -> bool:
 
 
 def check_is_instance_remote(root: UPathStr, db: str | None) -> bool:
-    # returns True for cloud SQLite
-    # and remote postgres
+    # returns True for cloud SQLite and remote postgres
     root_str = str(root)
     if not root_str.startswith("create-s3") and get_storage_type(root_str) == "local":
         return False
@@ -83,7 +82,8 @@ class InstanceSettings:
         schema_id: UUID | None = None,
         fine_grained_access: bool = False,
         db_permissions: str | None = None,
-        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite
+        _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite,
+        _is_clone: bool = False,
     ):
         from ._hub_utils import validate_db_arg
 
@@ -109,6 +109,7 @@ class InstanceSettings:
         self._db_permissions = db_permissions
         # if None then settings.user is used
         self._locker_user = _locker_user
+        self._is_clone = _is_clone
 
     def __repr__(self):
         """Rich string representation."""
@@ -434,7 +435,7 @@ class InstanceSettings:
 
     def _update_cloud_sqlite_file(self, unlock_cloud_sqlite: bool = True) -> None:
         """Upload the local sqlite file to the cloud file."""
-        if self._is_cloud_sqlite:
+        if self._is_cloud_sqlite and not self._is_clone:
             sqlite_file = self._sqlite_file
             logger.warning(
                 f"updating{' & unlocking' if unlock_cloud_sqlite else ''} cloud SQLite "
@@ -602,6 +603,14 @@ class InstanceSettings:
         disable_auto_connect(setup_django)(self, init=True)
 
     def _load_db(self) -> tuple[bool, str]:
+        """Load the database connection.
+
+        For cloud SQLite instances, downloads the database file from cloud storage.
+        For all instances, initializes Django ORM with the database connection.
+
+        Returns:
+            Tuple of (success: bool, error_message: str). Returns (True, "") on success.
+        """
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -615,8 +624,8 @@ class InstanceSettings:
                 return False, f"SQLite file {self._sqlite_file} does not exist"
         # we need the local sqlite to setup django
         self._update_local_sqlite_file()
-        # setting up django also performs a check for migrations & prints them
-        # as warnings
+
+        # setting up django also performs a check for migrations & prints them as warnings
         # this should fail, e.g., if the db is not reachable
         from lamindb_setup._check_setup import disable_auto_connect
 

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_load.py

@@ -69,8 +69,7 @@ def load_or_create_user_settings(api_key: str | None = None) -> UserSettings:
     """Return current user settings.
 
     Args:
-        api_key: if provided and there is no current user,
-            perform login and return the user settings.
+        api_key: if provided and there is no current user, perform login and return the user settings.
     """
     current_user_settings = current_user_settings_file()
     if not current_user_settings.exists():
@@ -125,6 +124,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         schema_id=None if store.schema_id in {None, "null"} else UUID(store.schema_id),
         fine_grained_access=store.fine_grained_access,
         db_permissions=_null_to_value(store.db_permissions),
+        _is_clone=store.is_clone,
     )
 
 

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_save.py

@@ -63,6 +63,7 @@ def save_settings(
                         "schema_id",
                         "fine_grained_access",
                         "db_permissions",
+                        "is_clone",
                     }:
                         settings_key = f"_{store_key.rstrip('_')}"
                     else:

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_storage.py

@@ -12,17 +12,14 @@ from lamin_utils import logger
 from lamindb_setup.errors import StorageAlreadyManaged
 
 from ._aws_options import (
-    HOSTED_REGIONS,
     LAMIN_ENDPOINTS,
     get_aws_options_manager,
 )
-from ._aws_storage import find_closest_aws_region
 from ._deprecated import deprecated
 from .hashing import hash_and_encode_as_b62
 from .upath import (
     LocalPathClasses,
     UPath,
-    _split_path_query,
     create_path,
     get_storage_region,
 )
@@ -58,12 +55,40 @@ def get_storage_type(root_as_str: str) -> StorageType:
     return convert.get(protocol, protocol)  # type: ignore
 
 
+def sanitize_root_user_input(root: UPathStr) -> UPath:
+    """Format a root path string."""
+    root_upath = root if isinstance(root, UPath) else UPath(root)
+    root_upath = root_upath.expanduser()
+    if isinstance(root_upath, LocalPathClasses):  # local paths
+        try:
+            (root_upath / ".lamindb").mkdir(parents=True, exist_ok=True)
+            root_upath = root_upath.resolve()
+        except Exception:
+            logger.warning(f"unable to create .lamindb/ folder in {root_upath}")
+    return root_upath
+
+
+def convert_sanitized_root_path_to_str(root_upath: UPath) -> str:
+    # embed endpoint_url into path string for storing and displaying
+    if root_upath.protocol == "s3":
+        endpoint_url = root_upath.storage_options.get("endpoint_url", None)
+        # LAMIN_ENDPOINTS include None
+        if endpoint_url not in LAMIN_ENDPOINTS:
+            return f"s3://{root_upath.path.rstrip('/')}?endpoint_url={endpoint_url}"
+    return root_upath.as_posix().rstrip("/")
+
+
+def convert_root_path_to_str(root: UPathStr) -> str:
+    """Format a root path string."""
+    sanitized_root_upath = sanitize_root_user_input(root)
+    return convert_sanitized_root_path_to_str(sanitized_root_upath)
+
+
 def mark_storage_root(
     root: UPathStr, uid: str, instance_id: UUID, instance_slug: str
 ) -> Literal["__marked__"] | str:
     # we need a file in folder-like storage locations on S3 to avoid
-    # permission errors from leveraging s3fs on an empty hosted storage location
-    # (path.fs.find raises a PermissionError)
+    # permission errors from leveraging s3fs on an empty hosted storage location (path.fs.find raises a PermissionError)
     # we also need it in case a storage location is ambiguous because a server / local environment
     # doesn't have a globally unique identifier, then we screen for this file to map the
     # path on a storage location in the registry
@@ -214,15 +239,7 @@ class StorageSettings:
     ):
         self._uid = uid
         self._uuid_ = uuid
-        self._root_init = UPath(root).expanduser()
-        if isinstance(self._root_init, LocalPathClasses):  # local paths
-            try:
-                (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
-                self._root_init = self._root_init.resolve()
-            except Exception:
-                logger.warning(
-                    f"unable to create .lamindb/ folder in {self._root_init}"
-                )
+        self._root_init: UPath = sanitize_root_user_input(root)
         self._root = None
         self._instance_id = instance_id
         # we don't yet infer region here to make init fast
@@ -337,13 +354,7 @@ class StorageSettings:
     @property
     def root_as_str(self) -> str:
         """Formatted root string."""
-        # embed endpoint_url into path string for storing and displaying
-        if self._root_init.protocol == "s3":
-            endpoint_url = self._root_init.storage_options.get("endpoint_url", None)
-            # LAMIN_ENDPOINTS include None
-            if endpoint_url not in LAMIN_ENDPOINTS:
-                return f"s3://{self._root_init.path.rstrip('/')}?endpoint_url={endpoint_url}"
-        return self._root_init.as_posix().rstrip("/")
+        return convert_sanitized_root_path_to_str(self._root_init)
 
     @property
     def cache_dir(

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/_settings_store.py

@@ -67,8 +67,8 @@ class InstanceSettingsStore(BaseSettings):
     owner: str
     name: str
     storage_root: str
-    storage_region: str | None  # take old type annotations here because pydantic
-    db: str | None  # doesn't like new types on 3.9 even with future annotations
+    storage_region: str | None
+    db: str | None
     schema_str: str | None
     schema_id: str | None = None
     fine_grained_access: bool = False
@@ -76,6 +76,7 @@ class InstanceSettingsStore(BaseSettings):
     id: str
     git_repo: str | None
     keep_artifacts_local: bool | None
+    is_clone: bool = False
     model_config = SettingsConfigDict(env_prefix="lamindb_instance_", env_file=".env")
 
 

{lamindb_setup-1.15.0 → lamindb_setup-1.15.2}/lamindb_setup/core/upath.py

@@ -908,12 +908,9 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str | None, str | None]:
     elif "blob_id" in stat:
         hash = b16_to_b64(stat["blob_id"])
         hash_type = "sha1"
-    # s3
-    # StorageClass is checked to be sure that it is indeed s3
-    # because http also has ETag
     elif "ETag" in stat:
         etag = stat["ETag"]
-        if "mimetype" in stat:
+        if "mimetype" in stat or ("url" in stat and stat["url"].startswith("http")):
             # http
             hash = hash_string(etag.strip('"'))
             hash_type = "md5-etag"