lamindb_setup 1.12.1__tar.gz → 1.13.0__tar.gz

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.12.1
+Version: 1.13.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
@@ -20,9 +20,8 @@ Requires-Dist: storage3!=0.11.2; python_version < '3.11'
 Requires-Dist: pyjwt<3.0.0
 Requires-Dist: psutil
 Requires-Dist: packaging
-Requires-Dist: urllib3<2 ; extra == "aws"
-Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
-Requires-Dist: s3fs>=2023.12.2,<=2025.7.0,!=2024.10.0 ; extra == "aws"
+Requires-Dist: aiobotocore[boto3]>=2.12.4,<3.0.0 ; extra == "aws"
+Requires-Dist: s3fs>=2023.12.2,<=2025.9.0,!=2024.10.0 ; extra == "aws"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"
 Requires-Dist: python-dotenv ; extra == "dev"
@@ -33,7 +32,7 @@ Requires-Dist: pytest-xdist ; extra == "dev"
 Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
 Requires-Dist: pandas ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: gcsfs>=2023.12.2,<=2025.7.0 ; extra == "gcp"
+Requires-Dist: gcsfs>=2023.12.2,<=2025.9.0 ; extra == "gcp"
 Project-URL: Home, https://github.com/laminlabs/lamindb-setup
 Provides-Extra: aws
 Provides-Extra: dev

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.12.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.13.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 import warnings

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/_connect_instance.py

@@ -23,7 +23,7 @@ from .core._settings import settings
 from .core._settings_instance import InstanceSettings
 from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
-from .core._settings_store import instance_settings_file, settings_dir
+from .core._settings_store import instance_settings_file
 from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
 from .core.django import reset_django
 from .errors import CannotSwitchDefaultInstance

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/_migrate.py

@@ -98,6 +98,27 @@ class migrate:
 
     @classmethod
     def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
+        import os
+
+        # NOTE: this is a temporary solution to avoid breaking tests
+        LAMIN_MIGRATE_ON_LAMBDA = (
+            os.environ.get("LAMIN_MIGRATE_ON_LAMBDA", "false") == "true"
+        )
+
+        if settings.instance.is_on_hub and LAMIN_MIGRATE_ON_LAMBDA:
+            response = httpx.post(
+                f"{settings.instance.api_url}/instances/{settings.instance._id}/migrate",
+                headers={"Authorization": f"Bearer {settings.user.access_token}"},
+            )
+            if response.status_code != 200:
+                raise Exception(f"Failed to migrate instance: {response.text}")
+        else:
+            cls._deploy(package_name=package_name, number=number)
+
+    @classmethod
+    def _deploy(
+        cls, package_name: str | None = None, number: int | None = None
+    ) -> None:
         """Deploy a migration."""
         from lamindb_setup._connect_instance import connect
         from lamindb_setup._schema_metadata import update_schema_in_hub
@@ -143,13 +164,6 @@ class migrate:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             if settings.instance.dialect != "sqlite":
                 update_schema_in_hub()
-                logger.warning(
-                    "clearing instance cache in hub; if this fails, re-run with latest lamindb version"
-                )
-                httpx.delete(
-                    f"{settings.instance.api_url}/cache/instances/{settings.instance._id.hex}",
-                    headers={"Authorization": f"Bearer {settings.user.access_token}"},
-                )
             call_with_fallback_auth(
                 update_instance,
                 instance_id=settings.instance._id.hex,

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/_schema_metadata.py

@@ -6,16 +6,6 @@ import json
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 
-from django.db.models import (
-    Field,
-    ForeignKey,
-    ForeignObjectRel,
-    ManyToManyField,
-    ManyToManyRel,
-    ManyToOneRel,
-    OneToOneField,
-    OneToOneRel,
-)
 from lamin_utils import logger
 from pydantic import BaseModel
 
@@ -34,6 +24,16 @@ except Exception:
 
 
 if TYPE_CHECKING:
+    from django.db.models import (
+        Field,
+        ForeignKey,
+        ForeignObjectRel,
+        ManyToManyField,
+        ManyToManyRel,
+        ManyToOneRel,
+        OneToOneField,
+        OneToOneRel,
+    )
     from supabase import Client
 
 
@@ -285,6 +285,9 @@ class _ModelHandler:
 
         column = None
         if relation_type not in ["many-to-many", "one-to-many"]:
+            # have to reload it here in case reset happened
+            from django.db.models import ForeignObjectRel
+
             if not isinstance(field, ForeignObjectRel):
                 column = field.column
 
@@ -316,6 +319,8 @@ class _ModelHandler:
 
     @staticmethod
     def _get_through_many_to_many(field_or_rel: ManyToManyField | ManyToManyRel):
+        # have to reload it here in case reset happened
+        from django.db.models import ManyToManyField, ManyToManyRel
         from lamindb.models import Registry
 
         if isinstance(field_or_rel, ManyToManyField):
@@ -349,6 +354,9 @@ class _ModelHandler:
     def _get_through(
         self, field_or_rel: ForeignKey | OneToOneField | ManyToOneRel | OneToOneRel
     ):
+        # have to reload it here in case reset happened
+        from django.db.models import ForeignObjectRel
+
         if isinstance(field_or_rel, ForeignObjectRel):
             rel_1 = field_or_rel.field.related_fields[0][0]
             rel_2 = field_or_rel.field.related_fields[0][1]

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/core/__init__.py

@@ -13,6 +13,7 @@ Settings:
 """
 
 from . import django, upath
+from ._clone import connect_local_sqlite, init_local_sqlite
 from ._deprecated import deprecated
 from ._docs import doc_args
 from ._settings import SetupSettings

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/core/_aws_options.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import logging
 import os
 import time
+from typing import Any
 
 from lamin_utils import logger
 from upath import UPath
@@ -55,11 +56,17 @@ class AWSOptionsManager:
         self._credentials_cache = {}
         self._parameters_cache = {}  # this is not refreshed
 
+        from aiobotocore.session import AioSession
         from s3fs import S3FileSystem
 
         # this is cached so will be resued with the connection initialized
+        # these options are set for paths in _path_inject_options
+        # here we set the same options to cache the filesystem
         fs = S3FileSystem(
-            cache_regions=True, use_listings_cache=True, version_aware=False
+            cache_regions=True,
+            use_listings_cache=True,
+            version_aware=False,
+            config_kwargs={"max_pool_connections": 64},
         )
 
         self._suppress_aiobotocore_traceback_logging()
@@ -82,6 +89,12 @@ class AWSOptionsManager:
             except Exception:
                 self.anon_public = True
 
+        empty_session = AioSession(profile="lamindb_empty_profile")
+        empty_session.full_config["profiles"]["lamindb_empty_profile"] = {}
+        # this is set downstream to avoid using local configs when we provide credentials
+        # or when we set anon=True
+        self.empty_session = empty_session
+
     def _find_root(self, path_str: str) -> str | None:
         roots = self._credentials_cache.keys()
         if path_str in roots:
@@ -109,35 +122,47 @@ class AWSOptionsManager:
     def _path_inject_options(
         self, path: UPath, credentials: dict, extra_parameters: dict | None = None
     ) -> UPath:
+        connection_options: dict[str, Any] = {}
+        storage_options = path.storage_options
         if credentials == {}:
-            # credentials were specified manually for the path
-            if "anon" in path.storage_options:
-                anon = path.storage_options["anon"]
-            elif path.fs.key is not None and path.fs.secret is not None:
-                anon = False
-            else:
+            # otherwise credentials were specified manually for the path
+            if "anon" not in storage_options and (
+                path.fs.key is None or path.fs.secret is None
+            ):
                 anon = self.anon
                 if not anon and self.anon_public and path.drive in PUBLIC_BUCKETS:
                     anon = True
-            connection_options = {"anon": anon}
+                if anon:
+                    connection_options["anon"] = anon
+                    connection_options["session"] = self.empty_session
         else:
-            connection_options = credentials
+            connection_options.update(credentials)
+            connection_options["session"] = self.empty_session
 
-        if "cache_regions" in path.storage_options:
-            connection_options["cache_regions"] = path.storage_options["cache_regions"]
+        if "cache_regions" in storage_options:
+            connection_options["cache_regions"] = storage_options["cache_regions"]
         else:
             connection_options["cache_regions"] = (
-                path.storage_options.get("endpoint_url", None) is None
+                storage_options.get("endpoint_url", None) is None
             )
         # we use cache to avoid some uneeded downloads or credential problems
         # see in upload_from
-        connection_options["use_listings_cache"] = path.storage_options.get(
+        connection_options["use_listings_cache"] = storage_options.get(
             "use_listings_cache", True
         )
         # normally we want to ignore objects vsrsions in a versioned bucket
-        connection_options["version_aware"] = path.storage_options.get(
+        connection_options["version_aware"] = storage_options.get(
             "version_aware", False
         )
+        # this is for better concurrency as the default batch_size is 128
+        # read https://github.com/laminlabs/lamindb-setup/pull/1146
+        if "config_kwargs" not in storage_options:
+            connection_options["config_kwargs"] = {"max_pool_connections": 64}
+        elif "max_pool_connections" not in (
+            config_kwargs := storage_options["config_kwargs"]
+        ):
+            config_kwargs["max_pool_connections"] = 64
+            connection_options["config_kwargs"] = config_kwargs
 
         if extra_parameters:
             connection_options.update(extra_parameters)
@@ -152,6 +177,7 @@ class AWSOptionsManager:
             if "r2.cloudflarestorage.com" in endpoint_url:
                 # fixed_upload_size should always be True for R2
                 # this option is needed for correct uploads to R2
+                # TODO: maybe set max_pool_connections=64 here also
                 path = UPath(path, fixed_upload_size=True)
             return path
         # trailing slash is needed to avoid returning incorrect results with .startswith

lamindb_setup-1.13.0/lamindb_setup/core/_clone.py

@@ -0,0 +1,93 @@
+"""Utilities to copy, clone and load Postgres instances as local SQLite databases.
+
+.. autosummary::
+   :toctree:
+
+   init_local_sqlite
+   connect_local_sqlite
+"""
+
+import os
+
+from lamindb_setup.core._settings_instance import InstanceSettings
+from lamindb_setup.core._settings_load import load_instance_settings
+from lamindb_setup.core._settings_store import instance_settings_file
+from lamindb_setup.core.django import reset_django
+
+
+def init_local_sqlite(
+    instance: str | None = None, copy_suffix: str | None = None
+) -> None:
+    """Initialize SQLite copy of an existing Postgres instance.
+
+    Creates a SQLite database with the same schema as the source Postgres instance.
+    The copy shares the same storage location as the original instance.
+
+    The copy is intended for read-only access to instance data without requiring a Postgres connection.
+    Data synchronization to complete the clone happens via a separate Lambda function.
+
+    Note that essential user, branch and storage tables are missing.
+    Therefore, it is not possible to store Artifacts without having replayed these records first.
+
+    Args:
+        instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
+            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
+            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+        copy_suffix: Optional suffix to append to the local clone name.
+    """
+    import lamindb_setup as ln_setup
+
+    if instance is None:  # pragma: no cover
+        instance = os.environ.get("LAMIN_CURRENT_INSTANCE")
+
+    if instance is None:
+        raise ValueError(
+            "No instance identifier provided and LAMIN_CURRENT_INSTANCE is not set"
+        )
+
+    if ln_setup.settings.instance is None:  # pragma: no cover
+        ln_setup.connect(instance)
+
+    name = (
+        f"{ln_setup.settings.instance.name}{copy_suffix}"
+        if copy_suffix is not None
+        else ln_setup.settings.instance.name
+    )
+    isettings = InstanceSettings(
+        id=ln_setup.settings.instance._id,
+        owner=ln_setup.settings.instance.owner,  # type: ignore
+        name=name,
+        storage=ln_setup.settings.storage,
+        db=None,
+        modules=",".join(ln_setup.settings.instance.modules),
+        is_on_hub=False,
+    )
+
+    isettings._persist(write_to_disk=True)
+
+    if not isettings._sqlite_file_local.exists():
+        # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
+        # Without this reset, the if not settings.configured check in setup_django() would skip reconfiguration,
+        # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
+        reset_django()
+        isettings._init_db()
+
+
+def connect_local_sqlite(instance: str) -> None:
+    """Load a SQLite instance of which a remote hub Postgres instance exists.
+
+    This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
+    The clone must first be created via `init_local_sqlite()`.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+    """
+    owner, name = instance.split("/")
+    settings_file = instance_settings_file(name=name, owner=owner)
+
+    if not settings_file.exists():
+        raise ValueError("SQLite clone not found. Run init_local_sqlite() first.")
+
+    isettings = load_instance_settings(settings_file)
+    isettings._persist(write_to_disk=False)
+    isettings._load_db()

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/core/_hub_core.py

@@ -13,6 +13,8 @@ from postgrest.exceptions import APIError
 
 from lamindb_setup._migrate import check_whether_migrations_in_sync
 
+from ._aws_options import HOSTED_REGIONS
+from ._aws_storage import find_closest_aws_region
 from ._hub_client import (
     call_with_fallback,
     call_with_fallback_auth,
@@ -420,6 +422,54 @@ def _init_instance_hub(
         logger.important(f"go to: https://lamin.ai/{slug}")
 
 
+def _get_default_bucket_for_instance(
+    instance_id: UUID | None, region: str | None, client: Client
+):
+    if instance_id is not None:
+        bucket_base = (
+            client.rpc(
+                "get_api_server_default_bucket_by_instance_id",
+                {"p_instance_id": instance_id.hex},
+            )
+            .execute()
+            .data
+        )
+        if bucket_base is not None:
+            return f"s3://{bucket_base}"
+
+    if os.getenv("LAMIN_ENV") in {None, "prod"}:
+        if region is None:
+            region = find_closest_aws_region()
+        elif region not in HOSTED_REGIONS:
+            raise ValueError(f"region has to be one of {HOSTED_REGIONS}")
+        root = f"s3://lamin-{region}"
+    else:
+        root = "s3://lamin-hosted-test"
+
+    return root
+
+
+# pass None if initializing an instance
+# this can be from the api server attached to the instance or the default bucket
+# that we use for instances with no api servers attached
+def get_default_bucket_for_instance(
+    instance_id: UUID | None, region: str | None = None, access_token: str | None = None
+):
+    if settings.user.handle != "anonymous" or access_token is not None:
+        return call_with_fallback_auth(
+            _get_default_bucket_for_instance,
+            instance_id=instance_id,
+            region=region,
+            access_token=access_token,
+        )
+    else:
+        return call_with_fallback(
+            _get_default_bucket_for_instance,
+            region=region,
+            instance_id=instance_id,
+        )
+
+
 def _connect_instance_hub(
     owner: str,  # account_handle
     name: str,  # instance_name

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/core/_settings.py

@@ -321,25 +321,28 @@ class SetupSettings:
 
     def __repr__(self) -> str:
         """Rich string representation."""
+        from lamin_utils import colors
+
         # do not show current setting representation when building docs
         if "sphinx" in sys.modules:
             return object.__repr__(self)
+
         repr = ""
         if self._instance_exists:
-            repr += "Current branch & space:\n"
+            instance_rep = self.instance.__repr__().split("\n")
+            repr += f"{colors.cyan('Instance:')} {instance_rep[0].replace('Instance: ', '')}\n"
+            repr += f" - work-dir: {self.work_dir}\n"
             repr += f" - branch: {self._read_branch_idlike_name()[1]}\n"
-            repr += f" - space: {self._read_space_idlike_name()[1]}\n"
-            repr += self.instance.__repr__()
+            repr += f" - space: {self._read_space_idlike_name()[1]}"
+            repr += f"\n{colors.yellow('Details:')}\n"
+            repr += "\n".join(instance_rep[1:])
         else:
-            repr += "Current instance: None"
-        repr += "\nConfig:\n"
-        repr += f" - private Django API: {self.private_django_api}\n"
-        repr += "Local directories:\n"
-        repr += f" - working directory: {self.work_dir}\n"
+            repr += f"{colors.cyan('Instance:')} None"
+        repr += f"\n{colors.blue('Cache & settings:')}\n"
         repr += f" - cache: {self.cache_dir.as_posix()}\n"
         repr += f" - user settings: {settings_dir.as_posix()}\n"
-        repr += f" - system settings: {system_settings_dir.as_posix()}\n"
-        repr += self.user.__repr__()
+        repr += f" - system settings: {system_settings_dir.as_posix()}"
+        repr += f"\n{colors.green('User:')} {self.user.handle}"
         return repr
 
 

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/core/_settings_instance.py

@@ -112,8 +112,8 @@ class InstanceSettings:
 
     def __repr__(self):
         """Rich string representation."""
-        representation = "Current instance:"
-        attrs = ["slug", "storage", "db", "modules", "git_repo"]
+        representation = f"Instance: {self.slug}"
+        attrs = ["storage", "db", "modules", "git_repo"]
         for attr in attrs:
             value = getattr(self, attr)
             if attr == "storage":
@@ -145,8 +145,8 @@ class InstanceSettings:
                 else:
                     db_print = value
                 representation += f"\n - {attr}: {db_print}"
-            elif attr == "modules":
-                representation += f"\n - {attr}: {value if value else '{}'}"
+            elif attr == "modules" and value:
+                representation += f"\n - {attr}: {', '.join(value)}"
             else:
                 representation += f"\n - {attr}: {value}"
         return representation
@@ -342,7 +342,12 @@ class InstanceSettings:
 
         Provide the full git repo URL.
         """
-        return self._git_repo
+        if self._git_repo is not None:
+            return self._git_repo
+        elif os.environ.get("LAMINDB_SYNC_GIT_REPO") is not None:
+            return sanitize_git_repo_url(os.environ["LAMINDB_SYNC_GIT_REPO"])
+        else:
+            return None
 
     @property
     def api_url(self) -> str | None:

{lamindb_setup-1.12.1 → lamindb_setup-1.13.0}/lamindb_setup/core/_settings_storage.py

@@ -101,7 +101,11 @@ def init_storage(
     StorageSettings,
     Literal["hub-record-not-created", "hub-record-retrieved", "hub-record-created"],
 ]:
-    from ._hub_core import delete_storage_record, init_storage_hub
+    from ._hub_core import (
+        delete_storage_record,
+        get_default_bucket_for_instance,
+        init_storage_hub,
+    )
 
     assert root is not None, "`root` argument can't be `None`"
 
@@ -117,20 +121,14 @@ def init_storage(
         # this means we constructed a hosted location of shape s3://bucket-name/uid
         # within LaminHub
         assert root_str.endswith(uid)
-    lamin_env = os.getenv("LAMIN_ENV")
     if root_str.startswith("create-s3"):
         if root_str != "create-s3":
             assert "--" in root_str, "example: `create-s3--eu-central-1`"
             region = root_str.replace("create-s3--", "")
-        if region is None:
-            region = find_closest_aws_region()
-        else:
-            if region not in HOSTED_REGIONS:
-                raise ValueError(f"region has to be one of {HOSTED_REGIONS}")
-        if lamin_env is None or lamin_env == "prod":
-            root = f"s3://lamin-{region}/{uid}"
-        else:
-            root = f"s3://lamin-hosted-test/{uid}"
+        bucket = get_default_bucket_for_instance(
+            None if init_instance else instance_id, region
+        )
+        root = f"{bucket}/{uid}"
     elif (input_protocol := fsspec.utils.get_protocol(root_str)) not in VALID_PROTOCOLS:
         valid_protocols = ("local",) + VALID_PROTOCOLS[1:]  # show local instead of file
         raise ValueError(