lamindb_setup 1.12.2__py3-none-any.whl → 1.13.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.12.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.13.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 import warnings

lamindb_setup/_connect_instance.py

@@ -23,7 +23,7 @@ from .core._settings import settings
 from .core._settings_instance import InstanceSettings
 from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
-from .core._settings_store import instance_settings_file, settings_dir
+from .core._settings_store import instance_settings_file
 from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
 from .core.django import reset_django
 from .errors import CannotSwitchDefaultInstance

lamindb_setup/_migrate.py

@@ -98,6 +98,27 @@ class migrate:
 
     @classmethod
     def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
+        import os
+
+        # NOTE: this is a temporary solution to avoid breaking tests
+        LAMIN_MIGRATE_ON_LAMBDA = (
+            os.environ.get("LAMIN_MIGRATE_ON_LAMBDA", "false") == "true"
+        )
+
+        if settings.instance.is_on_hub and LAMIN_MIGRATE_ON_LAMBDA:
+            response = httpx.post(
+                f"{settings.instance.api_url}/instances/{settings.instance._id}/migrate",
+                headers={"Authorization": f"Bearer {settings.user.access_token}"},
+            )
+            if response.status_code != 200:
+                raise Exception(f"Failed to migrate instance: {response.text}")
+        else:
+            cls._deploy(package_name=package_name, number=number)
+
+    @classmethod
+    def _deploy(
+        cls, package_name: str | None = None, number: int | None = None
+    ) -> None:
         """Deploy a migration."""
         from lamindb_setup._connect_instance import connect
         from lamindb_setup._schema_metadata import update_schema_in_hub
@@ -143,13 +164,6 @@ class migrate:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             if settings.instance.dialect != "sqlite":
                 update_schema_in_hub()
-                logger.warning(
-                    "clearing instance cache in hub; if this fails, re-run with latest lamindb version"
-                )
-                httpx.delete(
-                    f"{settings.instance.api_url}/cache/instances/{settings.instance._id.hex}",
-                    headers={"Authorization": f"Bearer {settings.user.access_token}"},
-                )
             call_with_fallback_auth(
                 update_instance,
                 instance_id=settings.instance._id.hex,

lamindb_setup/_schema_metadata.py

@@ -6,16 +6,6 @@ import json
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 
-from django.db.models import (
-    Field,
-    ForeignKey,
-    ForeignObjectRel,
-    ManyToManyField,
-    ManyToManyRel,
-    ManyToOneRel,
-    OneToOneField,
-    OneToOneRel,
-)
 from lamin_utils import logger
 from pydantic import BaseModel
 
@@ -34,6 +24,16 @@ except Exception:
 
 
 if TYPE_CHECKING:
+    from django.db.models import (
+        Field,
+        ForeignKey,
+        ForeignObjectRel,
+        ManyToManyField,
+        ManyToManyRel,
+        ManyToOneRel,
+        OneToOneField,
+        OneToOneRel,
+    )
     from supabase import Client
 
 
@@ -285,6 +285,9 @@ class _ModelHandler:
 
         column = None
         if relation_type not in ["many-to-many", "one-to-many"]:
+            # have to reload it here in case reset happened
+            from django.db.models import ForeignObjectRel
+
             if not isinstance(field, ForeignObjectRel):
                 column = field.column
 
@@ -316,6 +319,8 @@ class _ModelHandler:
 
     @staticmethod
     def _get_through_many_to_many(field_or_rel: ManyToManyField | ManyToManyRel):
+        # have to reload it here in case reset happened
+        from django.db.models import ManyToManyField, ManyToManyRel
         from lamindb.models import Registry
 
         if isinstance(field_or_rel, ManyToManyField):
@@ -349,6 +354,9 @@ class _ModelHandler:
     def _get_through(
         self, field_or_rel: ForeignKey | OneToOneField | ManyToOneRel | OneToOneRel
     ):
+        # have to reload it here in case reset happened
+        from django.db.models import ForeignObjectRel
+
         if isinstance(field_or_rel, ForeignObjectRel):
             rel_1 = field_or_rel.field.related_fields[0][0]
             rel_2 = field_or_rel.field.related_fields[0][1]

lamindb_setup/core/__init__.py

@@ -13,6 +13,7 @@ Settings:
 """
 
 from . import django, upath
+from ._clone import connect_local_sqlite, init_local_sqlite
 from ._deprecated import deprecated
 from ._docs import doc_args
 from ._settings import SetupSettings

lamindb_setup/core/_aws_options.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import logging
 import os
 import time
+from typing import Any
 
 from lamin_utils import logger
 from upath import UPath
@@ -55,11 +56,17 @@ class AWSOptionsManager:
         self._credentials_cache = {}
         self._parameters_cache = {}  # this is not refreshed
 
+        from aiobotocore.session import AioSession
         from s3fs import S3FileSystem
 
         # this is cached so will be resued with the connection initialized
+        # these options are set for paths in _path_inject_options
+        # here we set the same options to cache the filesystem
         fs = S3FileSystem(
-            cache_regions=True, use_listings_cache=True, version_aware=False
+            cache_regions=True,
+            use_listings_cache=True,
+            version_aware=False,
+            config_kwargs={"max_pool_connections": 64},
         )
 
         self._suppress_aiobotocore_traceback_logging()
@@ -82,6 +89,12 @@ class AWSOptionsManager:
             except Exception:
                 self.anon_public = True
 
+        empty_session = AioSession(profile="lamindb_empty_profile")
+        empty_session.full_config["profiles"]["lamindb_empty_profile"] = {}
+        # this is set downstream to avoid using local configs when we provide credentials
+        # or when we set anon=True
+        self.empty_session = empty_session
+
     def _find_root(self, path_str: str) -> str | None:
         roots = self._credentials_cache.keys()
         if path_str in roots:
@@ -109,35 +122,47 @@ class AWSOptionsManager:
     def _path_inject_options(
         self, path: UPath, credentials: dict, extra_parameters: dict | None = None
     ) -> UPath:
+        connection_options: dict[str, Any] = {}
+        storage_options = path.storage_options
         if credentials == {}:
-            # credentials were specified manually for the path
-            if "anon" in path.storage_options:
-                anon = path.storage_options["anon"]
-            elif path.fs.key is not None and path.fs.secret is not None:
-                anon = False
-            else:
+            # otherwise credentials were specified manually for the path
+            if "anon" not in storage_options and (
+                path.fs.key is None or path.fs.secret is None
+            ):
                 anon = self.anon
                 if not anon and self.anon_public and path.drive in PUBLIC_BUCKETS:
                     anon = True
-            connection_options = {"anon": anon}
+                if anon:
+                    connection_options["anon"] = anon
+                    connection_options["session"] = self.empty_session
         else:
-            connection_options = credentials
+            connection_options.update(credentials)
+            connection_options["session"] = self.empty_session
 
-        if "cache_regions" in path.storage_options:
-            connection_options["cache_regions"] = path.storage_options["cache_regions"]
+        if "cache_regions" in storage_options:
+            connection_options["cache_regions"] = storage_options["cache_regions"]
         else:
             connection_options["cache_regions"] = (
-                path.storage_options.get("endpoint_url", None) is None
+                storage_options.get("endpoint_url", None) is None
             )
         # we use cache to avoid some uneeded downloads or credential problems
         # see in upload_from
-        connection_options["use_listings_cache"] = path.storage_options.get(
+        connection_options["use_listings_cache"] = storage_options.get(
             "use_listings_cache", True
         )
         # normally we want to ignore objects vsrsions in a versioned bucket
-        connection_options["version_aware"] = path.storage_options.get(
+        connection_options["version_aware"] = storage_options.get(
             "version_aware", False
         )
+        # this is for better concurrency as the default batch_size is 128
+        # read https://github.com/laminlabs/lamindb-setup/pull/1146
+        if "config_kwargs" not in storage_options:
+            connection_options["config_kwargs"] = {"max_pool_connections": 64}
+        elif "max_pool_connections" not in (
+            config_kwargs := storage_options["config_kwargs"]
+        ):
+            config_kwargs["max_pool_connections"] = 64
+            connection_options["config_kwargs"] = config_kwargs
 
         if extra_parameters:
             connection_options.update(extra_parameters)
@@ -152,6 +177,7 @@ class AWSOptionsManager:
             if "r2.cloudflarestorage.com" in endpoint_url:
                 # fixed_upload_size should always be True for R2
                 # this option is needed for correct uploads to R2
+                # TODO: maybe set max_pool_connections=64 here also
                 path = UPath(path, fixed_upload_size=True)
             return path
         # trailing slash is needed to avoid returning incorrect results with .startswith

lamindb_setup/core/_clone.py

@@ -0,0 +1,93 @@
+"""Utilities to copy, clone and load Postgres instances as local SQLite databases.
+
+.. autosummary::
+   :toctree:
+
+   init_local_sqlite
+   connect_local_sqlite
+"""
+
+import os
+
+from lamindb_setup.core._settings_instance import InstanceSettings
+from lamindb_setup.core._settings_load import load_instance_settings
+from lamindb_setup.core._settings_store import instance_settings_file
+from lamindb_setup.core.django import reset_django
+
+
+def init_local_sqlite(
+    instance: str | None = None, copy_suffix: str | None = None
+) -> None:
+    """Initialize SQLite copy of an existing Postgres instance.
+
+    Creates a SQLite database with the same schema as the source Postgres instance.
+    The copy shares the same storage location as the original instance.
+
+    The copy is intended for read-only access to instance data without requiring a Postgres connection.
+    Data synchronization to complete the clone happens via a separate Lambda function.
+
+    Note that essential user, branch and storage tables are missing.
+    Therefore, it is not possible to store Artifacts without having replayed these records first.
+
+    Args:
+        instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
+            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
+            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+        copy_suffix: Optional suffix to append to the local clone name.
+    """
+    import lamindb_setup as ln_setup
+
+    if instance is None:  # pragma: no cover
+        instance = os.environ.get("LAMIN_CURRENT_INSTANCE")
+
+    if instance is None:
+        raise ValueError(
+            "No instance identifier provided and LAMIN_CURRENT_INSTANCE is not set"
+        )
+
+    if ln_setup.settings.instance is None:  # pragma: no cover
+        ln_setup.connect(instance)
+
+    name = (
+        f"{ln_setup.settings.instance.name}{copy_suffix}"
+        if copy_suffix is not None
+        else ln_setup.settings.instance.name
+    )
+    isettings = InstanceSettings(
+        id=ln_setup.settings.instance._id,
+        owner=ln_setup.settings.instance.owner,  # type: ignore
+        name=name,
+        storage=ln_setup.settings.storage,
+        db=None,
+        modules=",".join(ln_setup.settings.instance.modules),
+        is_on_hub=False,
+    )
+
+    isettings._persist(write_to_disk=True)
+
+    if not isettings._sqlite_file_local.exists():
+        # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
+        # Without this reset, the if not settings.configured check in setup_django() would skip reconfiguration,
+        # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
+        reset_django()
+        isettings._init_db()
+
+
+def connect_local_sqlite(instance: str) -> None:
+    """Load a SQLite instance of which a remote hub Postgres instance exists.
+
+    This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
+    The clone must first be created via `init_local_sqlite()`.
+
+    Args:
+        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
+    """
+    owner, name = instance.split("/")
+    settings_file = instance_settings_file(name=name, owner=owner)
+
+    if not settings_file.exists():
+        raise ValueError("SQLite clone not found. Run init_local_sqlite() first.")
+
+    isettings = load_instance_settings(settings_file)
+    isettings._persist(write_to_disk=False)
+    isettings._load_db()

lamindb_setup/core/_hub_core.py

@@ -13,6 +13,8 @@ from postgrest.exceptions import APIError
 
 from lamindb_setup._migrate import check_whether_migrations_in_sync
 
+from ._aws_options import HOSTED_REGIONS
+from ._aws_storage import find_closest_aws_region
 from ._hub_client import (
     call_with_fallback,
     call_with_fallback_auth,
@@ -420,6 +422,54 @@ def _init_instance_hub(
         logger.important(f"go to: https://lamin.ai/{slug}")
 
 
+def _get_default_bucket_for_instance(
+    instance_id: UUID | None, region: str | None, client: Client
+):
+    if instance_id is not None:
+        bucket_base = (
+            client.rpc(
+                "get_api_server_default_bucket_by_instance_id",
+                {"p_instance_id": instance_id.hex},
+            )
+            .execute()
+            .data
+        )
+        if bucket_base is not None:
+            return f"s3://{bucket_base}"
+
+    if os.getenv("LAMIN_ENV") in {None, "prod"}:
+        if region is None:
+            region = find_closest_aws_region()
+        elif region not in HOSTED_REGIONS:
+            raise ValueError(f"region has to be one of {HOSTED_REGIONS}")
+        root = f"s3://lamin-{region}"
+    else:
+        root = "s3://lamin-hosted-test"
+
+    return root
+
+
+# pass None if initializing an instance
+# this can be from the api server attached to the instance or the default bucket
+# that we use for instances with no api servers attached
+def get_default_bucket_for_instance(
+    instance_id: UUID | None, region: str | None = None, access_token: str | None = None
+):
+    if settings.user.handle != "anonymous" or access_token is not None:
+        return call_with_fallback_auth(
+            _get_default_bucket_for_instance,
+            instance_id=instance_id,
+            region=region,
+            access_token=access_token,
+        )
+    else:
+        return call_with_fallback(
+            _get_default_bucket_for_instance,
+            region=region,
+            instance_id=instance_id,
+        )
+
+
 def _connect_instance_hub(
     owner: str,  # account_handle
     name: str,  # instance_name

lamindb_setup/core/_settings.py

@@ -321,25 +321,28 @@ class SetupSettings:
 
     def __repr__(self) -> str:
         """Rich string representation."""
+        from lamin_utils import colors
+
         # do not show current setting representation when building docs
         if "sphinx" in sys.modules:
             return object.__repr__(self)
+
         repr = ""
         if self._instance_exists:
-            repr += "Current branch & space:\n"
+            instance_rep = self.instance.__repr__().split("\n")
+            repr += f"{colors.cyan('Instance:')} {instance_rep[0].replace('Instance: ', '')}\n"
+            repr += f" - work-dir: {self.work_dir}\n"
             repr += f" - branch: {self._read_branch_idlike_name()[1]}\n"
-            repr += f" - space: {self._read_space_idlike_name()[1]}\n"
-            repr += self.instance.__repr__()
+            repr += f" - space: {self._read_space_idlike_name()[1]}"
+            repr += f"\n{colors.yellow('Details:')}\n"
+            repr += "\n".join(instance_rep[1:])
         else:
-            repr += "Current instance: None"
-        repr += "\nConfig:\n"
-        repr += f" - private Django API: {self.private_django_api}\n"
-        repr += "Local directories:\n"
-        repr += f" - working directory: {self.work_dir}\n"
+            repr += f"{colors.cyan('Instance:')} None"
+        repr += f"\n{colors.blue('Cache & settings:')}\n"
         repr += f" - cache: {self.cache_dir.as_posix()}\n"
         repr += f" - user settings: {settings_dir.as_posix()}\n"
-        repr += f" - system settings: {system_settings_dir.as_posix()}\n"
-        repr += self.user.__repr__()
+        repr += f" - system settings: {system_settings_dir.as_posix()}"
+        repr += f"\n{colors.green('User:')} {self.user.handle}"
         return repr
 
 

lamindb_setup/core/_settings_instance.py

@@ -112,8 +112,8 @@ class InstanceSettings:
 
     def __repr__(self):
         """Rich string representation."""
-        representation = "Current instance:"
-        attrs = ["slug", "storage", "db", "modules", "git_repo"]
+        representation = f"Instance: {self.slug}"
+        attrs = ["storage", "db", "modules", "git_repo"]
         for attr in attrs:
             value = getattr(self, attr)
             if attr == "storage":
@@ -145,8 +145,8 @@ class InstanceSettings:
                 else:
                     db_print = value
                 representation += f"\n - {attr}: {db_print}"
-            elif attr == "modules":
-                representation += f"\n - {attr}: {value if value else '{}'}"
+            elif attr == "modules" and value:
+                representation += f"\n - {attr}: {', '.join(value)}"
             else:
                 representation += f"\n - {attr}: {value}"
         return representation
@@ -342,7 +342,12 @@ class InstanceSettings:
 
         Provide the full git repo URL.
         """
-        return self._git_repo
+        if self._git_repo is not None:
+            return self._git_repo
+        elif os.environ.get("LAMINDB_SYNC_GIT_REPO") is not None:
+            return sanitize_git_repo_url(os.environ["LAMINDB_SYNC_GIT_REPO"])
+        else:
+            return None
 
     @property
     def api_url(self) -> str | None:

lamindb_setup/core/_settings_storage.py

@@ -101,7 +101,11 @@ def init_storage(
     StorageSettings,
     Literal["hub-record-not-created", "hub-record-retrieved", "hub-record-created"],
 ]:
-    from ._hub_core import delete_storage_record, init_storage_hub
+    from ._hub_core import (
+        delete_storage_record,
+        get_default_bucket_for_instance,
+        init_storage_hub,
+    )
 
     assert root is not None, "`root` argument can't be `None`"
 
@@ -117,20 +121,14 @@ def init_storage(
         # this means we constructed a hosted location of shape s3://bucket-name/uid
         # within LaminHub
         assert root_str.endswith(uid)
-    lamin_env = os.getenv("LAMIN_ENV")
     if root_str.startswith("create-s3"):
         if root_str != "create-s3":
             assert "--" in root_str, "example: `create-s3--eu-central-1`"
             region = root_str.replace("create-s3--", "")
-        if region is None:
-            region = find_closest_aws_region()
-        else:
-            if region not in HOSTED_REGIONS:
-                raise ValueError(f"region has to be one of {HOSTED_REGIONS}")
-        if lamin_env is None or lamin_env == "prod":
-            root = f"s3://lamin-{region}/{uid}"
-        else:
-            root = f"s3://lamin-hosted-test/{uid}"
+        bucket = get_default_bucket_for_instance(
+            None if init_instance else instance_id, region
+        )
+        root = f"{bucket}/{uid}"
     elif (input_protocol := fsspec.utils.get_protocol(root_str)) not in VALID_PROTOCOLS:
         valid_protocols = ("local",) + VALID_PROTOCOLS[1:]  # show local instead of file
         raise ValueError(

lamindb_setup/core/django.py

@@ -7,8 +7,9 @@ import sys
 import importlib as il
 import jwt
 import time
+import threading
 from pathlib import Path
-import time
+from packaging import version
 from ._settings_instance import InstanceSettings, is_local_db_url
 
 from lamin_utils import logger
@@ -59,21 +60,21 @@ class DBTokenManager:
         from django.db.transaction import Atomic
 
         self.original_atomic_enter = Atomic.__enter__
+        self.atomic_is_patched = False
 
         self.tokens: dict[str, DBToken] = {}
 
     def get_connection(self, connection_name: str):
         from django.db import connections
 
-        connection = connections[connection_name]
-        assert connection.vendor == "postgresql"
-
-        return connection
+        return connections[connection_name]
 
     def set(self, token: DBToken, connection_name: str = "default"):
-        from django.db.transaction import Atomic
+        if connection_name in self.tokens:
+            return
 
-        connection = self.get_connection(connection_name)
+        from django.db.transaction import Atomic
+        from django.db.backends.signals import connection_created
 
         def set_token_wrapper(execute, sql, params, many, context):
             not_in_atomic_block = (
@@ -97,28 +98,50 @@ class DBTokenManager:
                 result.nextset()
             return result
 
-        connection.execute_wrappers.append(set_token_wrapper)
+        self.get_connection(connection_name).execute_wrappers.append(set_token_wrapper)
+
+        def connection_callback(sender, connection, **kwargs):
+            if (
+                connection.alias == connection_name
+                and set_token_wrapper not in connection.execute_wrappers
+            ):
+                connection.execute_wrappers.append(set_token_wrapper)
+
+        dispatch_uid = f"dbtokenmanager:{id(self)}:{connection_name}"
+        # emitted when a database connection is established
+        # not when a database wrapper is created
+        connection_created.connect(
+            connection_callback, dispatch_uid=dispatch_uid, weak=False
+        )
 
         self.tokens[connection_name] = token
 
-        # ensure we set the token only once for an outer atomic block
-        def __enter__(atomic):
-            self.original_atomic_enter(atomic)
-            connection_name = "default" if atomic.using is None else atomic.using
-            if connection_name in self.tokens:
-                # here we don't use the connection from the closure
-                # because Atomic is a single class to manage transactions for all connections
-                connection = self.get_connection(connection_name)
-                if len(connection.atomic_blocks) == 1:
-                    token = self.tokens[connection_name]
-                    # use raw psycopg2 connection here
-                    # atomic block ensures connection
-                    connection.connection.cursor().execute(token.token_query)
-
-        Atomic.__enter__ = __enter__
-        logger.debug("django.db.transaction.Atomic.__enter__ has been patched")
+        if not self.atomic_is_patched:
+            # ensure we set the token only once for an outer atomic block
+            def __enter__(atomic):
+                self.original_atomic_enter(atomic)
+                connection_name = "default" if atomic.using is None else atomic.using
+                if connection_name in self.tokens:
+                    # here we don't use the connection from the closure
+                    # because Atomic is a single class to manage transactions for all connections
+                    connection = self.get_connection(connection_name)
+                    if len(connection.atomic_blocks) == 1:
+                        token = self.tokens[connection_name]
+                        # use raw psycopg2 connection here
+                        # atomic block ensures connection
+                        connection.connection.cursor().execute(token.token_query)
+
+            Atomic.__enter__ = __enter__
+
+            self.atomic_is_patched = True
+            logger.debug("django.db.transaction.Atomic.__enter__ has been patched")
 
     def reset(self, connection_name: str = "default"):
+        if connection_name not in self.tokens:
+            return
+
+        from django.db.backends.signals import connection_created
+
         connection = self.get_connection(connection_name)
 
         connection.execute_wrappers = [
@@ -127,8 +150,17 @@ class DBTokenManager:
             if getattr(w, "__name__", None) != "set_token_wrapper"
         ]
 
+        dispatch_uid = f"dbtokenmanager:{id(self)}:{connection_name}"
+        connection_created.disconnect(dispatch_uid=dispatch_uid)
+
         self.tokens.pop(connection_name, None)
 
+        if not self.tokens:
+            from django.db.transaction import Atomic
+
+            Atomic.__enter__ = self.original_atomic_enter
+            self.atomic_is_patched = False
+
 
 db_token_manager = DBTokenManager()
 
@@ -244,6 +276,19 @@ def setup_django(
             "django.db.backends.base.base.BaseDatabaseWrapper.close_if_health_check_failed has been patched"
         )
 
+        disable_context: bool = False
+        if (
+            env_disable_context := os.getenv("LAMINDB_DISABLE_CONNECTION_CONTEXT")
+        ) is not None:
+            disable_context = env_disable_context == "true"
+        elif IS_RUN_FROM_IPYTHON:
+            from ipykernel import __version__ as ipykernel_version
+
+            disable_context = version.parse(ipykernel_version) >= version.parse("7.0.0")
+        if disable_context:
+            django.db.connections._connections = threading.local()
+            logger.debug("django.db.connections._connections has been patched")
+
         if isettings._fine_grained_access and isettings._db_permissions == "jwt":
             db_token = DBToken(isettings)
             db_token_manager.set(db_token)  # sets for the default connection
@@ -289,6 +334,10 @@ def reset_django():
 
     connections.close_all()
 
+    global db_token_manager
+
+    db_token_manager.reset()
+
     if getattr(settings, "_wrapped", None) is not None:
         settings._wrapped = None
 
@@ -307,7 +356,6 @@ def reset_django():
 
     il.invalidate_caches()
 
-    global db_token_manager
     db_token_manager = DBTokenManager()
 
     global IS_SETUP

lamindb_setup/core/upath.py

@@ -334,27 +334,27 @@ def upload_from(
         callback = ProgressCallback(local_path.name, "uploading")
         kwargs["callback"] = callback
 
+    protocol = self.protocol
+    cleanup_cache = False
     source: str | list[str] = local_path.as_posix()
     destination: str | list[str] = self.as_posix()
     if local_path_is_dir:
-        size: int = 0
-        files: list[str] = []
-        for file in (path for path in local_path.rglob("*") if path.is_file()):
-            size += file.stat().st_size
-            files.append(file.as_posix())
-        # see https://github.com/fsspec/s3fs/issues/897
-        # here we reduce batch_size for folders bigger than 8 GiB
-        # to avoid the problem in the issue
-        # the default batch size for this case is 128
-        if "batch_size" not in kwargs and size >= 8 * 2**30:
-            kwargs["batch_size"] = 64
-
         if not create_folder:
-            source = files
+            source = [
+                path.as_posix() for path in local_path.rglob("*") if path.is_file()
+            ]
             destination = fsspec.utils.other_paths(
-                files, self.as_posix(), exists=False, flatten=False
+                source, self.as_posix(), exists=False, flatten=False
             )
-    elif self.protocol == "s3" and "chunksize" not in kwargs:
+        elif protocol == "s3" and (bucket := self.drive) not in self.fs.dircache:
+            # the below lines are to avoid s3fs triggering create_bucket in upload if
+            # dirs are present, it allows to avoid the permission error
+            self.fs.dircache[bucket] = [{}]
+            assert isinstance(destination, str)
+            if not destination.endswith(TRAILING_SEP):
+                destination += "/"
+            cleanup_cache = True
+    elif protocol == "s3" and "chunksize" not in kwargs:
         size = local_path.stat().st_size
         MiB = 1024**2
         DEFAULT_CHUNKSIZE = 50 * MiB  # so in s3fs
@@ -364,21 +364,6 @@ def upload_from(
             rounded = math.ceil(raw / step) * step
             kwargs["chunksize"] = rounded
 
-    # the below lines are to avoid s3fs triggering create_bucket in upload if
-    # dirs are present, it allows to avoid the permission error
-    if self.protocol == "s3" and local_path_is_dir and create_folder:
-        bucket = self.drive
-        if bucket not in self.fs.dircache:
-            self.fs.dircache[bucket] = [{}]
-            assert isinstance(destination, str)
-            if not destination.endswith(TRAILING_SEP):  # type: ignore
-                destination += "/"
-            cleanup_cache = True
-        else:
-            cleanup_cache = False
-    else:
-        cleanup_cache = False
-
     self.fs.upload(source, destination, recursive=create_folder, **kwargs)
 
     if cleanup_cache:

{lamindb_setup-1.12.2.dist-info → lamindb_setup-1.13.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.12.2
+Version: 1.13.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
@@ -21,7 +21,7 @@ Requires-Dist: pyjwt<3.0.0
 Requires-Dist: psutil
 Requires-Dist: packaging
 Requires-Dist: aiobotocore[boto3]>=2.12.4,<3.0.0 ; extra == "aws"
-Requires-Dist: s3fs>=2023.12.2,<=2025.7.0,!=2024.10.0 ; extra == "aws"
+Requires-Dist: s3fs>=2023.12.2,<=2025.9.0,!=2024.10.0 ; extra == "aws"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"
 Requires-Dist: python-dotenv ; extra == "dev"
@@ -32,7 +32,7 @@ Requires-Dist: pytest-xdist ; extra == "dev"
 Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
 Requires-Dist: pandas ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: gcsfs>=2023.12.2,<=2025.7.0 ; extra == "gcp"
+Requires-Dist: gcsfs>=2023.12.2,<=2025.9.0 ; extra == "gcp"
 Project-URL: Home, https://github.com/laminlabs/lamindb-setup
 Provides-Extra: aws
 Provides-Extra: dev

{lamindb_setup-1.12.2.dist-info → lamindb_setup-1.13.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-lamindb_setup/__init__.py,sha256=3WL1Tb4ewlV8h1mrJAlhlYkKmBJJDDkVDTFLMeLC9hE,3112
+lamindb_setup/__init__.py,sha256=SV2P-RJAPRv_l-GGuqpUGnfTTGxuutqh1UgzIgNjQTM,3112
 lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=ToKMxsUq8dQBQh8baOrNVlSb1iC8h4zTg5dV8wMu0W4,6760
-lamindb_setup/_connect_instance.py,sha256=OSOGZEj_9JsLvZT9JsXGvbC96DY4akmbAOuqUKyjjiE,17717
+lamindb_setup/_connect_instance.py,sha256=QMlPxupGpTR0j1_akh4lJHOFND7izrFnGbzUTXIX2F8,17703
 lamindb_setup/_delete.py,sha256=KS3r-xGFuDmAbzPUy-9JR-YnPShYdaHjDRQrAmXQ0qM,5863
 lamindb_setup/_disconnect.py,sha256=FT8EpCm5XXDdhDH7QtAnkO3KPatq2HqT9VXGNjgJDbk,1232
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
@@ -10,41 +10,42 @@ lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9
 lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
 lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
 lamindb_setup/_init_instance.py,sha256=8ejD6zjV0eF7KR-DvnmDAVJb9Ty0hjaPtIkFbyLDvA0,14806
-lamindb_setup/_migrate.py,sha256=aOWE13LJOW55mC4QiYeCS5bJGSTRsRZPpUYz6e_xoFs,10773
+lamindb_setup/_migrate.py,sha256=nkTzzNlUMqCAPujIgYK-A6Eg08r6LQAhm3rtbNtKZnU,11212
 lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
-lamindb_setup/_schema_metadata.py,sha256=At_EAE9mMzMJIJ1mfiOZYXVgBaXRkWUW6a3fLz5Z_lY,15132
+lamindb_setup/_schema_metadata.py,sha256=af1Es7qFKGPRdNmk48384HiB2r-cDTdBPu0wB9qrga4,15526
 lamindb_setup/_set_managed_storage.py,sha256=y5YQASsWNYVWUYeLgh3N2YBETYP7mBtbpxe3X_Vgb5I,2699
 lamindb_setup/_setup_user.py,sha256=DapdzT3u0f5LN5W9W9A6PWw-n8ejcJciQtHN9b5lidA,5889
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/errors.py,sha256=qZTfSL0rpbY8AIG-Z4-3-_EbLW5zyo2CFEJrVU02-3A,1863
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=XlXLb4nmbc68uBj5Hp3xpDRezYGJIBZv6jAAqqN0p10,614
-lamindb_setup/core/__init__.py,sha256=5M4A6CVHBO_T5Rr9MeLaPW3WTk4-y00cgRYEgUJVU5U,410
-lamindb_setup/core/_aws_options.py,sha256=SadUhcLCRtvsy3Qvx6799Iv4_CJkb1rwWB6d5GIixHc,8080
+lamindb_setup/core/__init__.py,sha256=qPSHqJoRKoG9542Au7EfTRFUHBPrKaCoXmNNwQEonMY,470
+lamindb_setup/core/_aws_options.py,sha256=NtDLfR2BIz3MiR4rGrBu4uW70MFy2p3hjxCnN1sGDB8,9414
 lamindb_setup/core/_aws_storage.py,sha256=QEtV-riQrwfivcwqHnXBbkJ-9YyNEXL4fLoCmOHZ1BI,2003
+lamindb_setup/core/_clone.py,sha256=2NlXV04yykqg_k7z59C_kD1F1Hi4H-55H-JtNjhenQ0,3691
 lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
 lamindb_setup/core/_hub_client.py,sha256=J0x43at0zb0yWP-RoT2lyqaHV66ewUP3OiYVYQCjxe8,9974
-lamindb_setup/core/_hub_core.py,sha256=axnNugfAehXIB_GLJoJE2zQfsAfPNEw5UELnUvTLWHY,27279
+lamindb_setup/core/_hub_core.py,sha256=ii7IZ_CVw_-GRgFgfvnXhVL6KI3UiuGKnMeIwTgiMd4,28907
 lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=0nz3HKnBuXdDY4R2UJQts3ZVC7vROpsBAxWIgZNz800,14017
-lamindb_setup/core/_settings_instance.py,sha256=oLRH_BlQ6EshZtpXZ9v4qk_2bUZDG5Km4qjqpII3EPA,23675
+lamindb_setup/core/_settings.py,sha256=QrxSClRK2GgP1-xgNSi0I0hp-wCTUsCs40CnFsncBts,14175
+lamindb_setup/core/_settings_instance.py,sha256=7VXd1W88fgqEnAfzFQKUlDnTr3pmA_e8aIag7FqPrJI,23899
 lamindb_setup/core/_settings_load.py,sha256=j20cy3J56ZBHLDfB2A8oKjekNetMNsy0_W3eWD36pWI,5161
 lamindb_setup/core/_settings_save.py,sha256=jh412jXIAbIYvnSoW9riBFePRAa4vmPm-ScYD0smlnw,3292
-lamindb_setup/core/_settings_storage.py,sha256=pyU25hP5rQYjVe0tFPR8P6TzAYzu1NpT-PIbXoxfV18,15348
+lamindb_setup/core/_settings_storage.py,sha256=yFz0w7yB7_3UWdrndVHJ_BZyV4ONj2lPRaQn_cksFQw,15150
 lamindb_setup/core/_settings_store.py,sha256=ykJeBA9IODK4G_jrfBE9pb0c1xkfePkARPpb306DT08,2687
 lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
-lamindb_setup/core/django.py,sha256=kV8W3WZy5Rkhn4rDJv2GNoq8JYvX_8dLBHhDRZdSgwE,10542
+lamindb_setup/core/django.py,sha256=2HwhtfUEX4peSkczc0VSfA-CpfCGL4vNgkPe9Pwu5kw,12429
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=uk3LpDA7Jbk1GzUb8hCsxByg5cMYTjPusIvwyXe8g3Y,36023
-lamindb_setup-1.12.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.12.2.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.12.2.dist-info/METADATA,sha256=6x5IAxLh33hQYnOh9EIIckgewprGjwTotuxNmHAlzXI,1798
-lamindb_setup-1.12.2.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=J43wCLFLRxNAUFN1bAtm6y6Mgt168JKq-wnhhG048Us,35486
+lamindb_setup-1.13.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.13.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.13.0.dist-info/METADATA,sha256=80aqYu5D3oJTyPh7GbZwC29u0m8fWNRL_BNLiRkZSJM,1798
+lamindb_setup-1.13.0.dist-info/RECORD,,