lamindb_setup 1.0.2__py3-none-any.whl → 1.1.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -33,10 +33,12 @@ Modules & settings:
 
 """
 
-__version__ = "1.0.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.1.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 
+from packaging import version as packaging_version
+
 from . import core
 from ._check_setup import _check_instance_setup
 from ._close import close
@@ -50,6 +52,20 @@ from ._register_instance import register
 from ._setup_user import login, logout
 from .core._settings import settings
 
+# check that the version of s3fs is higher than the lower bound
+# needed because spatialdata installs old versions of s3fs
+try:
+    from s3fs import __version__ as s3fs_version
+
+    if packaging_version.parse(s3fs_version) < packaging_version.parse("2023.12.2"):
+        raise RuntimeError(
+            f"The version of s3fs you have ({s3fs_version}) is impompatible "
+            "with lamindb, please upgrade it: pip install s3fs>=2023.12.2"
+        )
+except ImportError:
+    # might be not installed
+    pass
+
 
 def _is_CI_environment() -> bool:
     ci_env_vars = [

lamindb_setup/_check_setup.py

@@ -2,13 +2,14 @@ from __future__ import annotations
 
 import functools
 import importlib as il
+import inspect
 import os
 from typing import TYPE_CHECKING
 
 from lamin_utils import logger
 
 from ._silence_loggers import silence_loggers
-from .core import django
+from .core import django as django_lamin
 from .core._settings import settings
 from .core._settings_store import current_instance_settings_file
 from .core.exceptions import DefaultMessageException
@@ -33,8 +34,12 @@ CURRENT_ISETTINGS: InstanceSettings | None = None
 IS_LOADING: bool = False
 
 
+class ModuleWasntConfigured(SystemExit):
+    pass
+
+
 # decorator to disable auto-connect when importing a module such as lamindb
-def _loading(func: Callable):
+def disable_auto_connect(func: Callable):
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         global IS_LOADING
@@ -70,14 +75,66 @@ def _get_current_instance_settings() -> InstanceSettings | None:
         return None
 
 
+def _normalize_module_name(module_name: str) -> str:
+    return module_name.replace("lnschema_", "").replace("_", "-")
+
+
+# checks that the provided modules is in the modules of the provided instance
+# or in the apps setup by django
+def _check_module_in_instance_modules(
+    module: str, isettings: InstanceSettings | None = None
+) -> None:
+    not_in_instance_msg = (
+        f"'{module}' is missing from this instance. "
+        "Please go to your instance settings page and add it under 'schema modules'."
+    )
+
+    if isettings is not None:
+        modules_raw = isettings.modules
+        modules = set(modules_raw).union(
+            _normalize_module_name(module) for module in modules_raw
+        )
+        if _normalize_module_name(module) not in modules and module not in modules:
+            raise ModuleWasntConfigured(not_in_instance_msg)
+        else:
+            return
+
+    from django.apps import apps
+
+    for app in apps.get_app_configs():
+        # app.name is always unnormalized module (python package) name
+        if module == app.name or module == _normalize_module_name(app.name):
+            return
+    raise ModuleWasntConfigured(not_in_instance_msg)
+
+
+# infer the name of the module that calls this function
+def _infer_callers_module_name() -> str | None:
+    stack = inspect.stack()
+    if len(stack) < 3:
+        return None
+    module = inspect.getmodule(stack[2][0])
+    return module.__name__.partition(".")[0] if module is not None else None
+
+
 # we make this a private function because in all the places it's used,
 # users should not see it
 def _check_instance_setup(from_module: str | None = None) -> bool:
-    if django.IS_SETUP:
+    if django_lamin.IS_SETUP:
         # reload logic here because module might not yet have been imported
         # upon first setup
-        if from_module is not None and from_module != "lamindb":
-            il.reload(il.import_module(from_module))
+        if from_module is not None:
+            if from_module != "lamindb":
+                _check_module_in_instance_modules(from_module)
+                il.reload(il.import_module(from_module))
+        else:
+            infer_module = _infer_callers_module_name()
+            if infer_module is not None and infer_module not in {
+                "lamindb",
+                "lamindb_setup",
+                "lamin_cli",
+            }:
+                _check_module_in_instance_modules(infer_module)
         return True
     silence_loggers()
     if os.environ.get("LAMINDB_MULTI_INSTANCE") == "true":
@@ -91,17 +148,19 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
         if (
             from_module is not None
             and settings.auto_connect
-            and not django.IS_SETUP
+            and not django_lamin.IS_SETUP
             and not IS_LOADING
         ):
-            if not from_module == "lamindb":
+            if from_module != "lamindb":
+                _check_module_in_instance_modules(from_module, isettings)
+
                 import lamindb
 
                 il.reload(il.import_module(from_module))
             else:
-                django.setup_django(isettings)
+                django_lamin.setup_django(isettings)
                 logger.important(f"connected lamindb: {isettings.slug}")
-        return django.IS_SETUP
+        return django_lamin.IS_SETUP
     else:
         if from_module is not None and settings.auto_connect:
             logger.warning(InstanceNotSetupError.default_message)

lamindb_setup/_connect_instance.py

@@ -271,6 +271,7 @@ def connect(instance: str | None = None, **kwargs) -> str | tuple | None:
             settings_dir / f"no_lnschema_core-{isettings.slug.replace('/', '--')}"
         )
         if not no_lnschema_core_file.exists():
+            # sqlite file for cloud sqlite instances is already updated here
             migrate_lnschema_core(
                 isettings, no_lnschema_core_file, write_file=_write_settings
             )
@@ -311,7 +312,8 @@ def connect(instance: str | None = None, **kwargs) -> str | tuple | None:
         load_from_isettings(isettings, user=_user, write_settings=_write_settings)
         if _reload_lamindb:
             importlib.reload(importlib.import_module("lamindb"))
-        logger.important(f"connected lamindb: {isettings.slug}")
+        else:
+            logger.important(f"connected lamindb: {isettings.slug}")
     except Exception as e:
         if isettings is not None:
             if _write_settings:
@@ -339,24 +341,11 @@ def migrate_lnschema_core(
     """Migrate lnschema_core tables to lamindb tables."""
     from urllib.parse import urlparse
 
-    if isettings.is_on_hub:
-        from lamindb_setup.core._hub_client import call_with_fallback_auth
-        from lamindb_setup.core._hub_crud import (
-            select_collaborator,
-        )
-
-        # double check that user is an admin, otherwise will fail below
-        # due to insufficient SQL permissions with cryptic error
-        collaborator = call_with_fallback_auth(
-            select_collaborator,
-            instance_id=settings.instance._id,
-            account_id=settings.user._uuid,
-        )
-        if collaborator is None or collaborator["role"] != "admin":
-            raise SystemExit(
-                "❌ Only admins can deploy migrations, please ensure that you're an"
-                f" admin: https://lamin.ai/{settings.instance.slug}/settings"
-            )
+    # we need to do this because the sqlite file should be already synced
+    # has no effect if not cloud sqlite
+    # errors if the sqlite file is not in the cloud and doesn't exist locally
+    # isettings.db syncs but doesn't error in this case due to error_no_origin=False
+    isettings._update_local_sqlite_file()
 
     parsed_uri = urlparse(isettings.db)
     db_type = parsed_uri.scheme
@@ -413,6 +402,24 @@ def migrate_lnschema_core(
                 if response != "y":
                     print("Aborted.")
                     quit()
+                if isettings.is_on_hub:
+                    from lamindb_setup.core._hub_client import call_with_fallback_auth
+                    from lamindb_setup.core._hub_crud import (
+                        select_collaborator,
+                    )
+
+                    # double check that user is an admin, otherwise will fail below
+                    # due to insufficient SQL permissions with cryptic error
+                    collaborator = call_with_fallback_auth(
+                        select_collaborator,
+                        instance_id=settings.instance._id,
+                        account_id=settings.user._uuid,
+                    )
+                    if collaborator is None or collaborator["role"] != "admin":
+                        raise SystemExit(
+                            "❌ Only admins can deploy migrations, please ensure that you're an"
+                            f" admin: https://lamin.ai/{settings.instance.slug}/settings"
+                        )
                 for table in tables_to_rename:
                     if db_type == "sqlite":
                         cur.execute(

lamindb_setup/_delete.py

@@ -7,7 +7,7 @@ from uuid import UUID
 from lamin_utils import logger
 
 from ._connect_instance import _connect_instance, get_owner_name_from_identifier
-from .core._aws_credentials import HOSTED_BUCKETS
+from .core._aws_options import HOSTED_BUCKETS
 from .core._hub_core import delete_instance as delete_instance_on_hub
 from .core._hub_core import get_storage_records_for_instance
 from .core._settings import settings

lamindb_setup/_init_instance.py

@@ -396,10 +396,5 @@ def infer_instance_name(
     if storage == "create-s3":
         raise ValueError("pass name to init if storage = 'create-s3'")
     storage_path = UPath(storage).resolve()
-    # not sure if name is ever ""
-    if storage_path.name != "":
-        name = storage_path.name
-    else:
-        # dedicated treatment of bucket names
-        name = storage_path.drive
+    name = storage_path.path.rstrip("/").split("/")[-1]
     return name.lower()

lamindb_setup/_migrate.py

@@ -5,7 +5,7 @@ from django.db.migrations.loader import MigrationLoader
 from lamin_utils import logger
 from packaging import version
 
-from ._check_setup import _check_instance_setup, _loading
+from ._check_setup import _check_instance_setup, disable_auto_connect
 from .core._settings import settings
 from .core.django import setup_django
 
@@ -62,7 +62,7 @@ class migrate:
     """
 
     @classmethod
-    @_loading
+    @disable_auto_connect
     def create(cls) -> None:
         """Create a migration."""
         if _check_instance_setup():
@@ -70,7 +70,7 @@ class migrate:
         setup_django(settings.instance, create_migrations=True)
 
     @classmethod
-    @_loading
+    @disable_auto_connect
     def deploy(cls) -> None:
         """Deploy a migration."""
         from ._schema_metadata import update_schema_in_hub
@@ -115,7 +115,7 @@ class migrate:
             )
 
     @classmethod
-    @_loading
+    @disable_auto_connect
     def check(cls) -> bool:
         """Check whether Registry definitions are in sync with migrations."""
         from django.core.management import call_command
@@ -132,7 +132,7 @@ class migrate:
         return True
 
     @classmethod
-    @_loading
+    @disable_auto_connect
     def squash(
         cls, package_name, migration_nr, start_migration_nr: str | None = None
     ) -> None:
@@ -148,7 +148,7 @@ class migrate:
             call_command("squashmigrations", package_name, migration_nr)
 
     @classmethod
-    @_loading
+    @disable_auto_connect
     def show(cls) -> None:
         """Show migrations."""
         from django.core.management import call_command

lamindb_setup/core/{_aws_credentials.py → _aws_options.py}

@@ -4,7 +4,7 @@ import os
 import time
 
 from lamin_utils import logger
-from upath.implementations.cloud import S3Path
+from upath import UPath
 
 HOSTED_REGIONS = [
     "eu-central-1",
@@ -25,22 +25,29 @@ def _keep_trailing_slash(path_str: str):
     return path_str if path_str[-1] == "/" else path_str + "/"
 
 
-AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
+AWS_CREDENTIALS_EXPIRATION: int = 11 * 60 * 60  # refresh credentials after 11 hours
 
 
 # set anon=True for these buckets if credentials fail for a public bucket
 # to be expanded
-PUBLIC_BUCKETS = ("cellxgene-data-public",)
+PUBLIC_BUCKETS: tuple[str] = ("cellxgene-data-public",)
 
 
-class AWSCredentialsManager:
+# s3-comaptible endpoints managed by lamin
+# None means the standard aws s3 endpoint
+LAMIN_ENDPOINTS: tuple[str | None] = (None,)
+
+
+class AWSOptionsManager:
     def __init__(self):
         self._credentials_cache = {}
 
         from s3fs import S3FileSystem
 
         # this is cached so will be resued with the connection initialized
-        fs = S3FileSystem(cache_regions=True)
+        fs = S3FileSystem(
+            cache_regions=True, use_listings_cache=True, version_aware=False
+        )
         try:
             fs.connect()
             self.anon: bool = fs.session._credentials is None
@@ -83,7 +90,7 @@ class AWSCredentialsManager:
     def _get_cached_credentials(self, root: str) -> dict:
         return self._credentials_cache[root]["credentials"]
 
-    def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
+    def _path_inject_options(self, path: UPath, credentials: dict) -> UPath:
         if credentials == {}:
             # credentials were specified manually for the path
             if "anon" in path.storage_options:
@@ -99,13 +106,27 @@ class AWSCredentialsManager:
             connection_options = credentials
 
         if "cache_regions" in path.storage_options:
-            cache_regions = path.storage_options["cache_regions"]
+            connection_options["cache_regions"] = path.storage_options["cache_regions"]
         else:
-            cache_regions = True
-
-        return S3Path(path, cache_regions=cache_regions, **connection_options)
-
-    def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
+            connection_options["cache_regions"] = (
+                path.storage_options.get("endpoint_url", None) is None
+            )
+        # we use cache to avoid some uneeded downloads or credential problems
+        # see in upload_from
+        connection_options["use_listings_cache"] = path.storage_options.get(
+            "use_listings_cache", True
+        )
+        # normally we want to ignore objects vsrsions in a versioned bucket
+        connection_options["version_aware"] = path.storage_options.get(
+            "version_aware", False
+        )
+
+        return UPath(path, **connection_options)
+
+    def enrich_path(self, path: UPath, access_token: str | None = None) -> UPath:
+        # ignore paths with non-lamin-managed endpoints
+        if path.storage_options.get("endpoint_url", None) not in LAMIN_ENDPOINTS:
+            return path
         # trailing slash is needed to avoid returning incorrect results
         # with .startswith
         # for example s3://lamindata-eu should not receive cache for s3://lamindata
@@ -160,13 +181,13 @@ class AWSCredentialsManager:
         return self._path_inject_options(path, credentials)
 
 
-_aws_credentials_manager: AWSCredentialsManager | None = None
+_aws_options_manager: AWSOptionsManager | None = None
 
 
-def get_aws_credentials_manager() -> AWSCredentialsManager:
-    global _aws_credentials_manager
+def get_aws_options_manager() -> AWSOptionsManager:
+    global _aws_options_manager
 
-    if _aws_credentials_manager is None:
-        _aws_credentials_manager = AWSCredentialsManager()
+    if _aws_options_manager is None:
+        _aws_options_manager = AWSOptionsManager()
 
-    return _aws_credentials_manager
+    return _aws_options_manager

lamindb_setup/core/_hub_client.py

@@ -66,7 +66,11 @@ def connect_hub(
 ) -> Client:
     env = Environment(fallback=fallback_env)
     if client_options is None:
-        client_options = ClientOptions(auto_refresh_token=False)
+        # function_client_timeout=5 by default
+        # increase to avoid rare timeouts for edge functions
+        client_options = ClientOptions(
+            auto_refresh_token=False, function_client_timeout=10
+        )
     return create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
 
 

lamindb_setup/core/_hub_core.py

@@ -311,6 +311,7 @@ def _init_instance(
             "db_database": db_dsn.db.database,
         }
         fields.update(db_fields)
+    slug = isettings.slug
     # I'd like the following to be an upsert, but this seems to violate RLS
     # Similarly, if we don't specify `returning="minimal"`, we'll violate RLS
     # we could make this idempotent by catching an error, but this seems dangerous
@@ -318,14 +319,13 @@ def _init_instance(
     try:
         client.table("instance").insert(fields, returning="minimal").execute()
     except APIError:
-        logger.warning(
-            f"instance already existed at: https://lamin.ai/{isettings.owner}/{isettings.name}"
-        )
+        logger.warning(f"instance already existed at: https://lamin.ai/{slug}")
         return None
     client.table("storage").update(
         {"instance_id": isettings._id.hex, "is_default": True}
     ).eq("id", isettings.storage._uuid.hex).execute()  # type: ignore
-    logger.important(f"go to: https://lamin.ai/{isettings.owner}/{isettings.name}")
+    if isettings.dialect != "sqlite" and isettings.is_remote:
+        logger.important(f"go to: https://lamin.ai/{slug}")
 
 
 def _connect_instance_hub(

lamindb_setup/core/_settings_instance.py

@@ -171,7 +171,8 @@ class InstanceSettings:
             return StorageSettings(record.root)
         elif not mute_warning:
             logger.warning(
-                f"none of the registered local storage locations were found in your environment: {local_records}"
+                "none of the registered local storage locations were found:\n    "
+                + "\n    ".join(r.root for r in all_local_records)
             )
             logger.important(
                 "please register a new local storage location via `ln.settings.storage_local = local_root_path` and re-load/connect the instance"
@@ -466,13 +467,11 @@ class InstanceSettings:
         settings._instance_settings = self
 
     def _init_db(self):
-        from lamindb_setup import _check_setup
+        from lamindb_setup._check_setup import disable_auto_connect
 
         from .django import setup_django
 
-        _check_setup.IS_LOADING = True
-        setup_django(self, init=True)
-        _check_setup.IS_LOADING = False
+        disable_auto_connect(setup_django)(self, init=True)
 
         from lamindb.models import Space
 
@@ -482,8 +481,6 @@ class InstanceSettings:
         )
 
     def _load_db(self) -> tuple[bool, str]:
-        from lamindb_setup import _check_setup
-
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -494,15 +491,15 @@ class InstanceSettings:
                     f" {legacy_file} to {self._sqlite_file}"
                 )
             return False, f"SQLite file {self._sqlite_file} does not exist"
-
-        from .django import setup_django
-
         # we need the local sqlite to setup django
-        self._update_local_sqlite_file(lock_cloud_sqlite=self._is_cloud_sqlite)
+        self._update_local_sqlite_file()
         # setting up django also performs a check for migrations & prints them
         # as warnings
         # this should fail, e.g., if the db is not reachable
-        _check_setup.IS_LOADING = True
-        setup_django(self)
-        _check_setup.IS_LOADING = False
+        from lamindb_setup._check_setup import disable_auto_connect
+
+        from .django import setup_django
+
+        disable_auto_connect(setup_django)(self)
+
         return True, ""

lamindb_setup/core/_settings_storage.py

@@ -10,13 +10,13 @@ from typing import TYPE_CHECKING, Any, Literal
 import fsspec
 from lamin_utils import logger
 
-from ._aws_credentials import HOSTED_REGIONS, get_aws_credentials_manager
-from ._aws_storage import find_closest_aws_region
-from .upath import (
-    LocalPathClasses,
-    UPath,
-    create_path,
+from ._aws_options import (
+    HOSTED_REGIONS,
+    LAMIN_ENDPOINTS,
+    get_aws_options_manager,
 )
+from ._aws_storage import find_closest_aws_region
+from .upath import LocalPathClasses, UPath, _split_path_query, create_path
 
 if TYPE_CHECKING:
     from uuid import UUID
@@ -44,15 +44,27 @@ def get_storage_region(path: UPathStr) -> str | None:
         from botocore.config import Config
         from botocore.exceptions import ClientError
 
-        # strip the prefix and any suffixes of the bucket name
-        bucket = path_str.replace("s3://", "").split("/")[0]
+        # check for endpoint_url in storage options if upath
+        if isinstance(path, UPath):
+            endpoint_url = path.storage_options.get("endpoint_url", None)
+        else:
+            endpoint_url = None
+        path_part = path_str.replace("s3://", "")
+        # check for endpoint_url in the path string
+        if "?" in path_part:
+            assert endpoint_url is None
+            path_part, query = _split_path_query(path_part)
+            endpoint_url = query.get("endpoint_url", [None])[0]
+        bucket = path_part.split("/")[0]
         session = botocore.session.get_session()
         credentials = session.get_credentials()
         if credentials is None or credentials.access_key is None:
             config = Config(signature_version=botocore.session.UNSIGNED)
         else:
             config = None
-        s3_client = session.create_client("s3", config=config)
+        s3_client = session.create_client(
+            "s3", endpoint_url=endpoint_url, config=config
+        )
         try:
             response = s3_client.head_bucket(Bucket=bucket)
         except ClientError as exc:
@@ -62,7 +74,7 @@ def get_storage_region(path: UPathStr) -> str | None:
         region = (
             response.get("ResponseMetadata", {})
             .get("HTTPHeaders", {})
-            .get("x-amz-bucket-region")
+            .get("x-amz-bucket-region", None)
         )
     else:
         region = None
@@ -116,9 +128,9 @@ def init_storage(
             if region not in HOSTED_REGIONS:
                 raise ValueError(f"region has to be one of {HOSTED_REGIONS}")
         if lamin_env is None or lamin_env == "prod":
-            root_str = f"s3://lamin-{region}/{uid}"
+            root = f"s3://lamin-{region}/{uid}"
         else:
-            root_str = f"s3://lamin-hosted-test/{uid}"
+            root = f"s3://lamin-hosted-test/{uid}"
     elif (input_protocol := fsspec.utils.get_protocol(root_str)) not in VALID_PROTOCOLS:
         valid_protocols = ("local",) + VALID_PROTOCOLS[1:]  # show local instead of file
         raise ValueError(
@@ -126,7 +138,7 @@ def init_storage(
         )
     ssettings = StorageSettings(
         uid=uid,
-        root=root_str,
+        root=root,
         region=region,
         instance_id=instance_id,
         access_token=access_token,
@@ -155,7 +167,7 @@ def init_storage(
         mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
     except Exception:
         logger.important(
-            f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root}"
+            f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root_as_str}"
         )
         # we have to check hub_record_status here because
         # _select_storage inside init_storage_hub also populates ssettings._uuid
@@ -190,7 +202,6 @@ class StorageSettings:
                 self._root_init = self._root_init.resolve()
             except Exception:
                 logger.warning(f"unable to create .lamindb folder in {self._root_init}")
-                pass
         self._root = None
         self._instance_id = instance_id
         # we don't yet infer region here to make init fast
@@ -255,7 +266,7 @@ class StorageSettings:
         elif getattr(self._root, "protocol", "") == "s3":
             # this is needed to be sure that the root always has nonexpired credentials
             # this just checks for time of the cached credentials in most cases
-            return get_aws_credentials_manager().enrich_path(
+            return get_aws_options_manager().enrich_path(
                 self._root, access_token=self.access_token
             )
         return self._root
@@ -275,6 +286,12 @@ class StorageSettings:
     @property
     def root_as_str(self) -> str:
         """Formatted root string."""
+        # embed endpoint_url into path string for storing and displaying
+        if self._root_init.protocol == "s3":
+            endpoint_url = self._root_init.storage_options.get("endpoint_url", None)
+            # LAMIN_ENDPOINTS include None
+            if endpoint_url not in LAMIN_ENDPOINTS:
+                return f"s3://{self._root_init.path.rstrip('/')}?endpoint_url={endpoint_url}"
         return self._root_init.as_posix().rstrip("/")
 
     @property

lamindb_setup/core/hashing.py

@@ -41,11 +41,6 @@ def b16_to_b64(s: str):
     return to_b64_str(base64.b16decode(s.strip('"'), casefold=True))
 
 
-def hash_string(string: str) -> str:
-    # as we're truncating (not here) at 22 b64, we choose md5 over sha512
-    return to_b64_str(hashlib.md5(string.encode("utf-8")).digest())
-
-
 # a lot to read about this: lamin-notes/2022/hashing
 def hash_set(s: set[str]) -> str:
     join_s = ":".join(sorted(s))
@@ -68,7 +63,9 @@ def hash_from_hashes_list(hashes: Iterable[str]) -> str:
     return to_b64_str(digest)[:HASH_LENGTH]
 
 
-def hash_code(file_path: UPathStr):
+# below is only used when comparing with git's sha1 hashes
+# we don't use it for our own hashes
+def hash_code(file_path: UPathStr) -> hashlib._Hash:
     with open(file_path, "rb") as fp:
         data = fp.read()
     data_size = len(data)
@@ -77,6 +74,16 @@ def hash_code(file_path: UPathStr):
     return hashlib.sha1(blob)
 
 
+def hash_small_bytes(data: bytes) -> str:
+    return to_b64_str(hashlib.md5(data).digest())
+
+
+# this is equivalent with hash_file for small files
+def hash_string(string: str) -> str:
+    # as we're truncating (not here) at 22 b64, we choose md5 over sha512
+    return to_b64_str(hashlib.md5(string.encode("utf-8")).digest())[:HASH_LENGTH]
+
+
 def hash_file(
     file_path: Path,
     file_size: int | None = None,

lamindb_setup/core/upath.py

@@ -11,15 +11,17 @@ from functools import partial
 from itertools import islice
 from pathlib import Path, PosixPath, PurePosixPath, WindowsPath
 from typing import TYPE_CHECKING, Any, Literal
+from urllib.parse import parse_qs, urlsplit
 
 import click
 import fsspec
 from lamin_utils import logger
 from upath import UPath
-from upath.implementations.cloud import CloudPath  # keep CloudPath!
+from upath.implementations.cloud import CloudPath, S3Path  # keep CloudPath!
 from upath.implementations.local import LocalPath
+from upath.registry import register_implementation
 
-from ._aws_credentials import HOSTED_BUCKETS, get_aws_credentials_manager
+from ._aws_options import HOSTED_BUCKETS, get_aws_options_manager
 from .hashing import HASH_LENGTH, b16_to_b64, hash_from_hashes_list, hash_string
 
 if TYPE_CHECKING:
@@ -360,7 +362,8 @@ def synchronize(
     print_progress: bool = False,
     callback: fsspec.callbacks.Callback | None = None,
     timestamp: float | None = None,
-):
+    just_check: bool = False,
+) -> bool:
     """Sync to a local destination path."""
     protocol = self.protocol
     # optimize the number of network requests
@@ -394,7 +397,7 @@ def synchronize(
         elif error_no_origin:
             warn_or_error += "\nIt is not possible to synchronize."
             raise FileNotFoundError(warn_or_error)
-        return None
+        return False
 
     # synchronization logic for directories
     # to synchronize directories, it should be possible to get modification times
@@ -427,6 +430,9 @@ def synchronize(
         else:
             destination_exists = False
             need_synchronize = True
+        # just check if synchronization is needed
+        if just_check:
+            return need_synchronize
         if need_synchronize:
             callback = ProgressCallback.requires_progress(
                 callback, print_progress, objectpath.name, "synchronizing"
@@ -456,13 +462,14 @@ def synchronize(
                             parent = file.parent
                             if next(parent.iterdir(), None) is None:
                                 parent.rmdir()
-        return None
+        return need_synchronize
 
     # synchronization logic for files
     callback = ProgressCallback.requires_progress(
         callback, print_progress, objectpath.name, "synchronizing"
     )
-    if objectpath.exists():
+    objectpath_exists = objectpath.exists()
+    if objectpath_exists:
         if cloud_mts != 0:
             local_mts_obj = objectpath.stat().st_mtime
             need_synchronize = cloud_mts > local_mts_obj
@@ -474,9 +481,17 @@ def synchronize(
             local_size_obj = objectpath.stat().st_size
             need_synchronize = cloud_size != local_size_obj
     else:
-        objectpath.parent.mkdir(parents=True, exist_ok=True)
+        if not just_check:
+            objectpath.parent.mkdir(parents=True, exist_ok=True)
         need_synchronize = True
+    # just check if synchronization is needed
+    if just_check:
+        return need_synchronize
     if need_synchronize:
+        # just to be sure that overwriting an existing file doesn't corrupt it
+        # we saw some frequent corruption on some systems for unclear reasons
+        if objectpath_exists:
+            objectpath.unlink()
         # hf has sync filesystem
         # on sync filesystems ChildProgressCallback.branched()
         # returns the default callback
@@ -490,6 +505,7 @@ def synchronize(
         # nothing happens if parent_update is not defined
         # because of Callback.no_op
         callback.parent_update()
+    return need_synchronize
 
 
 def modified(self) -> datetime | None:
@@ -739,16 +755,52 @@ warnings.filterwarnings(
 )
 
 
+# split query params from path string
+def _split_path_query(url: str) -> tuple[str, dict]:
+    split_result = urlsplit(url)
+    query = parse_qs(split_result.query)
+    path = split_result._replace(query="").geturl()
+    return path, query
+
+
+class S3QueryPath(S3Path):
+    @classmethod
+    def _transform_init_args(cls, args, protocol, storage_options):
+        args, protocol, storage_options = super()._transform_init_args(
+            args, protocol, storage_options
+        )
+        arg0 = args[0]
+        path, query = _split_path_query(str(arg0))
+        for param, param_values in query.items():
+            if len(param_values) > 1:
+                raise ValueError(f"Multiple values for {param} query parameter")
+            else:
+                param_value = param_values[0]
+                if param in storage_options and param_value != storage_options[param]:
+                    raise ValueError(
+                        f"Incompatible {param} in query and storage_options"
+                    )
+                storage_options.setdefault(param, param_value)
+        if hasattr(arg0, "storage_options"):
+            storage_options = {**arg0.storage_options, **storage_options}
+
+        return (path, *args[1:]), protocol, storage_options
+
+
+register_implementation("s3", S3QueryPath, clobber=True)
+
+
 def create_path(path: UPathStr, access_token: str | None = None) -> UPath:
     upath = UPath(path)
 
     if upath.protocol == "s3":
         # add managed credentials and other options for AWS s3 paths
-        return get_aws_credentials_manager().enrich_path(upath, access_token)
+        return get_aws_options_manager().enrich_path(upath, access_token)
 
     if upath.protocol in {"http", "https"}:
         # this is needed because by default aiohttp drops a connection after 5 min
         # so it is impossible to download large files
+        storage_options = {}
         client_kwargs = upath.storage_options.get("client_kwargs", {})
         if "timeout" not in client_kwargs:
             from aiohttp import ClientTimeout
@@ -757,7 +809,12 @@ def create_path(path: UPathStr, access_token: str | None = None) -> UPath:
                 **client_kwargs,
                 "timeout": ClientTimeout(sock_connect=30, sock_read=30),
             }
-            return UPath(upath, client_kwargs=client_kwargs)
+            storage_options["client_kwargs"] = client_kwargs
+        # see download_to for the reason
+        if "use_listings_cache" not in upath.storage_options:
+            storage_options["use_listings_cache"] = True
+        if len(storage_options) > 0:
+            return UPath(upath, **storage_options)
     return upath
 
 

{lamindb_setup-1.0.2.dist-info → lamindb_setup-1.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.0.2
+Version: 1.1.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
@@ -11,13 +11,15 @@ Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic-settings
 Requires-Dist: appdirs<2.0.0
 Requires-Dist: requests
-Requires-Dist: universal_pathlib==0.2.5
+Requires-Dist: universal_pathlib==0.2.6
 Requires-Dist: botocore<2.0.0
 Requires-Dist: supabase>=2.8.1,<=2.11.0
+Requires-Dist: storage3!=0.11.2; python_version < '3.11'
 Requires-Dist: psutil
+Requires-Dist: packaging
 Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
-Requires-Dist: s3fs>=2023.12.2,<=2024.10.0,!=2024.10.0 ; extra == "aws"
+Requires-Dist: s3fs>=2023.12.2,<=2025.2.0,!=2024.10.0 ; extra == "aws"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"
@@ -29,7 +31,7 @@ Requires-Dist: pytest-xdist ; extra == "dev"
 Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
 Requires-Dist: pandas ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: gcsfs>=2023.12.2,<=2024.10.0 ; extra == "gcp"
+Requires-Dist: gcsfs>=2023.12.2,<=2025.2.0 ; extra == "gcp"
 Project-URL: Home, https://github.com/laminlabs/lamindb-setup
 Provides-Extra: aws
 Provides-Extra: dev

{lamindb_setup-1.0.2.dist-info → lamindb_setup-1.1.0.dist-info}/RECORD

@@ -1,16 +1,16 @@
-lamindb_setup/__init__.py,sha256=C1Or-KZftjcYsNBbezefoC84rq0GVDE2IJEt4kNeVec,2127
+lamindb_setup/__init__.py,sha256=nG1aMZpbOPII-GtdqxGgMgeR-Lk_ltuPr_yAMiMd44o,2692
 lamindb_setup/_cache.py,sha256=aszT-zk3S5dTLKp5g1W-S_FPh2E5YVCALwWSGPJLWBM,1493
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
-lamindb_setup/_check_setup.py,sha256=DJ4lZp4N9Dcri8H7PEOl-YDoi4qqfQnXvwBuL4venAw,3425
+lamindb_setup/_check_setup.py,sha256=d1GS1Csy2G9AysfjoyVcNVY0lhHiWSwSLpfgdIQf35s,5477
 lamindb_setup/_close.py,sha256=pf8PHrtRBC6TycBtVAXzD9EZSGufoyp5M82o6zLHmt4,1240
-lamindb_setup/_connect_instance.py,sha256=1sA8i-vxZJhwOzobgqmzgsUE9BF6XaH593RDPoy6G0g,17572
-lamindb_setup/_delete.py,sha256=Mip5M9tCxyfsjzdcPCl6x9CQ0TkYTqKNNWDIcJ-KVMo,5677
+lamindb_setup/_connect_instance.py,sha256=eMiAjuMDuZ_x8N5LLHwz5O4VXv0nH6AzYW0tnnjUSug,18190
+lamindb_setup/_delete.py,sha256=4NqJkEA812VWRxTBW1o1h1YX0fF7sQwn08hj1BmvCPs,5673
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
 lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9k,522
 lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
 lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
-lamindb_setup/_init_instance.py,sha256=JV34P4ShinIdeUmHeoOvNE3WtzLbk37HcsWxfaEC_ks,13944
-lamindb_setup/_migrate.py,sha256=86ShCTLGnBT3HreYhua4B8YQTy6tyIOgpa0qlsdob1k,9093
+lamindb_setup/_init_instance.py,sha256=nZkqGqJQ_sF1-oQmnbpIjP1ExWUow5I5N7lsyX9m-io,13811
+lamindb_setup/_migrate.py,sha256=drf7Lsa8oDXBPBWMWMge0n5VyNPvsrCFAhcXlN91OxM,9165
 lamindb_setup/_register_instance.py,sha256=alQuYp2f8Ct8xvRC1gt8p_HZ0tqCd3gZD3kiPBLPpsI,1269
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=7ITlzIK32GHdhMq9e0GtPM3QbzJWhUvzPutiHrjjPk0,13986
@@ -18,30 +18,30 @@ lamindb_setup/_set_managed_storage.py,sha256=4tDxXQMt8Gw028uY3vIQxZQ7qBNXhQMc8sa
 lamindb_setup/_setup_user.py,sha256=-g7Xj6510BDyM8kuqAsVBZFwehlhBa_uWBSV1rPeuM8,4586
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=BxIVMX5HQq8oZ1OuY_saUEJz5Tdd7gaCPngxVu5iou4,417
-lamindb_setup/core/_aws_credentials.py,sha256=_wBWC10MGx3PW9UXGhsVNlq7YvCER3RhfRgAdlxEjNM,6120
+lamindb_setup/core/_aws_options.py,sha256=Evu8W9Xcr1_vYKo3Vfsv03dmD4x3itaPyWY8fBOrYcE,6954
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=HN7iUBdEgahw5e4NHCd1VJooUfieNb6GRzS5x8jU-q8,2549
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=cN19XbZmvLCxL_GKdOcKbedNRL7kR47vmLmA--NMv-U,6306
-lamindb_setup/core/_hub_core.py,sha256=qVGGsWVfP6GK9UzmEz1kuR_B8wFkgTstMJJoMHeUF0c,20007
+lamindb_setup/core/_hub_client.py,sha256=ywJ_HpGqwui6t_vsj-my80gYgSSgAiSFyZoRj03sBWk,6464
+lamindb_setup/core/_hub_core.py,sha256=OPByE76h6MQaCToT7AzbeH9Tr5-BbpIAh__oIiEl1VE,20021
 lamindb_setup/core/_hub_crud.py,sha256=IAuPZes1am8OFwtcf5jSRQPGG1eKwVTEsp9Li-uq0cQ,5377
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=KIn43HOhiRjkbTbddyJqv-WNTTa1bAizbM1tWXoXPBg,2869
 lamindb_setup/core/_settings.py,sha256=eslFO84vb5uRRfJ3r_uu4O8677l8lU5BbpZJMSAYw6A,8244
-lamindb_setup/core/_settings_instance.py,sha256=agzlSvKhvLyyNPf-Gw-M3FYn2ARpR1QEoCexhIHX5IU,19437
+lamindb_setup/core/_settings_instance.py,sha256=prEAS0MrSzPdxQTSjFoTzvGz6915GIL9v9RYhzPeoQ4,19362
 lamindb_setup/core/_settings_load.py,sha256=boeNntqIZ_DjelRBUAGp0ujc5akmbrrsk-LY28exa7E,4099
 lamindb_setup/core/_settings_save.py,sha256=rxGxgaK5i9exKqSJERQQyY1WZio20meoQJoYXlVW-1w,3138
-lamindb_setup/core/_settings_storage.py,sha256=dPIvbA6PkdjM8gsX6zxtH7VNMc4vhkuEO4luVMZY7RQ,12243
+lamindb_setup/core/_settings_storage.py,sha256=a3rRqntAX-dKZn6-TkY9pTPT8UPNUlwdni1tDNTJW84,13117
 lamindb_setup/core/_settings_store.py,sha256=WcsgOmgnu9gztcrhp-N4OONNZyxICHV8M0HdJllTaEo,2219
 lamindb_setup/core/_settings_user.py,sha256=lWqV3HmZCsEq2UsU_iVNW0p9ddsNg7-B6xOaMNH1aw0,1475
 lamindb_setup/core/_setup_bionty_sources.py,sha256=qTPMV5TEbNiTB81QqG2rSs6W8j8kQ7kVQMLOXRzAxBI,4004
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=i6TrT7HG0lqliPvZTlsZ_uplPaqhPBbabyfeR32SkA8,7107
 lamindb_setup/core/django.py,sha256=jSbckc_R39BhnANGhO5YFzKA8BHNANJDDgS5rwkDWmU,3828
 lamindb_setup/core/exceptions.py,sha256=4NpLUNUIfXYVTFX2FvLZF8RW34exk2Vn2X3G4YhnTRg,276
-lamindb_setup/core/hashing.py,sha256=EO7NSXMdKZLe9cHxCNHiqd5-6pVhv97uafqpoWV9FKc,3394
+lamindb_setup/core/hashing.py,sha256=M3Q1-ywnqh4Uy5zojbQfLju19HU0ySp8Oi7FGIJXfFI,3667
 lamindb_setup/core/types.py,sha256=zJii2le38BJUmsNVvzDrbzGYr0yaeb-9Rw9IKmsBr3k,523
-lamindb_setup/core/upath.py,sha256=iZIGnkvG6uZhjiq2qJ8v4pgABxw2rBwm_XSnZACzHVo,31075
-lamindb_setup-1.0.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.0.2.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.0.2.dist-info/METADATA,sha256=mUoczZRjbbv0JbQCETPFlJxYB-EW5h34XgFEWDYmL40,1690
-lamindb_setup-1.0.2.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=FUCBicCTAUPq-3lFwcDPngRLrNGfqQEn7l9ZCLjkz3Y,33335
+lamindb_setup-1.1.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.1.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.1.0.dist-info/METADATA,sha256=h799g4G2jS30UqZwG69XIIB_cWf46cPvnhisMsTOdSk,1770
+lamindb_setup-1.1.0.dist-info/RECORD,,