lamindb_setup 1.8.2__py3-none-any.whl → 1.9.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.8.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.9.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

lamindb_setup/_check_setup.py

@@ -147,6 +147,9 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
             else:
                 django_lamin.setup_django(isettings)
                 logger.important(f"connected lamindb: {isettings.slug}")
+                settings._instance_settings = (
+                    isettings  # update of local storage location
+                )
         return django_lamin.IS_SETUP
     else:
         if from_module is not None and settings.auto_connect:

lamindb_setup/_migrate.py

@@ -86,7 +86,7 @@ class migrate:
 
     @classmethod
     @disable_auto_connect
-    def deploy(cls) -> None:
+    def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
         """Deploy a migration."""
         from ._schema_metadata import update_schema_in_hub
 
@@ -116,7 +116,14 @@ class migrate:
             import lamindb
 
         # this sets up django and deploys the migrations
-        setup_django(settings.instance, deploy_migrations=True)
+        if package_name is not None and number is not None:
+            setup_django(
+                settings.instance,
+                deploy_migrations=True,
+                appname_number=(package_name, number),
+            )
+        else:
+            setup_django(settings.instance, deploy_migrations=True)
         # this populates the hub
         if settings.instance.is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")

lamindb_setup/core/_aws_options.py

@@ -22,7 +22,7 @@ else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 
-def _keep_trailing_slash(path_str: str):
+def _keep_trailing_slash(path_str: str) -> str:
     return path_str if path_str[-1] == "/" else path_str + "/"
 
 
@@ -54,6 +54,7 @@ class AWSOptionsManager:
 
     def __init__(self):
         self._credentials_cache = {}
+        self._parameters_cache = {}  # this is not refreshed
 
         from s3fs import S3FileSystem
 
@@ -106,7 +107,9 @@ class AWSOptionsManager:
     def _get_cached_credentials(self, root: str) -> dict:
         return self._credentials_cache[root]["credentials"]
 
-    def _path_inject_options(self, path: UPath, credentials: dict) -> UPath:
+    def _path_inject_options(
+        self, path: UPath, credentials: dict, extra_parameters: dict | None = None
+    ) -> UPath:
         if credentials == {}:
             # credentials were specified manually for the path
             if "anon" in path.storage_options:
@@ -137,6 +140,9 @@ class AWSOptionsManager:
             "version_aware", False
         )
 
+        if extra_parameters:
+            connection_options.update(extra_parameters)
+
         return UPath(path, **connection_options)
 
     def enrich_path(self, path: UPath, access_token: str | None = None) -> UPath:
@@ -158,7 +164,7 @@ class AWSOptionsManager:
         if root is not None:
             set_cache = False
             credentials = self._get_cached_credentials(root)
-
+            extra_parameters = self._parameters_cache.get(root)
             if access_token is not None:
                 set_cache = True
             elif credentials != {}:
@@ -172,17 +178,15 @@ class AWSOptionsManager:
             from ._hub_core import access_aws
             from ._settings import settings
 
-            if settings.user.handle != "anonymous" or access_token is not None:
-                storage_root_info = access_aws(path_str, access_token=access_token)
-            else:
-                storage_root_info = {"credentials": {}, "accessibility": {}}
-
+            storage_root_info = access_aws(path_str, access_token=access_token)
             accessibility = storage_root_info["accessibility"]
             is_managed = accessibility.get("is_managed", False)
             if is_managed:
                 credentials = storage_root_info["credentials"]
+                extra_parameters = accessibility["extra_parameters"]
             else:
                 credentials = {}
+                extra_parameters = None
 
             if access_token is None:
                 if "storage_root" in accessibility:
@@ -198,9 +202,13 @@ class AWSOptionsManager:
                         # write the bucket for everything else
                         root = path.drive
                     root = "s3://" + root
-                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 
-        return self._path_inject_options(path, credentials)
+                root = _keep_trailing_slash(root)
+                assert isinstance(root, str)
+                self._set_cached_credentials(root, credentials)
+                self._parameters_cache[root] = extra_parameters
+
+        return self._path_inject_options(path, credentials, extra_parameters)
 
 
 _aws_options_manager: AWSOptionsManager | None = None

lamindb_setup/core/_hub_client.py

@@ -124,6 +124,11 @@ def get_access_token(
             }
         )
         return auth_response.session.access_token
+    except Exception as e:
+        # we need to log the problem here because the exception is usually caught outside
+        # in call_with_fallback_auth
+        logger.warning(f"failed to update your lamindb access token: {e}")
+        raise e
     finally:
         hub.auth.sign_out(options={"scope": "local"})
 

lamindb_setup/core/_hub_core.py

@@ -460,9 +460,9 @@ def access_aws(storage_root: str, access_token: str | None = None) -> dict[str,
         storage_root_info = call_with_fallback_auth(
             _access_aws, storage_root=storage_root, access_token=access_token
         )
-        return storage_root_info
     else:
-        raise RuntimeError("Can only get access to AWS if authenticated.")
+        storage_root_info = call_with_fallback(_access_aws, storage_root=storage_root)
+    return storage_root_info
 
 
 def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
@@ -487,6 +487,8 @@ def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
         accessibility = storage_root_info["accessibility"]
         accessibility["storage_root"] = loaded_accessibility["storageRoot"]
         accessibility["is_managed"] = loaded_accessibility["isManaged"]
+        accessibility["extra_parameters"] = loaded_accessibility.get("extraParameters")
+
     return storage_root_info
 
 
@@ -497,8 +499,8 @@ def access_db(
     instance_slug: str
     instance_api_url: str | None
     if (
-        "LAMIN_TEST_DB_TOKEN" in os.environ
-        and (env_db_token := os.environ["LAMIN_TEST_DB_TOKEN"]) != ""
+        "LAMIN_DB_TOKEN" in os.environ
+        and (env_db_token := os.environ["LAMIN_DB_TOKEN"]) != ""
     ):
         return env_db_token
 

lamindb_setup/core/_settings.py

@@ -60,6 +60,9 @@ class SetupSettings:
 
     _cache_dir: Path | None = None
 
+    _branch = None  # do not have types here
+    _space = None  # do not have types here
+
     @property
     def _instance_settings_path(self) -> Path:
         return current_instance_settings_file()
@@ -112,12 +115,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def branch(self) -> Branch:
         """Default branch."""
-        from lamindb import Branch
+        if self._branch is None:
+            from lamindb import Branch
 
-        idlike, _ = self._read_branch_idlike_name()
-        return Branch.get(idlike)
+            idlike, _ = self._read_branch_idlike_name()
+            self._branch = Branch.get(idlike)
+        return self._branch
 
     @branch.setter
     def branch(self, value: str | Branch) -> None:
@@ -136,6 +143,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._branch_path.write_text(f"{branch_record.uid}\n{branch_record.name}")
+        self._branch = branch_record
 
     @property
     def _space_path(self) -> Path:
@@ -156,12 +164,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def space(self) -> Space:
         """Default space."""
-        from lamindb import Space
+        if self._space is None:
+            from lamindb import Space
 
-        idlike, _ = self._read_space_idlike_name()
-        return Space.get(idlike)
+            idlike, _ = self._read_space_idlike_name()
+            self._space = Space.get(idlike)
+        return self._space
 
     @space.setter
     def space(self, value: str | Space) -> None:
@@ -180,6 +192,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._space_path.write_text(f"{space_record.uid}\n{space_record.name}")
+        self._space = space_record
 
     @property
     def is_connected(self) -> bool:

lamindb_setup/core/_settings_instance.py

@@ -104,8 +104,20 @@ class InstanceSettings:
         for attr in attrs:
             value = getattr(self, attr)
             if attr == "storage":
-                representation += f"\n - storage root: {value.root_as_str}"
-                representation += f"\n - storage region: {value.region}"
+                if self.keep_artifacts_local:
+                    import lamindb as ln
+
+                    self._local_storage = ln.setup.settings.instance._local_storage
+                if self._local_storage is not None:
+                    value_local = self.local_storage
+                    representation += f"\n - local storage: {value_local.root_as_str} ({value_local.region})"
+                    representation += (
+                        f"\n - cloud storage: {value.root_as_str} ({value.region})"
+                    )
+                else:
+                    representation += (
+                        f"\n - storage: {value.root_as_str} ({value.region})"
+                    )
             elif attr == "db":
                 if self.dialect != "sqlite":
                     model = LaminDsnModel(db=value)
@@ -174,9 +186,10 @@ class InstanceSettings:
                         )
                         legacy_filepath.rename(marker_path)
                     else:
-                        raise ValueError(
+                        logger.warning(
                             f"local storage location '{root_path}' is corrupted, cannot find marker file with storage uid"
                         )
+                        continue
                 try:
                     uid = marker_path.read_text().splitlines()[0]
                 except PermissionError:
@@ -190,8 +203,9 @@ class InstanceSettings:
             if len(found) > 1:
                 found_display = "\n - ".join([f"{record.root}" for record in found])
                 logger.important(f"found locations:\n - {found_display}")
+            record = found[0]
             logger.important(f"defaulting to local storage: {record.root}")
-            return StorageSettings(record.root)
+            return StorageSettings(record.root, region=record.region)
         elif not mute_warning:
             start = LOCAL_STORAGE_MESSAGE[0].lower()
             logger.warning(f"{start}{LOCAL_STORAGE_MESSAGE[1:]}")
@@ -229,7 +243,9 @@ class InstanceSettings:
         Guide: :doc:`faq/keep-artifacts-local`
         """
         if not self.keep_artifacts_local:
-            raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
+            raise ValueError(
+                "`keep_artifacts_local` is False, switch via: ln.setup.settings.instance.keep_artifacts_local = True"
+            )
         if self._local_storage is None:
             self._local_storage = self._search_local_root()
         if self._local_storage is None:

lamindb_setup/core/_settings_user.py

@@ -39,7 +39,7 @@ class UserSettings:
     def __repr__(self) -> str:
         """Rich string representation."""
         representation = "Current user:"
-        attrs = ["handle", "email", "uid"]
+        attrs = ["handle", "uid"]
         for attr in attrs:
             value = getattr(self, attr)
             representation += f"\n - {attr}: {value}"

lamindb_setup/core/django.py

@@ -148,6 +148,7 @@ def setup_django(
     configure_only: bool = False,
     init: bool = False,
     view_schema: bool = False,
+    appname_number: tuple[str, int] | None = None,
 ):
     if IS_RUN_FROM_IPYTHON:
         os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
@@ -235,7 +236,11 @@ def setup_django(
         return None
 
     if deploy_migrations:
-        call_command("migrate", verbosity=2)
+        if appname_number is None:
+            call_command("migrate", verbosity=2)
+        else:
+            app_name, app_number = appname_number
+            call_command("migrate", app_name, app_number, verbosity=2)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     elif init:
         global IS_MIGRATING
@@ -247,7 +252,7 @@ def setup_django(
     IS_SETUP = True
 
     if isettings.keep_artifacts_local:
-        isettings._search_local_root()
+        isettings._local_storage = isettings._search_local_root()
 
 
 # THIS IS NOT SAFE

lamindb_setup/core/upath.py

@@ -393,8 +393,10 @@ def synchronize_to(
     """Sync to a local destination path."""
     destination = destination.resolve()
     protocol = origin.protocol
+    stat_kwargs = {"expand_info": True} if protocol == "hf" else {}
+    origin_str = str(origin)
     try:
-        cloud_info = origin.stat().as_info()
+        cloud_info = origin.fs.stat(origin_str, **stat_kwargs)
         exists = True
         is_dir = cloud_info["type"] == "directory"
     except FileNotFoundError:
@@ -441,7 +443,9 @@ def synchronize_to(
     if is_dir:
         cloud_stats = {
             file: get_modified(stat)
-            for file, stat in origin.fs.find(origin.as_posix(), detail=True).items()
+            for file, stat in origin.fs.find(
+                origin_str, detail=True, **stat_kwargs
+            ).items()
         }
         for cloud_path in cloud_stats:
             file_key = PurePosixPath(cloud_path).relative_to(origin.path).as_posix()

{lamindb_setup-1.8.2.dist-info → lamindb_setup-1.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.8.2
+Version: 1.9.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.8.2.dist-info → lamindb_setup-1.9.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-lamindb_setup/__init__.py,sha256=aROFrdb7k6cJGTETwMKBeRNp8LD5BurDRAfAp5lbAnM,2782
+lamindb_setup/__init__.py,sha256=Jbjaf2q8Kw_0mAn3bdeDDIxcH4BcJFFFj2AmGWVOlsc,2782
 lamindb_setup/_cache.py,sha256=5o749NuW6zi6uP4rmBtwxg7ifWpAHXVngzC0tEgXLgo,2776
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
-lamindb_setup/_check_setup.py,sha256=bXuqx2HEc178RM7gbKZQ65PEVJFu6uSOKiHAs_xz6GI,5575
+lamindb_setup/_check_setup.py,sha256=eeg7Vr7tUaTDObxq1X7J3TDPQZeitb_Uy6dxqa9xfzs,5707
 lamindb_setup/_connect_instance.py,sha256=PDvtAEHYJQVy-aMPNupN1u6PG9Rb_85JNKcjrOeHNy0,13478
 lamindb_setup/_delete.py,sha256=2KnZOqd5Kgr45XzjiDE9der35LODDUajZD6_hcurGtQ,5676
 lamindb_setup/_disconnect.py,sha256=p6tRLhixU4CuSxMKqzGTr-ovKmTRlZ8aID5dWQxOsg8,1092
@@ -10,7 +10,7 @@ lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9
 lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
 lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
 lamindb_setup/_init_instance.py,sha256=jNNE7ZWIl1V28YqfdbaiyZ_One6K2vv0RMubxdvvb74,15320
-lamindb_setup/_migrate.py,sha256=ya-15sc91i4JmEWI4j00T2892x8hdy2fSW-qz4IdxLs,9739
+lamindb_setup/_migrate.py,sha256=lkbEa_4qLxrQYSrLWKbXYQW32Yb1MBUa2hdfxCHnKUA,10047
 lamindb_setup/_register_instance.py,sha256=zmk7UrOF6qED_zTEaTM8GbZurMSP1SwddkRy7rpYmUY,1215
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=yMSuLZc-7iWUV7tETNuKmcDeMW0wtUr5ypKxsKC-m7k,14898
@@ -21,30 +21,30 @@ lamindb_setup/errors.py,sha256=H1UM-bii0U2vPyjprOBgZK4ijZJgzgCViyGWPd8v5yU,1493
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=XlXLb4nmbc68uBj5Hp3xpDRezYGJIBZv6jAAqqN0p10,614
 lamindb_setup/core/__init__.py,sha256=5M4A6CVHBO_T5Rr9MeLaPW3WTk4-y00cgRYEgUJVU5U,410
-lamindb_setup/core/_aws_options.py,sha256=JN6fJNcotdIuT-WkBRKDPdyDri9XmorEX2unbuC0KuQ,7738
+lamindb_setup/core/_aws_options.py,sha256=UbwnaEX2KcZBDOa2W2NbH62cNrKJQ78Hrn7zcszhQvU,8092
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=HN7iUBdEgahw5e4NHCd1VJooUfieNb6GRzS5x8jU-q8,2549
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=jICkfWW1eZoxh3ycviztBGqCJH53uVve5Xawbj8RZR4,8433
-lamindb_setup/core/_hub_core.py,sha256=Jf7Wfu59XF3Q6S-GgF6osDToBinQsUa33n55P7Cq-TQ,23919
+lamindb_setup/core/_hub_client.py,sha256=Pl3sEG2rasdJp4Rh5HNY0VsPVls-nfY0OxD5QzrYF9A,8678
+lamindb_setup/core/_hub_core.py,sha256=eOz7JcdpUz50BYkuOJDkdqnnBgXbamB-erLl1mAut0c,24006
 lamindb_setup/core/_hub_crud.py,sha256=Jz0d8wFKM1Pv9B9byyUJPlCIMkIzk56Jd-c3Awpm9Xw,5730
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=EtlxhtAdclS6rDRh5mrwh_q3gA7SJ1eF2rO1QtOYnnE,12949
-lamindb_setup/core/_settings_instance.py,sha256=40ty37SbCCc6pufi2455s4LcMtCbYRLLLz6P4rYwmiU,21198
+lamindb_setup/core/_settings.py,sha256=tCYkTwNRnxmgApmN5m9cjoBn4NBB5ZafS0cUVA1ipNM,13464
+lamindb_setup/core/_settings_instance.py,sha256=kkTbtOrezfZ97rXkyX8JvxVQxy2H6DSenmaAy9XoOS8,21900
 lamindb_setup/core/_settings_load.py,sha256=JWd0_hBy04xjKo-tH4y8C9RkaywjrmoT0PsKzVme0n4,5176
 lamindb_setup/core/_settings_save.py,sha256=XZx-vow7BT6y3JpRBB2UOJp2vwc7jOGea4wSgOPqjPU,3262
 lamindb_setup/core/_settings_storage.py,sha256=S9AvKLzJX0M_RsYcBKZB_P84CYtTY0hyeffYE3UqrQA,15478
 lamindb_setup/core/_settings_store.py,sha256=QmeWIGdIyq7UmjfHiEB_0xRD8hY-8-ZR2WntIKfwTKI,2714
-lamindb_setup/core/_settings_user.py,sha256=K2a6nQ0fhEiSb9mCY_p6ItNrHZ3J_j7EfO7CjZap9aA,1462
+lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
-lamindb_setup/core/django.py,sha256=dOt1OkUnZeYOo-LTjatQWQFh_MnjRf9IwwvRZhCwdZQ,9637
+lamindb_setup/core/django.py,sha256=jDwNJ3ZtkYezzNTGEW9NQZRwspc0akWQnCFWTgu3c1Q,9889
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=W47O9-Y205j29iWJ3RKKdomA587hGvoiD6_krASGFcM,35315
-lamindb_setup-1.8.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.8.2.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.8.2.dist-info/METADATA,sha256=8JhQoVWB9z_rwHXtAaGzxZoeEvFy5EmSM_PyFPp2B5A,1797
-lamindb_setup-1.8.2.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=reRwNJUOdeZ5n9m7Xi7RoNjmw0kqMPq_BkLPq6_fg-g,35468
+lamindb_setup-1.9.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.9.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.9.0.dist-info/METADATA,sha256=OmqlVQmuG8Iu_h9rPsfCgdfSo_6jH2aZJwq5jl6haH4,1797
+lamindb_setup-1.9.0.dist-info/RECORD,,