lamindb_setup 1.8.3__py3-none-any.whl → 1.9.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.8.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.9.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

lamindb_setup/_migrate.py

@@ -86,7 +86,7 @@ class migrate:
 
     @classmethod
     @disable_auto_connect
-    def deploy(cls) -> None:
+    def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
         """Deploy a migration."""
         from ._schema_metadata import update_schema_in_hub
 
@@ -116,7 +116,14 @@ class migrate:
             import lamindb
 
         # this sets up django and deploys the migrations
-        setup_django(settings.instance, deploy_migrations=True)
+        if package_name is not None and number is not None:
+            setup_django(
+                settings.instance,
+                deploy_migrations=True,
+                appname_number=(package_name, number),
+            )
+        else:
+            setup_django(settings.instance, deploy_migrations=True)
         # this populates the hub
         if settings.instance.is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")

lamindb_setup/core/_aws_options.py

@@ -22,7 +22,7 @@ else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 
-def _keep_trailing_slash(path_str: str):
+def _keep_trailing_slash(path_str: str) -> str:
     return path_str if path_str[-1] == "/" else path_str + "/"
 
 
@@ -54,6 +54,7 @@ class AWSOptionsManager:
 
     def __init__(self):
         self._credentials_cache = {}
+        self._parameters_cache = {}  # this is not refreshed
 
         from s3fs import S3FileSystem
 
@@ -106,7 +107,9 @@ class AWSOptionsManager:
     def _get_cached_credentials(self, root: str) -> dict:
         return self._credentials_cache[root]["credentials"]
 
-    def _path_inject_options(self, path: UPath, credentials: dict) -> UPath:
+    def _path_inject_options(
+        self, path: UPath, credentials: dict, extra_parameters: dict | None = None
+    ) -> UPath:
         if credentials == {}:
             # credentials were specified manually for the path
             if "anon" in path.storage_options:
@@ -137,6 +140,9 @@ class AWSOptionsManager:
             "version_aware", False
         )
 
+        if extra_parameters:
+            connection_options.update(extra_parameters)
+
         return UPath(path, **connection_options)
 
     def enrich_path(self, path: UPath, access_token: str | None = None) -> UPath:
@@ -158,7 +164,7 @@ class AWSOptionsManager:
         if root is not None:
             set_cache = False
             credentials = self._get_cached_credentials(root)
-
+            extra_parameters = self._parameters_cache.get(root)
             if access_token is not None:
                 set_cache = True
             elif credentials != {}:
@@ -177,8 +183,10 @@ class AWSOptionsManager:
             is_managed = accessibility.get("is_managed", False)
             if is_managed:
                 credentials = storage_root_info["credentials"]
+                extra_parameters = accessibility["extra_parameters"]
             else:
                 credentials = {}
+                extra_parameters = None
 
             if access_token is None:
                 if "storage_root" in accessibility:
@@ -194,9 +202,13 @@ class AWSOptionsManager:
                         # write the bucket for everything else
                         root = path.drive
                     root = "s3://" + root
-                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 
-        return self._path_inject_options(path, credentials)
+                root = _keep_trailing_slash(root)
+                assert isinstance(root, str)
+                self._set_cached_credentials(root, credentials)
+                self._parameters_cache[root] = extra_parameters
+
+        return self._path_inject_options(path, credentials, extra_parameters)
 
 
 _aws_options_manager: AWSOptionsManager | None = None

lamindb_setup/core/_hub_client.py

@@ -124,6 +124,11 @@ def get_access_token(
             }
         )
         return auth_response.session.access_token
+    except Exception as e:
+        # we need to log the problem here because the exception is usually caught outside
+        # in call_with_fallback_auth
+        logger.warning(f"failed to update your lamindb access token: {e}")
+        raise e
     finally:
         hub.auth.sign_out(options={"scope": "local"})
 

lamindb_setup/core/_hub_core.py

@@ -487,6 +487,8 @@ def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
         accessibility = storage_root_info["accessibility"]
         accessibility["storage_root"] = loaded_accessibility["storageRoot"]
         accessibility["is_managed"] = loaded_accessibility["isManaged"]
+        accessibility["extra_parameters"] = loaded_accessibility.get("extraParameters")
+
     return storage_root_info
 
 

lamindb_setup/core/_settings.py

@@ -60,6 +60,9 @@ class SetupSettings:
 
     _cache_dir: Path | None = None
 
+    _branch = None  # do not have types here
+    _space = None  # do not have types here
+
     @property
     def _instance_settings_path(self) -> Path:
         return current_instance_settings_file()
@@ -112,12 +115,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def branch(self) -> Branch:
         """Default branch."""
-        from lamindb import Branch
+        if self._branch is None:
+            from lamindb import Branch
 
-        idlike, _ = self._read_branch_idlike_name()
-        return Branch.get(idlike)
+            idlike, _ = self._read_branch_idlike_name()
+            self._branch = Branch.get(idlike)
+        return self._branch
 
     @branch.setter
     def branch(self, value: str | Branch) -> None:
@@ -136,6 +143,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._branch_path.write_text(f"{branch_record.uid}\n{branch_record.name}")
+        self._branch = branch_record
 
     @property
     def _space_path(self) -> Path:
@@ -156,12 +164,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def space(self) -> Space:
         """Default space."""
-        from lamindb import Space
+        if self._space is None:
+            from lamindb import Space
 
-        idlike, _ = self._read_space_idlike_name()
-        return Space.get(idlike)
+            idlike, _ = self._read_space_idlike_name()
+            self._space = Space.get(idlike)
+        return self._space
 
     @space.setter
     def space(self, value: str | Space) -> None:
@@ -180,6 +192,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._space_path.write_text(f"{space_record.uid}\n{space_record.name}")
+        self._space = space_record
 
     @property
     def is_connected(self) -> bool:

lamindb_setup/core/_settings_instance.py

@@ -186,9 +186,10 @@ class InstanceSettings:
                         )
                         legacy_filepath.rename(marker_path)
                     else:
-                        raise ValueError(
+                        logger.warning(
                             f"local storage location '{root_path}' is corrupted, cannot find marker file with storage uid"
                         )
+                        continue
                 try:
                     uid = marker_path.read_text().splitlines()[0]
                 except PermissionError:

lamindb_setup/core/django.py

@@ -148,6 +148,7 @@ def setup_django(
     configure_only: bool = False,
     init: bool = False,
     view_schema: bool = False,
+    appname_number: tuple[str, int] | None = None,
 ):
     if IS_RUN_FROM_IPYTHON:
         os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
@@ -235,7 +236,11 @@ def setup_django(
         return None
 
     if deploy_migrations:
-        call_command("migrate", verbosity=2)
+        if appname_number is None:
+            call_command("migrate", verbosity=2)
+        else:
+            app_name, app_number = appname_number
+            call_command("migrate", app_name, app_number, verbosity=2)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     elif init:
         global IS_MIGRATING

lamindb_setup/core/upath.py

@@ -393,8 +393,10 @@ def synchronize_to(
     """Sync to a local destination path."""
     destination = destination.resolve()
     protocol = origin.protocol
+    stat_kwargs = {"expand_info": True} if protocol == "hf" else {}
+    origin_str = str(origin)
     try:
-        cloud_info = origin.stat().as_info()
+        cloud_info = origin.fs.stat(origin_str, **stat_kwargs)
         exists = True
         is_dir = cloud_info["type"] == "directory"
     except FileNotFoundError:
@@ -441,7 +443,9 @@ def synchronize_to(
     if is_dir:
         cloud_stats = {
             file: get_modified(stat)
-            for file, stat in origin.fs.find(origin.as_posix(), detail=True).items()
+            for file, stat in origin.fs.find(
+                origin_str, detail=True, **stat_kwargs
+            ).items()
         }
         for cloud_path in cloud_stats:
             file_key = PurePosixPath(cloud_path).relative_to(origin.path).as_posix()

{lamindb_setup-1.8.3.dist-info → lamindb_setup-1.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.8.3
+Version: 1.9.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.8.3.dist-info → lamindb_setup-1.9.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=Pdh2-wDgY4x06M23rDIrY4XP4QrMck8SOSU00SAoSsw,2782
+lamindb_setup/__init__.py,sha256=Jbjaf2q8Kw_0mAn3bdeDDIxcH4BcJFFFj2AmGWVOlsc,2782
 lamindb_setup/_cache.py,sha256=5o749NuW6zi6uP4rmBtwxg7ifWpAHXVngzC0tEgXLgo,2776
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=eeg7Vr7tUaTDObxq1X7J3TDPQZeitb_Uy6dxqa9xfzs,5707
@@ -10,7 +10,7 @@ lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9
 lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
 lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
 lamindb_setup/_init_instance.py,sha256=jNNE7ZWIl1V28YqfdbaiyZ_One6K2vv0RMubxdvvb74,15320
-lamindb_setup/_migrate.py,sha256=ya-15sc91i4JmEWI4j00T2892x8hdy2fSW-qz4IdxLs,9739
+lamindb_setup/_migrate.py,sha256=lkbEa_4qLxrQYSrLWKbXYQW32Yb1MBUa2hdfxCHnKUA,10047
 lamindb_setup/_register_instance.py,sha256=zmk7UrOF6qED_zTEaTM8GbZurMSP1SwddkRy7rpYmUY,1215
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=yMSuLZc-7iWUV7tETNuKmcDeMW0wtUr5ypKxsKC-m7k,14898
@@ -21,17 +21,17 @@ lamindb_setup/errors.py,sha256=H1UM-bii0U2vPyjprOBgZK4ijZJgzgCViyGWPd8v5yU,1493
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=XlXLb4nmbc68uBj5Hp3xpDRezYGJIBZv6jAAqqN0p10,614
 lamindb_setup/core/__init__.py,sha256=5M4A6CVHBO_T5Rr9MeLaPW3WTk4-y00cgRYEgUJVU5U,410
-lamindb_setup/core/_aws_options.py,sha256=b8gAk9B1kjxhf73kTJpnM-uMre1koL0MPTCq9uwpEaE,7558
+lamindb_setup/core/_aws_options.py,sha256=UbwnaEX2KcZBDOa2W2NbH62cNrKJQ78Hrn7zcszhQvU,8092
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=HN7iUBdEgahw5e4NHCd1VJooUfieNb6GRzS5x8jU-q8,2549
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=jICkfWW1eZoxh3ycviztBGqCJH53uVve5Xawbj8RZR4,8433
-lamindb_setup/core/_hub_core.py,sha256=S3XjVA-bBchvyzTr-jNae4oLuN2Ho8A30me0MMswln0,23917
+lamindb_setup/core/_hub_client.py,sha256=Pl3sEG2rasdJp4Rh5HNY0VsPVls-nfY0OxD5QzrYF9A,8678
+lamindb_setup/core/_hub_core.py,sha256=eOz7JcdpUz50BYkuOJDkdqnnBgXbamB-erLl1mAut0c,24006
 lamindb_setup/core/_hub_crud.py,sha256=Jz0d8wFKM1Pv9B9byyUJPlCIMkIzk56Jd-c3Awpm9Xw,5730
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=EtlxhtAdclS6rDRh5mrwh_q3gA7SJ1eF2rO1QtOYnnE,12949
-lamindb_setup/core/_settings_instance.py,sha256=coGnDwQaqhviVV7wJnCWpL4uMz8IvY5sFALpYdh6wTc,21869
+lamindb_setup/core/_settings.py,sha256=tCYkTwNRnxmgApmN5m9cjoBn4NBB5ZafS0cUVA1ipNM,13464
+lamindb_setup/core/_settings_instance.py,sha256=kkTbtOrezfZ97rXkyX8JvxVQxy2H6DSenmaAy9XoOS8,21900
 lamindb_setup/core/_settings_load.py,sha256=JWd0_hBy04xjKo-tH4y8C9RkaywjrmoT0PsKzVme0n4,5176
 lamindb_setup/core/_settings_save.py,sha256=XZx-vow7BT6y3JpRBB2UOJp2vwc7jOGea4wSgOPqjPU,3262
 lamindb_setup/core/_settings_storage.py,sha256=S9AvKLzJX0M_RsYcBKZB_P84CYtTY0hyeffYE3UqrQA,15478
@@ -39,12 +39,12 @@ lamindb_setup/core/_settings_store.py,sha256=QmeWIGdIyq7UmjfHiEB_0xRD8hY-8-ZR2Wn
 lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
-lamindb_setup/core/django.py,sha256=MPBcapHL6mH_ebin32aales0JL3AMYqHXUut8ggVFtY,9664
+lamindb_setup/core/django.py,sha256=jDwNJ3ZtkYezzNTGEW9NQZRwspc0akWQnCFWTgu3c1Q,9889
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=W47O9-Y205j29iWJ3RKKdomA587hGvoiD6_krASGFcM,35315
-lamindb_setup-1.8.3.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.8.3.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.8.3.dist-info/METADATA,sha256=nua4OQ80w2e2G-gik39js5-KCXteq9hINyHknrw0NGI,1797
-lamindb_setup-1.8.3.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=reRwNJUOdeZ5n9m7Xi7RoNjmw0kqMPq_BkLPq6_fg-g,35468
+lamindb_setup-1.9.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.9.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.9.0.dist-info/METADATA,sha256=OmqlVQmuG8Iu_h9rPsfCgdfSo_6jH2aZJwq5jl6haH4,1797
+lamindb_setup-1.9.0.dist-info/RECORD,,