lamindb_setup 1.8.3__tar.gz → 1.9.0__tar.gz

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.8.3
+Version: 1.9.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.8.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.9.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/_migrate.py

@@ -86,7 +86,7 @@ class migrate:
 
     @classmethod
     @disable_auto_connect
-    def deploy(cls) -> None:
+    def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
         """Deploy a migration."""
         from ._schema_metadata import update_schema_in_hub
 
@@ -116,7 +116,14 @@ class migrate:
             import lamindb
 
         # this sets up django and deploys the migrations
-        setup_django(settings.instance, deploy_migrations=True)
+        if package_name is not None and number is not None:
+            setup_django(
+                settings.instance,
+                deploy_migrations=True,
+                appname_number=(package_name, number),
+            )
+        else:
+            setup_django(settings.instance, deploy_migrations=True)
         # this populates the hub
         if settings.instance.is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/core/_aws_options.py

@@ -22,7 +22,7 @@ else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 
-def _keep_trailing_slash(path_str: str):
+def _keep_trailing_slash(path_str: str) -> str:
     return path_str if path_str[-1] == "/" else path_str + "/"
 
 
@@ -54,6 +54,7 @@ class AWSOptionsManager:
 
     def __init__(self):
         self._credentials_cache = {}
+        self._parameters_cache = {}  # this is not refreshed
 
         from s3fs import S3FileSystem
 
@@ -106,7 +107,9 @@ class AWSOptionsManager:
     def _get_cached_credentials(self, root: str) -> dict:
         return self._credentials_cache[root]["credentials"]
 
-    def _path_inject_options(self, path: UPath, credentials: dict) -> UPath:
+    def _path_inject_options(
+        self, path: UPath, credentials: dict, extra_parameters: dict | None = None
+    ) -> UPath:
         if credentials == {}:
             # credentials were specified manually for the path
             if "anon" in path.storage_options:
@@ -137,6 +140,9 @@ class AWSOptionsManager:
             "version_aware", False
         )
 
+        if extra_parameters:
+            connection_options.update(extra_parameters)
+
         return UPath(path, **connection_options)
 
     def enrich_path(self, path: UPath, access_token: str | None = None) -> UPath:
@@ -158,7 +164,7 @@ class AWSOptionsManager:
         if root is not None:
             set_cache = False
             credentials = self._get_cached_credentials(root)
-
+            extra_parameters = self._parameters_cache.get(root)
             if access_token is not None:
                 set_cache = True
             elif credentials != {}:
@@ -177,8 +183,10 @@ class AWSOptionsManager:
             is_managed = accessibility.get("is_managed", False)
             if is_managed:
                 credentials = storage_root_info["credentials"]
+                extra_parameters = accessibility["extra_parameters"]
             else:
                 credentials = {}
+                extra_parameters = None
 
             if access_token is None:
                 if "storage_root" in accessibility:
@@ -194,9 +202,13 @@ class AWSOptionsManager:
                         # write the bucket for everything else
                         root = path.drive
                     root = "s3://" + root
-                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 
-        return self._path_inject_options(path, credentials)
+                root = _keep_trailing_slash(root)
+                assert isinstance(root, str)
+                self._set_cached_credentials(root, credentials)
+                self._parameters_cache[root] = extra_parameters
+
+        return self._path_inject_options(path, credentials, extra_parameters)
 
 
 _aws_options_manager: AWSOptionsManager | None = None

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/core/_hub_client.py

@@ -124,6 +124,11 @@ def get_access_token(
             }
         )
         return auth_response.session.access_token
+    except Exception as e:
+        # we need to log the problem here because the exception is usually caught outside
+        # in call_with_fallback_auth
+        logger.warning(f"failed to update your lamindb access token: {e}")
+        raise e
     finally:
         hub.auth.sign_out(options={"scope": "local"})
 

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/core/_hub_core.py

@@ -487,6 +487,8 @@ def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
         accessibility = storage_root_info["accessibility"]
         accessibility["storage_root"] = loaded_accessibility["storageRoot"]
         accessibility["is_managed"] = loaded_accessibility["isManaged"]
+        accessibility["extra_parameters"] = loaded_accessibility.get("extraParameters")
+
     return storage_root_info
 
 

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/core/_settings.py

@@ -60,6 +60,9 @@ class SetupSettings:
 
     _cache_dir: Path | None = None
 
+    _branch = None  # do not have types here
+    _space = None  # do not have types here
+
     @property
     def _instance_settings_path(self) -> Path:
         return current_instance_settings_file()
@@ -112,12 +115,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def branch(self) -> Branch:
         """Default branch."""
-        from lamindb import Branch
+        if self._branch is None:
+            from lamindb import Branch
 
-        idlike, _ = self._read_branch_idlike_name()
-        return Branch.get(idlike)
+            idlike, _ = self._read_branch_idlike_name()
+            self._branch = Branch.get(idlike)
+        return self._branch
 
     @branch.setter
     def branch(self, value: str | Branch) -> None:
@@ -136,6 +143,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._branch_path.write_text(f"{branch_record.uid}\n{branch_record.name}")
+        self._branch = branch_record
 
     @property
     def _space_path(self) -> Path:
@@ -156,12 +164,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def space(self) -> Space:
         """Default space."""
-        from lamindb import Space
+        if self._space is None:
+            from lamindb import Space
 
-        idlike, _ = self._read_space_idlike_name()
-        return Space.get(idlike)
+            idlike, _ = self._read_space_idlike_name()
+            self._space = Space.get(idlike)
+        return self._space
 
     @space.setter
     def space(self, value: str | Space) -> None:
@@ -180,6 +192,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._space_path.write_text(f"{space_record.uid}\n{space_record.name}")
+        self._space = space_record
 
     @property
     def is_connected(self) -> bool:

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/core/_settings_instance.py

@@ -186,9 +186,10 @@ class InstanceSettings:
                         )
                         legacy_filepath.rename(marker_path)
                     else:
-                        raise ValueError(
+                        logger.warning(
                             f"local storage location '{root_path}' is corrupted, cannot find marker file with storage uid"
                         )
+                        continue
                 try:
                     uid = marker_path.read_text().splitlines()[0]
                 except PermissionError:

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/core/django.py

@@ -148,6 +148,7 @@ def setup_django(
     configure_only: bool = False,
     init: bool = False,
     view_schema: bool = False,
+    appname_number: tuple[str, int] | None = None,
 ):
     if IS_RUN_FROM_IPYTHON:
         os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
@@ -235,7 +236,11 @@ def setup_django(
         return None
 
     if deploy_migrations:
-        call_command("migrate", verbosity=2)
+        if appname_number is None:
+            call_command("migrate", verbosity=2)
+        else:
+            app_name, app_number = appname_number
+            call_command("migrate", app_name, app_number, verbosity=2)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     elif init:
         global IS_MIGRATING

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/lamindb_setup/core/upath.py

@@ -393,8 +393,10 @@ def synchronize_to(
     """Sync to a local destination path."""
     destination = destination.resolve()
     protocol = origin.protocol
+    stat_kwargs = {"expand_info": True} if protocol == "hf" else {}
+    origin_str = str(origin)
     try:
-        cloud_info = origin.stat().as_info()
+        cloud_info = origin.fs.stat(origin_str, **stat_kwargs)
         exists = True
         is_dir = cloud_info["type"] == "directory"
     except FileNotFoundError:
@@ -441,7 +443,9 @@ def synchronize_to(
     if is_dir:
         cloud_stats = {
             file: get_modified(stat)
-            for file, stat in origin.fs.find(origin.as_posix(), detail=True).items()
+            for file, stat in origin.fs.find(
+                origin_str, detail=True, **stat_kwargs
+            ).items()
         }
         for cloud_path in cloud_stats:
             file_key = PurePosixPath(cloud_path).relative_to(origin.path).as_posix()

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/tests/hub-local/scripts/script-connect-fine-grained-access.py

@@ -13,6 +13,15 @@ ln_setup.connect("instance_test__access_v2")
 
 isettings = ln_setup.settings.instance
 
+# check extra parameters for s3 managed buckets
+# this is populated by create_instance imported from laminhub
+assert (
+    isettings.storage.root.storage_options["s3_additional_kwargs"][
+        "ServerSideEncryption"
+    ]
+    == "AES256"
+)
+
 assert isettings._fine_grained_access
 assert isettings._db_permissions == "jwt"
 assert isettings._api_url is not None

{lamindb_setup-1.8.3 → lamindb_setup-1.9.0}/tests/hub-local/test_all.py

@@ -182,7 +182,7 @@ def create_myinstance(create_testadmin1_session):  # -> Dict
 def create_instance_fine_grained_access(create_testadmin1_session):
     client, _ = create_testadmin1_session
 
-    instance = create_instance("instance_test", client=client)
+    instance = create_instance("instance_test", client=client, connect=False)
 
     yield instance