lamindb_setup 1.8.2__tar.gz → 1.9.0__tar.gz

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.8.2
+Version: 1.9.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/docs/hub-cloud/07-keep-artifacts-local.ipynb

@@ -48,7 +48,7 @@
     "    ln_setup.settings.instance.local_storage\n",
     "assert (\n",
     "    error.exconly()\n",
-    "    == \"ValueError: `keep_artifacts_local` is not enabled for this instance.\"\n",
+    "    == \"ValueError: `keep_artifacts_local` is False, switch via: ln.setup.settings.instance.keep_artifacts_local = True\"\n",
     ")"
    ]
   },
@@ -58,7 +58,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.settings.instance._keep_artifacts_local = True\n",
+    "ln_setup.settings.instance.keep_artifacts_local = True\n",
     "with pytest.raises(ValueError) as error:\n",
     "    ln_setup.settings.instance.local_storage\n",
     "assert error.exconly().startswith(\n",

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.8.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.9.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/_check_setup.py

@@ -147,6 +147,9 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
             else:
                 django_lamin.setup_django(isettings)
                 logger.important(f"connected lamindb: {isettings.slug}")
+                settings._instance_settings = (
+                    isettings  # update of local storage location
+                )
         return django_lamin.IS_SETUP
     else:
         if from_module is not None and settings.auto_connect:

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/_migrate.py

@@ -86,7 +86,7 @@ class migrate:
 
     @classmethod
     @disable_auto_connect
-    def deploy(cls) -> None:
+    def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
         """Deploy a migration."""
         from ._schema_metadata import update_schema_in_hub
 
@@ -116,7 +116,14 @@ class migrate:
             import lamindb
 
         # this sets up django and deploys the migrations
-        setup_django(settings.instance, deploy_migrations=True)
+        if package_name is not None and number is not None:
+            setup_django(
+                settings.instance,
+                deploy_migrations=True,
+                appname_number=(package_name, number),
+            )
+        else:
+            setup_django(settings.instance, deploy_migrations=True)
         # this populates the hub
         if settings.instance.is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/_aws_options.py

@@ -22,7 +22,7 @@ else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 
-def _keep_trailing_slash(path_str: str):
+def _keep_trailing_slash(path_str: str) -> str:
     return path_str if path_str[-1] == "/" else path_str + "/"
 
 
@@ -54,6 +54,7 @@ class AWSOptionsManager:
 
     def __init__(self):
         self._credentials_cache = {}
+        self._parameters_cache = {}  # this is not refreshed
 
         from s3fs import S3FileSystem
 
@@ -106,7 +107,9 @@ class AWSOptionsManager:
     def _get_cached_credentials(self, root: str) -> dict:
         return self._credentials_cache[root]["credentials"]
 
-    def _path_inject_options(self, path: UPath, credentials: dict) -> UPath:
+    def _path_inject_options(
+        self, path: UPath, credentials: dict, extra_parameters: dict | None = None
+    ) -> UPath:
         if credentials == {}:
             # credentials were specified manually for the path
             if "anon" in path.storage_options:
@@ -137,6 +140,9 @@ class AWSOptionsManager:
             "version_aware", False
         )
 
+        if extra_parameters:
+            connection_options.update(extra_parameters)
+
         return UPath(path, **connection_options)
 
     def enrich_path(self, path: UPath, access_token: str | None = None) -> UPath:
@@ -158,7 +164,7 @@ class AWSOptionsManager:
         if root is not None:
             set_cache = False
             credentials = self._get_cached_credentials(root)
-
+            extra_parameters = self._parameters_cache.get(root)
             if access_token is not None:
                 set_cache = True
             elif credentials != {}:
@@ -172,17 +178,15 @@ class AWSOptionsManager:
             from ._hub_core import access_aws
             from ._settings import settings
 
-            if settings.user.handle != "anonymous" or access_token is not None:
-                storage_root_info = access_aws(path_str, access_token=access_token)
-            else:
-                storage_root_info = {"credentials": {}, "accessibility": {}}
-
+            storage_root_info = access_aws(path_str, access_token=access_token)
             accessibility = storage_root_info["accessibility"]
             is_managed = accessibility.get("is_managed", False)
             if is_managed:
                 credentials = storage_root_info["credentials"]
+                extra_parameters = accessibility["extra_parameters"]
             else:
                 credentials = {}
+                extra_parameters = None
 
             if access_token is None:
                 if "storage_root" in accessibility:
@@ -198,9 +202,13 @@ class AWSOptionsManager:
                         # write the bucket for everything else
                         root = path.drive
                     root = "s3://" + root
-                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 
-        return self._path_inject_options(path, credentials)
+                root = _keep_trailing_slash(root)
+                assert isinstance(root, str)
+                self._set_cached_credentials(root, credentials)
+                self._parameters_cache[root] = extra_parameters
+
+        return self._path_inject_options(path, credentials, extra_parameters)
 
 
 _aws_options_manager: AWSOptionsManager | None = None

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/_hub_client.py

@@ -124,6 +124,11 @@ def get_access_token(
             }
         )
         return auth_response.session.access_token
+    except Exception as e:
+        # we need to log the problem here because the exception is usually caught outside
+        # in call_with_fallback_auth
+        logger.warning(f"failed to update your lamindb access token: {e}")
+        raise e
     finally:
         hub.auth.sign_out(options={"scope": "local"})
 

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/_hub_core.py

@@ -460,9 +460,9 @@ def access_aws(storage_root: str, access_token: str | None = None) -> dict[str,
         storage_root_info = call_with_fallback_auth(
             _access_aws, storage_root=storage_root, access_token=access_token
         )
-        return storage_root_info
     else:
-        raise RuntimeError("Can only get access to AWS if authenticated.")
+        storage_root_info = call_with_fallback(_access_aws, storage_root=storage_root)
+    return storage_root_info
 
 
 def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
@@ -487,6 +487,8 @@ def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
         accessibility = storage_root_info["accessibility"]
         accessibility["storage_root"] = loaded_accessibility["storageRoot"]
         accessibility["is_managed"] = loaded_accessibility["isManaged"]
+        accessibility["extra_parameters"] = loaded_accessibility.get("extraParameters")
+
     return storage_root_info
 
 
@@ -497,8 +499,8 @@ def access_db(
     instance_slug: str
     instance_api_url: str | None
     if (
-        "LAMIN_TEST_DB_TOKEN" in os.environ
-        and (env_db_token := os.environ["LAMIN_TEST_DB_TOKEN"]) != ""
+        "LAMIN_DB_TOKEN" in os.environ
+        and (env_db_token := os.environ["LAMIN_DB_TOKEN"]) != ""
     ):
         return env_db_token
 

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/_settings.py

@@ -60,6 +60,9 @@ class SetupSettings:
 
     _cache_dir: Path | None = None
 
+    _branch = None  # do not have types here
+    _space = None  # do not have types here
+
     @property
     def _instance_settings_path(self) -> Path:
         return current_instance_settings_file()
@@ -112,12 +115,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def branch(self) -> Branch:
         """Default branch."""
-        from lamindb import Branch
+        if self._branch is None:
+            from lamindb import Branch
 
-        idlike, _ = self._read_branch_idlike_name()
-        return Branch.get(idlike)
+            idlike, _ = self._read_branch_idlike_name()
+            self._branch = Branch.get(idlike)
+        return self._branch
 
     @branch.setter
     def branch(self, value: str | Branch) -> None:
@@ -136,6 +143,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._branch_path.write_text(f"{branch_record.uid}\n{branch_record.name}")
+        self._branch = branch_record
 
     @property
     def _space_path(self) -> Path:
@@ -156,12 +164,16 @@ class SetupSettings:
         return idlike, name
 
     @property
+    # TODO: refactor so that it returns a BranchMock object
+    # and we never need a DB request
     def space(self) -> Space:
         """Default space."""
-        from lamindb import Space
+        if self._space is None:
+            from lamindb import Space
 
-        idlike, _ = self._read_space_idlike_name()
-        return Space.get(idlike)
+            idlike, _ = self._read_space_idlike_name()
+            self._space = Space.get(idlike)
+        return self._space
 
     @space.setter
     def space(self, value: str | Space) -> None:
@@ -180,6 +192,7 @@ class SetupSettings:
         # we are sure that the current instance is setup because
         # it will error on lamindb import otherwise
         self._space_path.write_text(f"{space_record.uid}\n{space_record.name}")
+        self._space = space_record
 
     @property
     def is_connected(self) -> bool:

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/_settings_instance.py

@@ -104,8 +104,20 @@ class InstanceSettings:
         for attr in attrs:
             value = getattr(self, attr)
             if attr == "storage":
-                representation += f"\n - storage root: {value.root_as_str}"
-                representation += f"\n - storage region: {value.region}"
+                if self.keep_artifacts_local:
+                    import lamindb as ln
+
+                    self._local_storage = ln.setup.settings.instance._local_storage
+                if self._local_storage is not None:
+                    value_local = self.local_storage
+                    representation += f"\n - local storage: {value_local.root_as_str} ({value_local.region})"
+                    representation += (
+                        f"\n - cloud storage: {value.root_as_str} ({value.region})"
+                    )
+                else:
+                    representation += (
+                        f"\n - storage: {value.root_as_str} ({value.region})"
+                    )
             elif attr == "db":
                 if self.dialect != "sqlite":
                     model = LaminDsnModel(db=value)
@@ -174,9 +186,10 @@ class InstanceSettings:
                         )
                         legacy_filepath.rename(marker_path)
                     else:
-                        raise ValueError(
+                        logger.warning(
                             f"local storage location '{root_path}' is corrupted, cannot find marker file with storage uid"
                         )
+                        continue
                 try:
                     uid = marker_path.read_text().splitlines()[0]
                 except PermissionError:
@@ -190,8 +203,9 @@ class InstanceSettings:
             if len(found) > 1:
                 found_display = "\n - ".join([f"{record.root}" for record in found])
                 logger.important(f"found locations:\n - {found_display}")
+            record = found[0]
             logger.important(f"defaulting to local storage: {record.root}")
-            return StorageSettings(record.root)
+            return StorageSettings(record.root, region=record.region)
         elif not mute_warning:
             start = LOCAL_STORAGE_MESSAGE[0].lower()
             logger.warning(f"{start}{LOCAL_STORAGE_MESSAGE[1:]}")
@@ -229,7 +243,9 @@ class InstanceSettings:
         Guide: :doc:`faq/keep-artifacts-local`
         """
         if not self.keep_artifacts_local:
-            raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
+            raise ValueError(
+                "`keep_artifacts_local` is False, switch via: ln.setup.settings.instance.keep_artifacts_local = True"
+            )
         if self._local_storage is None:
             self._local_storage = self._search_local_root()
         if self._local_storage is None:

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/_settings_user.py

@@ -39,7 +39,7 @@ class UserSettings:
     def __repr__(self) -> str:
         """Rich string representation."""
         representation = "Current user:"
-        attrs = ["handle", "email", "uid"]
+        attrs = ["handle", "uid"]
         for attr in attrs:
             value = getattr(self, attr)
             representation += f"\n - {attr}: {value}"

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/django.py

@@ -148,6 +148,7 @@ def setup_django(
     configure_only: bool = False,
     init: bool = False,
     view_schema: bool = False,
+    appname_number: tuple[str, int] | None = None,
 ):
     if IS_RUN_FROM_IPYTHON:
         os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
@@ -235,7 +236,11 @@ def setup_django(
         return None
 
     if deploy_migrations:
-        call_command("migrate", verbosity=2)
+        if appname_number is None:
+            call_command("migrate", verbosity=2)
+        else:
+            app_name, app_number = appname_number
+            call_command("migrate", app_name, app_number, verbosity=2)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     elif init:
         global IS_MIGRATING
@@ -247,7 +252,7 @@ def setup_django(
     IS_SETUP = True
 
     if isettings.keep_artifacts_local:
-        isettings._search_local_root()
+        isettings._local_storage = isettings._search_local_root()
 
 
 # THIS IS NOT SAFE

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/lamindb_setup/core/upath.py

@@ -393,8 +393,10 @@ def synchronize_to(
     """Sync to a local destination path."""
     destination = destination.resolve()
     protocol = origin.protocol
+    stat_kwargs = {"expand_info": True} if protocol == "hf" else {}
+    origin_str = str(origin)
     try:
-        cloud_info = origin.stat().as_info()
+        cloud_info = origin.fs.stat(origin_str, **stat_kwargs)
         exists = True
         is_dir = cloud_info["type"] == "directory"
     except FileNotFoundError:
@@ -441,7 +443,9 @@ def synchronize_to(
     if is_dir:
         cloud_stats = {
             file: get_modified(stat)
-            for file, stat in origin.fs.find(origin.as_posix(), detail=True).items()
+            for file, stat in origin.fs.find(
+                origin_str, detail=True, **stat_kwargs
+            ).items()
         }
         for cloud_path in cloud_stats:
             file_key = PurePosixPath(cloud_path).relative_to(origin.path).as_posix()

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/tests/hub-local/scripts/script-connect-fine-grained-access.py

@@ -13,6 +13,15 @@ ln_setup.connect("instance_test__access_v2")
 
 isettings = ln_setup.settings.instance
 
+# check extra parameters for s3 managed buckets
+# this is populated by create_instance imported from laminhub
+assert (
+    isettings.storage.root.storage_options["s3_additional_kwargs"][
+        "ServerSideEncryption"
+    ]
+    == "AES256"
+)
+
 assert isettings._fine_grained_access
 assert isettings._db_permissions == "jwt"
 assert isettings._api_url is not None
@@ -52,5 +61,5 @@ with pytest.raises(RuntimeError):
 # check with providing access_token explicitly
 access_db(isettings, ln_setup.settings.user.access_token)
 # check specifying an env token via an env variable
-os.environ["LAMIN_TEST_DB_TOKEN"] = "test_db_token"
-assert access_db(isettings) == os.environ["LAMIN_TEST_DB_TOKEN"]
+os.environ["LAMIN_DB_TOKEN"] = "test_db_token"
+assert access_db(isettings) == os.environ["LAMIN_DB_TOKEN"]

{lamindb_setup-1.8.2 → lamindb_setup-1.9.0}/tests/hub-local/test_all.py

@@ -182,7 +182,7 @@ def create_myinstance(create_testadmin1_session):  # -> Dict
 def create_instance_fine_grained_access(create_testadmin1_session):
     client, _ = create_testadmin1_session
 
-    instance = create_instance("instance_test", client=client)
+    instance = create_instance("instance_test", client=client, connect=False)
 
     yield instance