lamindb_setup 0.81.4__tar.gz → 0.81.5__tar.gz

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 0.81.4
+Version: 0.81.5
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/docs/hub-prod/test-cloud-sync.ipynb

@@ -149,37 +149,6 @@
     "assert settings.paths.cloud_to_local_no_update(dir_sync.as_posix(), cache_key=\"dir_cache/key\") == settings.cache_dir / \"dir_cache/key\""
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "eda84820",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# for http urls\n",
-    "http_path = UPath(\"https://raw.githubusercontent.com/laminlabs/lamindb-setup/refs/heads/main/README.md\")\n",
-    "assert http_path.protocol == \"https\"\n",
-    "\n",
-    "http_stat = http_path.stat()\n",
-    "assert http_stat.st_size != 0\n",
-    "assert http_stat.st_mtime == 0\n",
-    "assert http_stat.as_info()[\"type\"] == \"file\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "df6a9be4",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "http_key = \"raw.githubusercontent.com/laminlabs/lamindb-setup/refs/heads/main/README.md\"\n",
-    "\n",
-    "assert settings.paths.cloud_to_local_no_update(http_path) == settings.cache_dir / http_key\n",
-    "assert settings.paths.cloud_to_local_no_update(str(http_path)) == settings.cache_dir / http_key\n",
-    "assert settings.paths.cloud_to_local_no_update(http_path, cache_key=\"check/README.md\") == settings.cache_dir / \"check/README.md\""
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "0b79f2f7",
@@ -222,57 +191,6 @@
     "dir_sync_local.rmdir()"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "d2246f90",
-   "metadata": {},
-   "source": [
-    "Test `cloud_to_local` for http"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bc1b7736",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "http_local = settings.paths.cloud_to_local(http_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ca9f0ba8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "assert isinstance(http_local, LocalPathClasses)\n",
-    "assert http_local.stat().st_size == http_path.stat().st_size"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6ae7cd2b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "http_local_mtime = http_local.stat().st_mtime\n",
-    "# no changes here because the file exists already\n",
-    "assert settings.paths.cloud_to_local(http_path).stat().st_mtime == http_local_mtime"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9da41c21",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "http_local.unlink()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "574c3f95",

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.81.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.81.5"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 import sys as _sys

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/_check_setup.py

@@ -52,6 +52,49 @@ def _get_current_instance_settings() -> InstanceSettings | None:
         return None
 
 
+def check_connecting_to_v1_instance(isettings: InstanceSettings):
+    from urllib.parse import urlparse
+
+    # we need to do this because the sqlite file should be already synced
+    # has no effect if not cloud sqlite
+    # errors if the sqlite file is not in the cloud and doesn't exist locally
+    # isettings.db syncs but doesn't error in this case due to error_no_origin=False
+    isettings._update_local_sqlite_file()
+
+    parsed_uri = urlparse(isettings.db)
+    db_type = parsed_uri.scheme
+
+    if db_type == "sqlite":
+        import sqlite3
+
+        conn = sqlite3.connect(parsed_uri.path)
+    elif db_type in ["postgresql", "postgres"]:
+        import psycopg2
+
+        conn = psycopg2.connect(isettings.db)
+    else:
+        raise ValueError("Unsupported database type. Use 'sqlite' or 'postgresql' URI.")
+
+    cur = conn.cursor()
+
+    if db_type == "sqlite":
+        cur.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name='lamindb_user'"
+        )
+        isv1 = cur.fetchone() is not None
+
+    else:  # postgres
+        cur.execute(
+            "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'lamindb_user')"
+        )
+        isv1 = cur.fetchone()[0]
+
+    if isv1:
+        raise SystemExit(
+            "This instance runs lamindb v1. Please run: pip install 'lamindb[bionty,wetlab,clinicore,ourprojects]'>1.0.0"
+        )
+
+
 # we make this a private function because in all the places it's used,
 # users should not see it
 def _check_instance_setup(
@@ -77,6 +120,7 @@ def _check_instance_setup(
     if isettings is not None:
         if reload_module and settings.auto_connect:
             if not django.IS_SETUP:
+                check_connecting_to_v1_instance(isettings)
                 django.setup_django(isettings)
                 if from_module is not None:
                     # this only reloads `from_module`

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/_migrate.py

@@ -28,14 +28,11 @@ def check_whether_migrations_in_sync(db_version_str: str):
         or installed_version.minor < db_version.minor
     ):
         db_version_lower = f"{db_version.major}.{db_version.minor}"
-        db_version_upper = f"{db_version.major}.{db_version.minor + 1}"
-        logger.warning(
-            f"the database ({db_version_str}) is ahead of your installed lamindb"
-            f" package ({installed_version_str})"
-        )
-        logger.important(
-            "please update lamindb: pip install"
-            f' "lamindb>={db_version_lower},<{db_version_upper}"'
+        raise SystemExit(
+            f"The database ({db_version_str}) is ahead of your installed lamindb"
+            f" package ({installed_version_str})\n"
+            "Please update lamindb: pip install"
+            f" 'lamindb[bionty,wetlab,clinicore,ourprojects]'>={db_version_lower}"
         )
     elif (
         installed_version.major > db_version.major

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/core/_hub_core.py

@@ -239,10 +239,6 @@ def _delete_instance(
             )
             root_string = storage_record["root"]
             # gate storage and instance deletion on empty storage location for
-            # normally auth.get_session() doesn't have access_token
-            # so this block is useless i think (Sergei)
-            # the token is received from user settings inside create_path
-            # might be needed in the hub though
             if client.auth.get_session() is not None:
                 access_token = client.auth.get_session().access_token
             else:
@@ -255,6 +251,7 @@ def _delete_instance(
             check_storage_is_empty(
                 root_path, account_for_sqlite_file=account_for_sqlite_file
             )
+    _update_instance_record(instance_with_storage["id"], {"storage_id": None}, client)
     # first delete the storage records because we will turn instance_id on
     # storage into a FK soon
     for storage_record in storage_records:

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/core/_hub_crud.py

@@ -12,23 +12,25 @@ def select_instance_by_owner_name(
     name: str,
     client: Client,
 ) -> dict | None:
-    # this won't find an instance without the default storage
-    data = (
-        client.table("instance")
-        .select(
-            "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
-            " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
+    try:
+        data = (
+            client.table("instance")
+            .select(
+                "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
+                " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
+            )
+            .eq("name", name)
+            .eq("account.handle", owner)
+            .eq("storage.is_default", True)
+            .execute()
+            .data
         )
-        .eq("name", name)
-        .eq("account.handle", owner)
-        .eq("storage.is_default", True)
-        .execute()
-        .data
-    )
+    except Exception:
+        return None
     if len(data) == 0:
         return None
     result = data[0]
-    # this is a list
+    # this is now a list
     # assume only one default storage
     result["storage"] = result["storage"][0]
     return result
@@ -87,22 +89,15 @@ def select_instance_by_id_with_storage(
     instance_id: str,
     client: Client,
 ):
-    # this won't find an instance without the default storage
-    data = (
+    response = (
         client.table("instance")
-        .select("*, storage!inner!storage_instance_id_359fca71_fk_instance_id(*)")
+        .select("*, storage!instance_storage_id_87963cc8_fk_storage_id(*)")
         .eq("id", instance_id)
-        .eq("storage.is_default", True)
         .execute()
-        .data
     )
-    if len(data) == 0:
+    if len(response.data) == 0:
         return None
-    result = data[0]
-    # this is a list
-    # assume only one default storage
-    result["storage"] = result["storage"][0]
-    return result
+    return response.data[0]
 
 
 def update_instance(instance_id: str, instance_fields: dict, client: Client):

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/core/_settings.py

@@ -200,14 +200,9 @@ class SetupPaths:
         # cache_key is ignored if filepath is a local path
         if not isinstance(filepath, LocalPathClasses):
             # settings is defined further in this file
-            if cache_key is None:
-                local_key = filepath.path  # type: ignore
-                protocol = filepath.protocol  # type: ignore
-                if protocol in {"http", "https"}:
-                    local_key = local_key.removeprefix(protocol + "://")
-            else:
-                local_key = cache_key
-            local_filepath = settings.cache_dir / local_key
+            local_filepath = settings.cache_dir / (
+                filepath.path if cache_key is None else cache_key  # type: ignore
+            )
         else:
             local_filepath = filepath
         return local_filepath

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/core/_settings_storage.py

@@ -7,7 +7,6 @@ import string
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 
-import fsspec
 from lamin_utils import logger
 
 from ._aws_credentials import HOSTED_REGIONS, get_aws_credentials_manager
@@ -25,10 +24,6 @@ if TYPE_CHECKING:
 
 IS_INITIALIZED_KEY = ".lamindb/_is_initialized"
 
-# a list of supported fsspec protocols
-# rename file to local before showing to a user
-VALID_PROTOCOLS = ("file", "gs", "s3", "hf", "http", "https")
-
 
 def base62(n_char: int) -> str:
     """Like nanoid without hyphen and underscore."""
@@ -119,11 +114,16 @@ def init_storage(
             root_str = f"s3://lamin-{region}/{uid}"
         else:
             root_str = f"s3://lamin-hosted-test/{uid}"
-    elif (input_protocol := fsspec.utils.get_protocol(root_str)) not in VALID_PROTOCOLS:
-        valid_protocols = ("local",) + VALID_PROTOCOLS[1:]  # show local instead of file
-        raise ValueError(
-            f"Protocol {input_protocol} is not supported, valid protocols are {', '.join(valid_protocols)}"
-        )
+    elif root_str.startswith(("gs://", "s3://", "hf://")):
+        pass
+    else:  # local path
+        try:
+            _ = Path(root_str)
+        except Exception as e:
+            logger.error(
+                "`storage` is not a valid local, GCP storage, AWS S3 path or Hugging Face path"
+            )
+            raise e
     ssettings = StorageSettings(
         uid=uid,
         root=root_str,
@@ -227,7 +227,7 @@ class StorageSettings:
 
     @property
     def record(self) -> Any:
-        """Storage record in the current instance."""
+        """Storage record in current instance."""
         if self._record is None:
             # dynamic import because of import order
             from lnschema_core.models import Storage
@@ -299,15 +299,14 @@ class StorageSettings:
         return self._region
 
     @property
-    def type(self) -> Literal["local", "s3", "gs", "hf", "http", "https"]:
+    def type(self) -> Literal["local", "s3", "gs"]:
         """AWS S3 vs. Google Cloud vs. local.
 
-        Returns the protocol as a string: "local", "s3", "gs", "http", "https".
+        Returns the protocol as a string: "local", "s3", "gs".
         """
         import fsspec
 
         convert = {"file": "local"}
-        # init_storage checks that the root protocol belongs to VALID_PROTOCOLS
         protocol = fsspec.utils.get_protocol(self.root_as_str)
         return convert.get(protocol, protocol)  # type: ignore
 
@@ -346,5 +345,5 @@ class StorageSettings:
         return self.root / filekey
 
     def local_filepath(self, filekey: UPathStr) -> UPath:
-        """Local (cache) filepath from filekey."""
+        """Local (cache) filepath from filekey: `local(filepath(...))`."""
         return self.cloud_to_local(self.key_to_filepath(filekey))

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/lamindb_setup/core/upath.py

@@ -190,13 +190,7 @@ class ProgressCallback(fsspec.callbacks.Callback):
         pass
 
     def update_relative_value(self, inc=1):
-        if inc != 0:
-            self.value += inc
-        # this is specific to http filesystem
-        # for some reason the last update is 0 always
-        # here 100% is forced manually in this case
-        elif self.value >= 0.999:
-            self.value = self.size
+        self.value += inc
         self.call()
 
     def branch(self, path_1, path_2, kwargs):
@@ -356,19 +350,27 @@ def synchronize(
         exists = True
         cloud_mts = timestamp
     else:
-        try:
-            cloud_stat = self.stat()
-            cloud_info = cloud_stat.as_info()
-            exists = True
-            is_dir = cloud_info["type"] == "directory"
-            if not is_dir:
-                # hf requires special treatment
-                if protocol == "hf":
-                    cloud_mts = cloud_info["last_commit"].date.timestamp()
-                else:
-                    cloud_mts = cloud_stat.st_mtime
-        except FileNotFoundError:
-            exists = False
+        # hf requires special treatment
+        if protocol == "hf":
+            try:
+                stat_hf = self.stat().as_info()
+                is_dir = stat_hf["type"] == "directory"
+                exists = True
+                if not is_dir:
+                    cloud_mts = stat_hf["last_commit"].date.timestamp()
+            except FileNotFoundError:
+                exists = False
+        else:
+            # perform only one network request to check existence, type and timestamp
+            try:
+                cloud_mts = self.modified.timestamp()
+                is_dir = False
+                exists = True
+            except FileNotFoundError:
+                exists = False
+            except IsADirectoryError:
+                is_dir = True
+                exists = True
 
     if not exists:
         warn_or_error = f"The original path {self} does not exist anymore."
@@ -384,7 +386,6 @@ def synchronize(
         return None
 
     # synchronization logic for directories
-    # to synchronize directories, it should be possible to get modification times
     if is_dir:
         files = self.fs.find(str(self), detail=True)
         if protocol == "s3":
@@ -450,16 +451,8 @@ def synchronize(
         callback, print_progress, objectpath.name, "synchronizing"
     )
     if objectpath.exists():
-        if cloud_mts != 0:
-            local_mts_obj = objectpath.stat().st_mtime
-            need_synchronize = cloud_mts > local_mts_obj
-        else:
-            # this is true for http for example
-            # where size is present but st_mtime is not
-            # we assume that any change without the change in size is unlikely
-            cloud_size = cloud_stat.st_size
-            local_size_obj = objectpath.stat().st_size
-            need_synchronize = cloud_size != local_size_obj
+        local_mts_obj = objectpath.stat().st_mtime  # type: ignore
+        need_synchronize = cloud_mts > local_mts_obj
     else:
         objectpath.parent.mkdir(parents=True, exist_ok=True)
         need_synchronize = True
@@ -471,8 +464,7 @@ def synchronize(
         self.download_to(
             objectpath, recursive=False, print_progress=False, callback=callback
         )
-        if cloud_mts != 0:
-            os.utime(objectpath, times=(cloud_mts, cloud_mts))
+        os.utime(objectpath, times=(cloud_mts, cloud_mts))
     else:
         # nothing happens if parent_update is not defined
         # because of Callback.no_op
@@ -747,9 +739,7 @@ def get_stat_file_cloud(stat: dict) -> tuple[int, str | None, str | None]:
         hash = b16_to_b64(stat["blob_id"])
         hash_type = "sha1"
     # s3
-    # StorageClass is checked to be sure that it is indeed s3
-    # because http also has ETag
-    elif "ETag" in stat and "StorageClass" in stat:
+    elif "ETag" in stat:
         etag = stat["ETag"]
         # small files
         if "-" not in etag:

{lamindb_setup-0.81.4 → lamindb_setup-0.81.5}/tests/hub-local/test_all.py

@@ -360,9 +360,3 @@ def test_init_storage_with_non_existing_bucket(create_testadmin1_session):
             )[0]
         )
     assert error.exconly().endswith("Not Found")
-
-
-def test_init_storage_incorrect_protocol():
-    with pytest.raises(ValueError) as error:
-        init_storage_base("incorrect-protocol://some-path/some-path-level")
-    assert "Protocol incorrect-protocol is not supported" in error.exconly()