lamindb_setup 0.72.1__tar.gz → 0.73.0__tar.gz

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/.github/workflows/build.yml

@@ -12,14 +12,13 @@ jobs:
   # tests only on production hub
   hub-prod:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 6
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.9" # consciously run one job on Python 3.9
+          python-version: "3.9" # run one job on 3.9
           cache: "pip"
-          cache-dependency-path: ".github/workflows/build.yml"
       - uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -39,10 +38,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        lamin_env:
-          - "staging"
-          - "prod"
-    timeout-minutes: 15
+        include:
+          - lamin_env: "prod"
+            python-version: "3.11"
+          - lamin_env: "staging"
+            python-version: "3.10" # test on 3.10
+    timeout-minutes: 6
     steps:
       - uses: aws-actions/configure-aws-credentials@v2
         with:
@@ -52,7 +53,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11" # consciously run one job on Python 3.11
+          python-version: ${{ matrix.python-version }}
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
       - name: checkout laminhub
@@ -92,12 +93,12 @@ jobs:
   # test user access to storage
   storage:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 6
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10" # consciously run one job on Python 3.10
+          python-version: "3.11" # we need to run everything for coverage on 3.11
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
       - run: pip install -U laminci
@@ -116,7 +117,7 @@ jobs:
   # test low-level hub functionality
   hub-local:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 6
     steps:
       - uses: aws-actions/configure-aws-credentials@v2
         with:
@@ -139,7 +140,7 @@ jobs:
         working-directory: laminhub/rest-hub/supabase
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11" # consciously run one job on Python 3.11
+          python-version: "3.11" # we need to run everything for coverage on 3.11
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
       - run: pip install -U laminci
@@ -165,9 +166,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           cache: "pip"
-          cache-dependency-path: ".github/workflows/build.yml"
       - run: |
           pip install coverage[toml]
           pip install --no-deps .

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/.pre-commit-config.yaml

@@ -49,4 +49,4 @@ repos:
     hooks:
       - id: pydocstyle
         args: # google style + __init__, see http://www.pydocstyle.org/en/stable/error_codes.html
-          - --ignore=D100,D101,D102,D103,D104,D106,D107,D203,D204,D213,D215,D400,D401,D403,D404,D406,D407,D408,D409,D412,D413
+          - --ignore=D100,D101,D102,D103,D104,D106,D107,D203,D204,D213,D215,D400,D401,D403,D404,D406,D407,D408,D409,D412,D413,D417

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.72.1
+Version: 0.73.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/docs/changelog.md

@@ -3,7 +3,14 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+🐛 Fix trailing slash in upload_from source | [773](https://github.com/laminlabs/lamindb-setup/pull/773) | [Koncopd](https://github.com/Koncopd) | 2024-05-23 | 0.73.0
+🚸 Make `upload_from()`, `download_to()`, and `view_tree()` more user friendly | [772](https://github.com/laminlabs/lamindb-setup/pull/772) | [falexwolf](https://github.com/falexwolf) | 2024-05-23 |
+✨ Resolve s3 bucket region even without access rights | [771](https://github.com/laminlabs/lamindb-setup/pull/771) | [Koncopd](https://github.com/Koncopd) | 2024-05-22 |
+💚 Coverage fix | [770](https://github.com/laminlabs/lamindb-setup/pull/770) | [falexwolf](https://github.com/falexwolf) | 2024-05-22 |
+👷 Fix coverage compute | [769](https://github.com/laminlabs/lamindb-setup/pull/769) | [falexwolf](https://github.com/falexwolf) | 2024-05-22 |
+🐛 Deal with migration errors when keep-artifacts-local is true | [767](https://github.com/laminlabs/lamindb-setup/pull/767) | [falexwolf](https://github.com/falexwolf) | 2024-05-20 | 0.72.2
 ♻️ Do not error if empty dict in `access_aws` | [764](https://github.com/laminlabs/lamindb-setup/pull/764) | [falexwolf](https://github.com/falexwolf) | 2024-05-19 | 0.72.1
+🐛 Keep training slash in aws cache keys | [766](https://github.com/laminlabs/lamindb-setup/pull/766) | [Koncopd](https://github.com/Koncopd) | 2024-05-19 |
 🐛 Check empty after storage record root init in delete | [763](https://github.com/laminlabs/lamindb-setup/pull/763) | [Koncopd](https://github.com/Koncopd) | 2024-05-18 |
 ✨ Call `access_aws` for all paths and cache | [762](https://github.com/laminlabs/lamindb-setup/pull/762) | [Koncopd](https://github.com/Koncopd) | 2024-05-18 | 0.72.0
 ⚡️ Speed-up file hash | [761](https://github.com/laminlabs/lamindb-setup/pull/761) | [Koncopd](https://github.com/Koncopd) | 2024-05-11 | 0.71.4

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/docs/hub-cloud/06-connect-hosted-instance.ipynb

@@ -86,10 +86,11 @@
    "outputs": [],
    "source": [
     "target_dir = root / \"test-dir-upload\"\n",
-    "target_dir.upload_from(test_dir, recursive=True)\n",
+    "target_dir.upload_from(test_dir)\n",
     "\n",
     "assert target_dir.is_dir()\n",
     "assert (target_dir / \"test-dir-upload\").exists()\n",
+    "assert (target_dir / \"test-dir-upload/file1\").exists()\n",
     "\n",
     "target_dir.rmdir()\n",
     "assert not target_dir.exists()"
@@ -101,10 +102,34 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "target_dir.upload_from(test_dir, dir_inplace=True)\n",
+    "# test trailing slash in target_dir\n",
+    "target_dir = root / \"test-dir-upload/\"\n",
+    "assert target_dir.path.endswith(\"/\")\n",
     "\n",
-    "assert target_dir.is_dir()\n",
-    "assert not (target_dir / \"test-dir-upload\").exists()"
+    "dest_dir = target_dir.upload_from(test_dir, create_folder=True)\n",
+    "\n",
+    "assert \"//\" not in dest_dir.path\n",
+    "assert dest_dir.as_posix() == (root / \"test-dir-upload/test-dir-upload\").as_posix()\n",
+    "assert dest_dir.is_dir()\n",
+    "\n",
+    "dest_dir.rmdir()\n",
+    "assert not dest_dir.exists()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "target_dir_check = root / \"test-dir-upload\"\n",
+    "assert not target_dir_check.exists()\n",
+    "\n",
+    "target_dir.upload_from(test_dir, create_folder=False)\n",
+    "\n",
+    "assert target_dir_check.is_dir()\n",
+    "assert (target_dir_check / \"file1\").exists()\n",
+    "assert not (target_dir_check / \"test-dir-upload\").exists()"
    ]
   },
   {

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.72.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.73.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/lamindb_setup/_init_instance.py

@@ -56,6 +56,7 @@ def register_storage_in_instance(ssettings: StorageSettings):
         "region": ssettings.region,
         "instance_uid": instance_uid,
         "created_by_id": current_user_id(),
+        "run": None,
     }
     if ssettings._uid is not None:
         defaults["uid"] = ssettings._uid

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/lamindb_setup/core/_aws_credentials.py

@@ -22,6 +22,10 @@ else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 
+def _keep_trailing_slash(path_str: str):
+    return path_str if path_str[-1] == "/" else path_str + "/"
+
+
 AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
 
 
@@ -81,7 +85,10 @@ class AWSCredentialsManager:
         return S3Path(path, cache_regions=cache_regions, **connection_options)
 
     def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
-        path_str = path.as_posix().rstrip("/")
+        # trailing slash is needed to avoid returning incorrect results
+        # with .startswith
+        # for example s3://lamindata-eu should not receive cache for s3://lamindata
+        path_str = _keep_trailing_slash(path.as_posix())
         root = self._find_root(path_str)
 
         if root is not None:
@@ -127,7 +134,7 @@ class AWSCredentialsManager:
                         # write the bucket for everything else
                         root = path._url.netloc
                     root = "s3://" + root
-                self._set_cached_credentials(root, credentials)
+                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 
         return self._path_inject_options(path, credentials)
 

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/lamindb_setup/core/_settings_instance.py

@@ -5,6 +5,7 @@ import shutil
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
+from django.db.utils import ProgrammingError
 from lamin_utils import logger
 
 from ._hub_client import call_with_fallback
@@ -106,8 +107,15 @@ class InstanceSettings:
             local_records = Storage.objects.filter(root=local_root)
         else:
             local_records = Storage.objects.filter(type="local")
+        all_local_records = local_records.all()
+        try:
+            # trigger an error in case of a migration issue
+            all_local_records.first()
+        except ProgrammingError:
+            logger.error("not able to load Storage registry: please migrate")
+            return None
         found = False
-        for record in local_records.all():
+        for record in all_local_records:
             root_path = Path(record.root)
             if root_path.exists():
                 marker_path = root_path / ".lamindb/_is_initialized"

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/lamindb_setup/core/_settings_storage.py

@@ -37,33 +37,36 @@ def base62(n_char: int) -> str:
     return id
 
 
-def get_storage_region(storage_root: UPathStr) -> str | None:
-    storage_root_str = str(storage_root)
-    if storage_root_str.startswith("s3://"):
-        import botocore.session as session
+def get_storage_region(path: UPathStr) -> str | None:
+    path_str = str(path)
+    if path_str.startswith("s3://"):
+        import botocore.session
         from botocore.config import Config
-        from botocore.exceptions import NoCredentialsError
+        from botocore.exceptions import ClientError
 
         # strip the prefix and any suffixes of the bucket name
-        bucket = storage_root_str.replace("s3://", "").split("/")[0]
-        s3_session = session.get_session()
-        s3_client = s3_session.create_client("s3")
+        bucket = path_str.replace("s3://", "").split("/")[0]
+        session = botocore.session.get_session()
+        credentials = session.get_credentials()
+        if credentials is None or credentials.access_key is None:
+            config = Config(signature_version=botocore.session.UNSIGNED)
+        else:
+            config = None
+        s3_client = session.create_client("s3", config=config)
         try:
             response = s3_client.head_bucket(Bucket=bucket)
-        except NoCredentialsError:  # deal with anonymous access
-            s3_client = s3_session.create_client(
-                "s3", config=Config(signature_version=session.UNSIGNED)
-            )
-            response = s3_client.head_bucket(Bucket=bucket)
-        storage_region = response["ResponseMetadata"].get("HTTPHeaders", {})[
-            "x-amz-bucket-region"
-        ]
-        # if we want to except botcore.exceptions.ClientError to reformat an
-        # error message, this is how to do test for the "NoSuchBucket" error:
-        #     exc.response["Error"]["Code"] == "NoSuchBucket"
+        except ClientError as exc:
+            response = getattr(exc, "response", {})
+            if response.get("Error", {}).get("Code") == "404":
+                raise exc
+        region = (
+            response.get("ResponseMetadata", {})
+            .get("HTTPHeaders", {})
+            .get("x-amz-bucket-region")
+        )
     else:
-        storage_region = None
-    return storage_region
+        region = None
+    return region
 
 
 def mark_storage_root(root: UPathStr, uid: str):

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/lamindb_setup/core/_setup_bionty_sources.py

@@ -53,6 +53,7 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
             kwargs["species"] = kwargs.pop("organism")
         elif hasattr(PublicSource, "organism") and "species" in kwargs:
             kwargs["organism"] = kwargs.pop("species")
+        kwargs["run"] = None  # can't yet access tracking information
         record = PublicSource(**kwargs)
         all_records.append(record)
 

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/lamindb_setup/core/upath.py

@@ -155,7 +155,10 @@ def create_mapper(
 
 
 def print_hook(size: int, value: int, objectname: str, action: str):
-    progress_in_percent = (value / size) * 100
+    if size == 0:
+        progress_in_percent = 100.0
+    else:
+        progress_in_percent = (value / size) * 100
     out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
     if "NBPRJ_TEST_NBPATH" not in os.environ:
         end = "\n" if progress_in_percent >= 100 else "\r"
@@ -238,50 +241,75 @@ class ChildProgressCallback(fsspec.callbacks.Callback):
         self.parent.update_relative_value(inc)
 
     def relative_update(self, inc=1):
-        self.parent_update(inc / self.size)
+        if self.size != 0:
+            self.parent_update(inc / self.size)
+        else:
+            self.parent_update(1)
 
 
-def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
-    """Download to a path."""
+def download_to(self, local_path: UPathStr, print_progress: bool = True, **kwargs):
+    """Download from self (a destination in the cloud) to the local path."""
+    if "recursive" not in kwargs:
+        kwargs["recursive"] = True
     if print_progress and "callback" not in kwargs:
         callback = ProgressCallback(
-            PurePosixPath(path).name, "downloading", adjust_size=True
+            PurePosixPath(local_path).name, "downloading", adjust_size=True
         )
         kwargs["callback"] = callback
 
-    self.fs.download(str(self), str(path), **kwargs)
+    self.fs.download(str(self), str(local_path), **kwargs)
 
 
 def upload_from(
     self,
-    path: UPathStr,
-    dir_inplace: bool = False,
-    print_progress: bool = False,
+    local_path: UPathStr,
+    create_folder: bool | None = None,
+    print_progress: bool = True,
     **kwargs,
-):
-    """Upload from a local path."""
-    path = Path(path)
-    path_is_dir = path.is_dir()
-    if not path_is_dir:
-        dir_inplace = False
+) -> UPath:
+    """Upload from the local path to `self` (a destination in the cloud).
+
+    If the local path is a directory, recursively upload its contents.
+
+    Args:
+        local_path: A local path of a file or directory.
+        create_folder: Only applies if `local_path` is a directory and then
+            defaults to `True`. If `True`, make a new folder in the destination
+            using the directory name of `local_path`. If `False`, upload the
+            contents of the directory to to the root-level of the destination.
+        print_progress: Print progress.
+
+    Returns:
+        The destination path.
+    """
+    local_path = Path(local_path)
+    local_path_is_dir = local_path.is_dir()
+    if create_folder is None:
+        create_folder = local_path_is_dir
+    if create_folder and not local_path_is_dir:
+        raise ValueError("create_folder can only be True if local_path is a directory")
 
     if print_progress and "callback" not in kwargs:
-        callback = ProgressCallback(path.name, "uploading")
+        callback = ProgressCallback(local_path.name, "uploading")
         kwargs["callback"] = callback
 
-    if dir_inplace:
-        source = [f for f in path.rglob("*") if f.is_file()]
-        destination = [str(self / f.relative_to(path)) for f in source]
+    if local_path_is_dir and not create_folder:
+        source = [f for f in local_path.rglob("*") if f.is_file()]
+        # convert_pathlike is needed to remove the trailing slash because
+        # UPath("s3://some-bucket/some-folder/") / "some-key"
+        # results in UPath("s3://some-bucket/some-folder//some-key")
+        # for upath 0.1.4
+        dest_root = convert_pathlike(self) if self._parts[-1] == "" else self
+        destination = [str(dest_root / f.relative_to(local_path)) for f in source]
         source = [str(f) for f in source]  # type: ignore
     else:
-        source = str(path)  # type: ignore
+        source = str(local_path)  # type: ignore
         destination = str(self)  # type: ignore
-    # this weird thing is to avoid s3fs triggering create_bucket in upload
-    # if dirs are present
-    # it allows to avoid permission error
-    if self.protocol != "s3" or not path_is_dir or dir_inplace:
-        cleanup_cache = False
-    else:
+
+    # the below lines are to avoid s3fs triggering create_bucket in upload if
+    # dirs are present it allows to avoid permission error
+    # would be easier to just
+    if self.protocol == "s3" and local_path_is_dir and create_folder:
         bucket = self._url.netloc
         if bucket not in self.fs.dircache:
             self.fs.dircache[bucket] = [{}]
@@ -290,14 +318,23 @@ def upload_from(
             cleanup_cache = True
         else:
             cleanup_cache = False
+    else:
+        cleanup_cache = False
 
-    self.fs.upload(source, destination, **kwargs)
+    self.fs.upload(source, destination, recursive=create_folder, **kwargs)
 
     if cleanup_cache:
         # normally this is invalidated after the upload but still better to check
         if bucket in self.fs.dircache:
             del self.fs.dircache[bucket]
 
+    if local_path_is_dir and create_folder:
+        # convert_pathlike is needed to remove the trailing slash
+        dest_root = convert_pathlike(self) if self._parts[-1] == "" else self
+        return dest_root / local_path.name
+    else:
+        return self
+
 
 def synchronize(
     self,
@@ -305,14 +342,14 @@ def synchronize(
     error_no_origin: bool = True,
     print_progress: bool = False,
     callback: fsspec.callbacks.Callback | None = None,
-    **kwargs,
+    timestamp: float | None = None,
 ):
     """Sync to a local destination path."""
     # optimize the number of network requests
-    if "timestamp" in kwargs:
+    if timestamp is not None:
         is_dir = False
         exists = True
-        cloud_mts = kwargs.pop("timestamp")
+        cloud_mts = timestamp
     else:
         # perform only one network request to check existence, type and timestamp
         try:
@@ -379,7 +416,7 @@ def synchronize(
                 destination = objectpath / file_key
                 child = callback.branched(origin, destination.as_posix())
                 UPath(origin, **self._kwargs).synchronize(
-                    destination, timestamp=timestamp, callback=child, **kwargs
+                    destination, callback=child, timestamp=timestamp
                 )
                 child.close()
             if destination_exists:
@@ -400,15 +437,16 @@ def synchronize(
     callback = ProgressCallback.requires_progress(
         callback, print_progress, objectpath.name, "synchronizing"
     )
-    kwargs["callback"] = callback
     if objectpath.exists():
-        local_mts = objectpath.stat().st_mtime  # type: ignore
-        need_synchronize = cloud_mts > local_mts
+        local_mts_obj = objectpath.stat().st_mtime  # type: ignore
+        need_synchronize = cloud_mts > local_mts_obj
     else:
         objectpath.parent.mkdir(parents=True, exist_ok=True)
         need_synchronize = True
     if need_synchronize:
-        self.download_to(objectpath, **kwargs)
+        self.download_to(
+            objectpath, recursive=False, print_progress=False, callback=callback
+        )
         os.utime(objectpath, times=(cloud_mts, cloud_mts))
     else:
         # nothing happens if parent_update is not defined
@@ -477,7 +515,7 @@ def compute_file_tree(
             if child_path.is_dir():
                 if include_dirs and child_path not in include_dirs:
                     continue
-                yield prefix + pointer + child_path.name
+                yield prefix + pointer + child_path.name + "/"
                 n_directories += 1
                 n_files_per_dir_and_type = defaultdict(lambda: 0)
                 extension = branch if pointer == tee else space

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/noxfile.py

@@ -47,7 +47,7 @@ uv pip install --system --no-deps {no_deps_packages}
     elif group == "hub-local":
         cmds = """uv pip install --system -e ./laminhub/rest-hub"""
     # current package
-    cmds += """\nuv pip install --system .[aws,dev] lamin-cli"""
+    cmds += """\nuv pip install --system -e '.[aws,dev]' lamin-cli"""
 
     [run(session, line) for line in cmds.splitlines()]
 

{lamindb_setup-0.72.1 → lamindb_setup-0.73.0}/tests/storage/test_storage_stats.py

@@ -1,17 +1,20 @@
 from __future__ import annotations
 
+from lamindb_setup.core._aws_credentials import HOSTED_REGIONS
+from lamindb_setup.core._settings_storage import get_storage_region
 from lamindb_setup.core.upath import UPath, compute_file_tree, get_stat_dir_cloud
 
 
 def test_get_stat_dir_cloud_aws():
-    string_path = "s3://lamindb-dev-datasets/iris_studies/study0_raw_images"
+    string_path = "s3://lamindata/iris_studies/study0_raw_images"
     path = UPath(string_path, anon=True)
     _, n_objects_file_tree = compute_file_tree(path)
     size, hash, hash_type, n_objects = get_stat_dir_cloud(path)
     assert n_objects == n_objects_file_tree
-    assert hash == "wVYKPpEsmmrqSpAZIRXCFg"
+    assert hash == "IVKGMfNwi8zKvnpaD_gG7w"
     assert hash_type == "md5-d"
-    assert size == 656692
+    assert size == 658465
+    assert n_objects == 51
 
 
 def test_get_stat_dir_cloud_gcp():
@@ -22,3 +25,8 @@ def test_get_stat_dir_cloud_gcp():
     assert hash == "6r5Hkce0UTy7X6gLeaqzBA"
     assert hash_type == "md5-d"
     assert size == 994441606
+
+
+def test_get_storage_region():
+    for region in HOSTED_REGIONS:
+        assert get_storage_region(f"s3://lamin-{region}") == region