lamindb 1.0.3__tar.gz → 1.0.5__tar.gz

{lamindb-1.0.3 → lamindb-1.0.5}/.github/workflows/build.yml

@@ -25,18 +25,25 @@ jobs:
               - 'tests/curators/test_curator.py'
 
       - id: set-matrix
+        shell: bash
         run: |
-          if [[ "${{ github.event_name }}" == "push" ]]; then
-            # On push, run all
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT
-          elif [[ "${{ steps.changes.outputs.curator }}" == "true" ]]; then
-            # If curator paths changed, include curator
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT
+          BASE_GROUPS=$(jq -n -c '["unit-core", "unit-storage", "tutorial", "guide", "biology", "faq", "storage", "cli"]')
+
+          if [[ "${{ github.event_name }}" == "push" || "${{ steps.changes.outputs.curator }}" == "true" ]]; then
+            # Run everything on push or when curator paths change
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: ($groups + ["curator"])}')
           else
-            # Otherwise, run all except curator
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\"]}" >> $GITHUB_OUTPUT
+            # Otherwise only run base groups
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: $groups}')
           fi
 
+          # Output as single line for GitHub Actions
+          echo "matrix=$(echo "$MATRIX" | jq -c .)" >> $GITHUB_OUTPUT
+
+          # Pretty print for debugging
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
   test:
     needs: pre-filter
     runs-on: ubuntu-latest
@@ -81,6 +88,8 @@ jobs:
 
       - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
 
+      - run: nox -s configure_coverage -- '${{needs.pre-filter.outputs.matrix}}'
+
       - name: install postgres
         if: ${{ matrix.group == 'faq' }}
         run: sudo apt-get install libpq-dev
@@ -100,7 +109,7 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
-      - run: nox -s "build(group='${{ matrix.group }}')"
+      - run: nox -s "test(group='${{ matrix.group }}')"
 
       - name: upload coverage
         uses: actions/upload-artifact@v4

{lamindb-1.0.3 → lamindb-1.0.5}/.gitignore

@@ -1,6 +1,7 @@
 __MACOSX/
 
 # LaminDB
+.coveragerc
 *.db
 *.lndb
 *.jpg

{lamindb-1.0.3 → lamindb-1.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.0.3
+Version: 1.0.5
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.13
@@ -9,7 +9,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: lamin_utils==0.13.10
-Requires-Dist: lamin_cli==1.0.4
+Requires-Dist: lamin_cli==1.0.7
 Requires-Dist: lamindb_setup[aws]==1.0.3
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -23,6 +23,7 @@ Requires-Dist: psycopg2-binary
 Requires-Dist: bionty==1.0.0 ; extra == "bionty"
 Requires-Dist: cellregistry ; extra == "cellregistry"
 Requires-Dist: clinicore==1.0.0 ; extra == "clinicore"
+Requires-Dist: tomlkit ; extra == "dev"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"

{lamindb-1.0.3 → lamindb-1.0.5}/docs/storage/add-replace-cache.ipynb

@@ -11,14 +11,53 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "dca2c05d",
+   "id": "bd29dc17",
    "metadata": {},
    "outputs": [],
    "source": [
     "import pytest\n",
-    "import lamindb as ln\n",
-    "\n",
-    "ln.setup.login(\"testuser1\")\n",
+    "import shutil\n",
+    "import lamindb as ln"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ac7266de",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln.setup.login(\"testuser1\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2c126065",
+   "metadata": {
+    "tags": [
+     "hide-output",
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    root_path = ln.UPath(\"s3://lamindb-ci/test-add-replace-cache\")\n",
+    "    if root_path.exists():\n",
+    "        root_path.rmdir()\n",
+    "    ln.setup.delete(\"testuser1/test-add-replace-cache\", force=True)\n",
+    "except BaseException:  # noqa: S110\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dca2c05d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "ln.setup.init(storage=\"s3://lamindb-ci/test-add-replace-cache\")"
    ]
   },
@@ -576,8 +615,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with pytest.raises(ValueError):\n",
-    "    artifact.replace(path_in_storage)"
+    "with pytest.raises(ValueError) as err:\n",
+    "    artifact.replace(path_in_storage)\n",
+    "assert err.exconly().startswith(\n",
+    "    \"ValueError: Can only replace with a local path not in any Storage.\"\n",
+    ")"
    ]
   },
   {
@@ -673,6 +715,139 @@
     "path_in_storage.unlink()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "d952ac58",
+   "metadata": {},
+   "source": [
+    "## Replace with folder artifacts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fd614d98",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "adata = ln.core.datasets.anndata_pbmc68k_reduced()\n",
+    "\n",
+    "adata.write_zarr(\"./test-files/pbmc68k.zarr\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0d35b755",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact = ln.Artifact(\"./test-files/pbmc68k.zarr\", key=\"pbmc68k.zarr\").save()\n",
+    "save_hash = artifact.hash\n",
+    "save_n_files = artifact.n_files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1045dd0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with pytest.raises(ValueError) as err:\n",
+    "    artifact.replace(\"./test-files/iris.csv\")\n",
+    "assert err.exconly().endswith(\"It is not allowed to replace a folder with a file.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4e20a364",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert save_hash is not None\n",
+    "assert artifact.path.is_dir()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d75282c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "adata.obs[\"add_new_col\"] = \"new\"\n",
+    "\n",
+    "adata.write_zarr(\"./test-files/pbmc68k_new.zarr\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7466df19",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact.replace(\"./test-files/pbmc68k_new.zarr\")\n",
+    "artifact.save()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bce3caef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert artifact.key == \"pbmc68k.zarr\"\n",
+    "assert artifact.hash != save_hash\n",
+    "assert artifact.n_files != save_n_files\n",
+    "assert artifact.path.is_dir()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6ca7eb57",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "shutil.rmtree(artifact.cache())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "06eadae1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with artifact.open() as store:\n",
+    "    assert \"add_new_col\" in store.obs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f05f770b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "shutil.rmtree(\"./test-files/pbmc68k.zarr\")\n",
+    "shutil.rmtree(\"./test-files/pbmc68k_new.zarr\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6e23dc87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact.delete(permanent=True, storage=True)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -702,7 +877,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.10.16"
   },
   "nbproject": {
    "id": "uBQMCcdYwEjA",

{lamindb-1.0.3 → lamindb-1.0.5}/docs/storage/upload.ipynb

@@ -11,14 +11,52 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "8eb097f4-4114-4a35-8764-ebcfe1e85bdc",
+   "id": "ce3356b1",
    "metadata": {},
    "outputs": [],
    "source": [
     "import lamindb as ln\n",
-    "import pytest\n",
-    "\n",
-    "ln.setup.login(\"testuser1\")\n",
+    "import pytest"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b6a334e8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln.setup.login(\"testuser1\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "382f6745",
+   "metadata": {
+    "tags": [
+     "hide-output",
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "try:\n",
+    "    root_path = ln.UPath(\"s3://lamindb-ci/test-upload\")\n",
+    "    if root_path.exists():\n",
+    "        root_path.rmdir()\n",
+    "    ln.setup.delete(\"testuser1/test-upload\")\n",
+    "except BaseException:  # noqa: S110\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8eb097f4-4114-4a35-8764-ebcfe1e85bdc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "ln.setup.init(storage=\"s3://lamindb-ci/test-upload\")"
    ]
   },
@@ -327,7 +365,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.16"
   },
   "nbproject": {
    "id": "psZgub4FOmzS",

{lamindb-1.0.3 → lamindb-1.0.5}/lamindb/__init__.py

@@ -53,7 +53,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.0.3"
+__version__ = "1.0.5"
 
 from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
 from lamindb_setup._check_setup import _check_instance_setup

{lamindb-1.0.3 → lamindb-1.0.5}/lamindb/_artifact.py

@@ -64,7 +64,7 @@ try:
 except ImportError:
 
     def zarr_is_adata(storepath):  # type: ignore
-        raise ImportError("Please install zarr: pip install zarr")
+        raise ImportError("Please install zarr: pip install zarr<=2.18.4")
 
 
 if TYPE_CHECKING:
@@ -845,7 +845,7 @@ def from_dir(
 # docstring handled through attach_func_to_class_method
 def replace(
     self,
-    data: UPathStr,
+    data: UPathStr | pd.DataFrame | AnnData | MuData,
     run: Run | None = None,
     format: str | None = None,
 ) -> None:
@@ -867,7 +867,18 @@ def replace(
 
     check_path_in_storage = privates["check_path_in_storage"]
     if check_path_in_storage:
-        raise ValueError("Can only replace with a local file not in any Storage.")
+        err_msg = (
+            "Can only replace with a local path not in any Storage. "
+            f"This data is in {Storage.objects.get(id=kwargs['storage_id'])}."
+        )
+        raise ValueError(err_msg)
+
+    _overwrite_versions = kwargs["_overwrite_versions"]
+    if self._overwrite_versions != _overwrite_versions:
+        err_msg = "It is not allowed to replace "
+        err_msg += "a folder" if self._overwrite_versions else "a file"
+        err_msg += " with " + ("a folder." if _overwrite_versions else "a file.")
+        raise ValueError(err_msg)
 
     if self.key is not None and not self._key_is_virtual:
         key_path = PurePosixPath(self.key)
@@ -902,6 +913,7 @@ def replace(
     self._hash_type = kwargs["_hash_type"]
     self.run_id = kwargs["run_id"]
     self.run = kwargs["run"]
+    self.n_files = kwargs["n_files"]
 
     self._local_filepath = privates["local_filepath"]
     self._cloud_filepath = privates["cloud_filepath"]
@@ -926,7 +938,15 @@ def open(
     if self._overwrite_versions and not self.is_latest:
         raise ValueError(inconsistent_state_msg)
     # ignore empty suffix for now
-    suffixes = ("", ".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma") + PYARROW_SUFFIXES
+    suffixes = (
+        "",
+        ".h5",
+        ".hdf5",
+        ".h5ad",
+        ".zarr",
+        ".anndata.zarr",
+        ".tiledbsoma",
+    ) + PYARROW_SUFFIXES
     if self.suffix not in suffixes:
         raise ValueError(
             "Artifact should have a zarr, h5, tiledbsoma object"