lamindb 1.0.4__tar.gz → 1.1.0__tar.gz

{lamindb-1.0.4 → lamindb-1.1.0}/.github/workflows/build.yml

@@ -22,28 +22,35 @@ jobs:
           filters: |
             curator:
               - 'lamindb/curators/**'
-              - 'tests/curators/test_curator.py'
+              - 'tests/curators/**'
 
       - id: set-matrix
+        shell: bash
         run: |
-          if [[ "${{ github.event_name }}" == "push" ]]; then
-            # On push, run all
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT
-          elif [[ "${{ steps.changes.outputs.curator }}" == "true" ]]; then
-            # If curator paths changed, include curator
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\",\"curator\"]}" >> $GITHUB_OUTPUT
+          BASE_GROUPS=$(jq -n -c '["unit-core", "unit-storage", "tutorial", "guide", "biology", "faq", "storage", "cli"]')
+
+          if [[ "${{ github.event_name }}" == "push" || "${{ steps.changes.outputs.curator }}" == "true" ]]; then
+            # Run everything on push or when curator paths change
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: ($groups + ["curator"])}')
           else
-            # Otherwise, run all except curator
-            echo "matrix={\"group\":[\"unit-core\",\"unit-storage\",\"tutorial\",\"guide\",\"biology\",\"faq\",\"storage\",\"cli\"]}" >> $GITHUB_OUTPUT
+            # Otherwise only run base groups
+            MATRIX=$(jq -n -c --argjson groups "$BASE_GROUPS" '{group: $groups}')
           fi
 
+          # Output as single line for GitHub Actions
+          echo "matrix=$(echo "$MATRIX" | jq -c .)" >> $GITHUB_OUTPUT
+
+          # Pretty print for debugging
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
   test:
     needs: pre-filter
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix: ${{fromJson(needs.pre-filter.outputs.matrix)}}
-    timeout-minutes: 15
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
         with:
@@ -54,8 +61,8 @@ jobs:
         with:
           python-version: |
             ${{ matrix.group == 'storage' && '3.10' ||
-                matrix.group == 'unit-storage' && '3.11' ||
-                '3.12' }}
+                '3.12'
+                }}
 
       - name: cache pre-commit
         uses: actions/cache@v4
@@ -81,6 +88,8 @@ jobs:
 
       - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
 
+      - run: nox -s configure_coverage -- '${{needs.pre-filter.outputs.matrix}}'
+
       - name: install postgres
         if: ${{ matrix.group == 'faq' }}
         run: sudo apt-get install libpq-dev
@@ -100,7 +109,7 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-east-1
 
-      - run: nox -s "build(group='${{ matrix.group }}')"
+      - run: nox -s "test(group='${{ matrix.group }}')"
 
       - name: upload coverage
         uses: actions/upload-artifact@v4

{lamindb-1.0.4 → lamindb-1.1.0}/.gitignore

@@ -1,6 +1,7 @@
 __MACOSX/
 
 # LaminDB
+.coveragerc
 *.db
 *.lndb
 *.jpg

{lamindb-1.0.4 → lamindb-1.1.0}/.gitmodules

@@ -13,3 +13,6 @@
 [submodule "sub/clinicore"]
 	path = sub/clinicore
 	url = https://github.com/laminlabs/clinicore
+[submodule "sub/cellxgene-lamin"]
+	path = sub/cellxgene-lamin
+	url = https://github.com/laminlabs/cellxgene-lamin.git

{lamindb-1.0.4 → lamindb-1.1.0}/.pre-commit-config.yaml

@@ -7,7 +7,7 @@ default_stages:
 minimum_pre_commit_version: 2.16.0
 repos:
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v4.0.0-alpha.4
+    rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
         exclude: |
@@ -15,7 +15,7 @@ repos:
             docs/changelog.md|.github/ISSUE_TEMPLATE/config.yml|tests/core/notebooks/basic-r-notebook.Rmd.cleaned.html
           )
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.6.1
+    rev: 0.8.1
     hooks:
       - id: nbstripout
         exclude: |
@@ -24,7 +24,7 @@ repos:
               docs/notes/
           )
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.2
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
@@ -47,13 +47,16 @@ repos:
               tests/core/notebooks/basic-r-notebook.Rmd.cleaned.html
             )
       - id: check-case-conflict
-  # - repo: https://github.com/pre-commit/mirrors-mypy
-  #   rev: v1.7.1
-  #   hooks:
-  #     - id: mypy
-  #       args: [--no-strict-optional, --ignore-missing-imports]
-  #       additional_dependencies: ["types-requests", "types-attrs"]
-  #       exclude: |
-  #         (?x)(
-  #             test_notebooks.py|script-to-test-versioning.py|tests/storage/conftest.py
-  #         )
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.14.1
+    hooks:
+      - id: mypy
+        args: [--no-strict-optional, --ignore-missing-imports]
+        additional_dependencies: ["types-requests", "types-attrs"]
+        exclude: |
+          (?x)(
+              test_notebooks.py|
+              script-to-test-versioning.py|
+              tests/storage/conftest.py|
+              tests/curators/conftest.py
+          )

{lamindb-1.0.4 → lamindb-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.0.4
+Version: 1.1.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.13
@@ -9,9 +9,11 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: lamin_utils==0.13.10
-Requires-Dist: lamin_cli==1.0.4
-Requires-Dist: lamindb_setup[aws]==1.0.3
+Requires-Dist: lamin_cli==1.1.0
+Requires-Dist: lamindb_setup[aws]==1.1.0
+Requires-Dist: pyyaml
 Requires-Dist: pyarrow
+Requires-Dist: pandera
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
 Requires-Dist: scipy<1.15.0
@@ -20,9 +22,10 @@ Requires-Dist: anndata>=0.8.0,<=0.11.3
 Requires-Dist: fsspec
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
-Requires-Dist: bionty==1.0.0 ; extra == "bionty"
+Requires-Dist: bionty==1.1.0 ; extra == "bionty"
 Requires-Dist: cellregistry ; extra == "cellregistry"
-Requires-Dist: clinicore==1.0.0 ; extra == "clinicore"
+Requires-Dist: clinicore==1.1.0 ; extra == "clinicore"
+Requires-Dist: tomlkit ; extra == "dev"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -34,14 +37,14 @@ Requires-Dist: mudata ; extra == "dev"
 Requires-Dist: nbproject_test>=0.6.0 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: readfcs>=1.1.9 ; extra == "fcs"
+Requires-Dist: readfcs>=2.0.1 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.5 ; extra == "jupyter"
 Requires-Dist: jupytext ; extra == "jupyter"
 Requires-Dist: nbconvert>=7.2.1 ; extra == "jupyter"
 Requires-Dist: mistune!=3.1.0 ; extra == "jupyter"
 Requires-Dist: omop ; extra == "omop"
-Requires-Dist: wetlab==1.0.1 ; extra == "wetlab"
+Requires-Dist: wetlab==1.1.0 ; extra == "wetlab"
 Requires-Dist: zarr>=2.16.0,<3.0.0a0 ; extra == "zarr"
 Project-URL: Home, https://github.com/laminlabs/lamindb
 Provides-Extra: bionty

{lamindb-1.0.4 → lamindb-1.1.0}/docs/arrays.ipynb

@@ -95,6 +95,13 @@
     "Note that it is also possible to register Hugging Face paths. For this `huggingface_hub` package should be installed."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We register a folder of `parquet` files as a single artifact."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -108,6 +115,31 @@
     "ln.Artifact(\"hf://datasets/Koncopd/lamindb-test/sharded_parquet\").save()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We also register a collection of individual `parquet` files."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact_shard1 = ln.Artifact(\n",
+    "    \"hf://datasets/Koncopd/lamindb-test/sharded_parquet/louvain=0/947eee0b064440c9b9910ca2eb89e608-0.parquet\"\n",
+    ").save()\n",
+    "artifact_shard2 = ln.Artifact(\n",
+    "    \"hf://datasets/Koncopd/lamindb-test/sharded_parquet/louvain=1/947eee0b064440c9b9910ca2eb89e608-0.parquet\"\n",
+    ").save()\n",
+    "\n",
+    "ln.Collection(\n",
+    "    [artifact_shard1, artifact_shard2], key=\"sharded_parquet_collection\"\n",
+    ").save()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -403,6 +435,49 @@
     "backed.head(5).to_pandas()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is also possible to open a collection of cloud artifacts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "collection = ln.Collection.get(key=\"sharded_parquet_collection\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backed = collection.open()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backed.to_table().to_pandas()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -434,7 +509,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.16"
   },
   "nbproject": {
    "id": "YVUCtH4GfQOy",

{lamindb-1.0.4 → lamindb-1.1.0}/docs/bio-registries.ipynb

@@ -31,7 +31,7 @@
    },
    "outputs": [],
    "source": [
-    "# !pip install 'lamindb[bionty]'\n",
+    "# pip install 'lamindb[bionty]'\n",
     "!lamin init --storage ./test-registries --modules bionty"
    ]
   },
@@ -49,19 +49,37 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Seed registries with public ontologies\n",
+    "## Import records from public ontologies\n",
     "\n",
-    "Let's first populate our {class}`~bionty.CellType` registry with the configured public ontology (Cell Ontology):"
+    "Let's first populate our {class}`~bionty.CellType` registry with the default public ontology (Cell Ontology)."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# [optional] inspect the available public ontology versions\n",
+    "bt.Source.df()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
-    "# check configured public ontology\n",
-    "bt.Source.filter(entity=\"bionty.CellType\", currently_used=True).one()"
+    "# [optional] inspect which version we're about to import\n",
+    "bt.Source.get(entity=\"bionty.CellType\", currently_used=True)"
    ]
   },
   {
@@ -78,7 +96,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "This is now your in-house CellType registry:"
+    "This is now your in-house cell type registry in which you can add & modify records as you like."
    ]
   },
   {
@@ -101,7 +119,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# similarly, let's populate the Gene registry with human and mouse genes\n",
+    "# let's also populate the Gene registry with human and mouse genes\n",
     "bt.Gene.import_source(organism=\"human\")\n",
     "bt.Gene.import_source(organism=\"mouse\")"
    ]
@@ -768,7 +786,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "lamindb",
+   "display_name": "py312",
    "language": "python",
    "name": "python3"
   },
@@ -782,7 +800,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.12.8"
   },
   "nbproject": {
    "id": "s7F7PezA1HdQ",