lamindb 0.76.1__tar.gz → 0.76.3__tar.gz

lamindb-0.76.3/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,94 @@
+name: Bug report
+description: Report something that is broken or incorrect
+labels: bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        **Note**: This is a public repository! Do not reveal any internal information.
+        Please read [this guide](https://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports) detailing how to provide the necessary information for us to reproduce your bug.
+        In brief:
+          * Please provide exact steps how to reproduce the bug in a clean Python environment if possible.
+
+  - type: textarea
+    id: report
+    attributes:
+      label: Report
+      description: A clear and concise description of what the bug is.
+      placeholder: |
+        This is a public repository!
+        Do not reveal any internal information.
+    validations:
+      required: true
+
+  - type: textarea
+    id: versions
+    attributes:
+      label: Version information
+      description: |
+        Please paste below the output of
+
+        ```bash
+        pip install session-info
+        ```
+
+        ```python
+        import session_info
+        session_info.show(html=False, dependencies=True)
+        ```
+      placeholder: |
+        -----
+        anndata             0.8.0rc2.dev27+ge524389
+        session_info        1.0.0
+        -----
+        asttokens           NA
+        awkward             1.8.0
+        backcall            0.2.0
+        cython_runtime      NA
+        dateutil            2.8.2
+        debugpy             1.6.0
+        decorator           5.1.1
+        entrypoints         0.4
+        executing           0.8.3
+        h5py                3.7.0
+        ipykernel           6.15.0
+        jedi                0.18.1
+        mpl_toolkits        NA
+        natsort             8.1.0
+        numpy               1.22.4
+        packaging           21.3
+        pandas              1.4.2
+        parso               0.8.3
+        pexpect             4.8.0
+        pickleshare         0.7.5
+        pkg_resources       NA
+        prompt_toolkit      3.0.29
+        psutil              5.9.1
+        ptyprocess          0.7.0
+        pure_eval           0.2.2
+        pydev_ipython       NA
+        pydevconsole        NA
+        pydevd              2.8.0
+        pydevd_file_utils   NA
+        pydevd_plugins      NA
+        pydevd_tracing      NA
+        pygments            2.12.0
+        pytz                2022.1
+        scipy               1.8.1
+        setuptools          62.5.0
+        setuptools_scm      NA
+        six                 1.16.0
+        stack_data          0.3.0
+        tornado             6.1
+        traitlets           5.3.0
+        wcwidth             0.2.5
+        zmq                 23.1.0
+        -----
+        IPython             8.4.0
+        jupyter_client      7.3.4
+        jupyter_core        4.10.0
+        -----
+        Python 3.9.13 | packaged by conda-forge | (main, May 27 2022, 16:58:50) [GCC 10.3.0]
+        Linux-5.18.6-arch1-1-x86_64-with-glibc2.35
+        -----
+        Session information updated at 2022-07-07 17:55

lamindb-0.76.3/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+    - name: LaminHub issues
+      url: https://github.com/laminlabs/laminhub-public
+      about: If you have issues with LaminHub, please report them here.
+    - name: Enterprise support
+      url: https://lamin.ai/contact
+      about: If you have non-technical inquiries, please contact us directly.

lamindb-0.76.3/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,19 @@
+name: Feature request
+description: Propose a new feature for lamindb
+labels: enhancement
+body:
+  - type: markdown
+    attributes:
+      value: |
+        **Note**: This is a public repository! Do not reveal any internal information.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Description of feature
+      description: Please describe your suggestion for a new feature. It might help to describe a problem or use case, plus any alternatives that you have considered.
+      placeholder: |
+        This is a public repository!
+        Do not reveal any internal information.
+    validations:
+      required: true

{lamindb-0.76.1 → lamindb-0.76.3}/.github/workflows/build.yml

@@ -14,14 +14,15 @@ jobs:
       fail-fast: false
       matrix:
         group:
-          - "unit"
+          - "unit-core"
+          - "unit-storage"
           - "tutorial"
           - "guide"
           - "biology"
           - "faq"
           - "storage"
           - "cli"
-    timeout-minutes: 7
+    timeout-minutes: 10
 
     steps:
       - uses: actions/checkout@v4
@@ -37,14 +38,14 @@ jobs:
         with:
           python-version: "3.11"
       - name: cache pre-commit
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
       - name: cache postgres
-        if: ${{ matrix.group == 'faq' || matrix.group == 'unit' }}
+        if: ${{ matrix.group == 'faq' || matrix.group == 'unit-core' || matrix.group == 'unit-storage' }}
         id: cache-postgres
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/postgres.tar
           key: cache-postgres-0
@@ -64,7 +65,7 @@ jobs:
       - run: nox -s lint
         if: ${{ matrix.group == 'tutorial' }} # choose a fast-running a group
       - run: nox -s "install_ci(group='${{ matrix.group }}')"
-      - uses: aws-actions/configure-aws-credentials@v1
+      - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -97,7 +98,7 @@ jobs:
           ssh-key: ${{ secrets.READ_LNDOCS }}
           path: lndocs
           ref: main
-      - uses: aws-actions/configure-aws-credentials@v1
+      - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

{lamindb-0.76.1 → lamindb-0.76.3}/.gitignore

@@ -61,7 +61,10 @@ test-search3
 test-search4
 test-search5
 default_storage
+default_storage_unit_core
+default_storage_unit_storage
 test.ipynb
+run-tests
 
 # General
 .DS_Store

{lamindb-0.76.1/sub/lnschema-core → lamindb-0.76.3}/.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
       - id: prettier
         exclude: |
           (?x)(
-            docs/changelog.md
+            docs/changelog.md|.github/ISSUE_TEMPLATE/config.yml
           )
   - repo: https://github.com/kynan/nbstripout
     rev: 0.6.1
@@ -48,9 +48,8 @@ repos:
     hooks:
       - id: mypy
         args: [--no-strict-optional, --ignore-missing-imports]
-        additional_dependencies:
-          ["types-pkg-resources", "types-requests", "types-attrs"]
+        additional_dependencies: ["types-requests", "types-attrs"]
         exclude: |
           (?x)(
-              test_notebooks.py
+              test_notebooks.py|script-to-test-versioning.py|tests/storage/conftest.py
           )

{lamindb-0.76.1 → lamindb-0.76.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.1
+Version: 0.76.3
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,9 +9,9 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.73.1
-Requires-Dist: lamindb_setup==0.76.5
-Requires-Dist: lamin_utils==0.13.2
+Requires-Dist: lnschema_core==0.73.3
+Requires-Dist: lamindb_setup==0.76.7
+Requires-Dist: lamin_utils==0.13.4
 Requires-Dist: lamin_cli==0.16.2
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
@@ -24,7 +24,7 @@ Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.48.2 ; extra == "bionty"
+Requires-Dist: bionty==0.49.0 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"
@@ -57,13 +57,5 @@ Provides-Extra: zarr
 
 # LaminDB - A data framework for biology
 
-- Manage data & metadata with a unified Python API ("lakehouse").
-- Track data lineage across notebooks & pipelines.
-- Integrate registries for experimental metadata & in-house ontologies.
-- Validate, standardize & annotate.
-- Collaborate across distributed databases.
-
-## Documentation
-
-Read the [docs](https://lamin.ai/docs).
+Read the [docs](https://docs.lamin.ai).
 

{lamindb-0.76.1 → lamindb-0.76.3}/README.md

@@ -4,12 +4,4 @@
 
 # LaminDB - A data framework for biology
 
-- Manage data & metadata with a unified Python API ("lakehouse").
-- Track data lineage across notebooks & pipelines.
-- Integrate registries for experimental metadata & in-house ontologies.
-- Validate, standardize & annotate.
-- Collaborate across distributed databases.
-
-## Documentation
-
-Read the [docs](https://lamin.ai/docs).
+Read the [docs](https://docs.lamin.ai).

{lamindb-0.76.1 → lamindb-0.76.3}/docs/arrays.ipynb

@@ -116,7 +116,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact = ln.Artifact.filter(key=\"lndb-storage/pbmc68k.h5ad\").one()"
+    "artifact = ln.Artifact.get(key=\"lndb-storage/pbmc68k.h5ad\")"
    ]
   },
   {
@@ -248,7 +248,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact = ln.Artifact.filter(key=\"lndb-storage/testfile.hdf5\").one()"
+    "artifact = ln.Artifact.get(key=\"lndb-storage/testfile.hdf5\")"
    ]
   },
   {

{lamindb-0.76.1 → lamindb-0.76.3}/docs/bio-registries.ipynb

@@ -162,7 +162,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "gdt_cell = bt.CellType.filter(ontology_id=\"CL:0000798\", created_by__handle=\"testuser1\").one()\n",
+    "gdt_cell = bt.CellType.get(ontology_id=\"CL:0000798\", created_by__handle=\"testuser1\")\n",
     "gdt_cell"
    ]
   },
@@ -644,7 +644,7 @@
    },
    "outputs": [],
    "source": [
-    "hepatocyte = bt.CellType.filter(name=\"hepatocyte\").one()\n",
+    "hepatocyte = bt.CellType.get(name=\"hepatocyte\")\n",
     "hepatocyte.source"
    ]
   },
@@ -685,7 +685,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "source = bt.Source.filter(entity=\"bionty.Organism\", name=\"ncbitaxon\").one()\n",
+    "source = bt.Source.get(entity=\"bionty.Organism\", name=\"ncbitaxon\")\n",
     "source"
    ]
   },
@@ -722,7 +722,7 @@
     "ln.save(records)\n",
     "\n",
     "# now we can query a iris organism and view its parents and children\n",
-    "iris = bt.Organism.filter(name=\"iris\").one()\n",
+    "iris = bt.Organism.get(name=\"iris\")\n",
     "iris.view_parents(with_children=True)"
    ]
   },

{lamindb-0.76.1 → lamindb-0.76.3}/docs/curate-df.ipynb

@@ -5,19 +5,21 @@
    "id": "1d2cd769",
    "metadata": {},
    "source": [
-    "# Curate DataFrame and AnnData objects\n",
+    "# Curate DataFrames and AnnDatas\n",
     "\n",
-    "When we talk about \"curating datasets\", we typically mean three distinct actions:\n",
+    "Curating datasets typically means three things:\n",
     "\n",
-    "1. Validate: ensure a dataset meets predefined validation criteria\n",
+    "1. Validate: ensure a dataset meets predefined _validation criteria_\n",
     "2. Standardize: transform a dataset so that it meets validation criteria, e.g., by fixing typos or using standardized identifiers\n",
     "3. Annotate: link a dataset against metadata records\n",
     "\n",
-    "## Key Concepts\n",
+    "In LaminDB, valid metadata is metadata that's stored in a metadata registry and _validation criteria_ merely defines a mapping onto a field of a registry.\n",
     "\n",
-    "- **Registries** store valid metadata records. For instance, if the string `\"Experiment 1\"` was registered as the `name` of a `ULabel` record, it's going to pass validation against `ULabel.name`.\n",
+    "```{admonition} Example\n",
     "\n",
-    "- **Artifacts**: These are the data objects that you manage with LaminDB. Artifacts can be validated & curated with metadata records from registries."
+    "`\"Experiment 1\"` is a valid value for `ULabel.name` if a record with this name exists in the {class}`~lamindb.ULabel` registry.\n",
+    "\n",
+    "```"
    ]
   },
   {
@@ -35,25 +37,12 @@
     "!lamin init --storage ./test-curate --schema bionty"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a691aae0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import lamindb as ln\n",
-    "import bionty as bt\n",
-    "import pandas as pd\n",
-    "import anndata as ad"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "946a3371",
    "metadata": {},
    "source": [
-    "## Validate and standardize metadata from a DataFrame"
+    "## Validate a DataFrame"
    ]
   },
   {
@@ -61,7 +50,7 @@
    "id": "52bbbac2",
    "metadata": {},
    "source": [
-    "Let's start with a DataFrame that we'd like to validate:"
+    "Let's start with a DataFrame that we'd like to validate."
    ]
   },
   {
@@ -75,6 +64,11 @@
    },
    "outputs": [],
    "source": [
+    "import lamindb as ln\n",
+    "import bionty as bt\n",
+    "import pandas as pd\n",
+    "\n",
+    "\n",
     "df = pd.DataFrame(\n",
     "    {\n",
     "        \"temperature\": [37.2, 36.3, 38.2],\n",
@@ -92,7 +86,7 @@
    "id": "c7d74af9",
    "metadata": {},
    "source": [
-    "First, let's define the validation criteria:"
+    "Define validation criteria and create a {class}`~lamindb.Curator` object."
    ]
   },
   {
@@ -106,33 +100,16 @@
    },
    "outputs": [],
    "source": [
-    "# define validation criteria for categorical variables\n",
-    "# in the dictionary, each key is a column name of the dataframe, and each value is a registry field onto which values are mapped\n",
+    "# in the dictionary, each key is a column name of the dataframe, and each value\n",
+    "# is a registry field onto which values are mapped\n",
     "categoricals = {\n",
     "    \"cell_type\": bt.CellType.name,\n",
     "    \"assay_ontology_id\": bt.ExperimentalFactor.ontology_id,\n",
     "    \"donor\": ln.ULabel.name,\n",
-    "}"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "6c293ddf",
-   "metadata": {},
-   "source": [
-    "Now, initialize a Curate object by passing data and its validation criteria:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "78424b30",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# create an Curate object to guide validation and annotation\n",
-    "# this object will use our DataFrame and the defined categorical criteria\n",
-    "curate = ln.Curate.from_df(df, categoricals=categoricals)"
+    "}\n",
+    "\n",
+    "# pass validation criteria\n",
+    "curate = ln.Curator.from_df(df, categoricals=categoricals)"
    ]
   },
   {
@@ -140,7 +117,7 @@
    "id": "804ae191",
    "metadata": {},
    "source": [
-    "The `validate()` method checks our data against the defined criteria. It identifies which values are already validated (exist in our registries) and which are new or potentially problematic."
+    "The {meth}`~lamindb.core.BaseCurator.validate` method checks our data against the defined criteria. It identifies which values are already validated (exist in our registries) and which are potentially problematic (do not yet exist in our registries)."
    ]
   },
   {
@@ -164,9 +141,9 @@
    "source": [
     "## Register new metadata values\n",
     "\n",
-    "If you see any \"non-validated\" values, you'll need to decide whether to add them to your registries or correct them in your data.\n",
+    "If you see \"non-validated\" values, you'll need to decide whether to add them to your registries or \"fix\" them in your dataset.\n",
     "\n",
-    "Because our current database instance is empty, here, we'll add values to the registries defined in the validation criteria."
+    "Because our current registries are still empty, we'll start by populating our {class}`~bionty.CellType` registry with values from a public ontology."
    ]
   },
   {
@@ -180,10 +157,36 @@
    },
    "outputs": [],
    "source": [
-    "# this adds cell types that were validated (via a public ontology)\n",
+    "# this adds cell types that were validated based on a public ontology\n",
     "curate.add_validated_from(\"cell_type\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "5bcfc09f",
+   "metadata": {},
+   "source": [
+    "If we call `validate()` again, we see that one cell type still doesn't pass validation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "104f9932",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "curate.validate()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8c2417c7",
+   "metadata": {},
+   "source": [
+    "Hence, let's understand which cell type in the public ontology might be the actual match."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -195,7 +198,7 @@
    },
    "outputs": [],
    "source": [
-    "# use a lookup object to get the correct spelling of categories from a public reference\n",
+    "# use a lookup object to get the correct spelling of categories from a public ontology\n",
     "lookup = curate.lookup(\"public\")\n",
     "lookup"
    ]
@@ -211,7 +214,8 @@
    },
    "outputs": [],
    "source": [
-    "cell_types = lookup[df.cell_type.name]\n",
+    "# here is an example for the cell_type column\n",
+    "cell_types = lookup[\"cell_type\"]\n",
     "cell_types.cerebral_cortex_pyramidal_neuron"
    ]
   },
@@ -226,12 +230,20 @@
    },
    "outputs": [],
    "source": [
-    "# curate the cell type\n",
+    "# fix the cell type\n",
     "df.cell_type = df.cell_type.replace({\"cerebral pyramidal neuron\": cell_types.cerebral_cortex_pyramidal_neuron.name})\n",
     "# now register curated and validated cell types\n",
     "curate.add_validated_from(df.cell_type.name)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "752142a1",
+   "metadata": {},
+   "source": [
+    "Now, do the same for `\"assay_ontology_id`\" and `\"donor\"`."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -243,7 +255,7 @@
    },
    "outputs": [],
    "source": [
-    "# this adds assays that were validated (via a public ontology)\n",
+    "# this adds assays that were validated based on a public ontology\n",
     "curate.add_validated_from(\"assay_ontology_id\")"
    ]
   },
@@ -259,7 +271,7 @@
    "outputs": [],
    "source": [
     "# this adds donors that were _not_ validated\n",
-    "curate.add_new_from(df.donor.name)"
+    "curate.add_new_from(\"donor\")"
    ]
   },
   {
@@ -283,7 +295,7 @@
    "id": "b9d09a10",
    "metadata": {},
    "source": [
-    "## Validate an AnnData object\n",
+    "## Validate an AnnData\n",
     "\n",
     "Here we addtionally specify which `var_index` to validate against."
    ]
@@ -299,6 +311,8 @@
    },
    "outputs": [],
    "source": [
+    "import anndata as ad\n",
+    "\n",
     "X = pd.DataFrame(\n",
     "    {\n",
     "        \"ENSG00000081059\": [1, 2, 3], \n",
@@ -326,7 +340,7 @@
    },
    "outputs": [],
    "source": [
-    "curate = ln.Curate.from_anndata(\n",
+    "curate = ln.Curator.from_anndata(\n",
     "    adata, \n",
     "    var_index=bt.Gene.ensembl_gene_id,  # validate var.index against Gene.ensembl_gene_id\n",
     "    categoricals=categoricals, \n",
@@ -350,10 +364,20 @@
   },
   {
    "cell_type": "markdown",
-   "id": "a8a7a653",
+   "id": "71a917d1",
+   "metadata": {},
+   "source": [
+    "Save the validated genes following the instruction:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e31dabcc",
    "metadata": {},
+   "outputs": [],
    "source": [
-    "## Curate data object to pass validation"
+    "curate.add_validated_from_var_index()"
    ]
   },
   {
@@ -368,7 +392,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "4e2fd290",
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "curate.non_validated"
@@ -379,7 +407,7 @@
    "id": "b2ec4aeb",
    "metadata": {},
    "source": [
-    "Subset anndata object to validated genes only:"
+    "Subset the `AnnData` to validated genes only:"
    ]
   },
   {
@@ -411,7 +439,7 @@
    },
    "outputs": [],
    "source": [
-    "curate = ln.Curate.from_anndata(\n",
+    "curate = ln.Curator.from_anndata(\n",
     "    adata_validated, \n",
     "    var_index=bt.Gene.ensembl_gene_id,  # validate var.index against Gene.ensembl_gene_id\n",
     "    categoricals=categoricals, \n",

{lamindb-0.76.1 → lamindb-0.76.3}/docs/faq/keep-artifacts-local.ipynb

@@ -112,7 +112,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "storage_record = ln.Storage.filter(root=ln.settings.storage_local.root).one()\n",
+    "storage_record = ln.Storage.get(root=ln.settings.storage_local.root)\n",
     "storage_record.description = \"Files stored locally in site X on server Y for reason ABC\"\n",
     "storage_record.save()\n",
     "ln.Storage.df()"

{lamindb-0.76.1 → lamindb-0.76.3}/docs/faq/key.ipynb

@@ -928,7 +928,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln.Artifact.filter(key__icontains=\"raw_feature_bc_matrix/matrix.mtx.gz\").one().path"
+    "ln.Artifact.get(key__icontains=\"raw_feature_bc_matrix/matrix.mtx.gz\").path"
    ]
   },
   {
@@ -937,7 +937,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact = ln.Artifact.filter(description=\"cellranger raw\").one()\n",
+    "artifact = ln.Artifact.get(description=\"cellranger raw\")\n",
     "artifact.path.glob(\"*\")"
    ]
   }