orfdata 0.2.0__tar.gz → 0.4.0__tar.gz

{orfdata-0.2.0/orfdata.egg-info → orfdata-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orfdata
-Version: 0.2.0
+Version: 0.4.0
 Summary: Reference open reading frames and experimental translation evidence on annotated transcripts
 Author: OpenVax
 License-Expression: Apache-2.0
@@ -11,6 +11,10 @@ Provides-Extra: pyensembl
 Requires-Dist: pyensembl>=2.27.2; extra == "pyensembl"
 Provides-Extra: build-data
 Requires-Dist: openpyxl>=3.1; extra == "build-data"
+Provides-Extra: data
+Requires-Dist: datacache<2,>=1.25.1; extra == "data"
+Provides-Extra: columnar
+Requires-Dist: pyarrow>=14; extra == "columnar"
 Provides-Extra: dev
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
@@ -18,6 +22,7 @@ Requires-Dist: build; extra == "dev"
 Requires-Dist: twine; extra == "dev"
 Requires-Dist: openpyxl>=3.1; extra == "dev"
 Requires-Dist: pyensembl>=2.27.2; extra == "dev"
+Requires-Dist: datacache<2,>=1.25.1; extra == "dev"
 Dynamic: license-file
 
 # orfdata
@@ -95,5 +100,7 @@ bind real human, mouse, dog and yeast reference slices; those fixtures are not
 new experimental ORF catalogues. [Species and reference identity](docs/species.md).
 
 Install with `pip install orfdata[pyensembl]`. See the
+[measurement API](docs/measurements.md) for native units, explicit samples and
+assays, and DataCache-managed downloads, and the
 [quantitative, multispecies data roadmap](https://github.com/openvax/orfdata/issues/3)
-for planned RNA abundance, ribosome occupancy and protein measurements.
+for upcoming RNA abundance, ribosome occupancy and protein data releases.

{orfdata-0.2.0 → orfdata-0.4.0}/README.md

@@ -73,5 +73,7 @@ bind real human, mouse, dog and yeast reference slices; those fixtures are not
 new experimental ORF catalogues. [Species and reference identity](docs/species.md).
 
 Install with `pip install orfdata[pyensembl]`. See the
+[measurement API](docs/measurements.md) for native units, explicit samples and
+assays, and DataCache-managed downloads, and the
 [quantitative, multispecies data roadmap](https://github.com/openvax/orfdata/issues/3)
-for planned RNA abundance, ribosome occupancy and protein measurements.
+for upcoming RNA abundance, ribosome occupancy and protein data releases.

orfdata-0.4.0/docs/measurements.md

@@ -0,0 +1,134 @@
+# RNA, ribosome and protein measurements
+
+ORFdata adds evidence to PyEnsembl's reference transcripts and proteins. Its
+quantitative layer stores measurements of genes, transcripts, reference ORFs,
+proteins, protein groups and peptides. A measurement stays at the resolution
+the experiment supports. Parent-gene RNA abundance is useful context for an ORF;
+it is not a separate measurement of each ORF in that gene.
+
+Install `orfdata[data]` to manage datasets. The initial 0.3 registry is empty;
+the [data roadmap](https://github.com/openvax/orfdata/issues/3) tracks source
+imports. The bundled human ORF evidence catalogue remains available offline.
+Its pooled PSM counts describe spectra, not sample-level protein abundance.
+
+## Discover, download and query
+
+```python
+from orfdata import DatasetRegistry, ScopedTarget
+
+registry = DatasetRegistry()  # Metadata only; no download.
+releases = registry.list(species="dog", assay="digest_ms")
+```
+
+For a registered release, use `registry.info(name, version)` for its immutable
+manifest, `registry.download(name, version)` to fetch it, and
+`registry.open(name, version)` to read it offline. `inspect` reports missing,
+available or invalid files without downloading or repairing them.
+`download_metadata` returns the artifact's access terms and file hashes/sizes.
+An invalid cache
+requires an explicit `download(..., force=True)`; old reader paths survive.
+Python download options such as timeout, retries and progress go to DataCache.
+
+```python
+# dataset = registry.open(name, version)
+# transcript is a PyEnsembl Transcript in the dataset's exact reference context.
+target = ScopedTarget.from_pyensembl(
+    transcript, kind="transcript", namespace="ensembl:110"
+)
+for measurement in dataset.measurements(target=target):
+    print(measurement.value, measurement.unit, measurement.status)
+```
+
+Use `ScopedTarget.from_orf(bound_reference_orf, namespace=source_release)` for a
+validated reference ORF. Namespaces and identifiers must match the source's
+mapping table. The default PyEnsembl target uses its unversioned `id`; request
+`versioned=True` when the source uses versioned identifiers. There is no implicit
+mapping across annotation releases, gene isoforms, ORFs or species. A variant or
+fusion product does not inherit a native reference protein measurement.
+
+The CLI uses the same operations:
+
+```sh
+orfdata datasets list --species yeast
+orfdata datasets info DATASET --version RELEASE
+orfdata datasets download DATASET --version RELEASE
+orfdata datasets inspect DATASET --version RELEASE
+orfdata datasets query DATASET --version RELEASE --quantity protein_abundance
+```
+
+Queries stream JSON lines. `--reference ref:SHA256` selects an exact context;
+target queries also require `--target`, `--kind` and `--namespace`. Unqualified
+sample/assay ID queries across multiple references are rejected. Existing
+catalogue commands such as `orfdata --gene TPST1` retain their behavior.
+
+## What each value means
+
+`Measurement` retains the dataset/version, reference-scoped target, sample and
+assay IDs, value, native unit, linear/log scale and log base, uncertainty and its
+definition, detection limit, status, origin and provenance. Measured zero,
+below detection, not measured, not mapped and missing are distinct. Censored or
+missing values use `None`; native sentinels can remain in provenance.
+
+RNA counts/TPM/FPKM, ribosome occupancy, source-defined translation efficiency,
+LFQ/iBAQ/TMT values, peptide intensity, PSM counts, copies per cell and turnover
+remain distinct. No conversion is automatic. Derived/predicted values name a
+method and retain source/input provenance. An estimate from calibration remains
+an estimate; an intensity/TPM ratio does not acquire molecule-ratio units.
+
+`Sample` records biological units, pools and source labels. `Assay` separately
+records technical replicates, runs, fractions, TMT plex/channel/bridge,
+normalization and calibration. A sample's strain/breed belongs in its attributes;
+it need not be the strain used to assemble its reference genome. `TargetMapping`
+retains all candidate assignments or an unmapped reason. A protein-group
+representative does not become a uniquely measured protein. See
+[pairing and orthology comparisons](pairing.md) for explicit experimental
+relationships and derived RNA/protein ratios.
+
+## Importing a source
+
+Define a `DatasetManifest` with exact reference contexts, raw source URLs,
+SHA-256 hashes and byte sizes, citations, file-specific access/redistribution
+terms, searched/measurable coverage, processing software/version/parameters and
+row accounting. Unavailable metadata stay unknown. Article licensing does not
+substitute for the data files' terms.
+
+An importer calls `QuantitativeDataset.create(path, manifest, samples=...,
+assays=..., measurements=..., mappings=...)`. These inputs may be generators;
+measurements are written incrementally. Every observation names a source asset
+or earlier input measurements. Derived inputs precede outputs in the same
+reference context. Duplicate IDs, broken foreign keys, wrong dataset releases
+and cross-reference links fail the import. Failed builds do not replace data.
+
+Use `build_dataset(manifest, builder, ...)` for a managed local import.
+`builder(source_paths, output_path)` reads the verified inputs and creates that
+SQLite dataset. DataCache publishes it atomically with input/output checksums
+and a processing receipt. `source_paths={asset_id: local_file}` supports manually
+acquired sources. Importing locally does not grant redistribution rights.
+
+A `DatasetRegistry(definitions, cache_root=...)` registers releases as:
+
+```python
+definitions = {
+    name: {
+        "default_version": version,
+        "versions": {version: {
+            "manifest": manifest.to_dict(),
+            "distribution": {"access_terms": terms, "redistributable": False},
+            "assets": {"dataset.sqlite": {
+                "url": url, "sha256": sha256, "size": size, "decompress": False,
+            }},
+        }},
+    }
+}
+```
+
+CLI users can supply this JSON with `orfdata datasets --registry FILE ...`.
+`ORFDATA_CACHE`, then `OPENVAX_DATA_CACHE`, then the platform's OpenVax cache
+select the root. ORFdata owns its `orfdata/` child. Release cache identities
+include manifest/reference and content hashes. SQLite partitions every table
+by reference key, with composite foreign keys; human, mouse, dog, yeast and
+arbitrary other taxa use the same schema. The core API needs neither pandas nor
+PyArrow. Optional `orfdata[columnar]` enables `dataset.export_parquet(path)`;
+query columns accompany full measurement JSON, preserving exact values and
+provenance even when a floating-point analysis column cannot represent an
+integer exactly.

orfdata-0.4.0/docs/pairing.md

@@ -0,0 +1,84 @@
+# Comparing RNA, ribosome and protein abundance
+
+Use declared experimental relationships to compare assays. A tissue label alone
+does not make an RNA sample and a protein sample a matched pair.
+
+```python
+for match in dataset.matched_measurements(allowed_bases=["same_material"]):
+    row = match.to_table_row()
+    print(row["rna_abundance"], row["protein_abundance"])
+```
+
+Rows include named RNA, ribosome occupancy, translation-efficiency, protein,
+peptide and spectrum-count columns, each containing original measurement
+records with units, missingness and provenance. Missing columns are empty lists.
+Multiple values remain multiple values; the API does not average them or create
+a Cartesian product of technical replicates. Each row retains its assay pairing
+and target relationship. A three-assay experiment can declare several pairwise
+comparisons; sharing an RNA measurement does not create independent replicates.
+
+Pairing bases are `same_material`, `adjacent_donor_tissue`,
+`biological_replicate`, `independent_cultures`, `tissue_aggregate` and
+`condition_aggregate`. The caller must choose the allowed bases. By default,
+only explicitly concurrent assays are returned; opt into `longitudinal` or
+`unknown` with `allowed_temporal`. Missing mates stay explicit. Technical runs,
+TMT channels and bridges remain in the assays and do not become biological
+sample counts.
+
+The CLI streams the same table rows:
+
+```sh
+orfdata datasets pairs DATASET --version RELEASE --basis same_material
+```
+
+## Which targets are being compared?
+
+`TargetRelation` links exact, reference-scoped targets with mapping evidence
+and cardinality. `same_target` compares one target across assays/samples.
+`canonical_translation` links a gene/transcript to its annotated protein.
+`parent_rna_context` links parent RNA to an ORF but keeps the RNA value attached
+to its original gene/transcript. `shared_group_member` records an ambiguous
+protein-group assignment; the group value stays a group value.
+
+Importers pass `AssayPair` and `TargetRelation` iterators as `pairings=` and
+`relations=` to `QuantitativeDataset.create`. IDs belong to the dataset release
+and reference context. Invalid assay references, contradictory known donor
+identities and conflicting one-to-one assignments fail the import. Original
+storage-version-1 datasets remain readable and have no declared relationships.
+
+## Ratios are derived measurements
+
+```python
+ratio = match.ratio(
+    selected_protein_measurement, selected_rna_measurement,
+    measurement_id="ratio-1", method="native protein/RNA ratio",
+    normalization={"protein": "source bridge normalization", "rna": "source TPM"},
+)
+```
+
+Select actual measurements from opposite sides of the match. Ratios require
+one-to-one target attribution and linear inputs. Parent-RNA/ORF and shared-group
+links cannot produce target-specific ratios. There is no implicit log conversion,
+unit conversion, replicate averaging or uncertainty propagation.
+
+An iBAQ/TPM ratio keeps `(iBAQ)/(TPM)` units. Copies-per-cell values also retain
+their native composite units and the calibration recorded on their source
+assays. Missing, censored or nonpositive denominators produce an unresolved
+`not_measured` result with its reason, never infinity or a measured zero. RPF/RNA
+ratios are source-defined translation-efficiency measures, not measured flux.
+All derived values retain their input IDs, pairing, target attribution and
+normalization assumptions.
+
+## Across species
+
+`compare_orthologs(left_dataset, right_dataset, mappings)` is separate and
+opt-in. Each `OrthologyMapping` pins its source file, release, both reference
+contexts, cardinality, confidence definition and evidence. The default accepts
+one-to-one canonical gene mappings; one-to-many relationships require explicit
+permission through `allowed_cardinalities`.
+
+The result contains each species' original observations. It neither merges
+identities nor transfers abundance or protein-existence evidence. ORF comparisons
+additionally require their own sequence, frame and positional evidence and
+`allow_orfs=True`; a gene ortholog cannot establish an orthologous ncORF.
+Human, mouse, dog, yeast and arbitrary other taxa use this same API.

{orfdata-0.2.0 → orfdata-0.4.0}/orfdata/__init__.py

@@ -12,8 +12,19 @@ from .catalog import (
     load_catalog,
     load_uorf_catalog,
 )
+from .dataset import QuantitativeDataset
+from .datasets import DatasetRegistry, build_dataset
 from .initiation_context import KOZAK_PREFERENCES, InitiationContext, annotate_initiation_context
+from .measurements import Assay, DatasetManifest, Measurement, Sample, SourceAsset, TargetMapping
 from .model import ORFModel, ReferenceORF
+from .pairing import (
+    AssayPair,
+    MatchedMeasurements,
+    OrthologyComparison,
+    OrthologyMapping,
+    TargetRelation,
+    compare_orthologs,
+)
 from .reference import ReferenceContext, ScopedTarget
 from .version import __version__
 
@@ -28,6 +39,21 @@ __all__ = [
     "ReferenceORF",
     "ReferenceContext",
     "ScopedTarget",
+    "QuantitativeDataset",
+    "DatasetRegistry",
+    "build_dataset",
+    "DatasetManifest",
+    "SourceAsset",
+    "Sample",
+    "Assay",
+    "Measurement",
+    "TargetMapping",
+    "AssayPair",
+    "TargetRelation",
+    "MatchedMeasurements",
+    "OrthologyMapping",
+    "OrthologyComparison",
+    "compare_orthologs",
     "load_catalog",
     "UORFCatalog",
     "UORFRecord",

{orfdata-0.2.0 → orfdata-0.4.0}/orfdata/cli.py

@@ -11,7 +11,14 @@ from .catalog import EVIDENCE_METHODS, UORFCatalog, load_catalog
 
 def run(argv=None, prog=None, *, catalog_loader=load_catalog):
     """Export reference records as JSON, TSV or FASTA."""
-    parser = argparse.ArgumentParser(description=__doc__, prog=prog)
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if argv and argv[0] == "datasets":
+        from .dataset_cli import run as dataset_run
+
+        return dataset_run(argv[1:], prog=(prog or "orfdata") + " datasets")
+    parser = argparse.ArgumentParser(
+        description=__doc__, prog=prog,
+        epilog="For quantitative data management, use 'orfdata datasets --help'.")
     parser.add_argument("--catalog", help="Use a compatible local JSON/JSON.gz catalogue")
     parser.add_argument("--gene", help="Exact gene symbol or Ensembl gene ID")
     parser.add_argument("--transcript", help="Exact source transcript ID (unversioned)")

orfdata-0.4.0/orfdata/data/datasets.json

@@ -0,0 +1 @@
+{}