orfdata 0.7.0__tar.gz → 0.8.0__tar.gz

{orfdata-0.7.0 → orfdata-0.8.0}/MANIFEST.in

@@ -2,3 +2,4 @@ include lint.sh test.sh deploy.sh
 recursive-include docs *.md
 recursive-include tests *.py *.json *.md *.tsv *.fasta
 recursive-include tests/data/copy *.xlsx *.gz *.soft *.xml *.pdf
+recursive-include tests/data/storage *.sqlite

{orfdata-0.7.0/orfdata.egg-info → orfdata-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orfdata
-Version: 0.7.0
+Version: 0.8.0
 Summary: Reference open reading frames and experimental translation evidence on annotated transcripts
 Author: OpenVax
 License-Expression: Apache-2.0

{orfdata-0.7.0 → orfdata-0.8.0}/docs/measurements.md

@@ -145,4 +145,5 @@ arbitrary other taxa use the same schema. The core API needs neither pandas nor
 PyArrow. Optional `orfdata[columnar]` enables `dataset.export_parquet(path)`;
 query columns accompany full measurement JSON, preserving exact values and
 provenance even when a floating-point analysis column cannot represent an
-integer exactly.
+integer exactly. [Compact storage and compatibility](storage.md) describes
+new SQLite v4 writes, legacy readers and actual-source size/query benchmarks.

orfdata-0.8.0/docs/storage.md

@@ -0,0 +1,45 @@
+# Quantitative storage
+
+ORFdata 0.8 writes compact SQLite storage version 4. `Measurement`,
+`QuantitativeDataset`, the registry and CLI keep the same public records and
+query semantics. Human, mouse, dog, yeast and arbitrary taxa retain separate
+reference-scoped identities.
+
+References, targets and assays are stored once and addressed by internal integer
+keys. Measurements retain their values and metadata; repeated required fields
+are reconstructed from indexed columns and the dataset manifest. Fields equal
+to the fixed version-4 defaults are omitted from stored JSON and restored on read.
+Zeros, nulls, censoring, uncertainty, nested provenance and derived inputs retain
+their meaning. Foreign keys prevent cross-reference target/assay assignments.
+Internal integer keys never appear as public biological identifiers.
+
+Published storage versions 1, 2 and 3 remain readable. There is no automatic
+migration or replacement of downloaded artifacts. A new source rebuild can use
+a different storage layout and therefore a different file checksum while
+returning the same records. Registry downloads always verify the exact published
+artifact's hash; source builds retain their own DataCache receipts.
+
+The implementation streams observations and bounds its target cache to 131,072
+entries. SQLite index statistics are populated during creation so a narrow
+target query can use the target index instead of scanning an entire reference.
+Raw SQL table layouts are internal; use the Python/CLI query and export APIs.
+
+## Actual-source benchmark
+
+The first 100,000 rows of the native Pfizer/GEO RNA table contain four dog
+libraries and 30,941 genes, including measured zeros. Source gzip SHA-256:
+`a22780f92ffd0a028e30fe3392d73d01f787eb5a62b2043e8d26940191cbac70`.
+This is a storage benchmark slice, not a newly released experimental dataset.
+
+| Measurement | Storage v3 | Storage v4 |
+|---|---:|---:|
+| Installed size | 121.6 MB | 38.6 MB |
+| Gzip size | 13.4 MB | 7.5 MB |
+| Build time | 8.90 s | 4.28 s |
+| One-gene query (four results) | 135 ms | 1.5 ms |
+| Peak build RSS | 39.8 MB | 52.1 MB |
+
+All 100,000 reconstructed observations were compared field for field. Timings
+are local measurements on Python 3.12, not cross-machine guarantees; the bounded
+target cache trades some memory for repeated-target lookup speed. Legacy tests
+also use actual databases written by ORFdata 0.3.0, 0.4.0 and 0.5.0.

{orfdata-0.7.0 → orfdata-0.8.0}/orfdata/dataset.py

@@ -5,6 +5,8 @@ import json
 import os
 import sqlite3
 from contextlib import closing
+from dataclasses import MISSING, fields
+from functools import lru_cache
 from pathlib import Path
 
 from .measurements import (
@@ -16,6 +18,7 @@ from .measurements import (
     MeasurementReference,
     Sample,
     TargetMapping,
+    _plain,
 )
 from .pairing import (
     PAIRING_BASES,
@@ -30,38 +33,50 @@ from .reference_mapping import MappedMeasurement, ReferenceMapping
 
 _SCHEMA = """
 CREATE TABLE metadata (name TEXT PRIMARY KEY, value TEXT NOT NULL);
-CREATE TABLE reference (key TEXT PRIMARY KEY, record TEXT NOT NULL);
+CREATE TABLE reference (i INTEGER PRIMARY KEY, key TEXT UNIQUE NOT NULL,
+    record TEXT NOT NULL, UNIQUE(key, i));
 CREATE TABLE target (
-    reference_key TEXT NOT NULL REFERENCES reference(key),
+    i INTEGER PRIMARY KEY, reference_i INTEGER NOT NULL,
+    reference_key TEXT NOT NULL,
     key TEXT NOT NULL, kind TEXT NOT NULL, namespace TEXT NOT NULL,
-    identifier TEXT NOT NULL, PRIMARY KEY(reference_key, key));
+    identifier TEXT NOT NULL, UNIQUE(reference_key, key), UNIQUE(reference_i, i),
+    FOREIGN KEY(reference_key, reference_i) REFERENCES reference(key, i));
 CREATE TABLE sample (
     reference_key TEXT NOT NULL REFERENCES reference(key),
     id TEXT NOT NULL, record TEXT NOT NULL, PRIMARY KEY(reference_key, id));
 CREATE TABLE assay (
+    i INTEGER PRIMARY KEY, reference_i INTEGER NOT NULL,
     reference_key TEXT NOT NULL, id TEXT NOT NULL, sample_id TEXT NOT NULL,
-    record TEXT NOT NULL, PRIMARY KEY(reference_key, id),
+    record TEXT NOT NULL, UNIQUE(reference_key, id), UNIQUE(reference_i, i),
     UNIQUE(reference_key, id, sample_id),
+    FOREIGN KEY(reference_key, reference_i) REFERENCES reference(key, i),
     FOREIGN KEY(reference_key, sample_id) REFERENCES sample(reference_key, id));
 CREATE TABLE mapping (
     reference_key TEXT NOT NULL REFERENCES reference(key),
     id TEXT NOT NULL, record TEXT NOT NULL, PRIMARY KEY(reference_key, id));
-CREATE TABLE measurement (
-    reference_key TEXT NOT NULL, id TEXT NOT NULL, target_key TEXT NOT NULL,
-    sample_id TEXT NOT NULL, assay_id TEXT NOT NULL, quantity TEXT NOT NULL,
-    status TEXT NOT NULL, record TEXT NOT NULL, PRIMARY KEY(reference_key, id),
-    FOREIGN KEY(reference_key, target_key) REFERENCES target(reference_key, key),
-    FOREIGN KEY(reference_key, assay_id, sample_id)
-        REFERENCES assay(reference_key, id, sample_id));
-CREATE TABLE input_measurement (
-    reference_key TEXT NOT NULL, output_id TEXT NOT NULL,
-    input_reference_key TEXT NOT NULL, input_id TEXT NOT NULL,
-    PRIMARY KEY(reference_key, output_id, input_reference_key, input_id),
-    FOREIGN KEY(reference_key, output_id) REFERENCES measurement(reference_key, id),
-    FOREIGN KEY(input_reference_key, input_id) REFERENCES measurement(reference_key, id));
+CREATE TABLE observation (
+    i INTEGER PRIMARY KEY, reference_i INTEGER NOT NULL,
+    id TEXT NOT NULL, target_i INTEGER NOT NULL, assay_i INTEGER NOT NULL,
+    quantity TEXT NOT NULL, status TEXT NOT NULL, record TEXT NOT NULL,
+    UNIQUE(reference_i, id),
+    FOREIGN KEY(reference_i, target_i) REFERENCES target(reference_i, i),
+    FOREIGN KEY(reference_i, assay_i) REFERENCES assay(reference_i, i));
+CREATE TABLE observation_input (
+    output_i INTEGER NOT NULL REFERENCES observation(i),
+    input_i INTEGER NOT NULL REFERENCES observation(i), PRIMARY KEY(output_i, input_i));
+CREATE VIEW measurement AS
+    SELECT r.key reference_key, m.id, t.key target_key, a.sample_id,
+           a.id assay_id, m.quantity, m.status, m.record
+    FROM observation m JOIN reference r ON r.i=m.reference_i
+    JOIN target t ON t.i=m.target_i JOIN assay a ON a.i=m.assay_i;
+CREATE VIEW input_measurement AS
+    SELECT r.key reference_key, o.id output_id, s.key input_reference_key, i.id input_id
+    FROM observation_input p JOIN observation o ON o.i=p.output_i
+    JOIN observation i ON i.i=p.input_i JOIN reference r ON r.i=o.reference_i
+    JOIN reference s ON s.i=i.reference_i;
 CREATE INDEX target_lookup ON target(reference_key, namespace, kind, identifier);
-CREATE INDEX observation_lookup ON measurement(reference_key, target_key, sample_id, assay_id);
-CREATE INDEX assay_lookup ON measurement(reference_key, assay_id, quantity, status);
+CREATE INDEX observation_lookup ON observation(reference_i, target_i, assay_i);
+CREATE INDEX assay_lookup ON observation(reference_i, assay_i, quantity, status);
 CREATE TABLE assay_pair (
     reference_key TEXT NOT NULL REFERENCES reference(key), id TEXT NOT NULL,
     right_reference_key TEXT NOT NULL REFERENCES reference(key),
@@ -100,6 +115,29 @@ def _target_key(target):
     return hashlib.sha256(_json(target.key).encode()).hexdigest()
 
 
+_STORED_FIELDS = tuple(f for f in fields(Measurement) if f.name not in {
+    "measurement_id", "target", "sample_id", "assay_id", "quantity", "status",
+    "dataset_id", "dataset_version"})
+_V4_DEFAULTS = {
+    "origin": "observed", "scale": "linear", "log_base": None,
+    "uncertainty": None, "uncertainty_type": None, "detection_limit": None,
+    "input_measurements": (), "input_references": (), "derivation": {}, "provenance": {},
+}
+
+
+def _measurement_record(observation):
+    # Schema defaults are fixed here, independent of future model defaults.
+    # Do not serialize the target
+    # and then discard it: its full reference is already stored once.
+    result = {}
+    for field in _STORED_FIELDS:
+        value = getattr(observation, field.name)
+        default = _V4_DEFAULTS.get(field.name, MISSING)
+        if default is MISSING or value != default:
+            result[field.name] = _plain(value)
+    return result
+
+
 class QuantitativeDataset:
     """A read-only release. Constructing or querying it never downloads data.
 
@@ -112,7 +150,7 @@ class QuantitativeDataset:
         self.path = Path(path).resolve()
         with closing(self._connect()) as connection:
             self._storage_version = connection.execute("PRAGMA user_version").fetchone()[0]
-            if self._storage_version not in (1, 2, 3):
+            if self._storage_version not in (1, 2, 3, 4):
                 raise ValueError("Unsupported quantitative database schema")
             row = connection.execute("SELECT value FROM metadata WHERE name='manifest'").fetchone()
             if row is None:
@@ -146,25 +184,46 @@ class QuantitativeDataset:
             connection = sqlite3.connect(path)
             connection.execute("PRAGMA foreign_keys=ON")
             connection.executescript(_SCHEMA)
-            connection.execute("PRAGMA user_version=3")
+            connection.execute("PRAGMA user_version=4")
             connection.execute("INSERT INTO metadata VALUES ('manifest', ?)",
                                (_json(manifest.to_dict()),))
+            reference_ids = {}
             for reference in manifest.references:
-                connection.execute("INSERT INTO reference VALUES (?, ?)",
-                                   (reference.key, _json(reference.to_dict())))
+                cursor = connection.execute("INSERT INTO reference (key, record) VALUES (?, ?)",
+                                            (reference.key, _json(reference.to_dict())))
+                reference_ids[reference.key] = cursor.lastrowid
+
+            @lru_cache(maxsize=131072)
+            def insert_target(target):
+                reference_key = target.reference.key
+                if reference_key not in reference_ids:
+                    raise sqlite3.IntegrityError("Target reference absent from dataset")
+                reference_i = reference_ids[reference_key]
+                key = _target_key(target)
+                connection.execute("INSERT OR IGNORE INTO target "
+                                   "(reference_i, reference_key, key, kind, namespace, identifier) "
+                                   "VALUES (?, ?, ?, ?, ?, ?)",
+                                   (reference_i, reference_key, key, target.kind, target.namespace, target.identifier))
+                target_i = connection.execute("SELECT i FROM target WHERE reference_key=? AND key=?",
+                                              (reference_key, key)).fetchone()[0]
+                return reference_i, target_i, key
             for sample in samples:
                 if not isinstance(sample, Sample):
                     raise TypeError("samples must contain Samples")
                 connection.execute("INSERT INTO sample VALUES (?, ?, ?)",
                                    (sample.reference.key, sample.sample_id, _json(sample.to_dict())))
+            assay_ids = {}
             for assay in assays:
                 if not isinstance(assay, Assay):
                     raise TypeError("assays must contain Assays")
                 if assay.kind not in manifest.assay_kinds:
                     raise ValueError("Assay kind absent from manifest coverage")
-                connection.execute("INSERT INTO assay VALUES (?, ?, ?, ?)",
-                                   (assay.reference.key, assay.assay_id, assay.sample_id,
-                                    _json(assay.to_dict())))
+                reference_i = reference_ids.get(assay.reference.key)
+                cursor = connection.execute("INSERT INTO assay "
+                                            "(reference_i, reference_key, id, sample_id, record) VALUES (?, ?, ?, ?, ?)",
+                                            (reference_i, assay.reference.key, assay.assay_id, assay.sample_id,
+                                             _json(assay.to_dict())))
+                assay_ids[(reference_i, assay.assay_id, assay.sample_id)] = cursor.lastrowid
             for pair in pairings:
                 if not isinstance(pair, AssayPair):
                     raise TypeError("pairings must contain AssayPairs")
@@ -194,9 +253,7 @@ class QuantitativeDataset:
                 if (relation.dataset_id, relation.dataset_version) != (manifest.dataset_id, manifest.version):
                     raise ValueError("Target relation belongs to a different dataset release")
                 for target in (relation.left, relation.right):
-                    connection.execute("INSERT OR IGNORE INTO target VALUES (?, ?, ?, ?, ?)",
-                                       (target.reference.key, _target_key(target), target.kind,
-                                        target.namespace, target.identifier))
+                    insert_target(target)
                 for index, (first, second) in enumerate(((relation.left, relation.right), (relation.right, relation.left))):
                     _check_one_to_one(relation_links[index],
                                       (relation.relation, first.key, second.reference.key, second.kind, second.namespace), second.key,
@@ -218,9 +275,7 @@ class QuantitativeDataset:
                 if (mapping.dataset_id, mapping.dataset_version) != (manifest.dataset_id, manifest.version):
                     raise ValueError("Reference mapping belongs to a different dataset release")
                 for target in (mapping.source, *mapping.targets):
-                    connection.execute("INSERT OR IGNORE INTO target VALUES (?, ?, ?, ?, ?)",
-                                       (target.reference.key, _target_key(target), target.kind,
-                                        target.namespace, target.identifier))
+                    insert_target(target)
                 connection.execute("INSERT INTO reference_mapping VALUES (?, ?, ?, ?, ?)",
                                    (mapping.source.reference.key, mapping.mapping_id, _target_key(mapping.source),
                                     mapping.target_reference.key, _json(mapping.to_dict())))
@@ -238,33 +293,39 @@ class QuantitativeDataset:
                     raise ValueError("Measurement provenance names an unknown source asset")
                 if source is None and not (observation.input_measurements or observation.input_references):
                     raise ValueError("Measurement requires a source_asset or input measurements")
-                reference_key = observation.target.reference.key
-                inputs = [(reference_key, identifier) for identifier in observation.input_measurements]
-                inputs.extend((value.reference.key, value.measurement_id) for value in observation.input_references)
+                reference_i, target_i, _ = insert_target(observation.target)
+                inputs = [(reference_i, identifier) for identifier in observation.input_measurements]
+                inputs.extend((reference_ids.get(value.reference.key), value.measurement_id)
+                              for value in observation.input_references)
+                input_ids = []
                 for input_reference, identifier in inputs:
-                    if connection.execute(
-                        "SELECT 1 FROM measurement WHERE reference_key=? AND id=?",
+                    found = connection.execute(
+                        "SELECT i FROM observation WHERE reference_i=? AND id=?",
                         (input_reference, identifier),
-                    ).fetchone() is None:
+                    ).fetchone()
+                    if found is None:
                         raise ValueError("Derived inputs must precede outputs in their declared reference contexts")
-                target = observation.target
-                target_key = _target_key(target)
-                connection.execute("INSERT OR IGNORE INTO target VALUES (?, ?, ?, ?, ?)",
-                                   (reference_key, target_key, target.kind,
-                                    target.namespace, target.identifier))
-                record = observation.to_dict()
-                record.pop("target")
-                connection.execute("INSERT INTO measurement VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
-                                   (reference_key, observation.measurement_id, target_key,
-                                    observation.sample_id, observation.assay_id,
-                                    observation.quantity, observation.status, _json(record)))
-                connection.executemany("INSERT INTO input_measurement VALUES (?, ?, ?, ?)",
-                                       ((reference_key, observation.measurement_id, input_reference, identifier)
-                                        for input_reference, identifier in inputs))
-            counts = {table: connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
+                    input_ids.append(found[0])
+                assay_i = assay_ids.get((reference_i, observation.assay_id, observation.sample_id))
+                if assay_i is None:
+                    raise sqlite3.IntegrityError("Observation assay/sample absent from its reference context")
+                cursor = connection.execute("INSERT INTO observation "
+                                            "(reference_i, id, target_i, assay_i, quantity, status, record) "
+                                            "VALUES (?, ?, ?, ?, ?, ?, ?)",
+                                            (reference_i, observation.measurement_id, target_i, assay_i,
+                                             observation.quantity, observation.status,
+                                             _json(_measurement_record(observation))))
+                connection.executemany("INSERT INTO observation_input VALUES (?, ?)",
+                                       ((cursor.lastrowid, input_i) for input_i in input_ids))
+            insert_target.cache_clear()
+            counts = {table: connection.execute(
+                f"SELECT count(*) FROM {'observation' if table == 'measurement' else table}").fetchone()[0]
                       for table in ("sample", "assay", "mapping", "target", "measurement",
                                     "assay_pair", "target_relation", "reference_mapping")}
             connection.execute("INSERT INTO metadata VALUES ('counts', ?)", (_json(counts),))
+            # Real cardinalities prevent the planner from scanning a whole
+            # reference in ID order for a narrow target query.
+            connection.execute("ANALYZE")
             connection.commit()
             connection.close()
             connection = None
@@ -430,7 +491,8 @@ class QuantitativeDataset:
         filters = dict(reference_key=scope, target_key=_target_key(target) if target else None,
                        sample_id=sample_id, assay_id=assay_id, quantity=quantity, status=status)
         selected = [(key, value) for key, value in filters.items() if value is not None]
-        query = ("SELECT m.record, m.reference_key, t.kind, t.namespace, t.identifier "
+        query = ("SELECT m.record, m.reference_key, t.kind, t.namespace, t.identifier, "
+                 "m.id, m.sample_id, m.assay_id, m.quantity, m.status "
                  "FROM measurement m JOIN target t ON m.reference_key=t.reference_key "
                  "AND m.target_key=t.key")
         if selected:
@@ -439,6 +501,11 @@ class QuantitativeDataset:
         with closing(self._connect()) as connection:
             for row in connection.execute(query, [value for _, value in selected]):
                 values = json.loads(row[0])
+                if self._storage_version == 4:
+                    values = dict(_V4_DEFAULTS, **values)
+                    values.update(measurement_id=row[5], sample_id=row[6], assay_id=row[7],
+                                  quantity=row[8], status=row[9], dataset_id=self.manifest.dataset_id,
+                                  dataset_version=self.manifest.version)
                 values["target"] = ScopedTarget(self._references[row[1]], row[2], row[4], row[3])
                 if "input_references" in values:
                     values["input_references"] = tuple(MeasurementReference.from_dict(value)

orfdata-0.8.0/orfdata/version.py

@@ -0,0 +1 @@
+__version__ = "0.8.0"

{orfdata-0.7.0 → orfdata-0.8.0/orfdata.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orfdata
-Version: 0.7.0
+Version: 0.8.0
 Summary: Reference open reading frames and experimental translation evidence on annotated transcripts
 Author: OpenVax
 License-Expression: Apache-2.0

{orfdata-0.7.0 → orfdata-0.8.0}/orfdata.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ docs/pfizer-protein.md
 docs/reference.md
 docs/source-references.md
 docs/species.md
+docs/storage.md
 orfdata/__init__.py
 orfdata/build.py
 orfdata/catalog.py
@@ -56,6 +57,7 @@ tests/test_pfizer.py
 tests/test_reference_context.py
 tests/test_reference_mapping.py
 tests/test_reference_orf.py
+tests/test_storage.py
 tests/test_uorf_catalog.py
 tests/data/copy/GSM1905474.soft
 tests/data/copy/GSM1905474_KL9_fpkm.txt.gz
@@ -92,4 +94,12 @@ tests/data/references/canis_lupus_familiaris.json
 tests/data/references/context-keys-v0.4.json
 tests/data/references/homo_sapiens.json
 tests/data/references/mus_musculus.json
-tests/data/references/saccharomyces_cerevisiae.json
+tests/data/references/saccharomyces_cerevisiae.json
+tests/data/storage/README.md
+tests/data/storage/generate.py
+tests/data/storage/v1.json
+tests/data/storage/v1.sqlite
+tests/data/storage/v2.json
+tests/data/storage/v2.sqlite
+tests/data/storage/v3.json
+tests/data/storage/v3.sqlite

orfdata-0.8.0/tests/data/storage/README.md

@@ -0,0 +1,15 @@
+These synthetic databases were generated by the **released** ORFdata versions
+0.3.0 (storage 1), 0.4.0 (storage 2) and 0.5.0 (storage 3), using `generate.py`.
+They test actual legacy layouts; changing a new database's `user_version` would
+not produce a valid old-format fixture.
+
+Each contains four species with colliding local sample/assay/target/measurement
+IDs, measured zero, a censored value, uncertainty and derived input lineage.
+Versions 2/3 include pairings; version 3 includes reference mappings. No records
+are experimental evidence. The JSON files retain the original writer's records.
+
+Example regeneration from a checkout of the appropriate release:
+
+```sh
+PYTHONPATH=/path/to/orfdata-0.3.0 python generate.py /path/to/fixtures 1
+```

orfdata-0.8.0/tests/data/storage/generate.py

@@ -0,0 +1,74 @@
+"""Run from released 0.3.0/0.4.0/0.5.0 to generate genuine legacy databases."""
+
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+import orfdata
+from orfdata import (
+    Assay,
+    DatasetManifest,
+    Measurement,
+    QuantitativeDataset,
+    ReferenceContext,
+    Sample,
+    ScopedTarget,
+    SourceAsset,
+)
+
+output = Path(sys.argv[1])
+version = int(sys.argv[2])
+references = tuple(ReferenceContext(taxon, name, assembly, "ensembl", "110")
+                   for taxon, name, assembly in ((9606, "human", "GRCh38"),
+                                                 (10090, "mouse", "GRCm39"),
+                                                 (9615, "dog", "ROS_Cfam_1.0"),
+                                                 (4932, "yeast", "R64-1-1")))
+source = SourceAsset("input.tsv", "https://example.invalid/storage-fixture.tsv",
+                     hashlib.sha256(b"synthetic fixture\n").hexdigest(), 18,
+                     "Synthetic test data; Apache-2.0", redistributable=True)
+manifest = DatasetManifest("legacy-fixture", "1", "Synthetic legacy storage validation",
+                           references, (source,), ("Synthetic, not experimental evidence",),
+                           ("rna_seq", "digest_ms"), {"scope": "four deliberately colliding species"},
+                           {"writer": "orfdata", "writer_version": orfdata.__version__})
+samples, assays, observations, pairs, relations, mappings = [], [], [], [], [], []
+for index, reference in enumerate(references):
+    samples.append(Sample("sample", reference, biological_unit_id="culture"))
+    assays.extend((Assay("rna", "sample", "rna_seq", reference), Assay("protein", "sample", "digest_ms", reference)))
+    gene = ScopedTarget(reference, "gene", "same-id", "fixture")
+    protein = ScopedTarget(reference, "protein", "same-id", "fixture")
+    observations.extend((
+        Measurement("rna-value", gene, "sample", "rna", "rna_abundance", index, "TPM",
+                    manifest.dataset_id, manifest.version, provenance={"source_asset": "input.tsv", "row": index}),
+        Measurement("protein-value", protein, "sample", "protein", "protein_abundance",
+                    None if index == 1 else 10.25 + index, "iBAQ", manifest.dataset_id, manifest.version,
+                    status="below_detection" if index == 1 else "measured", detection_limit=20 if index == 1 else None,
+                    uncertainty=0.5, uncertainty_type="standard_deviation",
+                    provenance={"source_asset": "input.tsv", "row": index, "native": "<20" if index == 1 else "observed"}),
+        Measurement("derived", gene, "sample", "rna", "rna_abundance", 1, "fixture-units",
+                    manifest.dataset_id, manifest.version, origin="derived", input_measurements=("rna-value",),
+                    derivation={"method": "synthetic lineage fixture"}),
+    ))
+    if version >= 2:
+        from orfdata import AssayPair, TargetRelation
+
+        pairs.append(AssayPair("pair", reference, manifest.dataset_id, manifest.version,
+                               "rna", "protein", "same_material", {"source_asset": "input.tsv"}, "concurrent"))
+        relations.append(TargetRelation("relation", manifest.dataset_id, manifest.version, gene, protein,
+                                        "canonical_translation", "one_to_one", {"source_asset": "input.tsv"}))
+    if version >= 3:
+        from orfdata import ReferenceMapping
+
+        mappings.append(ReferenceMapping("map", manifest.dataset_id, manifest.version,
+                                         protein, reference, (ScopedTarget(reference, "protein", "mapped-id", "fixture"),),
+                                         "unique", "synthetic identifier match", {"sequence": "not validated"},
+                                         provenance={"source_asset": "input.tsv"}))
+kwargs = dict(samples=samples, assays=assays, measurements=observations)
+if version >= 2:
+    kwargs.update(pairings=pairs, relations=relations)
+if version >= 3:
+    kwargs.update(reference_mappings=mappings)
+dataset = QuantitativeDataset.create(output / f"v{version}.sqlite", manifest, **kwargs)
+expected = dict(writer_version=orfdata.__version__, manifest=manifest.to_dict(),
+                measurements=[m.to_dict() for m in observations], counts=dataset.counts)
+(output / f"v{version}.json").write_text(json.dumps(expected, indent=2) + "\n")