pedigree-graph 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pedigree_graph-0.6.0/.github/workflows/publish.yml +65 -0
- pedigree_graph-0.6.0/.gitignore +38 -0
- pedigree_graph-0.6.0/CHANGELOG.md +157 -0
- pedigree_graph-0.6.0/CONTEXT.md +67 -0
- pedigree_graph-0.6.0/LICENSE +21 -0
- pedigree_graph-0.6.0/LIMITATIONS.md +183 -0
- pedigree_graph-0.6.0/PKG-INFO +190 -0
- pedigree_graph-0.6.0/README.md +158 -0
- pedigree_graph-0.6.0/benchmarks/.gitignore +2 -0
- pedigree_graph-0.6.0/benchmarks/profile_pedigree_graph.py +551 -0
- pedigree_graph-0.6.0/docs/adr/0001-dp-kinship-bench-split-rejected.md +133 -0
- pedigree_graph-0.6.0/docs/adr/0002-pair-engines-read-only-collaborators.md +66 -0
- pedigree_graph-0.6.0/docs/adr/0003-relationship-plan-documents-not-drives-engines.md +51 -0
- pedigree_graph-0.6.0/docs/adr/0004-registry-aligned-max-degree-semantics.md +67 -0
- pedigree_graph-0.6.0/docs/adr/0005-exact-on-demand-pairwise-kinship.md +101 -0
- pedigree_graph-0.6.0/docs/architecture.md +64 -0
- pedigree_graph-0.6.0/docs/code-quality-review/README.md +27 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-001-fix-pair-coordinate-space-mismatch-after-from-subsample.md +49 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-002-replace-loose-constructor-inputs-and-dense-id-remapping-with-a-validated-input-model.md +47 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-003-decompose-pedigreegraph-move-relationship-engines-out-of-core-py.md +53 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-004-make-relationship-extraction-semantics-a-single-explicit-plan-instead-of-duplicated-degree-branches.md +42 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-005-replace-stringly-typed-effective-size-payload-dicts-with-explicit-typed-models.md +43 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-006-split-effective-size-py-into-focused-estimator-modules.md +35 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-007-simplify-or-fully-support-assemble-csc-s-extra-contract.md +35 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-008-decompose-kinship-kernel-py-and-shrink-dp-kinship-s-state-machine.md +47 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-009-reassess-the-experimental-bfs-engine-after-core-relationship-semantics-are-centralized.md +30 -0
- pedigree_graph-0.6.0/docs/code-quality-review/pgq-010-add-architecture-guardrails-for-large-files-and-hidden-coordinate-type-contracts.md +46 -0
- pedigree_graph-0.6.0/docs/code-quality-review.md +16 -0
- pedigree_graph-0.6.0/pedigree_graph/__init__.py +77 -0
- pedigree_graph-0.6.0/pedigree_graph/_bfs_engine.py +584 -0
- pedigree_graph-0.6.0/pedigree_graph/_bfs_kernel.py +96 -0
- pedigree_graph-0.6.0/pedigree_graph/_cohort_utils.py +116 -0
- pedigree_graph-0.6.0/pedigree_graph/_core.py +1216 -0
- pedigree_graph-0.6.0/pedigree_graph/_effective_size.py +220 -0
- pedigree_graph-0.6.0/pedigree_graph/_inbreeding_kernel.py +168 -0
- pedigree_graph-0.6.0/pedigree_graph/_kinship_allocator.py +390 -0
- pedigree_graph-0.6.0/pedigree_graph/_kinship_csc.py +91 -0
- pedigree_graph-0.6.0/pedigree_graph/_kinship_depth.py +120 -0
- pedigree_graph-0.6.0/pedigree_graph/_kinship_dp.py +877 -0
- pedigree_graph-0.6.0/pedigree_graph/_kinship_kernel.py +60 -0
- pedigree_graph-0.6.0/pedigree_graph/_kinship_pairwise.py +436 -0
- pedigree_graph-0.6.0/pedigree_graph/_lineage_kernel.py +111 -0
- pedigree_graph-0.6.0/pedigree_graph/_ne_caballero_toro.py +343 -0
- pedigree_graph-0.6.0/pedigree_graph/_ne_common.py +62 -0
- pedigree_graph-0.6.0/pedigree_graph/_ne_family_size.py +396 -0
- pedigree_graph-0.6.0/pedigree_graph/_ne_founders.py +188 -0
- pedigree_graph-0.6.0/pedigree_graph/_ne_hill.py +236 -0
- pedigree_graph-0.6.0/pedigree_graph/_ne_rates.py +200 -0
- pedigree_graph-0.6.0/pedigree_graph/_ne_results.py +327 -0
- pedigree_graph-0.6.0/pedigree_graph/_pair_extractor.py +543 -0
- pedigree_graph-0.6.0/pedigree_graph/_pair_utils.py +140 -0
- pedigree_graph-0.6.0/pedigree_graph/_registry.py +188 -0
- pedigree_graph-0.6.0/pedigree_graph/_streaming_counter.py +299 -0
- pedigree_graph-0.6.0/pedigree_graph/experimental.py +23 -0
- pedigree_graph-0.6.0/pedigree_graph/py.typed +0 -0
- pedigree_graph-0.6.0/pedigree_graph.egg-info/PKG-INFO +190 -0
- pedigree_graph-0.6.0/pedigree_graph.egg-info/SOURCES.txt +87 -0
- pedigree_graph-0.6.0/pedigree_graph.egg-info/dependency_links.txt +1 -0
- pedigree_graph-0.6.0/pedigree_graph.egg-info/requires.txt +14 -0
- pedigree_graph-0.6.0/pedigree_graph.egg-info/scm_file_list.json +84 -0
- pedigree_graph-0.6.0/pedigree_graph.egg-info/scm_version.json +8 -0
- pedigree_graph-0.6.0/pedigree_graph.egg-info/top_level.txt +1 -0
- pedigree_graph-0.6.0/pyproject.toml +97 -0
- pedigree_graph-0.6.0/setup.cfg +4 -0
- pedigree_graph-0.6.0/tests/conftest.py +132 -0
- pedigree_graph-0.6.0/tests/data/small_pedigree.parquet +0 -0
- pedigree_graph-0.6.0/tests/test_architecture_guardrails.py +67 -0
- pedigree_graph-0.6.0/tests/test_bfs_kernel.py +89 -0
- pedigree_graph-0.6.0/tests/test_cohort_utils.py +105 -0
- pedigree_graph-0.6.0/tests/test_count_pairs_streaming.py +327 -0
- pedigree_graph-0.6.0/tests/test_effective_size.py +1016 -0
- pedigree_graph-0.6.0/tests/test_effective_size_scaling.py +844 -0
- pedigree_graph-0.6.0/tests/test_experimental.py +465 -0
- pedigree_graph-0.6.0/tests/test_from_arrays_sex.py +97 -0
- pedigree_graph-0.6.0/tests/test_inbreeding_kernel.py +152 -0
- pedigree_graph-0.6.0/tests/test_inbreeding_properties.py +49 -0
- pedigree_graph-0.6.0/tests/test_kinship_kernel.py +596 -0
- pedigree_graph-0.6.0/tests/test_kinship_properties.py +107 -0
- pedigree_graph-0.6.0/tests/test_n_ancestors.py +74 -0
- pedigree_graph-0.6.0/tests/test_n_descendants.py +129 -0
- pedigree_graph-0.6.0/tests/test_ne_common_properties.py +111 -0
- pedigree_graph-0.6.0/tests/test_ne_properties.py +149 -0
- pedigree_graph-0.6.0/tests/test_pair_engines.py +100 -0
- pedigree_graph-0.6.0/tests/test_pair_engines_properties.py +78 -0
- pedigree_graph-0.6.0/tests/test_pair_extraction_properties.py +61 -0
- pedigree_graph-0.6.0/tests/test_pair_kinship_registry_properties.py +101 -0
- pedigree_graph-0.6.0/tests/test_pair_utils_properties.py +111 -0
- pedigree_graph-0.6.0/tests/test_pedigree_graph.py +1691 -0
- pedigree_graph-0.6.0/tests/test_relationship_plan.py +96 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
name: Build distributions
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- name: Check out repository
|
|
16
|
+
uses: actions/checkout@v7
|
|
17
|
+
with:
|
|
18
|
+
fetch-depth: 0 # setuptools-scm derives the version from the tag history
|
|
19
|
+
|
|
20
|
+
- name: Set up Python
|
|
21
|
+
uses: actions/setup-python@v7
|
|
22
|
+
with:
|
|
23
|
+
python-version: "3.13"
|
|
24
|
+
cache: pip
|
|
25
|
+
|
|
26
|
+
- name: Install the build frontend
|
|
27
|
+
run: python -m pip install build
|
|
28
|
+
|
|
29
|
+
- name: Build sdist and wheel
|
|
30
|
+
run: python -m build
|
|
31
|
+
|
|
32
|
+
- name: Verify the built version matches the tag
|
|
33
|
+
run: |
|
|
34
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
35
|
+
built="$(ls dist/pedigree_graph-*.tar.gz | sed -E 's|.*/pedigree_graph-(.+)\.tar\.gz$|\1|')"
|
|
36
|
+
if [ "$tag" != "$built" ]; then
|
|
37
|
+
echo "::error::tag ${GITHUB_REF_NAME} implies version ${tag}, but setuptools-scm built ${built}"
|
|
38
|
+
exit 1
|
|
39
|
+
fi
|
|
40
|
+
echo "Built version ${built} matches tag ${GITHUB_REF_NAME}"
|
|
41
|
+
|
|
42
|
+
- name: Upload distributions
|
|
43
|
+
uses: actions/upload-artifact@v7
|
|
44
|
+
with:
|
|
45
|
+
name: dist
|
|
46
|
+
path: dist/
|
|
47
|
+
|
|
48
|
+
publish:
|
|
49
|
+
name: Publish to PyPI
|
|
50
|
+
needs: build
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
environment:
|
|
53
|
+
name: pypi
|
|
54
|
+
url: https://pypi.org/p/pedigree-graph
|
|
55
|
+
permissions:
|
|
56
|
+
id-token: write # trusted publishing (OIDC) — no API token stored
|
|
57
|
+
steps:
|
|
58
|
+
- name: Download distributions
|
|
59
|
+
uses: actions/download-artifact@v8
|
|
60
|
+
with:
|
|
61
|
+
name: dist
|
|
62
|
+
path: dist/
|
|
63
|
+
|
|
64
|
+
- name: Publish to PyPI
|
|
65
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.so
|
|
6
|
+
.Python
|
|
7
|
+
*.egg
|
|
8
|
+
*.egg-info/
|
|
9
|
+
build/
|
|
10
|
+
dist/
|
|
11
|
+
.eggs/
|
|
12
|
+
|
|
13
|
+
# Virtual envs
|
|
14
|
+
env/
|
|
15
|
+
venv/
|
|
16
|
+
.venv/
|
|
17
|
+
|
|
18
|
+
# Testing / coverage
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.hypothesis/
|
|
21
|
+
.coverage
|
|
22
|
+
.coverage.*
|
|
23
|
+
htmlcov/
|
|
24
|
+
.tox/
|
|
25
|
+
coverage.xml
|
|
26
|
+
|
|
27
|
+
# Editors
|
|
28
|
+
.vscode/
|
|
29
|
+
.idea/
|
|
30
|
+
*.swp
|
|
31
|
+
*.swo
|
|
32
|
+
.DS_Store
|
|
33
|
+
|
|
34
|
+
# Numba cache
|
|
35
|
+
__pycache__/
|
|
36
|
+
|
|
37
|
+
# Setuptools-scm
|
|
38
|
+
_version.py
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
This file tracks public-API changes per release. For per-commit
|
|
4
|
+
history, see `git log`. Historical release notes prior to v0.5.0
|
|
5
|
+
live on the corresponding GitHub release pages.
|
|
6
|
+
|
|
7
|
+
## v0.6.0
|
|
8
|
+
|
|
9
|
+
- **First release published to PyPI.** `pip install pedigree-graph` now
|
|
10
|
+
works, retiring the `git+https://...@vX.Y.Z` install form that consumers
|
|
11
|
+
had to carry because the project was unavailable on the index.
|
|
12
|
+
Distributions are built and uploaded by a tag-triggered GitHub Actions
|
|
13
|
+
workflow using PyPI trusted publishing (OIDC), so no API token is
|
|
14
|
+
stored in the repository or in CI secrets. Downstream packages pinning
|
|
15
|
+
a git URL can move to a version range such as
|
|
16
|
+
`pedigree-graph>=0.6,<0.7`; the bound is worth keeping tight because
|
|
17
|
+
`PAIR_KINSHIP` and `extract_pairs` are consumed directly by simace,
|
|
18
|
+
fitace, and pedsum.
|
|
19
|
+
- **Packaging metadata completed for the index listing.** `readme`,
|
|
20
|
+
`license` (SPDX `MIT`), `license-files`, `authors`, `classifiers`, and
|
|
21
|
+
`[project.urls]` are now declared, so the PyPI page renders the README
|
|
22
|
+
and links back to the repository and this changelog. The build backend
|
|
23
|
+
floor moved from `setuptools>=64` to `setuptools>=77`, which is where
|
|
24
|
+
PEP 639 SPDX license support lands.
|
|
25
|
+
- **`py.typed` marker added.** The package now advertises inline type
|
|
26
|
+
information under PEP 561, so type checkers read its annotations when
|
|
27
|
+
it is installed as a wheel rather than as an editable source checkout.
|
|
28
|
+
Runtime code is unchanged.
|
|
29
|
+
|
|
30
|
+
## v0.5.4
|
|
31
|
+
|
|
32
|
+
- **Self-kinship diagonal fixed for inbred individuals when rows are not
|
|
33
|
+
in generation-monotonic order.** The matrix-DP kinship kernel assumed
|
|
34
|
+
every relative discovered during a row's merge walk had a smaller row
|
|
35
|
+
index, but `from_arrays` only requires topological order (parents
|
|
36
|
+
before children). When a relative at an earlier generation had a
|
|
37
|
+
higher row index, the diagonal append broke the row's sorted order and
|
|
38
|
+
the binary search reading `phi(mother, father)` silently returned 0 —
|
|
39
|
+
so the self-kinship diagonal read `(1+0)/2` instead of `(1+F)/2`, and
|
|
40
|
+
the GRM diagonal consumed downstream was wrong for inbred individuals.
|
|
41
|
+
Off-diagonals and the pairwise `compute_pair_kinship` path were
|
|
42
|
+
unaffected, as was any pedigree loaded in generation order (the common
|
|
43
|
+
case, which now hits a zero-overhead fast path).
|
|
44
|
+
|
|
45
|
+
- **Pair-set subtraction and id validation no longer go through
|
|
46
|
+
`np.unique`/`np.isin`.** Three internal hot spots were rewritten with
|
|
47
|
+
sort/searchsorted or diff-based equivalents, verified output-identical
|
|
48
|
+
on real data and under differential fuzzing (5,000 randomized trials
|
|
49
|
+
per site, including empties, duplicates, and ids near the int64
|
|
50
|
+
pair-key bound):
|
|
51
|
+
- `PedigreeGraph._subtract_pairs` now sorts the remove keys and
|
|
52
|
+
binary-searches candidates instead of `np.isin` (~6.9x on a 300k-N
|
|
53
|
+
pedigree; halves `sibling_pairs()` wall time). Same pattern —
|
|
54
|
+
and rationale — as `extract_from_sparse` already used.
|
|
55
|
+
- The duplicate-id check in `_validate_id_column` counts equal
|
|
56
|
+
adjacent elements after a sort instead of `len(np.unique)` (~37x).
|
|
57
|
+
- `pairs_from_groups` detects group boundaries by diffing its
|
|
58
|
+
already-sorted key array instead of re-running `np.unique` on it.
|
|
59
|
+
|
|
60
|
+
Returned pairs, counts, orderings, and error messages are unchanged —
|
|
61
|
+
this change is performance-only.
|
|
62
|
+
|
|
63
|
+
## v0.5.3
|
|
64
|
+
|
|
65
|
+
- **`count_pairs_streaming` warns when a cousin/collateral residual
|
|
66
|
+
underflows.** The scalar engine derives `H1C`, `1C1R`, `1C2R`, and
|
|
67
|
+
`H1C1R` by inclusion–exclusion — subtracting closer-relationship
|
|
68
|
+
contributions with fixed coefficients that are exact only on non-inbred,
|
|
69
|
+
single-mating pedigrees. On inbred or structurally complex real
|
|
70
|
+
pedigrees those corrections can over-count, driving the raw residual
|
|
71
|
+
negative; it was then silently clamped to `0`, indistinguishable from a
|
|
72
|
+
true absence (e.g. millions of `1C` but `H1C == 0`). The clamp now logs a
|
|
73
|
+
`WARNING` naming the code and the underflow magnitude and points to the
|
|
74
|
+
matrix engine (`extract_pairs`) for an exact count. Returned counts are
|
|
75
|
+
unchanged — only the diagnostic is new.
|
|
76
|
+
|
|
77
|
+
## v0.5.2
|
|
78
|
+
|
|
79
|
+
- **`count_pairs_streaming()` releases its transient matrices on exit.**
|
|
80
|
+
The scalar streaming counter builds the adjacency powers `_A`…`_A5` and
|
|
81
|
+
now drops them via `_release_pair_matrices()` before returning, exactly
|
|
82
|
+
as `extract_pairs()` already did. Previously they stayed resident for
|
|
83
|
+
the graph's lifetime, inflating peak memory of any later inbreeding / Ne
|
|
84
|
+
/ lineage work on the same graph (~400–520 MiB on a 1M-row pedigree).
|
|
85
|
+
The counts remain cached and the matrices rebuild lazily via
|
|
86
|
+
`_ensure_parent_csr()` if pair work runs again, so callers that reached
|
|
87
|
+
into the private `_release_pair_matrices()` after a streaming call (e.g.
|
|
88
|
+
pedsum `summarize`) can drop that workaround. Fixes #4.
|
|
89
|
+
|
|
90
|
+
## v0.5.0
|
|
91
|
+
|
|
92
|
+
- **Registry-aligned `max_degree` semantics.** `extract_pairs`,
|
|
93
|
+
`count_pairs`, and `count_pairs_streaming` now include exactly the
|
|
94
|
+
relationship categories whose `REL_REGISTRY[code].degree` is less
|
|
95
|
+
than or equal to the cutoff. `max_degree=0` is MZ-only,
|
|
96
|
+
`max_degree=2` stops before 1st cousins, and `max_degree=3`
|
|
97
|
+
includes 1st cousins plus the other degree-3 categories. The public
|
|
98
|
+
defaults changed from `2` to `3` to preserve the old default behavior
|
|
99
|
+
of including 1st cousins.
|
|
100
|
+
|
|
101
|
+
- **`PedigreeGraph.compute_n_ancestors()`** — new cached method.
|
|
102
|
+
Returns the per-individual count of *distinct* strict ancestors
|
|
103
|
+
(`int32`, length `n`). Backed by a sparse boolean transitive
|
|
104
|
+
closure of the parent graph; memory scales with the total closure
|
|
105
|
+
size. Suitable for pedigrees up to ~1M rows on commodity hardware;
|
|
106
|
+
deeper / wider pedigrees may need a future retirement-style DP
|
|
107
|
+
variant.
|
|
108
|
+
|
|
109
|
+
- **`PedigreeGraph.compute_n_descendants()`** — new cached method.
|
|
110
|
+
Returns the per-individual descendant *path count* (`int32`, length
|
|
111
|
+
`n`). In non-inbred pedigrees this equals the unique-descendant
|
|
112
|
+
count; in inbred pedigrees it over-counts a descendant reachable via
|
|
113
|
+
multiple ancestor paths. Matches the convention used historically
|
|
114
|
+
by `pedsum` (`compute_descendants`) and by the matrix engine's GP /
|
|
115
|
+
Av / 1C pair counts. Raises `OverflowError` if any per-individual
|
|
116
|
+
path count exceeds `int32` max (the kernel accumulates in `int64`
|
|
117
|
+
and the cast happens after a bounds check, so deeply inbred
|
|
118
|
+
pedigrees cannot silently wrap).
|
|
119
|
+
|
|
120
|
+
- **`PedigreeGraph.from_arrays(...)`** — accepts a new optional `sex`
|
|
121
|
+
kwarg (`np.ndarray | None`). When omitted, behaviour is unchanged
|
|
122
|
+
(sex defaults to zeros). Existing callers do not need updates.
|
|
123
|
+
|
|
124
|
+
- **Defensive warning for the `sex`-default foot-gun.**
|
|
125
|
+
``ne_sex_ratio`` and ``ne_variance_family_size`` now emit a
|
|
126
|
+
``RuntimeWarning`` when ``pg.sex`` is uniformly 0 or 1 — almost
|
|
127
|
+
always a sign that the caller forgot to pass ``sex=`` to
|
|
128
|
+
``from_arrays`` and is consuming silently-degenerate (single-sex)
|
|
129
|
+
Ne results. The estimator return values are unchanged (``ne=None``);
|
|
130
|
+
the warning is the new diagnostic. Kinship-only callers
|
|
131
|
+
(relationship-pair extraction, GRMs, PA-FGRS) are not affected
|
|
132
|
+
because they don't invoke the sex-aware estimators.
|
|
133
|
+
|
|
134
|
+
- New private kernel module `pedigree_graph/_lineage_kernel.py` houses
|
|
135
|
+
the descendant (numba-JIT) and ancestor (scipy sparse) primitives.
|
|
136
|
+
|
|
137
|
+
- **`PedigreeGraph.count_pairs_streaming(max_degree=2, scope="full")`**
|
|
138
|
+
— new method. Memory-bounded relationship pair counts via pure
|
|
139
|
+
scalar arithmetic; no pair-key arrays are ever materialized. Peak
|
|
140
|
+
memory is O(N) regardless of pedigree density. Returns all 23
|
|
141
|
+
codes from `REL_REGISTRY`. Bit-identical to `count_pairs` for
|
|
142
|
+
the 10 simple codes (`MZ`, `MO`, `FO`, `FS`, `MHS`, `PHS`, `GP`,
|
|
143
|
+
`GGP`, `GGGP`, `G3GP`); approximate (~1% on deep low-inbreeding
|
|
144
|
+
pedigrees) for the 13 cousin / collateral codes (`Av`, `1C`,
|
|
145
|
+
`H1C`, `HAv`, `GAv`, `GGAv`, `G3Av`, `HGAv`, `HGGAv`, `1C1R`,
|
|
146
|
+
`H1C1R`, `1C2R`, `2C`). The scalar path is **full-graph only**:
|
|
147
|
+
`scope='subsample'` raises `NotImplementedError` on graphs built
|
|
148
|
+
via `from_subsample` (use `count_pairs` for subsample-restricted
|
|
149
|
+
counts). See `LIMITATIONS.md` for the full precision contract.
|
|
150
|
+
Benchmark: 5 seconds on a 783K-row stallion-heavy livestock
|
|
151
|
+
pedigree where both matrix and BFS engines OOM at 30 GB.
|
|
152
|
+
|
|
153
|
+
- **`max_degree` validation** — `extract_pairs`, `count_pairs`, and
|
|
154
|
+
`count_pairs_streaming` now reject `max_degree` outside `[0, 5]`
|
|
155
|
+
with `ValueError`. Degree 0 is accepted (cheap codes MZ / MO /
|
|
156
|
+
FO / FS are computed regardless; the cap controls the expensive
|
|
157
|
+
matrix products at degree 2 and above).
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# pedigree-graph
|
|
2
|
+
|
|
3
|
+
Sparse-matrix pedigree relationship extraction and kinship computation. A
|
|
4
|
+
pedigree is a parent→child DAG of individuals; this context is the vocabulary
|
|
5
|
+
for the relationships between individuals and the two coordinate systems used
|
|
6
|
+
to name them.
|
|
7
|
+
|
|
8
|
+
## Language
|
|
9
|
+
|
|
10
|
+
### Coordinate spaces
|
|
11
|
+
|
|
12
|
+
**Graph-space**:
|
|
13
|
+
An individual's row index within the full pedigree the graph was built over.
|
|
14
|
+
_Avoid_: full index, absolute index, internal index
|
|
15
|
+
|
|
16
|
+
**Caller-space**:
|
|
17
|
+
An individual's row index within the subsample the caller supplied, which may
|
|
18
|
+
order or omit individuals differently from the full pedigree.
|
|
19
|
+
_Avoid_: subsample index, df index, external index
|
|
20
|
+
|
|
21
|
+
### Relationships
|
|
22
|
+
|
|
23
|
+
**Relationship pair**:
|
|
24
|
+
An unordered pair of individuals sharing a relationship category. Some pair
|
|
25
|
+
arrays preserve relationship orientation (for example descendant→ancestor for
|
|
26
|
+
lineal relationships); pair-key encodings canonicalize as `(lo, hi)` with
|
|
27
|
+
`lo < hi`.
|
|
28
|
+
_Avoid_: edge, link, tuple; assuming every pair array is already canonical
|
|
29
|
+
|
|
30
|
+
**Relationship category**:
|
|
31
|
+
A class of relationship identified by a short code (e.g. `FS`, `MHS`, `1C`),
|
|
32
|
+
defined by `(up, down, n_ancestors)` — meioses up to the common ancestor(s),
|
|
33
|
+
meioses back down, and whether the connecting ancestor is a single individual
|
|
34
|
+
(half / lineal) or a mated pair (full).
|
|
35
|
+
_Avoid_: relationship type (when the code is meant), kind
|
|
36
|
+
|
|
37
|
+
**Degree**:
|
|
38
|
+
The kinship distance of a relationship category — `0` for MZ twins, `1` for
|
|
39
|
+
parent-offspring and full sibs, and so on. A degree cutoff includes relationship
|
|
40
|
+
categories whose degree is less than or equal to the cutoff.
|
|
41
|
+
|
|
42
|
+
**Nominal kinship**:
|
|
43
|
+
The kinship coefficient implied by a relationship category's `(up, down,
|
|
44
|
+
n_ancestors)` formula, assuming a single relationship path and no inbreeding
|
|
45
|
+
or co-coalescence.
|
|
46
|
+
_Avoid_: exact kinship
|
|
47
|
+
|
|
48
|
+
**Exact pairwise kinship**:
|
|
49
|
+
The kinship coefficient for a particular pair of individuals after summing all
|
|
50
|
+
pedigree paths, including inbreeding, MZ co-coalescence, and duplicate
|
|
51
|
+
relationship paths such as double cousins.
|
|
52
|
+
_Avoid_: nominal kinship
|
|
53
|
+
|
|
54
|
+
## Relationships
|
|
55
|
+
|
|
56
|
+
- A **relationship pair** holds two individuals and belongs to one **relationship category**; canonical ordering is a storage/encoding choice, not part of the relationship itself.
|
|
57
|
+
- Every individual index is expressed in either **graph-space** or **caller-space**; the same individual generally has a different index in each.
|
|
58
|
+
- A pair returned to a caller is in **caller-space**; the kinship matrix is indexed in **graph-space**. Converting between the two is required whenever both meet.
|
|
59
|
+
|
|
60
|
+
## Example dialogue
|
|
61
|
+
|
|
62
|
+
> **Reviewer:** "`extract_pairs` gave me pair `(1, 0)` for the MZ twins — why is its kinship `0.0`?"
|
|
63
|
+
> **Author:** "Those indices are **caller-space** — you reversed the subsample. The kinship matrix is **graph-space**, so indexing it with caller indices reads the wrong cell. The pair has to be mapped back to graph-space first."
|
|
64
|
+
|
|
65
|
+
## Flagged ambiguities
|
|
66
|
+
|
|
67
|
+
- "index" alone is ambiguous between **graph-space** and **caller-space** — always qualify which space, since the same individual differs between them and conflating them caused a kinship-lookup bug (PGQ-001).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ryan Waples
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# Limitations
|
|
2
|
+
|
|
3
|
+
Current scaling and correctness limitations of the relationship-pair
|
|
4
|
+
engines. Read before reaching for `extract_pairs` / `count_pairs` on
|
|
5
|
+
pair-dense pedigrees.
|
|
6
|
+
|
|
7
|
+
## Pair counting is O(answer size) for the matrix and BFS engines
|
|
8
|
+
|
|
9
|
+
The matrix and BFS engines both materialise every pair as an
|
|
10
|
+
``(idx1, idx2)`` array before counting it:
|
|
11
|
+
|
|
12
|
+
- ``extract_pairs()`` returns ``dict[code, (np.ndarray, np.ndarray)]``.
|
|
13
|
+
- ``count_pairs()`` is a thin facade — it runs ``extract_pairs()`` and
|
|
14
|
+
returns ``{code: len(idx_array)}``.
|
|
15
|
+
|
|
16
|
+
Memory is therefore proportional to the total relationship-pair count,
|
|
17
|
+
**not** the pedigree size.
|
|
18
|
+
|
|
19
|
+
### Worst case: prolific-stallion livestock pedigrees
|
|
20
|
+
|
|
21
|
+
Stallion-driven half-sib density is the engine's hard wall. On a real
|
|
22
|
+
horse-breed pedigree (783K individuals, one all-time-great stallion
|
|
23
|
+
siring 2,500 horses, top-sire grand-offspring set ~50K):
|
|
24
|
+
|
|
25
|
+
- Paternal half-sib pair count = ~156 M
|
|
26
|
+
- ``_half_sib_matrix`` materialisation = ~312 M nonzeros (~3 GB) just
|
|
27
|
+
for the symmetric MHS+PHS matrix
|
|
28
|
+
- Per-grandparent grandchild bucket for ``_cousin_pairs`` enumeration
|
|
29
|
+
reaches ``C(50K, 2) ≈ 1.25 B`` candidate pairs through one stallion
|
|
30
|
+
grandparent (~10 GB of int64 keys)
|
|
31
|
+
- ``_A2 @ _A3.T`` for 1C1R / H1C1R has row nnz ~2,500 (one stallion's
|
|
32
|
+
great-grandchildren spread); chunk sizes blow up at default
|
|
33
|
+
``chunk_rows``
|
|
34
|
+
|
|
35
|
+
Both engines OOM on this pedigree well before producing a count, even
|
|
36
|
+
on 30 GB hosts. Matrix OOMs in ``A_f @ A_f.T`` (PHS sparse product);
|
|
37
|
+
BFS OOMs in the numba cousin enumeration kernel.
|
|
38
|
+
|
|
39
|
+
### Operational workarounds
|
|
40
|
+
|
|
41
|
+
1. ``pedsum --no-pairs`` skips the pair-counting stage entirely. The
|
|
42
|
+
horse pedigree completes in ~30s with 1 GB peak RSS this way,
|
|
43
|
+
producing every other section (size structure, family, mating,
|
|
44
|
+
lineage, founder contribution, inbreeding, effective size) but
|
|
45
|
+
returning stub values for the 23 relationship counts (``pairs: {}``
|
|
46
|
+
and ``relationship_summary.computed: false``).
|
|
47
|
+
|
|
48
|
+
2. ``PedigreeGraph.count_pairs_streaming()`` — pure-scalar per-anchor
|
|
49
|
+
arithmetic, O(N) memory, exact on 10 simple codes, approximate
|
|
50
|
+
within ~1% on 13 cousin / collateral codes for deep low-inbreeding
|
|
51
|
+
pedigrees. See the precision contract below.
|
|
52
|
+
|
|
53
|
+
## ``count_pairs_streaming`` precision contract
|
|
54
|
+
|
|
55
|
+
The scalar path is **full-graph only** — ``scope='subsample'`` raises
|
|
56
|
+
``NotImplementedError`` on graphs constructed via ``from_subsample``.
|
|
57
|
+
Use ``count_pairs`` for subsample-restricted counts.
|
|
58
|
+
|
|
59
|
+
- **Exact** on 10 codes (bit-identical to ``count_pairs`` on every
|
|
60
|
+
input):
|
|
61
|
+
``MZ``, ``MO``, ``FO``, ``FS``, ``MHS``, ``PHS``,
|
|
62
|
+
``GP``, ``GGP``, ``GGGP``, ``G3GP``.
|
|
63
|
+
|
|
64
|
+
- **Approximate** on 13 cousin / collateral codes:
|
|
65
|
+
``Av``, ``1C``, ``H1C``, ``HAv``, ``GAv``, ``GGAv``, ``G3Av``,
|
|
66
|
+
``HGAv``, ``HGGAv``, ``1C1R``, ``H1C1R``, ``1C2R``, ``2C``.
|
|
67
|
+
|
|
68
|
+
Scalar formulas assume each individual has the full complement of
|
|
69
|
+
known grandparents at the relevant depth, so constants like
|
|
70
|
+
``4*FS`` in the ``H1C`` correction over-subtract on shallow
|
|
71
|
+
pedigrees; ``H1C`` may clamp to ``0`` on depth ≤ 3. Twin parents
|
|
72
|
+
and sib-mating offspring also push the formulas off bit-identity
|
|
73
|
+
because the inclusion-exclusion terms assume neither pattern.
|
|
74
|
+
|
|
75
|
+
On the synthetic ``small_pedigree`` fixture (3000 rows, depth 3,
|
|
76
|
+
~0.5% sib-mating, 10 twin pairs): ``Av`` off by 3, ``HAv`` off by
|
|
77
|
+
11, ``1C`` off by 30, ``H1C`` clamped to 0. On deep livestock
|
|
78
|
+
pedigrees (depth ≥ 5, low inbreeding) the formulas are accurate to
|
|
79
|
+
better than 1%.
|
|
80
|
+
|
|
81
|
+
The horse-pedigree benchmark (N=783K, mean F=0.007) completes in
|
|
82
|
+
~5 seconds with peak RSS ~730 MB.
|
|
83
|
+
|
|
84
|
+
## Cousin-code matrix/BFS divergence on inbred input
|
|
85
|
+
|
|
86
|
+
Independent of scaling, the ``matrix`` and ``bfs`` engines give
|
|
87
|
+
**different counts** on inbred pedigrees for the four
|
|
88
|
+
cousin-multiplicity codes: ``1C1R``, ``H1C1R``, ``1C2R``, ``2C``.
|
|
89
|
+
|
|
90
|
+
- **Matrix engine** uses path multiplicity: ``M.data >= 2`` (full) or
|
|
91
|
+
``M.data == 1`` (half) thresholds on ``_A2 @ _A3.T`` / ``_A2 @ _A4.T``
|
|
92
|
+
/ ``_A3 @ _A3.T``. A pair sharing an ancestor via two distinct
|
|
93
|
+
paths is counted twice in the matrix entry; the threshold
|
|
94
|
+
classifies based on this multiplicity.
|
|
95
|
+
- **BFS engine** uses distinct-shared-ancestor semantics: a pair
|
|
96
|
+
sharing N distinct ancestors at the relevant depth is counted once
|
|
97
|
+
regardless of paths.
|
|
98
|
+
|
|
99
|
+
The divergence is documented in
|
|
100
|
+
``tests/test_experimental.py:171`` (the ``inbred_with_cousins_pedigree``
|
|
101
|
+
fixture) and asserted in ``test_inbred_with_cousins_cousin_codes_diverge``.
|
|
102
|
+
|
|
103
|
+
``extract_pairs(scope="full")`` returns matrix-engine values by
|
|
104
|
+
default. Callers needing BFS-distinct semantics on inbred input must
|
|
105
|
+
use ``pedigree_graph.experimental.count_pairs_bfs`` and accept the
|
|
106
|
+
matrix-vs-BFS difference for those four codes.
|
|
107
|
+
|
|
108
|
+
## ``int8`` overflow risk in BFS ``P_k`` boolean matmul
|
|
109
|
+
|
|
110
|
+
The BFS engine (`pedigree_graph.experimental.count_pairs_bfs`) uses
|
|
111
|
+
``np.int8`` for ``P_k.data`` during the boolean matmul stages.
|
|
112
|
+
Theoretically vulnerable to silent path-count overflow under extreme
|
|
113
|
+
consanguinity: more than 127 distinct paths to a single ``(i, X)`` pair
|
|
114
|
+
before the ``M.data[:] = 1`` clamp will wrap.
|
|
115
|
+
|
|
116
|
+
Empirically not seen on any tested pedigree. Switch to ``int32`` if it
|
|
117
|
+
ever bites — the change is a one-line dtype swap at the matmul site;
|
|
118
|
+
memory cost is 4× on the intermediate matrices.
|
|
119
|
+
|
|
120
|
+
## ``compute_n_ancestors`` memory scales with ``sum_i n_ancestors[i]``
|
|
121
|
+
|
|
122
|
+
``PedigreeGraph.compute_n_ancestors`` is a sparse boolean transitive
|
|
123
|
+
closure of the parent graph (``_lineage_kernel._compute_n_ancestors``).
|
|
124
|
+
Memory scales with ``sum_i n_ancestors[i]``, so very deep / very wide
|
|
125
|
+
pedigrees can hit RAM limits:
|
|
126
|
+
|
|
127
|
+
- N=100K, G=10, random mating → 2.2 s, peak RSS ~0.5 GB.
|
|
128
|
+
- N=10M with saturated ancestry → extrapolates beyond commodity hardware.
|
|
129
|
+
|
|
130
|
+
A retirement-style DP (analogous to the F kernel's row-retirement
|
|
131
|
+
optimisation in ``_kinship_kernel``) would bound peak memory to the
|
|
132
|
+
live frontier rather than the cumulative ancestor set. Deferred until
|
|
133
|
+
a user hits the wall.
|
|
134
|
+
|
|
135
|
+
## Half-founders and missing parents
|
|
136
|
+
|
|
137
|
+
Both engines accept half-founders (one parent known, one missing).
|
|
138
|
+
The sibling group-by filters to known parents only:
|
|
139
|
+
|
|
140
|
+
- ``FS`` requires BOTH parents known on both individuals.
|
|
141
|
+
- ``MHS`` only considers individuals with mother known.
|
|
142
|
+
- ``PHS`` only considers individuals with father known.
|
|
143
|
+
|
|
144
|
+
This matches the standard convention but can surprise callers who
|
|
145
|
+
expect half-founders to contribute to half-sib counts on the
|
|
146
|
+
"missing" side. They don't.
|
|
147
|
+
|
|
148
|
+
## Subsample-restricted counts are O(full pair count)
|
|
149
|
+
|
|
150
|
+
``PedigreeGraph.from_subsample(...)`` builds a graph that returns
|
|
151
|
+
subsample-filtered pair arrays from ``extract_pairs``, but the
|
|
152
|
+
underlying enumeration runs over the FULL pedigree first (raw counts
|
|
153
|
+
saved, then sample mask applied). Memory is bounded by full-pedigree
|
|
154
|
+
pair counts, not the subsample.
|
|
155
|
+
|
|
156
|
+
For a 10% subsample of a stallion-heavy pedigree, this is still
|
|
157
|
+
OOM-prone because the full-pedigree intermediate doesn't shrink.
|
|
158
|
+
|
|
159
|
+
## What this file does NOT cover
|
|
160
|
+
|
|
161
|
+
- Lineal-code counting limitations (none significant — ``_A^k.nnz`` is
|
|
162
|
+
O(N · depth) and tractable to N=10M+).
|
|
163
|
+
- F (inbreeding coefficient) scaling — covered by
|
|
164
|
+
``pedigree_graph._kinship_kernel`` and its own row-retirement
|
|
165
|
+
optimisation work.
|
|
166
|
+
- Effective size estimator scaling — covered by
|
|
167
|
+
``pedigree_graph._effective_size`` and the ``skip_ne_coancestry``
|
|
168
|
+
knob.
|
|
169
|
+
- BFS engine internal limitations — see the ``int8`` overflow section
|
|
170
|
+
above for the path-count overflow case, and GitHub issues
|
|
171
|
+
[#2 (numba kernel parallelisation)](https://github.com/rwaples/pedigree-graph/issues/2)
|
|
172
|
+
and [#3 (10M+ scaling test)](https://github.com/rwaples/pedigree-graph/issues/3)
|
|
173
|
+
for open performance / scalability questions.
|
|
174
|
+
|
|
175
|
+
## Last updated
|
|
176
|
+
|
|
177
|
+
2026-05-20 — ``int8`` overflow risk and ``compute_n_ancestors``
|
|
178
|
+
scalability sections added; BFS internal follow-ups re-homed from
|
|
179
|
+
retired ``external/pedsum/STATUS.md`` to GitHub issues #2 and #3.
|
|
180
|
+
|
|
181
|
+
2026-05-19 — ``count_pairs_streaming`` precision contract
|
|
182
|
+
reconciled; ``Av`` documented as approximate; stale
|
|
183
|
+
"count-only-experiment-didn't-ship" narrative removed.
|