pedigree-graph 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. pedigree_graph-0.6.0/.github/workflows/publish.yml +65 -0
  2. pedigree_graph-0.6.0/.gitignore +38 -0
  3. pedigree_graph-0.6.0/CHANGELOG.md +157 -0
  4. pedigree_graph-0.6.0/CONTEXT.md +67 -0
  5. pedigree_graph-0.6.0/LICENSE +21 -0
  6. pedigree_graph-0.6.0/LIMITATIONS.md +183 -0
  7. pedigree_graph-0.6.0/PKG-INFO +190 -0
  8. pedigree_graph-0.6.0/README.md +158 -0
  9. pedigree_graph-0.6.0/benchmarks/.gitignore +2 -0
  10. pedigree_graph-0.6.0/benchmarks/profile_pedigree_graph.py +551 -0
  11. pedigree_graph-0.6.0/docs/adr/0001-dp-kinship-bench-split-rejected.md +133 -0
  12. pedigree_graph-0.6.0/docs/adr/0002-pair-engines-read-only-collaborators.md +66 -0
  13. pedigree_graph-0.6.0/docs/adr/0003-relationship-plan-documents-not-drives-engines.md +51 -0
  14. pedigree_graph-0.6.0/docs/adr/0004-registry-aligned-max-degree-semantics.md +67 -0
  15. pedigree_graph-0.6.0/docs/adr/0005-exact-on-demand-pairwise-kinship.md +101 -0
  16. pedigree_graph-0.6.0/docs/architecture.md +64 -0
  17. pedigree_graph-0.6.0/docs/code-quality-review/README.md +27 -0
  18. pedigree_graph-0.6.0/docs/code-quality-review/pgq-001-fix-pair-coordinate-space-mismatch-after-from-subsample.md +49 -0
  19. pedigree_graph-0.6.0/docs/code-quality-review/pgq-002-replace-loose-constructor-inputs-and-dense-id-remapping-with-a-validated-input-model.md +47 -0
  20. pedigree_graph-0.6.0/docs/code-quality-review/pgq-003-decompose-pedigreegraph-move-relationship-engines-out-of-core-py.md +53 -0
  21. pedigree_graph-0.6.0/docs/code-quality-review/pgq-004-make-relationship-extraction-semantics-a-single-explicit-plan-instead-of-duplicated-degree-branches.md +42 -0
  22. pedigree_graph-0.6.0/docs/code-quality-review/pgq-005-replace-stringly-typed-effective-size-payload-dicts-with-explicit-typed-models.md +43 -0
  23. pedigree_graph-0.6.0/docs/code-quality-review/pgq-006-split-effective-size-py-into-focused-estimator-modules.md +35 -0
  24. pedigree_graph-0.6.0/docs/code-quality-review/pgq-007-simplify-or-fully-support-assemble-csc-s-extra-contract.md +35 -0
  25. pedigree_graph-0.6.0/docs/code-quality-review/pgq-008-decompose-kinship-kernel-py-and-shrink-dp-kinship-s-state-machine.md +47 -0
  26. pedigree_graph-0.6.0/docs/code-quality-review/pgq-009-reassess-the-experimental-bfs-engine-after-core-relationship-semantics-are-centralized.md +30 -0
  27. pedigree_graph-0.6.0/docs/code-quality-review/pgq-010-add-architecture-guardrails-for-large-files-and-hidden-coordinate-type-contracts.md +46 -0
  28. pedigree_graph-0.6.0/docs/code-quality-review.md +16 -0
  29. pedigree_graph-0.6.0/pedigree_graph/__init__.py +77 -0
  30. pedigree_graph-0.6.0/pedigree_graph/_bfs_engine.py +584 -0
  31. pedigree_graph-0.6.0/pedigree_graph/_bfs_kernel.py +96 -0
  32. pedigree_graph-0.6.0/pedigree_graph/_cohort_utils.py +116 -0
  33. pedigree_graph-0.6.0/pedigree_graph/_core.py +1216 -0
  34. pedigree_graph-0.6.0/pedigree_graph/_effective_size.py +220 -0
  35. pedigree_graph-0.6.0/pedigree_graph/_inbreeding_kernel.py +168 -0
  36. pedigree_graph-0.6.0/pedigree_graph/_kinship_allocator.py +390 -0
  37. pedigree_graph-0.6.0/pedigree_graph/_kinship_csc.py +91 -0
  38. pedigree_graph-0.6.0/pedigree_graph/_kinship_depth.py +120 -0
  39. pedigree_graph-0.6.0/pedigree_graph/_kinship_dp.py +877 -0
  40. pedigree_graph-0.6.0/pedigree_graph/_kinship_kernel.py +60 -0
  41. pedigree_graph-0.6.0/pedigree_graph/_kinship_pairwise.py +436 -0
  42. pedigree_graph-0.6.0/pedigree_graph/_lineage_kernel.py +111 -0
  43. pedigree_graph-0.6.0/pedigree_graph/_ne_caballero_toro.py +343 -0
  44. pedigree_graph-0.6.0/pedigree_graph/_ne_common.py +62 -0
  45. pedigree_graph-0.6.0/pedigree_graph/_ne_family_size.py +396 -0
  46. pedigree_graph-0.6.0/pedigree_graph/_ne_founders.py +188 -0
  47. pedigree_graph-0.6.0/pedigree_graph/_ne_hill.py +236 -0
  48. pedigree_graph-0.6.0/pedigree_graph/_ne_rates.py +200 -0
  49. pedigree_graph-0.6.0/pedigree_graph/_ne_results.py +327 -0
  50. pedigree_graph-0.6.0/pedigree_graph/_pair_extractor.py +543 -0
  51. pedigree_graph-0.6.0/pedigree_graph/_pair_utils.py +140 -0
  52. pedigree_graph-0.6.0/pedigree_graph/_registry.py +188 -0
  53. pedigree_graph-0.6.0/pedigree_graph/_streaming_counter.py +299 -0
  54. pedigree_graph-0.6.0/pedigree_graph/experimental.py +23 -0
  55. pedigree_graph-0.6.0/pedigree_graph/py.typed +0 -0
  56. pedigree_graph-0.6.0/pedigree_graph.egg-info/PKG-INFO +190 -0
  57. pedigree_graph-0.6.0/pedigree_graph.egg-info/SOURCES.txt +87 -0
  58. pedigree_graph-0.6.0/pedigree_graph.egg-info/dependency_links.txt +1 -0
  59. pedigree_graph-0.6.0/pedigree_graph.egg-info/requires.txt +14 -0
  60. pedigree_graph-0.6.0/pedigree_graph.egg-info/scm_file_list.json +84 -0
  61. pedigree_graph-0.6.0/pedigree_graph.egg-info/scm_version.json +8 -0
  62. pedigree_graph-0.6.0/pedigree_graph.egg-info/top_level.txt +1 -0
  63. pedigree_graph-0.6.0/pyproject.toml +97 -0
  64. pedigree_graph-0.6.0/setup.cfg +4 -0
  65. pedigree_graph-0.6.0/tests/conftest.py +132 -0
  66. pedigree_graph-0.6.0/tests/data/small_pedigree.parquet +0 -0
  67. pedigree_graph-0.6.0/tests/test_architecture_guardrails.py +67 -0
  68. pedigree_graph-0.6.0/tests/test_bfs_kernel.py +89 -0
  69. pedigree_graph-0.6.0/tests/test_cohort_utils.py +105 -0
  70. pedigree_graph-0.6.0/tests/test_count_pairs_streaming.py +327 -0
  71. pedigree_graph-0.6.0/tests/test_effective_size.py +1016 -0
  72. pedigree_graph-0.6.0/tests/test_effective_size_scaling.py +844 -0
  73. pedigree_graph-0.6.0/tests/test_experimental.py +465 -0
  74. pedigree_graph-0.6.0/tests/test_from_arrays_sex.py +97 -0
  75. pedigree_graph-0.6.0/tests/test_inbreeding_kernel.py +152 -0
  76. pedigree_graph-0.6.0/tests/test_inbreeding_properties.py +49 -0
  77. pedigree_graph-0.6.0/tests/test_kinship_kernel.py +596 -0
  78. pedigree_graph-0.6.0/tests/test_kinship_properties.py +107 -0
  79. pedigree_graph-0.6.0/tests/test_n_ancestors.py +74 -0
  80. pedigree_graph-0.6.0/tests/test_n_descendants.py +129 -0
  81. pedigree_graph-0.6.0/tests/test_ne_common_properties.py +111 -0
  82. pedigree_graph-0.6.0/tests/test_ne_properties.py +149 -0
  83. pedigree_graph-0.6.0/tests/test_pair_engines.py +100 -0
  84. pedigree_graph-0.6.0/tests/test_pair_engines_properties.py +78 -0
  85. pedigree_graph-0.6.0/tests/test_pair_extraction_properties.py +61 -0
  86. pedigree_graph-0.6.0/tests/test_pair_kinship_registry_properties.py +101 -0
  87. pedigree_graph-0.6.0/tests/test_pair_utils_properties.py +111 -0
  88. pedigree_graph-0.6.0/tests/test_pedigree_graph.py +1691 -0
  89. pedigree_graph-0.6.0/tests/test_relationship_plan.py +96 -0
@@ -0,0 +1,65 @@
1
+ name: Publish
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ build:
12
+ name: Build distributions
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - name: Check out repository
16
+ uses: actions/checkout@v7
17
+ with:
18
+ fetch-depth: 0 # setuptools-scm derives the version from the tag history
19
+
20
+ - name: Set up Python
21
+ uses: actions/setup-python@v7
22
+ with:
23
+ python-version: "3.13"
24
+ cache: pip
25
+
26
+ - name: Install the build frontend
27
+ run: python -m pip install build
28
+
29
+ - name: Build sdist and wheel
30
+ run: python -m build
31
+
32
+ - name: Verify the built version matches the tag
33
+ run: |
34
+ tag="${GITHUB_REF_NAME#v}"
35
+ built="$(ls dist/pedigree_graph-*.tar.gz | sed -E 's|.*/pedigree_graph-(.+)\.tar\.gz$|\1|')"
36
+ if [ "$tag" != "$built" ]; then
37
+ echo "::error::tag ${GITHUB_REF_NAME} implies version ${tag}, but setuptools-scm built ${built}"
38
+ exit 1
39
+ fi
40
+ echo "Built version ${built} matches tag ${GITHUB_REF_NAME}"
41
+
42
+ - name: Upload distributions
43
+ uses: actions/upload-artifact@v7
44
+ with:
45
+ name: dist
46
+ path: dist/
47
+
48
+ publish:
49
+ name: Publish to PyPI
50
+ needs: build
51
+ runs-on: ubuntu-latest
52
+ environment:
53
+ name: pypi
54
+ url: https://pypi.org/p/pedigree-graph
55
+ permissions:
56
+ id-token: write # trusted publishing (OIDC) — no API token stored
57
+ steps:
58
+ - name: Download distributions
59
+ uses: actions/download-artifact@v8
60
+ with:
61
+ name: dist
62
+ path: dist/
63
+
64
+ - name: Publish to PyPI
65
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,38 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ *.egg
8
+ *.egg-info/
9
+ build/
10
+ dist/
11
+ .eggs/
12
+
13
+ # Virtual envs
14
+ env/
15
+ venv/
16
+ .venv/
17
+
18
+ # Testing / coverage
19
+ .pytest_cache/
20
+ .hypothesis/
21
+ .coverage
22
+ .coverage.*
23
+ htmlcov/
24
+ .tox/
25
+ coverage.xml
26
+
27
+ # Editors
28
+ .vscode/
29
+ .idea/
30
+ *.swp
31
+ *.swo
32
+ .DS_Store
33
+
34
+ # Numba cache
35
+ __pycache__/
36
+
37
+ # Setuptools-scm
38
+ _version.py
@@ -0,0 +1,157 @@
1
+ # Changelog
2
+
3
+ This file tracks public-API changes per release. For per-commit
4
+ history, see `git log`. Historical release notes prior to v0.5.0
5
+ live on the corresponding GitHub release pages.
6
+
7
+ ## v0.6.0
8
+
9
+ - **First release published to PyPI.** `pip install pedigree-graph` now
10
+ works, retiring the `git+https://...@vX.Y.Z` install form that consumers
11
+ had to carry because the project was unavailable on the index.
12
+ Distributions are built and uploaded by a tag-triggered GitHub Actions
13
+ workflow using PyPI trusted publishing (OIDC), so no API token is
14
+ stored in the repository or in CI secrets. Downstream packages pinning
15
+ a git URL can move to a version range such as
16
+ `pedigree-graph>=0.6,<0.7`; the bound is worth keeping tight because
17
+ `PAIR_KINSHIP` and `extract_pairs` are consumed directly by simace,
18
+ fitace, and pedsum.
19
+ - **Packaging metadata completed for the index listing.** `readme`,
20
+ `license` (SPDX `MIT`), `license-files`, `authors`, `classifiers`, and
21
+ `[project.urls]` are now declared, so the PyPI page renders the README
22
+ and links back to the repository and this changelog. The build backend
23
+ floor moved from `setuptools>=64` to `setuptools>=77`, which is where
24
+ PEP 639 SPDX license support lands.
25
+ - **`py.typed` marker added.** The package now advertises inline type
26
+ information under PEP 561, so type checkers read its annotations when
27
+ it is installed as a wheel rather than as an editable source checkout.
28
+ Runtime code is unchanged.
29
+
30
+ ## v0.5.4
31
+
32
+ - **Self-kinship diagonal fixed for inbred individuals when rows are not
33
+ in generation-monotonic order.** The matrix-DP kinship kernel assumed
34
+ every relative discovered during a row's merge walk had a smaller row
35
+ index, but `from_arrays` only requires topological order (parents
36
+ before children). When a relative at an earlier generation had a
37
+ higher row index, the diagonal append broke the row's sorted order and
38
+ the binary search reading `phi(mother, father)` silently returned 0 —
39
+ so the self-kinship diagonal read `(1+0)/2` instead of `(1+F)/2`, and
40
+ the GRM diagonal consumed downstream was wrong for inbred individuals.
41
+ Off-diagonals and the pairwise `compute_pair_kinship` path were
42
+ unaffected, as was any pedigree loaded in generation order (the common
43
+ case, which now hits a zero-overhead fast path).
44
+
45
+ - **Pair-set subtraction and id validation no longer go through
46
+ `np.unique`/`np.isin`.** Three internal hot spots were rewritten with
47
+ sort/searchsorted or diff-based equivalents, verified output-identical
48
+ on real data and under differential fuzzing (5,000 randomized trials
49
+ per site, including empties, duplicates, and ids near the int64
50
+ pair-key bound):
51
+ - `PedigreeGraph._subtract_pairs` now sorts the remove keys and
52
+ binary-searches candidates instead of `np.isin` (~6.9x on a 300k-N
53
+ pedigree; halves `sibling_pairs()` wall time). Same pattern —
54
+ and rationale — as `extract_from_sparse` already used.
55
+ - The duplicate-id check in `_validate_id_column` counts equal
56
+ adjacent elements after a sort instead of `len(np.unique)` (~37x).
57
+ - `pairs_from_groups` detects group boundaries by diffing its
58
+ already-sorted key array instead of re-running `np.unique` on it.
59
+
60
+ Returned pairs, counts, orderings, and error messages are unchanged —
61
+ this change is performance-only.
62
+
63
+ ## v0.5.3
64
+
65
+ - **`count_pairs_streaming` warns when a cousin/collateral residual
66
+ underflows.** The scalar engine derives `H1C`, `1C1R`, `1C2R`, and
67
+ `H1C1R` by inclusion–exclusion — subtracting closer-relationship
68
+ contributions with fixed coefficients that are exact only on non-inbred,
69
+ single-mating pedigrees. On inbred or structurally complex real
70
+ pedigrees those corrections can over-count, driving the raw residual
71
+ negative; it was then silently clamped to `0`, indistinguishable from a
72
+ true absence (e.g. millions of `1C` but `H1C == 0`). The clamp now logs a
73
+ `WARNING` naming the code and the underflow magnitude and points to the
74
+ matrix engine (`extract_pairs`) for an exact count. Returned counts are
75
+ unchanged — only the diagnostic is new.
76
+
77
+ ## v0.5.2
78
+
79
+ - **`count_pairs_streaming()` releases its transient matrices on exit.**
80
+ The scalar streaming counter builds the adjacency powers `_A`…`_A5` and
81
+ now drops them via `_release_pair_matrices()` before returning, exactly
82
+ as `extract_pairs()` already did. Previously they stayed resident for
83
+ the graph's lifetime, inflating peak memory of any later inbreeding / Ne
84
+ / lineage work on the same graph (~400–520 MiB on a 1M-row pedigree).
85
+ The counts remain cached and the matrices rebuild lazily via
86
+ `_ensure_parent_csr()` if pair work runs again, so callers that reached
87
+ into the private `_release_pair_matrices()` after a streaming call (e.g.
88
+ pedsum `summarize`) can drop that workaround. Fixes #4.
89
+
90
+ ## v0.5.0
91
+
92
+ - **Registry-aligned `max_degree` semantics.** `extract_pairs`,
93
+ `count_pairs`, and `count_pairs_streaming` now include exactly the
94
+ relationship categories whose `REL_REGISTRY[code].degree` is less
95
+ than or equal to the cutoff. `max_degree=0` is MZ-only,
96
+ `max_degree=2` stops before 1st cousins, and `max_degree=3`
97
+ includes 1st cousins plus the other degree-3 categories. The public
98
+ defaults changed from `2` to `3` to preserve the old default behavior
99
+ of including 1st cousins.
100
+
101
+ - **`PedigreeGraph.compute_n_ancestors()`** — new cached method.
102
+ Returns the per-individual count of *distinct* strict ancestors
103
+ (`int32`, length `n`). Backed by a sparse boolean transitive
104
+ closure of the parent graph; memory scales with the total closure
105
+ size. Suitable for pedigrees up to ~1M rows on commodity hardware;
106
+ deeper / wider pedigrees may need a future retirement-style DP
107
+ variant.
108
+
109
+ - **`PedigreeGraph.compute_n_descendants()`** — new cached method.
110
+ Returns the per-individual descendant *path count* (`int32`, length
111
+ `n`). In non-inbred pedigrees this equals the unique-descendant
112
+ count; in inbred pedigrees it over-counts a descendant reachable via
113
+ multiple ancestor paths. Matches the convention used historically
114
+ by `pedsum` (`compute_descendants`) and by the matrix engine's GP /
115
+ Av / 1C pair counts. Raises `OverflowError` if any per-individual
116
+ path count exceeds `int32` max (the kernel accumulates in `int64`
117
+ and the cast happens after a bounds check, so deeply inbred
118
+ pedigrees cannot silently wrap).
119
+
120
+ - **`PedigreeGraph.from_arrays(...)`** — accepts a new optional `sex`
121
+ kwarg (`np.ndarray | None`). When omitted, behaviour is unchanged
122
+ (sex defaults to zeros). Existing callers do not need updates.
123
+
124
+ - **Defensive warning for the `sex`-default foot-gun.**
125
+ ``ne_sex_ratio`` and ``ne_variance_family_size`` now emit a
126
+ ``RuntimeWarning`` when ``pg.sex`` is uniformly 0 or 1 — almost
127
+ always a sign that the caller forgot to pass ``sex=`` to
128
+ ``from_arrays`` and is consuming silently-degenerate (single-sex)
129
+ Ne results. The estimator return values are unchanged (``ne=None``);
130
+ the warning is the new diagnostic. Kinship-only callers
131
+ (relationship-pair extraction, GRMs, PA-FGRS) are not affected
132
+ because they don't invoke the sex-aware estimators.
133
+
134
+ - New private kernel module `pedigree_graph/_lineage_kernel.py` houses
135
+ the descendant (numba-JIT) and ancestor (scipy sparse) primitives.
136
+
137
+ - **`PedigreeGraph.count_pairs_streaming(max_degree=2, scope="full")`**
138
+ — new method. Memory-bounded relationship pair counts via pure
139
+ scalar arithmetic; no pair-key arrays are ever materialized. Peak
140
+ memory is O(N) regardless of pedigree density. Returns all 23
141
+ codes from `REL_REGISTRY`. Bit-identical to `count_pairs` for
142
+ the 10 simple codes (`MZ`, `MO`, `FO`, `FS`, `MHS`, `PHS`, `GP`,
143
+ `GGP`, `GGGP`, `G3GP`); approximate (~1% on deep low-inbreeding
144
+ pedigrees) for the 13 cousin / collateral codes (`Av`, `1C`,
145
+ `H1C`, `HAv`, `GAv`, `GGAv`, `G3Av`, `HGAv`, `HGGAv`, `1C1R`,
146
+ `H1C1R`, `1C2R`, `2C`). The scalar path is **full-graph only**:
147
+ `scope='subsample'` raises `NotImplementedError` on graphs built
148
+ via `from_subsample` (use `count_pairs` for subsample-restricted
149
+ counts). See `LIMITATIONS.md` for the full precision contract.
150
+ Benchmark: 5 seconds on a 783K-row stallion-heavy livestock
151
+ pedigree where both matrix and BFS engines OOM at 30 GB.
152
+
153
+ - **`max_degree` validation** — `extract_pairs`, `count_pairs`, and
154
+ `count_pairs_streaming` now reject `max_degree` outside `[0, 5]`
155
+ with `ValueError`. Degree 0 is accepted (cheap codes MZ / MO /
156
+ FO / FS are computed regardless; the cap controls the expensive
157
+ matrix products at degree 2 and above).
@@ -0,0 +1,67 @@
1
+ # pedigree-graph
2
+
3
+ Sparse-matrix pedigree relationship extraction and kinship computation. A
4
+ pedigree is a parent→child DAG of individuals; this context is the vocabulary
5
+ for the relationships between individuals and the two coordinate systems used
6
+ to name them.
7
+
8
+ ## Language
9
+
10
+ ### Coordinate spaces
11
+
12
+ **Graph-space**:
13
+ An individual's row index within the full pedigree the graph was built over.
14
+ _Avoid_: full index, absolute index, internal index
15
+
16
+ **Caller-space**:
17
+ An individual's row index within the subsample the caller supplied, which may
18
+ order or omit individuals differently from the full pedigree.
19
+ _Avoid_: subsample index, df index, external index
20
+
21
+ ### Relationships
22
+
23
+ **Relationship pair**:
24
+ An unordered pair of individuals sharing a relationship category. Some pair
25
+ arrays preserve relationship orientation (for example descendant→ancestor for
26
+ lineal relationships); pair-key encodings canonicalize as `(lo, hi)` with
27
+ `lo < hi`.
28
+ _Avoid_: edge, link, tuple; assuming every pair array is already canonical
29
+
30
+ **Relationship category**:
31
+ A class of relationship identified by a short code (e.g. `FS`, `MHS`, `1C`),
32
+ defined by `(up, down, n_ancestors)` — meioses up to the common ancestor(s),
33
+ meioses back down, and whether the connecting ancestor is a single individual
34
+ (half / lineal) or a mated pair (full).
35
+ _Avoid_: relationship type (when the code is meant), kind
36
+
37
+ **Degree**:
38
+ The kinship distance of a relationship category — `0` for MZ twins, `1` for
39
+ parent-offspring and full sibs, and so on. A degree cutoff includes relationship
40
+ categories whose degree is less than or equal to the cutoff.
41
+
42
+ **Nominal kinship**:
43
+ The kinship coefficient implied by a relationship category's `(up, down,
44
+ n_ancestors)` formula, assuming a single relationship path and no inbreeding
45
+ or co-coalescence.
46
+ _Avoid_: exact kinship
47
+
48
+ **Exact pairwise kinship**:
49
+ The kinship coefficient for a particular pair of individuals after summing all
50
+ pedigree paths, including inbreeding, MZ co-coalescence, and duplicate
51
+ relationship paths such as double cousins.
52
+ _Avoid_: nominal kinship
53
+
54
+ ## Relationships
55
+
56
+ - A **relationship pair** holds two individuals and belongs to one **relationship category**; canonical ordering is a storage/encoding choice, not part of the relationship itself.
57
+ - Every individual index is expressed in either **graph-space** or **caller-space**; the same individual generally has a different index in each.
58
+ - A pair returned to a caller is in **caller-space**; the kinship matrix is indexed in **graph-space**. Converting between the two is required whenever both meet.
59
+
60
+ ## Example dialogue
61
+
62
+ > **Reviewer:** "`extract_pairs` gave me pair `(1, 0)` for the MZ twins — why is its kinship `0.0`?"
63
+ > **Author:** "Those indices are **caller-space** — you reversed the subsample. The kinship matrix is **graph-space**, so indexing it with caller indices reads the wrong cell. The pair has to be mapped back to graph-space first."
64
+
65
+ ## Flagged ambiguities
66
+
67
+ - "index" alone is ambiguous between **graph-space** and **caller-space** — always qualify which space, since the same individual differs between them and conflating them caused a kinship-lookup bug (PGQ-001).
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ryan Waples
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,183 @@
1
+ # Limitations
2
+
3
+ Current scaling and correctness limitations of the relationship-pair
4
+ engines. Read before reaching for `extract_pairs` / `count_pairs` on
5
+ pair-dense pedigrees.
6
+
7
+ ## Pair counting is O(answer size) for the matrix and BFS engines
8
+
9
+ The matrix and BFS engines both materialise every pair as an
10
+ ``(idx1, idx2)`` array before counting it:
11
+
12
+ - ``extract_pairs()`` returns ``dict[code, (np.ndarray, np.ndarray)]``.
13
+ - ``count_pairs()`` is a thin facade — it runs ``extract_pairs()`` and
14
+ returns ``{code: len(idx_array)}``.
15
+
16
+ Memory is therefore proportional to the total relationship-pair count,
17
+ **not** the pedigree size.
18
+
19
+ ### Worst case: prolific-stallion livestock pedigrees
20
+
21
+ Stallion-driven half-sib density is the engine's hard wall. On a real
22
+ horse-breed pedigree (783K individuals, one all-time-great stallion
23
+ siring 2,500 horses, top-sire grand-offspring set ~50K):
24
+
25
+ - Paternal half-sib pair count = ~156 M
26
+ - ``_half_sib_matrix`` materialisation = ~312 M nonzeros (~3 GB) just
27
+ for the symmetric MHS+PHS matrix
28
+ - Per-grandparent grandchild bucket for ``_cousin_pairs`` enumeration
29
+ reaches ``C(50K, 2) ≈ 1.25 B`` candidate pairs through one stallion
30
+ grandparent (~10 GB of int64 keys)
31
+ - ``_A2 @ _A3.T`` for 1C1R / H1C1R has row nnz ~2,500 (one stallion's
32
+ great-grandchildren spread); chunk sizes blow up at default
33
+ ``chunk_rows``
34
+
35
+ Both engines OOM on this pedigree well before producing a count, even
36
+ on 30 GB hosts. Matrix OOMs in ``A_f @ A_f.T`` (PHS sparse product);
37
+ BFS OOMs in the numba cousin enumeration kernel.
38
+
39
+ ### Operational workarounds
40
+
41
+ 1. ``pedsum --no-pairs`` skips the pair-counting stage entirely. The
42
+ horse pedigree completes in ~30s with 1 GB peak RSS this way,
43
+ producing every other section (size structure, family, mating,
44
+ lineage, founder contribution, inbreeding, effective size) but
45
+ returning stub values for the 23 relationship counts (``pairs: {}``
46
+ and ``relationship_summary.computed: false``).
47
+
48
+ 2. ``PedigreeGraph.count_pairs_streaming()`` — pure-scalar per-anchor
49
+ arithmetic, O(N) memory, exact on 10 simple codes, approximate
50
+ within ~1% on 13 cousin / collateral codes for deep low-inbreeding
51
+ pedigrees. See the precision contract below.
52
+
53
+ ## ``count_pairs_streaming`` precision contract
54
+
55
+ The scalar path is **full-graph only** — ``scope='subsample'`` raises
56
+ ``NotImplementedError`` on graphs constructed via ``from_subsample``.
57
+ Use ``count_pairs`` for subsample-restricted counts.
58
+
59
+ - **Exact** on 10 codes (bit-identical to ``count_pairs`` on every
60
+ input):
61
+ ``MZ``, ``MO``, ``FO``, ``FS``, ``MHS``, ``PHS``,
62
+ ``GP``, ``GGP``, ``GGGP``, ``G3GP``.
63
+
64
+ - **Approximate** on 13 cousin / collateral codes:
65
+ ``Av``, ``1C``, ``H1C``, ``HAv``, ``GAv``, ``GGAv``, ``G3Av``,
66
+ ``HGAv``, ``HGGAv``, ``1C1R``, ``H1C1R``, ``1C2R``, ``2C``.
67
+
68
+ Scalar formulas assume each individual has the full complement of
69
+ known grandparents at the relevant depth, so constants like
70
+ ``4*FS`` in the ``H1C`` correction over-subtract on shallow
71
+ pedigrees; ``H1C`` may clamp to ``0`` on depth ≤ 3. Twin parents
72
+ and sib-mating offspring also push the formulas off bit-identity
73
+ because the inclusion-exclusion terms assume neither pattern.
74
+
75
+ On the synthetic ``small_pedigree`` fixture (3000 rows, depth 3,
76
+ ~0.5% sib-mating, 10 twin pairs): ``Av`` off by 3, ``HAv`` off by
77
+ 11, ``1C`` off by 30, ``H1C`` clamped to 0. On deep livestock
78
+ pedigrees (depth ≥ 5, low inbreeding) the formulas are accurate to
79
+ better than 1%.
80
+
81
+ The horse-pedigree benchmark (N=783K, mean F=0.007) completes in
82
+ ~5 seconds with peak RSS ~730 MB.
83
+
84
+ ## Cousin-code matrix/BFS divergence on inbred input
85
+
86
+ Independent of scaling, the ``matrix`` and ``bfs`` engines give
87
+ **different counts** on inbred pedigrees for the four
88
+ cousin-multiplicity codes: ``1C1R``, ``H1C1R``, ``1C2R``, ``2C``.
89
+
90
+ - **Matrix engine** uses path multiplicity: ``M.data >= 2`` (full) or
91
+ ``M.data == 1`` (half) thresholds on ``_A2 @ _A3.T`` / ``_A2 @ _A4.T``
92
+ / ``_A3 @ _A3.T``. A pair sharing an ancestor via two distinct
93
+ paths is counted twice in the matrix entry; the threshold
94
+ classifies based on this multiplicity.
95
+ - **BFS engine** uses distinct-shared-ancestor semantics: a pair
96
+ sharing N distinct ancestors at the relevant depth is counted once
97
+ regardless of paths.
98
+
99
+ The divergence is documented in
100
+ ``tests/test_experimental.py:171`` (the ``inbred_with_cousins_pedigree``
101
+ fixture) and asserted in ``test_inbred_with_cousins_cousin_codes_diverge``.
102
+
103
+ ``extract_pairs(scope="full")`` returns matrix-engine values by
104
+ default. Callers needing BFS-distinct semantics on inbred input must
105
+ use ``pedigree_graph.experimental.count_pairs_bfs`` and accept the
106
+ matrix-vs-BFS difference for those four codes.
107
+
108
+ ## ``int8`` overflow risk in BFS ``P_k`` boolean matmul
109
+
110
+ The BFS engine (`pedigree_graph.experimental.count_pairs_bfs`) uses
111
+ ``np.int8`` for ``P_k.data`` during the boolean matmul stages.
112
+ Theoretically vulnerable to silent path-count overflow under extreme
113
+ consanguinity: more than 127 distinct paths to a single ``(i, X)`` pair
114
+ before the ``M.data[:] = 1`` clamp will wrap.
115
+
116
+ Empirically not seen on any tested pedigree. Switch to ``int32`` if it
117
+ ever bites — the change is a one-line dtype swap at the matmul site;
118
+ memory cost is 4× on the intermediate matrices.
119
+
120
+ ## ``compute_n_ancestors`` memory scales with ``sum_i n_ancestors[i]``
121
+
122
+ ``PedigreeGraph.compute_n_ancestors`` is a sparse boolean transitive
123
+ closure of the parent graph (``_lineage_kernel._compute_n_ancestors``).
124
+ Memory scales with ``sum_i n_ancestors[i]``, so very deep / very wide
125
+ pedigrees can hit RAM limits:
126
+
127
+ - N=100K, G=10, random mating → 2.2 s, peak RSS ~0.5 GB.
128
+ - N=10M with saturated ancestry → extrapolates beyond commodity hardware.
129
+
130
+ A retirement-style DP (analogous to the F kernel's row-retirement
131
+ optimisation in ``_kinship_kernel``) would bound peak memory to the
132
+ live frontier rather than the cumulative ancestor set. Deferred until
133
+ a user hits the wall.
134
+
135
+ ## Half-founders and missing parents
136
+
137
+ Both engines accept half-founders (one parent known, one missing).
138
+ The sibling group-by filters to known parents only:
139
+
140
+ - ``FS`` requires BOTH parents known on both individuals.
141
+ - ``MHS`` only considers individuals with mother known.
142
+ - ``PHS`` only considers individuals with father known.
143
+
144
+ This matches the standard convention but can surprise callers who
145
+ expect half-founders to contribute to half-sib counts on the
146
+ "missing" side. They don't.
147
+
148
+ ## Subsample-restricted counts are O(full pair count)
149
+
150
+ ``PedigreeGraph.from_subsample(...)`` builds a graph that returns
151
+ subsample-filtered pair arrays from ``extract_pairs``, but the
152
+ underlying enumeration runs over the FULL pedigree first (raw counts
153
+ saved, then sample mask applied). Memory is bounded by full-pedigree
154
+ pair counts, not the subsample.
155
+
156
+ For a 10% subsample of a stallion-heavy pedigree, this is still
157
+ OOM-prone because the full-pedigree intermediate doesn't shrink.
158
+
159
+ ## What this file does NOT cover
160
+
161
+ - Lineal-code counting limitations (none significant — ``_A^k.nnz`` is
162
+ O(N · depth) and tractable to N=10M+).
163
+ - F (inbreeding coefficient) scaling — covered by
164
+ ``pedigree_graph._kinship_kernel`` and its own row-retirement
165
+ optimisation work.
166
+ - Effective size estimator scaling — covered by
167
+ ``pedigree_graph._effective_size`` and the ``skip_ne_coancestry``
168
+ knob.
169
+ - BFS engine internal limitations — see the ``int8`` overflow section
170
+ above for the path-count overflow case, and GitHub issues
171
+ [#2 (numba kernel parallelisation)](https://github.com/rwaples/pedigree-graph/issues/2)
172
+ and [#3 (10M+ scaling test)](https://github.com/rwaples/pedigree-graph/issues/3)
173
+ for open performance / scalability questions.
174
+
175
+ ## Last updated
176
+
177
+ 2026-05-20 — ``int8`` overflow risk and ``compute_n_ancestors``
178
+ scalability sections added; BFS internal follow-ups re-homed from
179
+ retired ``external/pedsum/STATUS.md`` to GitHub issues #2 and #3.
180
+
181
+ 2026-05-19 — ``count_pairs_streaming`` precision contract
182
+ reconciled; ``Av`` documented as approximate; stale
183
+ "count-only-experiment-didn't-ship" narrative removed.