dnakit 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. dnakit/__init__.py +108 -0
  2. dnakit/_version.py +5 -0
  3. dnakit/alignment/__init__.py +6 -0
  4. dnakit/alignment/pairwise.py +512 -0
  5. dnakit/alignment/results.py +188 -0
  6. dnakit/backends/__init__.py +80 -0
  7. dnakit/backends/external.py +517 -0
  8. dnakit/backends/registry.py +207 -0
  9. dnakit/batch.py +452 -0
  10. dnakit/cache/__init__.py +5 -0
  11. dnakit/cache/store.py +424 -0
  12. dnakit/chunking.py +920 -0
  13. dnakit/cli/__init__.py +5 -0
  14. dnakit/cli/__main__.py +6 -0
  15. dnakit/cli/app.py +814 -0
  16. dnakit/cli/workflow.py +94 -0
  17. dnakit/config/__init__.py +5 -0
  18. dnakit/config/loader.py +381 -0
  19. dnakit/core/__init__.py +80 -0
  20. dnakit/core/_json.py +206 -0
  21. dnakit/core/backend_info.py +70 -0
  22. dnakit/core/collection.py +163 -0
  23. dnakit/core/coordinates.py +295 -0
  24. dnakit/core/enums.py +95 -0
  25. dnakit/core/facade.py +719 -0
  26. dnakit/core/feature.py +90 -0
  27. dnakit/core/gap.py +82 -0
  28. dnakit/core/issues.py +62 -0
  29. dnakit/core/provenance.py +305 -0
  30. dnakit/core/record.py +161 -0
  31. dnakit/core/results.py +134 -0
  32. dnakit/core/sequence.py +327 -0
  33. dnakit/datasets/__init__.py +100 -0
  34. dnakit/datasets/_advanced_shared.py +198 -0
  35. dnakit/datasets/_metadata.py +33 -0
  36. dnakit/datasets/advanced_deduplicate.py +183 -0
  37. dnakit/datasets/advanced_split.py +302 -0
  38. dnakit/datasets/clustering.py +286 -0
  39. dnakit/datasets/config.py +531 -0
  40. dnakit/datasets/deduplicate.py +303 -0
  41. dnakit/datasets/evaluation.py +244 -0
  42. dnakit/datasets/filter.py +135 -0
  43. dnakit/datasets/neural_clustering.py +221 -0
  44. dnakit/datasets/results.py +520 -0
  45. dnakit/datasets/split.py +450 -0
  46. dnakit/descriptors/__init__.py +102 -0
  47. dnakit/descriptors/_dinucleotide.py +345 -0
  48. dnakit/descriptors/_shared.py +180 -0
  49. dnakit/descriptors/all.py +917 -0
  50. dnakit/descriptors/basic.py +193 -0
  51. dnakit/descriptors/codon.py +195 -0
  52. dnakit/descriptors/complexity.py +201 -0
  53. dnakit/descriptors/entropy.py +101 -0
  54. dnakit/descriptors/homopolymer.py +105 -0
  55. dnakit/descriptors/kmer.py +88 -0
  56. dnakit/descriptors/results.py +286 -0
  57. dnakit/descriptors/schema.py +414 -0
  58. dnakit/descriptors/window.py +203 -0
  59. dnakit/download/__init__.py +54 -0
  60. dnakit/download/catalogs.py +166 -0
  61. dnakit/download/ena.py +131 -0
  62. dnakit/download/encode.py +103 -0
  63. dnakit/download/files.py +469 -0
  64. dnakit/download/indexes.py +233 -0
  65. dnakit/download/metadata.py +164 -0
  66. dnakit/download/models.py +227 -0
  67. dnakit/download/ncbi.py +333 -0
  68. dnakit/download/sequence.py +133 -0
  69. dnakit/download/ucsc.py +99 -0
  70. dnakit/evaluation/__init__.py +81 -0
  71. dnakit/evaluation/_shared.py +288 -0
  72. dnakit/evaluation/collection.py +270 -0
  73. dnakit/evaluation/config.py +591 -0
  74. dnakit/evaluation/distribution.py +287 -0
  75. dnakit/evaluation/frechet.py +223 -0
  76. dnakit/evaluation/generative.py +381 -0
  77. dnakit/evaluation/reference.py +409 -0
  78. dnakit/evaluation/results.py +137 -0
  79. dnakit/evaluation/scorecard.py +211 -0
  80. dnakit/evaluation/sequence.py +403 -0
  81. dnakit/evaluation/synthesis.py +249 -0
  82. dnakit/exceptions.py +189 -0
  83. dnakit/fingerprints/__init__.py +109 -0
  84. dnakit/fingerprints/_shared.py +116 -0
  85. dnakit/fingerprints/advanced.py +829 -0
  86. dnakit/fingerprints/bit.py +229 -0
  87. dnakit/fingerprints/encoding.py +351 -0
  88. dnakit/fingerprints/kmer.py +170 -0
  89. dnakit/fingerprints/results.py +531 -0
  90. dnakit/fingerprints/sketch.py +246 -0
  91. dnakit/io/__init__.py +112 -0
  92. dnakit/io/_advanced_common.py +105 -0
  93. dnakit/io/_formats.py +994 -0
  94. dnakit/io/annotations.py +1179 -0
  95. dnakit/io/api.py +759 -0
  96. dnakit/io/config.py +315 -0
  97. dnakit/io/genbank.py +523 -0
  98. dnakit/io/indexed.py +1146 -0
  99. dnakit/io/metadata.py +452 -0
  100. dnakit/io/parquet.py +56 -0
  101. dnakit/io/results.py +131 -0
  102. dnakit/io/source.py +138 -0
  103. dnakit/io/tables.py +1237 -0
  104. dnakit/molbio/__init__.py +101 -0
  105. dnakit/molbio/_shared.py +266 -0
  106. dnakit/molbio/assembly.py +279 -0
  107. dnakit/molbio/crispr.py +181 -0
  108. dnakit/molbio/optimization.py +897 -0
  109. dnakit/molbio/primer3_design.py +667 -0
  110. dnakit/molbio/primers.py +899 -0
  111. dnakit/molbio/restriction.py +573 -0
  112. dnakit/molbio/results.py +583 -0
  113. dnakit/ops/__init__.py +588 -0
  114. dnakit/ops/_common.py +201 -0
  115. dnakit/ops/circular.py +193 -0
  116. dnakit/ops/concat.py +204 -0
  117. dnakit/ops/direction.py +27 -0
  118. dnakit/ops/edit.py +443 -0
  119. dnakit/ops/evolution.py +991 -0
  120. dnakit/ops/mutation.py +151 -0
  121. dnakit/ops/records.py +1012 -0
  122. dnakit/ops/translation.py +257 -0
  123. dnakit/patterns/__init__.py +81 -0
  124. dnakit/patterns/_shared.py +282 -0
  125. dnakit/patterns/coding.py +360 -0
  126. dnakit/patterns/crispr.py +337 -0
  127. dnakit/patterns/motif.py +721 -0
  128. dnakit/patterns/regions.py +341 -0
  129. dnakit/patterns/repeats.py +480 -0
  130. dnakit/patterns/restriction.py +281 -0
  131. dnakit/patterns/results.py +411 -0
  132. dnakit/py.typed +1 -0
  133. dnakit/references/__init__.py +27 -0
  134. dnakit/references/models.py +155 -0
  135. dnakit/references/ncbi.py +785 -0
  136. dnakit/representations/__init__.py +29 -0
  137. dnakit/representations/backends.py +928 -0
  138. dnakit/representations/checkpoints.py +356 -0
  139. dnakit/representations/extraction.py +220 -0
  140. dnakit/representations/models.py +395 -0
  141. dnakit/representations/results.py +79 -0
  142. dnakit/search/__init__.py +132 -0
  143. dnakit/search/_http.py +271 -0
  144. dnakit/search/_shared.py +108 -0
  145. dnakit/search/blast.py +516 -0
  146. dnakit/search/ena.py +245 -0
  147. dnakit/search/encode.py +102 -0
  148. dnakit/search/ensembl.py +874 -0
  149. dnakit/search/models.py +159 -0
  150. dnakit/search/ncbi.py +756 -0
  151. dnakit/search/ucsc.py +388 -0
  152. dnakit/secondary_structure/__init__.py +38 -0
  153. dnakit/secondary_structure/dotbracket.py +410 -0
  154. dnakit/secondary_structure/nupack.py +589 -0
  155. dnakit/secondary_structure/results.py +142 -0
  156. dnakit/similarity/__init__.py +125 -0
  157. dnakit/similarity/_shared.py +152 -0
  158. dnakit/similarity/approximate.py +258 -0
  159. dnakit/similarity/compare.py +162 -0
  160. dnakit/similarity/dashing.py +936 -0
  161. dnakit/similarity/distance.py +279 -0
  162. dnakit/similarity/index.py +365 -0
  163. dnakit/similarity/matrix.py +254 -0
  164. dnakit/similarity/results.py +676 -0
  165. dnakit/similarity/search.py +286 -0
  166. dnakit/similarity/sketch.py +96 -0
  167. dnakit/similarity/vector.py +662 -0
  168. dnakit/standardize/__init__.py +77 -0
  169. dnakit/standardize/_shared.py +100 -0
  170. dnakit/standardize/config.py +411 -0
  171. dnakit/standardize/gaps.py +1060 -0
  172. dnakit/standardize/normalize.py +565 -0
  173. dnakit/standardize/results.py +247 -0
  174. dnakit/standardize/validate.py +487 -0
  175. dnakit/structure3d/__init__.py +47 -0
  176. dnakit/structure3d/_shared.py +47 -0
  177. dnakit/structure3d/geometry.py +773 -0
  178. dnakit/structure3d/pdb.py +253 -0
  179. dnakit/structure3d/results.py +233 -0
  180. dnakit/structure3d/threedna.py +174 -0
  181. dnakit/thermodynamics/__init__.py +112 -0
  182. dnakit/thermodynamics/_shared.py +92 -0
  183. dnakit/thermodynamics/backends.py +653 -0
  184. dnakit/thermodynamics/calculations.py +837 -0
  185. dnakit/thermodynamics/config.py +175 -0
  186. dnakit/thermodynamics/equilibrium.py +340 -0
  187. dnakit/thermodynamics/optics.py +385 -0
  188. dnakit/thermodynamics/parameters.py +79 -0
  189. dnakit/thermodynamics/results.py +445 -0
  190. dnakit/visualization/__init__.py +43 -0
  191. dnakit/visualization/_svg.py +185 -0
  192. dnakit/visualization/advanced.py +314 -0
  193. dnakit/visualization/config.py +255 -0
  194. dnakit/visualization/export.py +349 -0
  195. dnakit/visualization/matrix.py +279 -0
  196. dnakit/visualization/report.py +431 -0
  197. dnakit/visualization/results.py +236 -0
  198. dnakit/visualization/sequence.py +459 -0
  199. dnakit/workflows/__init__.py +45 -0
  200. dnakit/workflows/manifest.py +365 -0
  201. dnakit/workflows/runner.py +1548 -0
  202. dnakit/workflows/schema.py +898 -0
  203. dnakit-0.1.0.dev0.dist-info/METADATA +126 -0
  204. dnakit-0.1.0.dev0.dist-info/RECORD +210 -0
  205. dnakit-0.1.0.dev0.dist-info/WHEEL +5 -0
  206. dnakit-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  207. dnakit-0.1.0.dev0.dist-info/licenses/DISCLAIMER.md +26 -0
  208. dnakit-0.1.0.dev0.dist-info/licenses/LICENSE +21 -0
  209. dnakit-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.md +73 -0
  210. dnakit-0.1.0.dev0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,303 @@
1
+ """Exact and reverse-complement-aware record deduplication."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Hashable, Iterable
6
+ from typing import TypeAlias
7
+
8
+ from dnakit.core import DNARecord, DNASequence, DNASet, Gap, Topology
9
+ from dnakit.core._json import JSONValue
10
+ from dnakit.exceptions import ConfigurationError
11
+
12
+ from ._metadata import metadata_value_key
13
+ from .config import DedupEquivalence, DeduplicationConfig
14
+ from .results import DedupAction, DedupGroup, DeduplicationResult, Orientation
15
+
16
+ SequenceKey: TypeAlias = tuple[tuple[str | Gap, ...], object, object]
17
+ EquivalenceKey: TypeAlias = SequenceKey | frozenset[SequenceKey] | tuple[str, object, object]
18
+
19
+
20
+ def _least_rotation(symbols: str) -> tuple[str, int]:
21
+ """Return the lexicographically least rotation and its first offset."""
22
+
23
+ if not symbols:
24
+ return symbols, 0
25
+ length = len(symbols)
26
+ doubled = symbols + symbols
27
+ left, right, matched = 0, 1, 0
28
+ while left < length and right < length and matched < length:
29
+ left_symbol = doubled[left + matched]
30
+ right_symbol = doubled[right + matched]
31
+ if left_symbol == right_symbol:
32
+ matched += 1
33
+ continue
34
+ if left_symbol > right_symbol:
35
+ left += matched + 1
36
+ if left <= right:
37
+ left = right + 1
38
+ else:
39
+ right += matched + 1
40
+ if right <= left:
41
+ right = left + 1
42
+ matched = 0
43
+ best_offset = min(left, right)
44
+ return doubled[best_offset : best_offset + length], best_offset
45
+
46
+
47
+ def _sequence_key(sequence: DNASequence) -> SequenceKey:
48
+ # Alphabet is deliberately omitted: ACGT content has the same identity
49
+ # whether its validated declaration is strict or IUPAC.
50
+ return (sequence.parts, sequence.topology, sequence.strandedness)
51
+
52
+
53
+ def _equivalence_key(sequence: DNASequence, equivalence: DedupEquivalence) -> EquivalenceKey:
54
+ forward = _sequence_key(sequence)
55
+ if equivalence == "exact":
56
+ return forward
57
+ if equivalence in {"circular", "circular_reverse_complement"}:
58
+ if sequence.topology is not Topology.CIRCULAR:
59
+ raise ConfigurationError(
60
+ "Circular equivalence requires every sequence to declare circular topology.",
61
+ code="CIRCULAR_DEDUPLICATION_TOPOLOGY_REQUIRED",
62
+ )
63
+ if sequence.is_gapped:
64
+ raise ConfigurationError(
65
+ "Circular rotation equivalence does not support Gap objects.",
66
+ code="CIRCULAR_DEDUPLICATION_GAP_UNSUPPORTED",
67
+ )
68
+ canonical, _ = _least_rotation(sequence.symbols)
69
+ if equivalence == "circular_reverse_complement":
70
+ reverse, _ = _least_rotation(sequence.reverse_complement().symbols)
71
+ canonical = min(canonical, reverse)
72
+ return canonical, sequence.topology, sequence.strandedness
73
+ return frozenset((forward, _sequence_key(sequence.reverse_complement())))
74
+
75
+
76
+ def _materialize(records: Iterable[DNARecord]) -> tuple[DNARecord, ...]:
77
+ try:
78
+ materialized = tuple(records)
79
+ except TypeError as exc:
80
+ raise ConfigurationError(
81
+ "records must be an iterable of DNARecord objects.",
82
+ code="INVALID_DATASET_RECORDS",
83
+ ) from exc
84
+ for index, record in enumerate(materialized):
85
+ if not isinstance(record, DNARecord):
86
+ raise ConfigurationError(
87
+ "Every dataset item must be a DNARecord.",
88
+ code="INVALID_DATASET_RECORD",
89
+ context={"input_index": index, "type": type(record).__name__},
90
+ )
91
+ return materialized
92
+
93
+
94
+ def _quality_score(record: DNARecord) -> float:
95
+ quality = record.letter_annotations.get("phred_quality")
96
+ if not quality:
97
+ return float("-inf")
98
+ return sum(quality) / len(quality)
99
+
100
+
101
+ def _representative_index(
102
+ members: tuple[int, ...], records: tuple[DNARecord, ...], policy: str
103
+ ) -> int:
104
+ if policy == "first":
105
+ return members[0]
106
+ if policy == "last":
107
+ return members[-1]
108
+ return max(members, key=lambda index: (_quality_score(records[index]), -index))
109
+
110
+
111
+ def _conflict(
112
+ members: tuple[int, ...],
113
+ records: tuple[DNARecord, ...],
114
+ field: str | None,
115
+ ) -> tuple[bool, tuple[JSONValue, ...], int]:
116
+ if field is None:
117
+ return False, (), 0
118
+ values: list[JSONValue] = []
119
+ missing = 0
120
+ for index in members:
121
+ if field not in records[index].metadata:
122
+ missing += 1
123
+ else:
124
+ values.append(records[index].metadata[field])
125
+ unique_by_key = {metadata_value_key(value): value for value in values}
126
+ unique_values = tuple(unique_by_key[key] for key in sorted(unique_by_key, key=repr))
127
+ distinct_count = len(unique_values) + int(missing > 0)
128
+ return distinct_count > 1, unique_values, missing
129
+
130
+
131
+ def _merge_metadata(representative: DNARecord, members: tuple[DNARecord, ...]) -> DNARecord:
132
+ merged: dict[str, object] = dict(representative.metadata)
133
+ keys = {key for member in members for key in member.metadata}
134
+ for key in sorted(keys):
135
+ present = [member.metadata[key] for member in members if key in member.metadata]
136
+ if not present:
137
+ continue
138
+ first_key = metadata_value_key(present[0])
139
+ if all(metadata_value_key(value) == first_key for value in present[1:]):
140
+ merged.setdefault(key, present[0])
141
+ return DNARecord(
142
+ representative.sequence,
143
+ representative.id,
144
+ description=representative.description,
145
+ features=representative.features,
146
+ metadata=merged,
147
+ letter_annotations=representative.letter_annotations,
148
+ )
149
+
150
+
151
+ def _orientations(
152
+ representative: DNARecord,
153
+ members: tuple[int, ...],
154
+ records: tuple[DNARecord, ...],
155
+ equivalence: DedupEquivalence,
156
+ ) -> tuple[Orientation, ...]:
157
+ representative_key = _sequence_key(representative.sequence)
158
+ reverse_key = _sequence_key(representative.sequence.reverse_complement())
159
+ values: list[Orientation] = []
160
+ for index in members:
161
+ member_key = _sequence_key(records[index].sequence)
162
+ orientation: Orientation = "forward"
163
+ if equivalence == "reverse_complement" and member_key != representative_key:
164
+ if member_key != reverse_key:
165
+ raise AssertionError("Member does not match its reverse-complement group.")
166
+ orientation = "reverse_complement"
167
+ values.append(orientation)
168
+ return tuple(values)
169
+
170
+
171
+ def _circular_transforms(
172
+ representative: DNARecord,
173
+ members: tuple[int, ...],
174
+ records: tuple[DNARecord, ...],
175
+ equivalence: DedupEquivalence,
176
+ ) -> tuple[tuple[Orientation, ...], tuple[int | None, ...]]:
177
+ if equivalence not in {"circular", "circular_reverse_complement"}:
178
+ return _orientations(representative, members, records, equivalence), ()
179
+ orientations: list[Orientation] = []
180
+ offsets: list[int | None] = []
181
+ for index in members:
182
+ sequence = records[index].sequence
183
+ forward, forward_offset = _least_rotation(sequence.symbols)
184
+ orientation: Orientation = "forward"
185
+ offset = forward_offset
186
+ if equivalence == "circular_reverse_complement":
187
+ reverse, reverse_offset = _least_rotation(sequence.reverse_complement().symbols)
188
+ if reverse < forward:
189
+ orientation = "reverse_complement"
190
+ offset = reverse_offset
191
+ orientations.append(orientation)
192
+ offsets.append(offset)
193
+ return tuple(orientations), tuple(offsets)
194
+
195
+
196
+ def deduplicate(
197
+ records: Iterable[DNARecord],
198
+ *,
199
+ equivalence: DedupEquivalence = "exact",
200
+ config: DeduplicationConfig | None = None,
201
+ ) -> DeduplicationResult:
202
+ """Deduplicate normalized records while preserving a complete group audit."""
203
+
204
+ if equivalence not in {
205
+ "exact",
206
+ "reverse_complement",
207
+ "circular",
208
+ "circular_reverse_complement",
209
+ }:
210
+ raise ConfigurationError(
211
+ "Unknown deduplication equivalence.",
212
+ code="INVALID_DEDUPLICATION_EQUIVALENCE",
213
+ context={"equivalence": equivalence},
214
+ )
215
+ resolved = DeduplicationConfig() if config is None else config
216
+ if not isinstance(resolved, DeduplicationConfig):
217
+ raise ConfigurationError(
218
+ "config must be DeduplicationConfig or None.",
219
+ code="INVALID_DEDUPLICATION_CONFIG",
220
+ )
221
+ materialized = _materialize(records)
222
+ grouped: dict[Hashable, list[int]] = {}
223
+ for index, record in enumerate(materialized):
224
+ grouped.setdefault(_equivalence_key(record.sequence, equivalence), []).append(index)
225
+
226
+ output: list[DNARecord] = []
227
+ audit_groups: list[DedupGroup] = []
228
+ conflict_count = 0
229
+ for group_index, raw_members in enumerate(grouped.values()):
230
+ members = tuple(raw_members)
231
+ representative_index = _representative_index(
232
+ members, materialized, resolved.representative_policy
233
+ )
234
+ representative = materialized[representative_index]
235
+ orientations, rotation_offsets = _circular_transforms(
236
+ representative, members, materialized, equivalence
237
+ )
238
+ conflict, conflict_values, missing_count = _conflict(
239
+ members, materialized, resolved.conflict_field
240
+ )
241
+ if conflict:
242
+ conflict_count += 1
243
+ if resolved.conflict_policy == "error":
244
+ raise ConfigurationError(
245
+ "A duplicate group contains conflicting metadata values.",
246
+ code="DEDUPLICATION_METADATA_CONFLICT",
247
+ context={
248
+ "group_index": group_index,
249
+ "member_ids": [materialized[index].id for index in members],
250
+ "conflict_field": resolved.conflict_field,
251
+ },
252
+ hint="Choose drop_group, keep_representative, or keep_all explicitly.",
253
+ )
254
+ action: DedupAction = "deduplicated"
255
+ if conflict and resolved.conflict_policy == "drop_group":
256
+ action = "dropped"
257
+ elif conflict and resolved.conflict_policy == "keep_all":
258
+ action = "kept_all"
259
+ output.extend(materialized[index] for index in members)
260
+ else:
261
+ chosen = representative
262
+ if resolved.merge_metadata:
263
+ chosen = _merge_metadata(
264
+ representative, tuple(materialized[index] for index in members)
265
+ )
266
+ output.append(chosen)
267
+ audit_groups.append(
268
+ DedupGroup(
269
+ group_index=group_index,
270
+ representative_id=representative.id,
271
+ member_ids=tuple(materialized[index].id for index in members),
272
+ orientations=orientations,
273
+ conflict=conflict,
274
+ conflict_values=conflict_values,
275
+ missing_conflict_value_count=missing_count,
276
+ action=action,
277
+ rotation_offsets=rotation_offsets,
278
+ rotation_offset_definition=(
279
+ "zero-based left-rotation offset to the lexicographically least "
280
+ "representation after the reported orientation"
281
+ if equivalence in {"circular", "circular_reverse_complement"}
282
+ else None
283
+ ),
284
+ )
285
+ )
286
+ dataset = DNASet(output)
287
+ return DeduplicationResult(
288
+ records=dataset,
289
+ groups=tuple(audit_groups),
290
+ equivalence=equivalence,
291
+ representative_policy=resolved.representative_policy,
292
+ conflict_field=resolved.conflict_field,
293
+ conflict_policy=resolved.conflict_policy,
294
+ merge_metadata=resolved.merge_metadata,
295
+ input_count=len(materialized),
296
+ output_count=len(dataset),
297
+ duplicate_count=len(materialized) - len(audit_groups),
298
+ conflicted_group_count=conflict_count,
299
+ removed_count=len(materialized) - len(dataset),
300
+ )
301
+
302
+
303
+ __all__ = ["deduplicate"]
@@ -0,0 +1,244 @@
1
+ """Exhaustive bounded leakage detection and split-quality metrics."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Mapping, Sequence
7
+
8
+ from dnakit.core import DNA, DNASet
9
+ from dnakit.core._json import FrozenDict
10
+ from dnakit.exceptions import ConfigurationError
11
+
12
+ from ._advanced_shared import pair_similarity, validate_pairwise_sequences
13
+ from ._metadata import metadata_value_key
14
+ from .config import LeakageConfig
15
+ from .results import (
16
+ LeakageEvent,
17
+ LeakageReport,
18
+ SplitAssignment,
19
+ SplitQualityResult,
20
+ )
21
+
22
+
23
+ def _validate_splits(
24
+ splits: Mapping[str, DNA | DNASet],
25
+ ) -> tuple[tuple[str, DNASet], ...]:
26
+ if not isinstance(splits, Mapping) or len(splits) < 2:
27
+ raise ConfigurationError("splits must map at least two names to DNA values.")
28
+ items = tuple(splits.items())
29
+ if any(
30
+ not isinstance(name, str) or not name.strip() or not isinstance(dataset, (DNA, DNASet))
31
+ for name, dataset in items
32
+ ):
33
+ raise ConfigurationError("Split names must be non-empty and values must be DNA.")
34
+ return tuple(
35
+ (name, dataset.dataset if isinstance(dataset, DNA) else dataset) for name, dataset in items
36
+ )
37
+
38
+
39
+ def detect_leakage(
40
+ splits: Mapping[str, DNA | DNASet],
41
+ *,
42
+ config: LeakageConfig | None = None,
43
+ ) -> LeakageReport:
44
+ """Exhaustively compare every cross-split pair under hard record/pair/event limits."""
45
+
46
+ resolved = LeakageConfig() if config is None else config
47
+ if not isinstance(resolved, LeakageConfig):
48
+ raise ConfigurationError("config must be LeakageConfig.")
49
+ items = _validate_splits(splits)
50
+ total = sum(len(dataset) for _, dataset in items)
51
+ if total > resolved.max_records:
52
+ raise ConfigurationError(
53
+ "Leakage input exceeds max_records.",
54
+ code="LEAKAGE_RECORD_LIMIT",
55
+ context={"record_count": total, "max_records": resolved.max_records},
56
+ )
57
+ pair_total = sum(
58
+ len(left) * len(right)
59
+ for position, (_, left) in enumerate(items)
60
+ for _, right in items[position + 1 :]
61
+ )
62
+ if pair_total > resolved.max_cross_pairs:
63
+ raise ConfigurationError(
64
+ "Leakage cross-pair workload exceeds max_cross_pairs.",
65
+ code="LEAKAGE_PAIR_LIMIT",
66
+ context={"cross_pair_count": pair_total, "max_cross_pairs": resolved.max_cross_pairs},
67
+ )
68
+ all_records = tuple(record for _, dataset in items for record in dataset)
69
+ validate_pairwise_sequences(all_records, operation="detect_leakage")
70
+ events: list[LeakageEvent] = []
71
+ exact_count = 0
72
+ high_count = 0
73
+ for position, (left_name, left_set) in enumerate(items):
74
+ for right_name, right_set in items[position + 1 :]:
75
+ for left_index, left in enumerate(left_set):
76
+ for right_index, right in enumerate(right_set):
77
+ exact = left.sequence.symbols == right.sequence.symbols
78
+ similarity = pair_similarity(
79
+ left,
80
+ right,
81
+ method=resolved.method,
82
+ k=resolved.k,
83
+ canonical=resolved.canonical,
84
+ max_alignment_cells=resolved.max_alignment_cells,
85
+ )
86
+ if exact or similarity >= resolved.threshold:
87
+ exact_count += int(exact)
88
+ high_count += int(not exact and similarity >= resolved.threshold)
89
+ if len(events) >= resolved.max_events:
90
+ raise ConfigurationError(
91
+ "Leakage event count exceeds max_events; "
92
+ "no partial report returned.",
93
+ code="LEAKAGE_EVENT_LIMIT",
94
+ context={"max_events": resolved.max_events},
95
+ )
96
+ events.append(
97
+ LeakageEvent(
98
+ left_name,
99
+ right_name,
100
+ left.id,
101
+ right.id,
102
+ left_index,
103
+ right_index,
104
+ similarity,
105
+ exact,
106
+ )
107
+ )
108
+ strictness = (
109
+ "exhaustive cross-pair exact dynamic-programming comparison"
110
+ if resolved.method in {"identity", "edit"}
111
+ else "exhaustive cross-pair deterministic k-mer/fingerprint comparison; "
112
+ "may miss biologically similar pairs not captured by configured features"
113
+ )
114
+ return LeakageReport(
115
+ tuple(events),
116
+ resolved.method,
117
+ resolved.threshold,
118
+ strictness,
119
+ exact_count,
120
+ high_count,
121
+ pair_total,
122
+ resolved.max_records,
123
+ resolved.max_cross_pairs,
124
+ resolved.max_events,
125
+ False,
126
+ resolved.k,
127
+ resolved.canonical,
128
+ resolved.max_alignment_cells,
129
+ )
130
+
131
+
132
+ def evaluate_split_quality(
133
+ records: DNA | DNASet,
134
+ assignments: Sequence[SplitAssignment],
135
+ *,
136
+ target_ratios: Mapping[str, float],
137
+ label_key: str | None = None,
138
+ group_keys: Sequence[str] = (),
139
+ leakage_report: LeakageReport | None = None,
140
+ ) -> SplitQualityResult:
141
+ """Evaluate ratio error, label total variation, group leakage, and pair leakage."""
142
+
143
+ if not isinstance(records, (DNA, DNASet)):
144
+ raise ConfigurationError("records must be DNA or DNASet.")
145
+ dataset = records.dataset if isinstance(records, DNA) else records
146
+ assignment_tuple = tuple(assignments)
147
+ if len(assignment_tuple) != len(dataset):
148
+ raise ConfigurationError("assignments must align one-to-one with records.")
149
+ by_index: dict[int, str] = {}
150
+ for item in assignment_tuple:
151
+ if not isinstance(item, SplitAssignment) or item.input_index in by_index:
152
+ raise ConfigurationError("assignments contain invalid or duplicate input indices.")
153
+ if item.input_index >= len(dataset) or dataset[item.input_index].id != item.record_id:
154
+ raise ConfigurationError("assignment index/ID does not align with records.")
155
+ by_index[item.input_index] = item.split
156
+ validated_ratios: dict[str, float] = {}
157
+ for name, value in target_ratios.items():
158
+ if (
159
+ not isinstance(name, str)
160
+ or not name.strip()
161
+ or isinstance(value, bool)
162
+ or not isinstance(value, (int, float))
163
+ or not math.isfinite(value)
164
+ or value <= 0.0
165
+ ):
166
+ raise ConfigurationError(
167
+ "target_ratios must map non-empty names to finite positive values."
168
+ )
169
+ validated_ratios[name] = float(value)
170
+ split_names = tuple(validated_ratios)
171
+ if set(item.split for item in assignment_tuple) - set(split_names):
172
+ raise ConfigurationError("assignments contain names absent from target_ratios.")
173
+ total_ratio = math.fsum(validated_ratios.values())
174
+ if not math.isclose(total_ratio, 1.0, rel_tol=0.0, abs_tol=1e-9):
175
+ raise ConfigurationError("target_ratios must sum to one.")
176
+ counts = {name: sum(item.split == name for item in assignment_tuple) for name in split_names}
177
+ total = len(dataset)
178
+ achieved = {name: counts[name] / total if total else 0.0 for name in split_names}
179
+ deviations = {name: abs(achieved[name] - validated_ratios[name]) for name in split_names}
180
+ label_tv: dict[str, float] = {name: 0.0 for name in split_names}
181
+ if label_key is not None and total:
182
+ global_counts: dict[object, int] = {}
183
+ split_counts: dict[str, dict[object, int]] = {name: {} for name in split_names}
184
+ for index, record in enumerate(dataset):
185
+ if label_key not in record.metadata:
186
+ raise ConfigurationError("A record is missing label metadata.")
187
+ key = metadata_value_key(record.metadata[label_key])
188
+ global_counts[key] = global_counts.get(key, 0) + 1
189
+ name = by_index[index]
190
+ split_counts[name][key] = split_counts[name].get(key, 0) + 1
191
+ for name in split_names:
192
+ denominator = counts[name]
193
+ if denominator:
194
+ label_tv[name] = 0.5 * math.fsum(
195
+ abs(split_counts[name].get(key, 0) / denominator - value / total)
196
+ for key, value in global_counts.items()
197
+ )
198
+ validated_group_keys = tuple(group_keys)
199
+ if any(not isinstance(key, str) or not key.strip() for key in validated_group_keys):
200
+ raise ConfigurationError("group_keys must contain non-empty strings.")
201
+ if len(validated_group_keys) != len(set(validated_group_keys)):
202
+ raise ConfigurationError("group_keys must be unique.")
203
+ if label_key is not None and (not isinstance(label_key, str) or not label_key.strip()):
204
+ raise ConfigurationError("label_key must be non-empty or None.")
205
+ if leakage_report is not None and not isinstance(leakage_report, LeakageReport):
206
+ raise ConfigurationError("leakage_report must be LeakageReport or None.")
207
+ group_leaks = 0
208
+ for key in validated_group_keys:
209
+ seen: dict[object, set[str]] = {}
210
+ for index, record in enumerate(dataset):
211
+ if key not in record.metadata:
212
+ raise ConfigurationError("A record is missing group metadata.")
213
+ group_value = metadata_value_key(record.metadata[key])
214
+ seen.setdefault(group_value, set()).add(by_index[index])
215
+ group_leaks += sum(len(splits) > 1 for splits in seen.values())
216
+ max_ratio = max(deviations.values(), default=0.0)
217
+ max_label = max(label_tv.values(), default=0.0)
218
+ leakage_count = None if leakage_report is None else len(leakage_report.events)
219
+ leakage_penalty = 0.0 if leakage_count is None else min(1.0, leakage_count / max(total, 1))
220
+ score = max(0.0, 1.0 - max_ratio - max_label - min(1.0, group_leaks) - leakage_penalty)
221
+ return SplitQualityResult(
222
+ total,
223
+ FrozenDict(counts),
224
+ FrozenDict(validated_ratios),
225
+ FrozenDict(achieved),
226
+ FrozenDict(deviations),
227
+ max_ratio,
228
+ label_key,
229
+ FrozenDict(label_tv),
230
+ validated_group_keys,
231
+ group_leaks,
232
+ leakage_count,
233
+ score,
234
+ "max(0,1-max_ratio_deviation-max_label_TV-min(1,group_leaks)-leakage_events/max(N,1))",
235
+ FrozenDict(
236
+ {
237
+ "empty_split_label_tv": 0.0,
238
+ "group_leak_definition": "one typed metadata group present in more than one split",
239
+ }
240
+ ),
241
+ )
242
+
243
+
244
+ __all__ = ["detect_leakage", "evaluate_split_quality"]
@@ -0,0 +1,135 @@
1
+ """Explicit metadata-based exclusion for immutable DNA record sets."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterable
6
+ from typing import Literal, TypeAlias, overload
7
+
8
+ from dnakit.core import DNA, DNARecord, DNASet
9
+ from dnakit.exceptions import ConfigurationError, InputFormatError
10
+
11
+ MetadataExclusionField: TypeAlias = Literal["chromosome", "species"]
12
+ MetadataValues: TypeAlias = object | Iterable[object]
13
+ MetadataDataset: TypeAlias = DNA | DNASet
14
+
15
+ _EXCLUSION_FIELDS = frozenset({"chromosome", "species"})
16
+
17
+
18
+ def _copy_set(dataset: DNASet, records: Iterable[DNARecord]) -> DNASet:
19
+ return DNASet(
20
+ records,
21
+ name=dataset.name,
22
+ source=dataset.source,
23
+ version=dataset.version,
24
+ metadata=dataset.metadata,
25
+ )
26
+
27
+
28
+ def _materialize_values(values: MetadataValues) -> tuple[object, ...]:
29
+ if isinstance(values, (str, bytes)) or not isinstance(values, Iterable):
30
+ materialized = (values,)
31
+ else:
32
+ materialized = tuple(values)
33
+ if not materialized:
34
+ raise ConfigurationError(
35
+ "At least one metadata value must be excluded.",
36
+ code="EMPTY_METADATA_EXCLUSION_VALUES",
37
+ )
38
+ return materialized
39
+
40
+
41
+ @overload
42
+ def exclude_by_metadata(
43
+ dataset: DNA,
44
+ field: MetadataExclusionField,
45
+ values: MetadataValues,
46
+ ) -> DNA: ...
47
+
48
+
49
+ @overload
50
+ def exclude_by_metadata(
51
+ dataset: DNASet,
52
+ field: MetadataExclusionField,
53
+ values: MetadataValues,
54
+ ) -> DNASet: ...
55
+
56
+
57
+ def exclude_by_metadata(
58
+ dataset: MetadataDataset,
59
+ field: MetadataExclusionField,
60
+ values: MetadataValues,
61
+ ) -> DNA | DNASet:
62
+ """Return a copy without records whose chromosome or species matches.
63
+
64
+ Matching is exact and compares the stored JSON-compatible metadata value.
65
+ Every record must contain ``field``; missing fields raise before any record
66
+ is removed so that an incomplete dataset cannot be silently filtered.
67
+ """
68
+
69
+ if not isinstance(dataset, (DNA, DNASet)):
70
+ raise TypeError("dataset must be DNA or DNASet.")
71
+ source = dataset.dataset if isinstance(dataset, DNA) else dataset
72
+ if not isinstance(field, str) or field not in _EXCLUSION_FIELDS:
73
+ raise ConfigurationError(
74
+ "Metadata exclusion field must be 'chromosome' or 'species'.",
75
+ code="INVALID_METADATA_EXCLUSION_FIELD",
76
+ context={"field": field},
77
+ )
78
+ excluded_values = _materialize_values(values)
79
+ missing_ids = tuple(record.id for record in source if field not in record.metadata)
80
+ if missing_ids:
81
+ raise InputFormatError(
82
+ "Every DNASet record must contain the metadata field used for exclusion.",
83
+ code="MISSING_METADATA_FIELD_FOR_EXCLUSION",
84
+ context={
85
+ "field": field,
86
+ "missing_record_ids": missing_ids[:10],
87
+ "missing_count": len(missing_ids),
88
+ },
89
+ hint=f"Populate metadata[{field!r}] for every record before exclusion.",
90
+ )
91
+ selected = tuple(
92
+ (index, record)
93
+ for index, record in enumerate(source)
94
+ if not any(record.metadata[field] == value for value in excluded_values)
95
+ )
96
+ if isinstance(dataset, DNA):
97
+ return dataset._derive(selected)
98
+ return _copy_set(dataset, (record for _, record in selected))
99
+
100
+
101
+ @overload
102
+ def exclude_chromosomes(dataset: DNA, chromosomes: MetadataValues) -> DNA: ...
103
+
104
+
105
+ @overload
106
+ def exclude_chromosomes(dataset: DNASet, chromosomes: MetadataValues) -> DNASet: ...
107
+
108
+
109
+ def exclude_chromosomes(dataset: MetadataDataset, chromosomes: MetadataValues) -> DNA | DNASet:
110
+ """Return a copy without records from the selected chromosome value(s)."""
111
+
112
+ return exclude_by_metadata(dataset, "chromosome", chromosomes)
113
+
114
+
115
+ @overload
116
+ def exclude_species(dataset: DNA, species: MetadataValues) -> DNA: ...
117
+
118
+
119
+ @overload
120
+ def exclude_species(dataset: DNASet, species: MetadataValues) -> DNASet: ...
121
+
122
+
123
+ def exclude_species(dataset: MetadataDataset, species: MetadataValues) -> DNA | DNASet:
124
+ """Return a copy without records from the selected species value(s)."""
125
+
126
+ return exclude_by_metadata(dataset, "species", species)
127
+
128
+
129
+ __all__ = [
130
+ "MetadataExclusionField",
131
+ "MetadataValues",
132
+ "exclude_by_metadata",
133
+ "exclude_chromosomes",
134
+ "exclude_species",
135
+ ]