dnakit 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dnakit/__init__.py +108 -0
- dnakit/_version.py +5 -0
- dnakit/alignment/__init__.py +6 -0
- dnakit/alignment/pairwise.py +512 -0
- dnakit/alignment/results.py +188 -0
- dnakit/backends/__init__.py +80 -0
- dnakit/backends/external.py +517 -0
- dnakit/backends/registry.py +207 -0
- dnakit/batch.py +452 -0
- dnakit/cache/__init__.py +5 -0
- dnakit/cache/store.py +424 -0
- dnakit/chunking.py +920 -0
- dnakit/cli/__init__.py +5 -0
- dnakit/cli/__main__.py +6 -0
- dnakit/cli/app.py +814 -0
- dnakit/cli/workflow.py +94 -0
- dnakit/config/__init__.py +5 -0
- dnakit/config/loader.py +381 -0
- dnakit/core/__init__.py +80 -0
- dnakit/core/_json.py +206 -0
- dnakit/core/backend_info.py +70 -0
- dnakit/core/collection.py +163 -0
- dnakit/core/coordinates.py +295 -0
- dnakit/core/enums.py +95 -0
- dnakit/core/facade.py +719 -0
- dnakit/core/feature.py +90 -0
- dnakit/core/gap.py +82 -0
- dnakit/core/issues.py +62 -0
- dnakit/core/provenance.py +305 -0
- dnakit/core/record.py +161 -0
- dnakit/core/results.py +134 -0
- dnakit/core/sequence.py +327 -0
- dnakit/datasets/__init__.py +100 -0
- dnakit/datasets/_advanced_shared.py +198 -0
- dnakit/datasets/_metadata.py +33 -0
- dnakit/datasets/advanced_deduplicate.py +183 -0
- dnakit/datasets/advanced_split.py +302 -0
- dnakit/datasets/clustering.py +286 -0
- dnakit/datasets/config.py +531 -0
- dnakit/datasets/deduplicate.py +303 -0
- dnakit/datasets/evaluation.py +244 -0
- dnakit/datasets/filter.py +135 -0
- dnakit/datasets/neural_clustering.py +221 -0
- dnakit/datasets/results.py +520 -0
- dnakit/datasets/split.py +450 -0
- dnakit/descriptors/__init__.py +102 -0
- dnakit/descriptors/_dinucleotide.py +345 -0
- dnakit/descriptors/_shared.py +180 -0
- dnakit/descriptors/all.py +917 -0
- dnakit/descriptors/basic.py +193 -0
- dnakit/descriptors/codon.py +195 -0
- dnakit/descriptors/complexity.py +201 -0
- dnakit/descriptors/entropy.py +101 -0
- dnakit/descriptors/homopolymer.py +105 -0
- dnakit/descriptors/kmer.py +88 -0
- dnakit/descriptors/results.py +286 -0
- dnakit/descriptors/schema.py +414 -0
- dnakit/descriptors/window.py +203 -0
- dnakit/download/__init__.py +54 -0
- dnakit/download/catalogs.py +166 -0
- dnakit/download/ena.py +131 -0
- dnakit/download/encode.py +103 -0
- dnakit/download/files.py +469 -0
- dnakit/download/indexes.py +233 -0
- dnakit/download/metadata.py +164 -0
- dnakit/download/models.py +227 -0
- dnakit/download/ncbi.py +333 -0
- dnakit/download/sequence.py +133 -0
- dnakit/download/ucsc.py +99 -0
- dnakit/evaluation/__init__.py +81 -0
- dnakit/evaluation/_shared.py +288 -0
- dnakit/evaluation/collection.py +270 -0
- dnakit/evaluation/config.py +591 -0
- dnakit/evaluation/distribution.py +287 -0
- dnakit/evaluation/frechet.py +223 -0
- dnakit/evaluation/generative.py +381 -0
- dnakit/evaluation/reference.py +409 -0
- dnakit/evaluation/results.py +137 -0
- dnakit/evaluation/scorecard.py +211 -0
- dnakit/evaluation/sequence.py +403 -0
- dnakit/evaluation/synthesis.py +249 -0
- dnakit/exceptions.py +189 -0
- dnakit/fingerprints/__init__.py +109 -0
- dnakit/fingerprints/_shared.py +116 -0
- dnakit/fingerprints/advanced.py +829 -0
- dnakit/fingerprints/bit.py +229 -0
- dnakit/fingerprints/encoding.py +351 -0
- dnakit/fingerprints/kmer.py +170 -0
- dnakit/fingerprints/results.py +531 -0
- dnakit/fingerprints/sketch.py +246 -0
- dnakit/io/__init__.py +112 -0
- dnakit/io/_advanced_common.py +105 -0
- dnakit/io/_formats.py +994 -0
- dnakit/io/annotations.py +1179 -0
- dnakit/io/api.py +759 -0
- dnakit/io/config.py +315 -0
- dnakit/io/genbank.py +523 -0
- dnakit/io/indexed.py +1146 -0
- dnakit/io/metadata.py +452 -0
- dnakit/io/parquet.py +56 -0
- dnakit/io/results.py +131 -0
- dnakit/io/source.py +138 -0
- dnakit/io/tables.py +1237 -0
- dnakit/molbio/__init__.py +101 -0
- dnakit/molbio/_shared.py +266 -0
- dnakit/molbio/assembly.py +279 -0
- dnakit/molbio/crispr.py +181 -0
- dnakit/molbio/optimization.py +897 -0
- dnakit/molbio/primer3_design.py +667 -0
- dnakit/molbio/primers.py +899 -0
- dnakit/molbio/restriction.py +573 -0
- dnakit/molbio/results.py +583 -0
- dnakit/ops/__init__.py +588 -0
- dnakit/ops/_common.py +201 -0
- dnakit/ops/circular.py +193 -0
- dnakit/ops/concat.py +204 -0
- dnakit/ops/direction.py +27 -0
- dnakit/ops/edit.py +443 -0
- dnakit/ops/evolution.py +991 -0
- dnakit/ops/mutation.py +151 -0
- dnakit/ops/records.py +1012 -0
- dnakit/ops/translation.py +257 -0
- dnakit/patterns/__init__.py +81 -0
- dnakit/patterns/_shared.py +282 -0
- dnakit/patterns/coding.py +360 -0
- dnakit/patterns/crispr.py +337 -0
- dnakit/patterns/motif.py +721 -0
- dnakit/patterns/regions.py +341 -0
- dnakit/patterns/repeats.py +480 -0
- dnakit/patterns/restriction.py +281 -0
- dnakit/patterns/results.py +411 -0
- dnakit/py.typed +1 -0
- dnakit/references/__init__.py +27 -0
- dnakit/references/models.py +155 -0
- dnakit/references/ncbi.py +785 -0
- dnakit/representations/__init__.py +29 -0
- dnakit/representations/backends.py +928 -0
- dnakit/representations/checkpoints.py +356 -0
- dnakit/representations/extraction.py +220 -0
- dnakit/representations/models.py +395 -0
- dnakit/representations/results.py +79 -0
- dnakit/search/__init__.py +132 -0
- dnakit/search/_http.py +271 -0
- dnakit/search/_shared.py +108 -0
- dnakit/search/blast.py +516 -0
- dnakit/search/ena.py +245 -0
- dnakit/search/encode.py +102 -0
- dnakit/search/ensembl.py +874 -0
- dnakit/search/models.py +159 -0
- dnakit/search/ncbi.py +756 -0
- dnakit/search/ucsc.py +388 -0
- dnakit/secondary_structure/__init__.py +38 -0
- dnakit/secondary_structure/dotbracket.py +410 -0
- dnakit/secondary_structure/nupack.py +589 -0
- dnakit/secondary_structure/results.py +142 -0
- dnakit/similarity/__init__.py +125 -0
- dnakit/similarity/_shared.py +152 -0
- dnakit/similarity/approximate.py +258 -0
- dnakit/similarity/compare.py +162 -0
- dnakit/similarity/dashing.py +936 -0
- dnakit/similarity/distance.py +279 -0
- dnakit/similarity/index.py +365 -0
- dnakit/similarity/matrix.py +254 -0
- dnakit/similarity/results.py +676 -0
- dnakit/similarity/search.py +286 -0
- dnakit/similarity/sketch.py +96 -0
- dnakit/similarity/vector.py +662 -0
- dnakit/standardize/__init__.py +77 -0
- dnakit/standardize/_shared.py +100 -0
- dnakit/standardize/config.py +411 -0
- dnakit/standardize/gaps.py +1060 -0
- dnakit/standardize/normalize.py +565 -0
- dnakit/standardize/results.py +247 -0
- dnakit/standardize/validate.py +487 -0
- dnakit/structure3d/__init__.py +47 -0
- dnakit/structure3d/_shared.py +47 -0
- dnakit/structure3d/geometry.py +773 -0
- dnakit/structure3d/pdb.py +253 -0
- dnakit/structure3d/results.py +233 -0
- dnakit/structure3d/threedna.py +174 -0
- dnakit/thermodynamics/__init__.py +112 -0
- dnakit/thermodynamics/_shared.py +92 -0
- dnakit/thermodynamics/backends.py +653 -0
- dnakit/thermodynamics/calculations.py +837 -0
- dnakit/thermodynamics/config.py +175 -0
- dnakit/thermodynamics/equilibrium.py +340 -0
- dnakit/thermodynamics/optics.py +385 -0
- dnakit/thermodynamics/parameters.py +79 -0
- dnakit/thermodynamics/results.py +445 -0
- dnakit/visualization/__init__.py +43 -0
- dnakit/visualization/_svg.py +185 -0
- dnakit/visualization/advanced.py +314 -0
- dnakit/visualization/config.py +255 -0
- dnakit/visualization/export.py +349 -0
- dnakit/visualization/matrix.py +279 -0
- dnakit/visualization/report.py +431 -0
- dnakit/visualization/results.py +236 -0
- dnakit/visualization/sequence.py +459 -0
- dnakit/workflows/__init__.py +45 -0
- dnakit/workflows/manifest.py +365 -0
- dnakit/workflows/runner.py +1548 -0
- dnakit/workflows/schema.py +898 -0
- dnakit-0.1.0.dev0.dist-info/METADATA +126 -0
- dnakit-0.1.0.dev0.dist-info/RECORD +210 -0
- dnakit-0.1.0.dev0.dist-info/WHEEL +5 -0
- dnakit-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- dnakit-0.1.0.dev0.dist-info/licenses/DISCLAIMER.md +26 -0
- dnakit-0.1.0.dev0.dist-info/licenses/LICENSE +21 -0
- dnakit-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.md +73 -0
- dnakit-0.1.0.dev0.dist-info/top_level.txt +1 -0
dnakit/io/_formats.py
ADDED
|
@@ -0,0 +1,994 @@
|
|
|
1
|
+
"""Streaming FASTA, FASTQ, and structured table codecs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Iterable, Iterator, Mapping
|
|
8
|
+
from typing import Any, TextIO, cast
|
|
9
|
+
|
|
10
|
+
from dnakit.core._json import to_json_compatible
|
|
11
|
+
from dnakit.core.coordinates import CompoundLocation, Interval, Location, UnresolvedLocation
|
|
12
|
+
from dnakit.core.enums import DNAAlphabet, Strandedness, Topology
|
|
13
|
+
from dnakit.core.feature import DNAFeature
|
|
14
|
+
from dnakit.core.gap import Gap
|
|
15
|
+
from dnakit.core.record import DNARecord
|
|
16
|
+
from dnakit.core.sequence import DNASequence
|
|
17
|
+
from dnakit.exceptions import ConfigurationError, DNAKitError, InputFormatError
|
|
18
|
+
|
|
19
|
+
from .config import ReadConfig, WriteConfig
|
|
20
|
+
|
|
21
|
+
_PHRED_KEY = "phred_quality"
|
|
22
|
+
_RECORD_SCHEMA_VERSION = "dnakit.record.v1"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _json_object_without_duplicates(pairs: list[tuple[str, object]]) -> dict[str, object]:
|
|
26
|
+
result: dict[str, object] = {}
|
|
27
|
+
for key, value in pairs:
|
|
28
|
+
if key in result:
|
|
29
|
+
raise ValueError(f"duplicate JSON object key: {key}")
|
|
30
|
+
result[key] = value
|
|
31
|
+
return result
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _reject_json_constant(value: str) -> object:
|
|
35
|
+
raise ValueError(f"non-standard JSON constant: {value}")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _validate_json_structure(
|
|
39
|
+
value: object,
|
|
40
|
+
config: ReadConfig,
|
|
41
|
+
*,
|
|
42
|
+
context: Mapping[str, object] | None = None,
|
|
43
|
+
) -> None:
|
|
44
|
+
nodes = 0
|
|
45
|
+
stack: list[tuple[object, int]] = [(value, 1)]
|
|
46
|
+
while stack:
|
|
47
|
+
current, depth = stack.pop()
|
|
48
|
+
nodes += 1
|
|
49
|
+
if depth > config.max_json_depth or nodes > config.max_json_nodes:
|
|
50
|
+
raise InputFormatError(
|
|
51
|
+
"JSON structure exceeds its configured depth or node limit.",
|
|
52
|
+
code="JSON_STRUCTURE_LIMIT_EXCEEDED",
|
|
53
|
+
context={
|
|
54
|
+
**dict(context or {}),
|
|
55
|
+
"max_json_depth": config.max_json_depth,
|
|
56
|
+
"max_json_nodes": config.max_json_nodes,
|
|
57
|
+
},
|
|
58
|
+
)
|
|
59
|
+
if isinstance(current, Mapping):
|
|
60
|
+
nodes += len(current)
|
|
61
|
+
if nodes > config.max_json_nodes:
|
|
62
|
+
raise InputFormatError(
|
|
63
|
+
"JSON structure exceeds its configured depth or node limit.",
|
|
64
|
+
code="JSON_STRUCTURE_LIMIT_EXCEEDED",
|
|
65
|
+
context={
|
|
66
|
+
**dict(context or {}),
|
|
67
|
+
"max_json_depth": config.max_json_depth,
|
|
68
|
+
"max_json_nodes": config.max_json_nodes,
|
|
69
|
+
},
|
|
70
|
+
)
|
|
71
|
+
stack.extend((item, depth + 1) for item in current.values())
|
|
72
|
+
elif isinstance(current, list):
|
|
73
|
+
stack.extend((item, depth + 1) for item in current)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _strict_json_loads(
|
|
77
|
+
text: str,
|
|
78
|
+
config: ReadConfig,
|
|
79
|
+
*,
|
|
80
|
+
message: str,
|
|
81
|
+
code: str,
|
|
82
|
+
context: Mapping[str, object] | None = None,
|
|
83
|
+
) -> object:
|
|
84
|
+
try:
|
|
85
|
+
value = json.loads(
|
|
86
|
+
text,
|
|
87
|
+
object_pairs_hook=_json_object_without_duplicates,
|
|
88
|
+
parse_constant=_reject_json_constant,
|
|
89
|
+
)
|
|
90
|
+
except RecursionError as exc:
|
|
91
|
+
raise InputFormatError(
|
|
92
|
+
"JSON structure exceeds its configured depth or node limit.",
|
|
93
|
+
code="JSON_STRUCTURE_LIMIT_EXCEEDED",
|
|
94
|
+
context={
|
|
95
|
+
**dict(context or {}),
|
|
96
|
+
"max_json_depth": config.max_json_depth,
|
|
97
|
+
"max_json_nodes": config.max_json_nodes,
|
|
98
|
+
},
|
|
99
|
+
) from exc
|
|
100
|
+
except (json.JSONDecodeError, ValueError) as exc:
|
|
101
|
+
details = dict(context or {})
|
|
102
|
+
if isinstance(exc, json.JSONDecodeError):
|
|
103
|
+
details.setdefault("line_number", exc.lineno)
|
|
104
|
+
details["column_number"] = exc.colno
|
|
105
|
+
raise InputFormatError(message, code=code, context=details) from exc
|
|
106
|
+
_validate_json_structure(value, config, context=context)
|
|
107
|
+
return value
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _input_error(
|
|
111
|
+
message: str,
|
|
112
|
+
*,
|
|
113
|
+
code: str,
|
|
114
|
+
line_number: int | None = None,
|
|
115
|
+
record_id: str | None = None,
|
|
116
|
+
) -> InputFormatError:
|
|
117
|
+
context: dict[str, object] = {}
|
|
118
|
+
if line_number is not None:
|
|
119
|
+
context["line_number"] = line_number
|
|
120
|
+
if record_id is not None:
|
|
121
|
+
context["record_id"] = record_id
|
|
122
|
+
return InputFormatError(message, code=code, context=context)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _sequence(
|
|
126
|
+
text: str,
|
|
127
|
+
config: ReadConfig,
|
|
128
|
+
*,
|
|
129
|
+
record_id: str,
|
|
130
|
+
line_number: int,
|
|
131
|
+
alphabet: DNAAlphabet | str | None = None,
|
|
132
|
+
topology: Topology | str | None = None,
|
|
133
|
+
strandedness: Strandedness | str | None = None,
|
|
134
|
+
) -> DNASequence:
|
|
135
|
+
if len(text) > config.max_sequence_symbols:
|
|
136
|
+
raise _input_error(
|
|
137
|
+
"Record sequence exceeds max_sequence_symbols.",
|
|
138
|
+
code="SEQUENCE_SYMBOL_LIMIT_EXCEEDED",
|
|
139
|
+
line_number=line_number,
|
|
140
|
+
record_id=record_id,
|
|
141
|
+
)
|
|
142
|
+
value = text.upper() if config.uppercase else text
|
|
143
|
+
try:
|
|
144
|
+
return DNASequence(
|
|
145
|
+
value,
|
|
146
|
+
alphabet=config.alphabet if alphabet is None else alphabet,
|
|
147
|
+
topology=config.topology if topology is None else topology,
|
|
148
|
+
strandedness=config.strandedness if strandedness is None else strandedness,
|
|
149
|
+
)
|
|
150
|
+
except DNAKitError as exc:
|
|
151
|
+
raise _input_error(
|
|
152
|
+
"Record contains symbols incompatible with the configured DNA alphabet.",
|
|
153
|
+
code="INVALID_SEQUENCE_CONTENT",
|
|
154
|
+
line_number=line_number,
|
|
155
|
+
record_id=record_id,
|
|
156
|
+
) from exc
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _header(value: str, *, line_number: int, marker: str) -> tuple[str, str]:
|
|
160
|
+
body = value[1:].strip()
|
|
161
|
+
if not body:
|
|
162
|
+
raise _input_error(
|
|
163
|
+
f"{marker} header must contain a record ID.",
|
|
164
|
+
code=f"EMPTY_{marker}_HEADER",
|
|
165
|
+
line_number=line_number,
|
|
166
|
+
)
|
|
167
|
+
fields = body.split(maxsplit=1)
|
|
168
|
+
return fields[0], fields[1] if len(fields) == 2 else ""
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _serialized_header(record: DNARecord, *, format: str) -> str:
|
|
172
|
+
if any(symbol.isspace() for symbol in record.id):
|
|
173
|
+
raise InputFormatError(
|
|
174
|
+
f"{format} record IDs must not contain whitespace.",
|
|
175
|
+
code="INVALID_SEQUENCE_HEADER",
|
|
176
|
+
context={"record_id": record.id, "format": format},
|
|
177
|
+
)
|
|
178
|
+
if "\n" in record.description or "\r" in record.description:
|
|
179
|
+
raise InputFormatError(
|
|
180
|
+
f"{format} descriptions must not contain line breaks.",
|
|
181
|
+
code="INVALID_SEQUENCE_HEADER",
|
|
182
|
+
context={"record_id": record.id, "format": format},
|
|
183
|
+
)
|
|
184
|
+
return record.id if not record.description else f"{record.id} {record.description}"
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def iter_fasta(handle: TextIO, config: ReadConfig) -> Iterator[DNARecord]:
|
|
188
|
+
"""Parse multiline FASTA records without materializing the file."""
|
|
189
|
+
|
|
190
|
+
record_id: str | None = None
|
|
191
|
+
description = ""
|
|
192
|
+
chunks: list[str] = []
|
|
193
|
+
header_line = 0
|
|
194
|
+
record_line_count = 0
|
|
195
|
+
symbol_count = 0
|
|
196
|
+
for line_number, raw_line in enumerate(handle, start=1):
|
|
197
|
+
line = raw_line.rstrip("\r\n")
|
|
198
|
+
if len(line) > config.max_field_size:
|
|
199
|
+
raise _input_error(
|
|
200
|
+
"FASTA line exceeds max_field_size.",
|
|
201
|
+
code="SEQUENCE_LINE_TOO_LONG",
|
|
202
|
+
line_number=line_number,
|
|
203
|
+
)
|
|
204
|
+
if line.startswith(">"):
|
|
205
|
+
if record_id is not None:
|
|
206
|
+
yield DNARecord(
|
|
207
|
+
_sequence(
|
|
208
|
+
"".join(chunks),
|
|
209
|
+
config,
|
|
210
|
+
record_id=record_id,
|
|
211
|
+
line_number=header_line,
|
|
212
|
+
),
|
|
213
|
+
record_id,
|
|
214
|
+
description=description,
|
|
215
|
+
)
|
|
216
|
+
record_id, description = _header(line, line_number=line_number, marker="FASTA")
|
|
217
|
+
chunks = []
|
|
218
|
+
symbol_count = 0
|
|
219
|
+
header_line = line_number
|
|
220
|
+
record_line_count = 1
|
|
221
|
+
else:
|
|
222
|
+
record_line_count += 1
|
|
223
|
+
if record_line_count > config.max_record_lines:
|
|
224
|
+
raise _input_error(
|
|
225
|
+
"FASTA record exceeds max_record_lines.",
|
|
226
|
+
code="SEQUENCE_RECORD_LINE_LIMIT_EXCEEDED",
|
|
227
|
+
line_number=line_number,
|
|
228
|
+
record_id=record_id,
|
|
229
|
+
)
|
|
230
|
+
if not line:
|
|
231
|
+
continue
|
|
232
|
+
if record_id is None:
|
|
233
|
+
raise _input_error(
|
|
234
|
+
"FASTA sequence data appeared before the first header.",
|
|
235
|
+
code="FASTA_MISSING_HEADER",
|
|
236
|
+
line_number=line_number,
|
|
237
|
+
)
|
|
238
|
+
symbol_count += len(line)
|
|
239
|
+
if symbol_count > config.max_sequence_symbols:
|
|
240
|
+
raise _input_error(
|
|
241
|
+
"FASTA record exceeds max_sequence_symbols.",
|
|
242
|
+
code="SEQUENCE_SYMBOL_LIMIT_EXCEEDED",
|
|
243
|
+
line_number=line_number,
|
|
244
|
+
record_id=record_id,
|
|
245
|
+
)
|
|
246
|
+
chunks.append(line)
|
|
247
|
+
if record_id is not None:
|
|
248
|
+
yield DNARecord(
|
|
249
|
+
_sequence("".join(chunks), config, record_id=record_id, line_number=header_line),
|
|
250
|
+
record_id,
|
|
251
|
+
description=description,
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def iter_fastq(handle: TextIO, config: ReadConfig) -> Iterator[DNARecord]:
|
|
256
|
+
"""Parse strict four-line FASTQ records and validate Phred values."""
|
|
257
|
+
|
|
258
|
+
line_number = 0
|
|
259
|
+
while True:
|
|
260
|
+
header = handle.readline()
|
|
261
|
+
if not header:
|
|
262
|
+
return
|
|
263
|
+
line_number += 1
|
|
264
|
+
if len(header.rstrip("\r\n")) > config.max_field_size:
|
|
265
|
+
raise _input_error(
|
|
266
|
+
"FASTQ header exceeds max_field_size.",
|
|
267
|
+
code="SEQUENCE_LINE_TOO_LONG",
|
|
268
|
+
line_number=line_number,
|
|
269
|
+
)
|
|
270
|
+
if not header.startswith("@"):
|
|
271
|
+
raise _input_error(
|
|
272
|
+
"FASTQ record must begin with '@'.",
|
|
273
|
+
code="FASTQ_MISSING_HEADER",
|
|
274
|
+
line_number=line_number,
|
|
275
|
+
)
|
|
276
|
+
record_id, description = _header(
|
|
277
|
+
header.rstrip("\r\n"), line_number=line_number, marker="FASTQ"
|
|
278
|
+
)
|
|
279
|
+
sequence_line = handle.readline()
|
|
280
|
+
plus_line = handle.readline()
|
|
281
|
+
quality_line = handle.readline()
|
|
282
|
+
if not sequence_line or not plus_line or not quality_line:
|
|
283
|
+
raise _input_error(
|
|
284
|
+
"FASTQ record ended before all four lines were present.",
|
|
285
|
+
code="TRUNCATED_FASTQ_RECORD",
|
|
286
|
+
line_number=line_number,
|
|
287
|
+
record_id=record_id,
|
|
288
|
+
)
|
|
289
|
+
sequence_line_number = line_number + 1
|
|
290
|
+
line_number += 3
|
|
291
|
+
for offset, value in enumerate((sequence_line, plus_line, quality_line), start=1):
|
|
292
|
+
if len(value.rstrip("\r\n")) > config.max_field_size:
|
|
293
|
+
raise _input_error(
|
|
294
|
+
"FASTQ line exceeds max_field_size.",
|
|
295
|
+
code="SEQUENCE_LINE_TOO_LONG",
|
|
296
|
+
line_number=sequence_line_number + offset - 1,
|
|
297
|
+
record_id=record_id,
|
|
298
|
+
)
|
|
299
|
+
sequence_text = sequence_line.rstrip("\r\n")
|
|
300
|
+
plus = plus_line.rstrip("\r\n")
|
|
301
|
+
quality_text = quality_line.rstrip("\r\n")
|
|
302
|
+
if len(sequence_text) > config.max_sequence_symbols:
|
|
303
|
+
raise _input_error(
|
|
304
|
+
"FASTQ record exceeds max_sequence_symbols.",
|
|
305
|
+
code="SEQUENCE_SYMBOL_LIMIT_EXCEEDED",
|
|
306
|
+
line_number=sequence_line_number,
|
|
307
|
+
record_id=record_id,
|
|
308
|
+
)
|
|
309
|
+
if not plus.startswith("+"):
|
|
310
|
+
raise _input_error(
|
|
311
|
+
"FASTQ separator line must begin with '+'.",
|
|
312
|
+
code="FASTQ_MISSING_SEPARATOR",
|
|
313
|
+
line_number=line_number - 1,
|
|
314
|
+
record_id=record_id,
|
|
315
|
+
)
|
|
316
|
+
repeated_header = plus[1:].strip()
|
|
317
|
+
if repeated_header and repeated_header.split(maxsplit=1)[0] != record_id:
|
|
318
|
+
raise _input_error(
|
|
319
|
+
"FASTQ '+' header ID does not match the '@' header ID.",
|
|
320
|
+
code="FASTQ_HEADER_MISMATCH",
|
|
321
|
+
line_number=line_number - 1,
|
|
322
|
+
record_id=record_id,
|
|
323
|
+
)
|
|
324
|
+
sequence = _sequence(
|
|
325
|
+
sequence_text,
|
|
326
|
+
config,
|
|
327
|
+
record_id=record_id,
|
|
328
|
+
line_number=sequence_line_number,
|
|
329
|
+
)
|
|
330
|
+
if len(quality_text) != sequence.symbol_length:
|
|
331
|
+
raise _input_error(
|
|
332
|
+
"FASTQ quality length must equal sequence symbol_length.",
|
|
333
|
+
code="FASTQ_QUALITY_LENGTH_MISMATCH",
|
|
334
|
+
line_number=line_number,
|
|
335
|
+
record_id=record_id,
|
|
336
|
+
)
|
|
337
|
+
quality_codepoints = tuple(ord(symbol) for symbol in quality_text)
|
|
338
|
+
quality = tuple(value - config.phred_offset for value in quality_codepoints)
|
|
339
|
+
if any(
|
|
340
|
+
codepoint > 126 or value < 0 or value > 93
|
|
341
|
+
for codepoint, value in zip(quality_codepoints, quality, strict=True)
|
|
342
|
+
):
|
|
343
|
+
raise _input_error(
|
|
344
|
+
"FASTQ quality characters must be printable ASCII and encode Phred values "
|
|
345
|
+
"from 0 through 93.",
|
|
346
|
+
code="FASTQ_QUALITY_OUT_OF_RANGE",
|
|
347
|
+
line_number=line_number,
|
|
348
|
+
record_id=record_id,
|
|
349
|
+
)
|
|
350
|
+
yield DNARecord(
|
|
351
|
+
sequence,
|
|
352
|
+
record_id,
|
|
353
|
+
description=description,
|
|
354
|
+
letter_annotations={_PHRED_KEY: quality},
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _require_object(value: object, *, record_index: int) -> Mapping[str, object]:
|
|
359
|
+
if not isinstance(value, Mapping):
|
|
360
|
+
raise InputFormatError(
|
|
361
|
+
"Each structured record must be a JSON object.",
|
|
362
|
+
code="INVALID_STRUCTURED_RECORD",
|
|
363
|
+
context={"record_index": record_index},
|
|
364
|
+
)
|
|
365
|
+
return cast(Mapping[str, object], value)
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _record_from_mapping(
|
|
369
|
+
row: Mapping[str, object], config: ReadConfig, *, record_index: int
|
|
370
|
+
) -> DNARecord:
|
|
371
|
+
sequence_value = row.get(config.sequence_column)
|
|
372
|
+
parts_value = row.get(config.parts_column)
|
|
373
|
+
if parts_value is None and not isinstance(sequence_value, str):
|
|
374
|
+
raise InputFormatError(
|
|
375
|
+
"Structured input requires a string sequence column or a parts array.",
|
|
376
|
+
code="MISSING_SEQUENCE_COLUMN",
|
|
377
|
+
context={"record_index": record_index, "column": config.sequence_column},
|
|
378
|
+
)
|
|
379
|
+
if sequence_value is not None and not isinstance(sequence_value, str):
|
|
380
|
+
raise InputFormatError(
|
|
381
|
+
"Structured input sequence must be text or null.",
|
|
382
|
+
code="INVALID_SEQUENCE_COLUMN",
|
|
383
|
+
context={"record_index": record_index, "column": config.sequence_column},
|
|
384
|
+
)
|
|
385
|
+
if isinstance(sequence_value, str) and len(sequence_value) > config.max_sequence_symbols:
|
|
386
|
+
raise InputFormatError(
|
|
387
|
+
"Structured record exceeds max_sequence_symbols.",
|
|
388
|
+
code="SEQUENCE_SYMBOL_LIMIT_EXCEEDED",
|
|
389
|
+
context={"record_index": record_index, "column": config.sequence_column},
|
|
390
|
+
)
|
|
391
|
+
schema_version = row.get("schema_version")
|
|
392
|
+
if schema_version not in {None, "", _RECORD_SCHEMA_VERSION}:
|
|
393
|
+
raise InputFormatError(
|
|
394
|
+
"Structured input uses an unsupported DNAKit record schema.",
|
|
395
|
+
code="UNSUPPORTED_RECORD_SCHEMA",
|
|
396
|
+
context={"record_index": record_index, "schema_version": schema_version},
|
|
397
|
+
)
|
|
398
|
+
id_value = row.get(config.id_column)
|
|
399
|
+
if id_value is None or id_value == "":
|
|
400
|
+
record_id = f"{config.anonymous_id_prefix}{record_index + 1}"
|
|
401
|
+
elif isinstance(id_value, str):
|
|
402
|
+
record_id = id_value
|
|
403
|
+
else:
|
|
404
|
+
raise InputFormatError(
|
|
405
|
+
"Structured input record ID must be text.",
|
|
406
|
+
code="INVALID_RECORD_ID_COLUMN",
|
|
407
|
+
context={"record_index": record_index, "column": config.id_column},
|
|
408
|
+
)
|
|
409
|
+
description_value = row.get(config.description_column, "")
|
|
410
|
+
if description_value is None:
|
|
411
|
+
description = ""
|
|
412
|
+
elif isinstance(description_value, str):
|
|
413
|
+
description = description_value
|
|
414
|
+
else:
|
|
415
|
+
raise InputFormatError(
|
|
416
|
+
"Structured input description must be text.",
|
|
417
|
+
code="INVALID_DESCRIPTION_COLUMN",
|
|
418
|
+
context={"record_index": record_index, "column": config.description_column},
|
|
419
|
+
)
|
|
420
|
+
metadata_value = row.get(config.metadata_column, {})
|
|
421
|
+
annotations_value = row.get(config.letter_annotations_column, {})
|
|
422
|
+
features_value = row.get(config.features_column, ())
|
|
423
|
+
alphabet_value = row.get("alphabet")
|
|
424
|
+
topology_value = row.get("topology")
|
|
425
|
+
strandedness_value = row.get("strandedness")
|
|
426
|
+
alphabet_value = None if alphabet_value == "" else alphabet_value
|
|
427
|
+
topology_value = None if topology_value == "" else topology_value
|
|
428
|
+
strandedness_value = None if strandedness_value == "" else strandedness_value
|
|
429
|
+
for name, value in (
|
|
430
|
+
("alphabet", alphabet_value),
|
|
431
|
+
("topology", topology_value),
|
|
432
|
+
("strandedness", strandedness_value),
|
|
433
|
+
):
|
|
434
|
+
if value is not None and not isinstance(value, str):
|
|
435
|
+
raise InputFormatError(
|
|
436
|
+
f"Structured input {name} must be text or null.",
|
|
437
|
+
code="INVALID_SEQUENCE_TYPE_COLUMN",
|
|
438
|
+
context={"record_index": record_index, "column": name},
|
|
439
|
+
)
|
|
440
|
+
if metadata_value is None:
|
|
441
|
+
metadata_value = {}
|
|
442
|
+
if annotations_value is None:
|
|
443
|
+
annotations_value = {}
|
|
444
|
+
if features_value is None:
|
|
445
|
+
features_value = ()
|
|
446
|
+
if isinstance(parts_value, str):
|
|
447
|
+
parts_value = (
|
|
448
|
+
_strict_json_loads(
|
|
449
|
+
parts_value,
|
|
450
|
+
config,
|
|
451
|
+
message=f"Column {config.parts_column!r} must contain a JSON array.",
|
|
452
|
+
code="INVALID_JSON_COLUMN",
|
|
453
|
+
context={"record_index": record_index, "column": config.parts_column},
|
|
454
|
+
)
|
|
455
|
+
if parts_value
|
|
456
|
+
else None
|
|
457
|
+
)
|
|
458
|
+
for column, value in (
|
|
459
|
+
(config.metadata_column, metadata_value),
|
|
460
|
+
(config.letter_annotations_column, annotations_value),
|
|
461
|
+
(config.features_column, features_value),
|
|
462
|
+
):
|
|
463
|
+
if isinstance(value, str):
|
|
464
|
+
value = (
|
|
465
|
+
_strict_json_loads(
|
|
466
|
+
value,
|
|
467
|
+
config,
|
|
468
|
+
message=f"Column {column!r} must contain valid JSON.",
|
|
469
|
+
code="INVALID_JSON_COLUMN",
|
|
470
|
+
context={"record_index": record_index, "column": column},
|
|
471
|
+
)
|
|
472
|
+
if value
|
|
473
|
+
else ([] if column == config.features_column else {})
|
|
474
|
+
)
|
|
475
|
+
if column == config.metadata_column:
|
|
476
|
+
metadata_value = value
|
|
477
|
+
elif column == config.letter_annotations_column:
|
|
478
|
+
annotations_value = value
|
|
479
|
+
else:
|
|
480
|
+
features_value = value
|
|
481
|
+
if not isinstance(metadata_value, Mapping) or not isinstance(annotations_value, Mapping):
|
|
482
|
+
raise InputFormatError(
|
|
483
|
+
"Metadata and letter_annotations must be objects.",
|
|
484
|
+
code="INVALID_STRUCTURED_METADATA",
|
|
485
|
+
context={"record_index": record_index},
|
|
486
|
+
)
|
|
487
|
+
if parts_value is None:
|
|
488
|
+
assert isinstance(sequence_value, str)
|
|
489
|
+
sequence = _sequence(
|
|
490
|
+
sequence_value,
|
|
491
|
+
config,
|
|
492
|
+
record_id=record_id,
|
|
493
|
+
line_number=record_index + 1,
|
|
494
|
+
alphabet=cast(str | None, alphabet_value),
|
|
495
|
+
topology=cast(str | None, topology_value),
|
|
496
|
+
strandedness=cast(str | None, strandedness_value),
|
|
497
|
+
)
|
|
498
|
+
else:
|
|
499
|
+
sequence = _sequence_from_parts(
|
|
500
|
+
parts_value,
|
|
501
|
+
sequence_text=sequence_value,
|
|
502
|
+
config=config,
|
|
503
|
+
record_id=record_id,
|
|
504
|
+
record_index=record_index,
|
|
505
|
+
alphabet=cast(str | None, alphabet_value),
|
|
506
|
+
topology=cast(str | None, topology_value),
|
|
507
|
+
strandedness=cast(str | None, strandedness_value),
|
|
508
|
+
)
|
|
509
|
+
features = _features_from_value(features_value, record_index=record_index)
|
|
510
|
+
try:
|
|
511
|
+
return DNARecord(
|
|
512
|
+
sequence,
|
|
513
|
+
record_id,
|
|
514
|
+
description=description,
|
|
515
|
+
features=features,
|
|
516
|
+
metadata=metadata_value,
|
|
517
|
+
letter_annotations=cast(Mapping[str, Iterable[int | float]], annotations_value),
|
|
518
|
+
)
|
|
519
|
+
except DNAKitError as exc:
|
|
520
|
+
raise InputFormatError(
|
|
521
|
+
"Structured record violates the DNARecord schema.",
|
|
522
|
+
code="INVALID_STRUCTURED_RECORD",
|
|
523
|
+
context={"record_index": record_index, "record_id": record_id},
|
|
524
|
+
) from exc
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _sequence_from_parts(
|
|
528
|
+
value: object,
|
|
529
|
+
*,
|
|
530
|
+
sequence_text: str | None,
|
|
531
|
+
config: ReadConfig,
|
|
532
|
+
record_id: str,
|
|
533
|
+
record_index: int,
|
|
534
|
+
alphabet: str | None,
|
|
535
|
+
topology: str | None,
|
|
536
|
+
strandedness: str | None,
|
|
537
|
+
) -> DNASequence:
|
|
538
|
+
if not isinstance(value, list):
|
|
539
|
+
raise InputFormatError(
|
|
540
|
+
"Structured input parts must be an array.",
|
|
541
|
+
code="INVALID_SEQUENCE_PARTS_COLUMN",
|
|
542
|
+
context={"record_index": record_index},
|
|
543
|
+
)
|
|
544
|
+
parts: list[str | Gap] = []
|
|
545
|
+
symbol_count = 0
|
|
546
|
+
try:
|
|
547
|
+
for item in value:
|
|
548
|
+
if not isinstance(item, Mapping):
|
|
549
|
+
raise TypeError("each sequence part must be an object")
|
|
550
|
+
kind = item.get("kind")
|
|
551
|
+
if kind == "symbols":
|
|
552
|
+
symbols = item.get("symbols")
|
|
553
|
+
if not isinstance(symbols, str):
|
|
554
|
+
raise TypeError("a symbol part requires string symbols")
|
|
555
|
+
symbol_count += len(symbols)
|
|
556
|
+
if symbol_count > config.max_sequence_symbols:
|
|
557
|
+
raise InputFormatError(
|
|
558
|
+
"Structured record exceeds max_sequence_symbols.",
|
|
559
|
+
code="SEQUENCE_SYMBOL_LIMIT_EXCEEDED",
|
|
560
|
+
context={"record_index": record_index, "record_id": record_id},
|
|
561
|
+
)
|
|
562
|
+
parts.append(symbols.upper() if config.uppercase else symbols)
|
|
563
|
+
elif kind == "gap":
|
|
564
|
+
raw_evidence = item.get("evidence", [])
|
|
565
|
+
raw_metadata = item.get("metadata", {})
|
|
566
|
+
if not isinstance(raw_evidence, list) or not isinstance(raw_metadata, Mapping):
|
|
567
|
+
raise TypeError("gap evidence and metadata have invalid types")
|
|
568
|
+
parts.append(
|
|
569
|
+
Gap(
|
|
570
|
+
cast(int | None, item.get("length")),
|
|
571
|
+
kind=cast(str, item.get("gap_kind", "unknown")),
|
|
572
|
+
crossable=cast(bool | None, item.get("crossable")),
|
|
573
|
+
evidence=cast(list[str], raw_evidence),
|
|
574
|
+
metadata=cast(Mapping[str, object], raw_metadata),
|
|
575
|
+
)
|
|
576
|
+
)
|
|
577
|
+
else:
|
|
578
|
+
raise TypeError("sequence part kind must be symbols or gap")
|
|
579
|
+
resolved = DNASequence(
|
|
580
|
+
parts,
|
|
581
|
+
alphabet=config.alphabet if alphabet is None else alphabet,
|
|
582
|
+
topology=config.topology if topology is None else topology,
|
|
583
|
+
strandedness=config.strandedness if strandedness is None else strandedness,
|
|
584
|
+
)
|
|
585
|
+
except InputFormatError:
|
|
586
|
+
raise
|
|
587
|
+
except (DNAKitError, TypeError, ValueError) as exc:
|
|
588
|
+
raise InputFormatError(
|
|
589
|
+
"Structured input contains invalid sequence parts.",
|
|
590
|
+
code="INVALID_SEQUENCE_PARTS_COLUMN",
|
|
591
|
+
context={"record_index": record_index, "record_id": record_id},
|
|
592
|
+
) from exc
|
|
593
|
+
if sequence_text is not None:
|
|
594
|
+
expected = sequence_text.upper() if config.uppercase else sequence_text
|
|
595
|
+
if expected != resolved.symbols:
|
|
596
|
+
raise InputFormatError(
|
|
597
|
+
"Structured sequence text does not match the symbols in parts.",
|
|
598
|
+
code="SEQUENCE_PARTS_MISMATCH",
|
|
599
|
+
context={"record_index": record_index, "record_id": record_id},
|
|
600
|
+
)
|
|
601
|
+
return resolved
|
|
602
|
+
|
|
603
|
+
|
|
604
|
+
def _interval_from_value(value: object) -> Interval:
|
|
605
|
+
if not isinstance(value, Mapping):
|
|
606
|
+
raise TypeError("interval must be an object")
|
|
607
|
+
return Interval(cast(int, value.get("start")), cast(int, value.get("end")))
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _location_from_value(value: object) -> Location:
|
|
611
|
+
if not isinstance(value, Mapping):
|
|
612
|
+
raise TypeError("feature location must be an object")
|
|
613
|
+
kind = value.get("kind")
|
|
614
|
+
if kind == "interval":
|
|
615
|
+
return _interval_from_value(value)
|
|
616
|
+
raw_parts = value.get("parts")
|
|
617
|
+
if kind == "compound":
|
|
618
|
+
if not isinstance(raw_parts, list):
|
|
619
|
+
raise TypeError("compound location parts must be an array")
|
|
620
|
+
return CompoundLocation(_interval_from_value(part) for part in raw_parts)
|
|
621
|
+
if kind == "unresolved":
|
|
622
|
+
reason = value.get("reason")
|
|
623
|
+
raw_anchors = value.get("anchors", [])
|
|
624
|
+
if not isinstance(reason, str) or not isinstance(raw_anchors, list):
|
|
625
|
+
raise TypeError("unresolved location requires a reason and anchor array")
|
|
626
|
+
return UnresolvedLocation(
|
|
627
|
+
reason,
|
|
628
|
+
(_interval_from_value(anchor) for anchor in raw_anchors),
|
|
629
|
+
)
|
|
630
|
+
raise TypeError("feature location kind must be interval, compound, or unresolved")
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def _features_from_value(value: object, *, record_index: int) -> tuple[DNAFeature, ...]:
|
|
634
|
+
if not isinstance(value, (list, tuple)):
|
|
635
|
+
raise InputFormatError(
|
|
636
|
+
"Structured input features must be an array.",
|
|
637
|
+
code="INVALID_FEATURES_COLUMN",
|
|
638
|
+
context={"record_index": record_index},
|
|
639
|
+
)
|
|
640
|
+
features: list[DNAFeature] = []
|
|
641
|
+
try:
|
|
642
|
+
for raw_feature in value:
|
|
643
|
+
if not isinstance(raw_feature, Mapping):
|
|
644
|
+
raise TypeError("each feature must be an object")
|
|
645
|
+
qualifiers = raw_feature.get("qualifiers", {})
|
|
646
|
+
if not isinstance(qualifiers, Mapping):
|
|
647
|
+
raise TypeError("feature qualifiers must be an object")
|
|
648
|
+
features.append(
|
|
649
|
+
DNAFeature(
|
|
650
|
+
cast(str, raw_feature.get("type")),
|
|
651
|
+
_location_from_value(raw_feature.get("location")),
|
|
652
|
+
id=cast(str | None, raw_feature.get("id")),
|
|
653
|
+
strand=cast(str, raw_feature.get("strand", "unknown")),
|
|
654
|
+
label=cast(str | None, raw_feature.get("label")),
|
|
655
|
+
score=cast(float | None, raw_feature.get("score")),
|
|
656
|
+
phase=cast(int | None, raw_feature.get("phase")),
|
|
657
|
+
qualifiers=cast(Mapping[str, object], qualifiers),
|
|
658
|
+
source=cast(str | None, raw_feature.get("source")),
|
|
659
|
+
)
|
|
660
|
+
)
|
|
661
|
+
except (DNAKitError, TypeError, ValueError) as exc:
|
|
662
|
+
raise InputFormatError(
|
|
663
|
+
"Structured input contains an invalid feature.",
|
|
664
|
+
code="INVALID_FEATURES_COLUMN",
|
|
665
|
+
context={"record_index": record_index},
|
|
666
|
+
) from exc
|
|
667
|
+
return tuple(features)
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def iter_delimited(handle: TextIO, config: ReadConfig, *, delimiter: str) -> Iterator[DNARecord]:
|
|
671
|
+
previous_limit = csv.field_size_limit()
|
|
672
|
+
try:
|
|
673
|
+
csv.field_size_limit(config.max_field_size)
|
|
674
|
+
reader = csv.DictReader(handle, delimiter=delimiter)
|
|
675
|
+
fieldnames = reader.fieldnames
|
|
676
|
+
if fieldnames is None or (
|
|
677
|
+
config.sequence_column not in fieldnames and config.parts_column not in fieldnames
|
|
678
|
+
):
|
|
679
|
+
raise InputFormatError(
|
|
680
|
+
"Delimited input must have a header containing sequence or parts.",
|
|
681
|
+
code="MISSING_SEQUENCE_COLUMN",
|
|
682
|
+
context={"column": config.sequence_column},
|
|
683
|
+
)
|
|
684
|
+
if any(not isinstance(name, str) or not name for name in fieldnames) or len(
|
|
685
|
+
set(fieldnames)
|
|
686
|
+
) != len(fieldnames):
|
|
687
|
+
raise InputFormatError(
|
|
688
|
+
"Delimited input header names must be non-empty and unique.",
|
|
689
|
+
code="INVALID_TABLE_HEADER",
|
|
690
|
+
context={"fieldnames": fieldnames},
|
|
691
|
+
)
|
|
692
|
+
for index, row in enumerate(reader):
|
|
693
|
+
if None in row:
|
|
694
|
+
raise InputFormatError(
|
|
695
|
+
"Delimited input row contains more fields than its header.",
|
|
696
|
+
code="EXTRA_TABLE_FIELDS",
|
|
697
|
+
context={"record_index": index, "line_number": reader.line_num},
|
|
698
|
+
)
|
|
699
|
+
yield _record_from_mapping(row, config, record_index=index)
|
|
700
|
+
except csv.Error as exc:
|
|
701
|
+
raise InputFormatError(
|
|
702
|
+
"Delimited input exceeds the configured field-size limit or is malformed.",
|
|
703
|
+
code="CSV_FIELD_ERROR",
|
|
704
|
+
context={"max_field_size": config.max_field_size, "reason": str(exc)},
|
|
705
|
+
) from exc
|
|
706
|
+
finally:
|
|
707
|
+
csv.field_size_limit(previous_limit)
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def iter_json_lines(handle: TextIO, config: ReadConfig) -> Iterator[DNARecord]:
|
|
711
|
+
record_index = 0
|
|
712
|
+
for line_number, line in enumerate(handle, start=1):
|
|
713
|
+
if len(line.rstrip("\r\n")) > config.max_field_size:
|
|
714
|
+
raise _input_error(
|
|
715
|
+
"JSON Lines record exceeds max_field_size.",
|
|
716
|
+
code="STRUCTURED_LINE_TOO_LONG",
|
|
717
|
+
line_number=line_number,
|
|
718
|
+
)
|
|
719
|
+
if not line.strip():
|
|
720
|
+
continue
|
|
721
|
+
value = _strict_json_loads(
|
|
722
|
+
line,
|
|
723
|
+
config,
|
|
724
|
+
message="JSON Lines input contains invalid JSON.",
|
|
725
|
+
code="INVALID_JSON",
|
|
726
|
+
context={"line_number": line_number},
|
|
727
|
+
)
|
|
728
|
+
yield _record_from_mapping(
|
|
729
|
+
_require_object(value, record_index=record_index),
|
|
730
|
+
config,
|
|
731
|
+
record_index=record_index,
|
|
732
|
+
)
|
|
733
|
+
record_index += 1
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def iter_json_array(handle: TextIO, config: ReadConfig) -> Iterator[DNARecord]:
|
|
737
|
+
"""Parse the standard JSON array representation.
|
|
738
|
+
|
|
739
|
+
The parser materializes at most ``max_input_bytes`` and separately bounds
|
|
740
|
+
JSON depth and node count. Use JSONL, CSV, TSV, FASTA, or FASTQ for streaming.
|
|
741
|
+
"""
|
|
742
|
+
|
|
743
|
+
value = _strict_json_loads(
|
|
744
|
+
handle.read(),
|
|
745
|
+
config,
|
|
746
|
+
message="JSON input contains invalid JSON.",
|
|
747
|
+
code="INVALID_JSON",
|
|
748
|
+
)
|
|
749
|
+
if not isinstance(value, list):
|
|
750
|
+
raise InputFormatError(
|
|
751
|
+
"JSON record input must be an array of objects.",
|
|
752
|
+
code="INVALID_JSON_ROOT",
|
|
753
|
+
)
|
|
754
|
+
for index, item in enumerate(value):
|
|
755
|
+
yield _record_from_mapping(
|
|
756
|
+
_require_object(item, record_index=index), config, record_index=index
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def record_to_mapping(record: DNARecord) -> dict[str, Any]:
|
|
761
|
+
"""Return the portable MVP record schema used by JSON and delimited I/O."""
|
|
762
|
+
|
|
763
|
+
return {
|
|
764
|
+
"schema_version": _RECORD_SCHEMA_VERSION,
|
|
765
|
+
"id": record.id,
|
|
766
|
+
"sequence": record.sequence.symbols,
|
|
767
|
+
"parts": [_sequence_part_to_mapping(part) for part in record.sequence.parts],
|
|
768
|
+
"alphabet": record.sequence.alphabet.value,
|
|
769
|
+
"topology": record.sequence.topology.value,
|
|
770
|
+
"strandedness": record.sequence.strandedness.value,
|
|
771
|
+
"description": record.description,
|
|
772
|
+
"features": [_feature_to_mapping(feature) for feature in record.features],
|
|
773
|
+
"metadata": to_json_compatible(record.metadata),
|
|
774
|
+
"letter_annotations": to_json_compatible(record.letter_annotations),
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def _sequence_part_to_mapping(part: str | Gap) -> dict[str, Any]:
|
|
779
|
+
if isinstance(part, str):
|
|
780
|
+
return {"kind": "symbols", "symbols": part}
|
|
781
|
+
return {
|
|
782
|
+
"kind": "gap",
|
|
783
|
+
"length": part.length,
|
|
784
|
+
"gap_kind": part.kind.value,
|
|
785
|
+
"crossable": part.crossable,
|
|
786
|
+
"evidence": list(part.evidence),
|
|
787
|
+
"metadata": to_json_compatible(part.metadata),
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
|
|
791
|
+
def _location_to_mapping(location: Location) -> dict[str, Any]:
|
|
792
|
+
if isinstance(location, Interval):
|
|
793
|
+
return {"kind": "interval", "start": location.start, "end": location.end}
|
|
794
|
+
if isinstance(location, CompoundLocation):
|
|
795
|
+
return {
|
|
796
|
+
"kind": "compound",
|
|
797
|
+
"parts": [{"start": part.start, "end": part.end} for part in location.parts],
|
|
798
|
+
}
|
|
799
|
+
return {
|
|
800
|
+
"kind": "unresolved",
|
|
801
|
+
"reason": location.reason,
|
|
802
|
+
"anchors": [{"start": anchor.start, "end": anchor.end} for anchor in location.anchors],
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def _feature_to_mapping(feature: DNAFeature) -> dict[str, Any]:
|
|
807
|
+
return {
|
|
808
|
+
"type": feature.type,
|
|
809
|
+
"location": _location_to_mapping(feature.location),
|
|
810
|
+
"id": feature.id,
|
|
811
|
+
"strand": feature.strand.value,
|
|
812
|
+
"label": feature.label,
|
|
813
|
+
"score": feature.score,
|
|
814
|
+
"phase": feature.phase,
|
|
815
|
+
"qualifiers": to_json_compatible(feature.qualifiers),
|
|
816
|
+
"source": feature.source,
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _reject_implicit_feature_loss(record: DNARecord, *, format: str, config: WriteConfig) -> None:
|
|
821
|
+
if record.features and config.feature_policy == "error":
|
|
822
|
+
raise InputFormatError(
|
|
823
|
+
f"{format} cannot represent DNARecord features without information loss.",
|
|
824
|
+
code="FEATURE_LOSS_NOT_ALLOWED",
|
|
825
|
+
context={"record_id": record.id, "feature_count": len(record.features)},
|
|
826
|
+
hint="Use JSON/JSONL/CSV/TSV or set feature_policy='drop' explicitly.",
|
|
827
|
+
)
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _reject_implicit_gap_loss(record: DNARecord, *, format: str) -> None:
|
|
831
|
+
if record.sequence.is_gapped:
|
|
832
|
+
raise InputFormatError(
|
|
833
|
+
f"{format} cannot represent explicit Gap objects without information loss.",
|
|
834
|
+
code="GAP_LOSS_NOT_ALLOWED",
|
|
835
|
+
context={"record_id": record.id},
|
|
836
|
+
hint="Use JSON, JSONL, CSV, or TSV for lossless Gap persistence.",
|
|
837
|
+
)
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def write_fasta(handle: TextIO, records: Iterable[DNARecord], config: WriteConfig) -> int:
|
|
841
|
+
count = 0
|
|
842
|
+
for record in records:
|
|
843
|
+
_reject_implicit_gap_loss(record, format="FASTA")
|
|
844
|
+
_reject_implicit_feature_loss(record, format="FASTA", config=config)
|
|
845
|
+
header = _serialized_header(record, format="FASTA")
|
|
846
|
+
sequence = record.sequence.to_string()
|
|
847
|
+
handle.write(f">{header}\n")
|
|
848
|
+
for offset in range(0, len(sequence), config.line_width):
|
|
849
|
+
handle.write(sequence[offset : offset + config.line_width] + "\n")
|
|
850
|
+
count += 1
|
|
851
|
+
return count
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def write_fastq(handle: TextIO, records: Iterable[DNARecord], config: WriteConfig) -> int:
|
|
855
|
+
count = 0
|
|
856
|
+
for record in records:
|
|
857
|
+
_reject_implicit_gap_loss(record, format="FASTQ")
|
|
858
|
+
_reject_implicit_feature_loss(record, format="FASTQ", config=config)
|
|
859
|
+
sequence = record.sequence.to_string()
|
|
860
|
+
values = record.letter_annotations.get(_PHRED_KEY)
|
|
861
|
+
if values is None:
|
|
862
|
+
raise InputFormatError(
|
|
863
|
+
"FASTQ output requires letter_annotations['phred_quality'].",
|
|
864
|
+
code="FASTQ_QUALITY_MISSING",
|
|
865
|
+
context={"record_id": record.id},
|
|
866
|
+
)
|
|
867
|
+
if len(values) != len(sequence):
|
|
868
|
+
raise InputFormatError(
|
|
869
|
+
"FASTQ quality length must equal sequence symbol_length.",
|
|
870
|
+
code="FASTQ_QUALITY_LENGTH_MISMATCH",
|
|
871
|
+
context={"record_id": record.id},
|
|
872
|
+
)
|
|
873
|
+
quality_characters: list[str] = []
|
|
874
|
+
for value in values:
|
|
875
|
+
if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 93:
|
|
876
|
+
raise InputFormatError(
|
|
877
|
+
"FASTQ output Phred values must be integers from 0 through 93.",
|
|
878
|
+
code="FASTQ_QUALITY_OUT_OF_RANGE",
|
|
879
|
+
context={"record_id": record.id, "value": value},
|
|
880
|
+
)
|
|
881
|
+
codepoint = value + config.phred_offset
|
|
882
|
+
if codepoint > 126:
|
|
883
|
+
raise InputFormatError(
|
|
884
|
+
"FASTQ quality value cannot be encoded with the configured Phred offset.",
|
|
885
|
+
code="FASTQ_QUALITY_OUT_OF_RANGE",
|
|
886
|
+
context={
|
|
887
|
+
"record_id": record.id,
|
|
888
|
+
"value": value,
|
|
889
|
+
"phred_offset": config.phred_offset,
|
|
890
|
+
},
|
|
891
|
+
)
|
|
892
|
+
quality_characters.append(chr(codepoint))
|
|
893
|
+
header = _serialized_header(record, format="FASTQ")
|
|
894
|
+
handle.write(f"@{header}\n{sequence}\n+\n{''.join(quality_characters)}\n")
|
|
895
|
+
count += 1
|
|
896
|
+
return count
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def write_delimited(
|
|
900
|
+
handle: TextIO,
|
|
901
|
+
records: Iterable[DNARecord],
|
|
902
|
+
config: WriteConfig,
|
|
903
|
+
*,
|
|
904
|
+
delimiter: str,
|
|
905
|
+
) -> int:
|
|
906
|
+
del config
|
|
907
|
+
fieldnames = [
|
|
908
|
+
"schema_version",
|
|
909
|
+
"id",
|
|
910
|
+
"sequence",
|
|
911
|
+
"parts",
|
|
912
|
+
"alphabet",
|
|
913
|
+
"topology",
|
|
914
|
+
"strandedness",
|
|
915
|
+
"description",
|
|
916
|
+
"features",
|
|
917
|
+
"metadata",
|
|
918
|
+
"letter_annotations",
|
|
919
|
+
]
|
|
920
|
+
writer = csv.DictWriter(handle, fieldnames=fieldnames, delimiter=delimiter, lineterminator="\n")
|
|
921
|
+
writer.writeheader()
|
|
922
|
+
count = 0
|
|
923
|
+
for record in records:
|
|
924
|
+
row = record_to_mapping(record)
|
|
925
|
+
row["parts"] = json.dumps(row["parts"], ensure_ascii=False, sort_keys=True)
|
|
926
|
+
row["features"] = json.dumps(row["features"], ensure_ascii=False, sort_keys=True)
|
|
927
|
+
row["metadata"] = json.dumps(row["metadata"], ensure_ascii=False, sort_keys=True)
|
|
928
|
+
row["letter_annotations"] = json.dumps(
|
|
929
|
+
row["letter_annotations"], ensure_ascii=False, sort_keys=True
|
|
930
|
+
)
|
|
931
|
+
writer.writerow(row)
|
|
932
|
+
count += 1
|
|
933
|
+
return count
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def write_json_lines(handle: TextIO, records: Iterable[DNARecord], config: WriteConfig) -> int:
|
|
937
|
+
if config.json_indent is not None:
|
|
938
|
+
raise ConfigurationError(
|
|
939
|
+
"JSONL requires one compact JSON object per physical line.",
|
|
940
|
+
code="JSONL_INDENT_NOT_ALLOWED",
|
|
941
|
+
hint="Use format='json' when pretty-printed multi-line output is required.",
|
|
942
|
+
)
|
|
943
|
+
count = 0
|
|
944
|
+
for record in records:
|
|
945
|
+
payload = record_to_mapping(record)
|
|
946
|
+
handle.write(
|
|
947
|
+
json.dumps(
|
|
948
|
+
payload,
|
|
949
|
+
ensure_ascii=False,
|
|
950
|
+
sort_keys=True,
|
|
951
|
+
separators=(",", ":"),
|
|
952
|
+
)
|
|
953
|
+
+ "\n"
|
|
954
|
+
)
|
|
955
|
+
count += 1
|
|
956
|
+
return count
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
def write_json_array(handle: TextIO, records: Iterable[DNARecord], config: WriteConfig) -> int:
|
|
960
|
+
handle.write("[\n" if config.json_indent is not None else "[")
|
|
961
|
+
count = 0
|
|
962
|
+
for record in records:
|
|
963
|
+
if count:
|
|
964
|
+
handle.write(",\n" if config.json_indent is not None else ",")
|
|
965
|
+
payload = record_to_mapping(record)
|
|
966
|
+
encoded = json.dumps(
|
|
967
|
+
payload,
|
|
968
|
+
ensure_ascii=False,
|
|
969
|
+
sort_keys=True,
|
|
970
|
+
separators=None if config.json_indent is not None else (",", ":"),
|
|
971
|
+
indent=config.json_indent,
|
|
972
|
+
)
|
|
973
|
+
if config.json_indent is not None:
|
|
974
|
+
encoded = "\n".join(f" {line}" for line in encoded.splitlines())
|
|
975
|
+
handle.write(encoded)
|
|
976
|
+
count += 1
|
|
977
|
+
if config.json_indent is not None:
|
|
978
|
+
handle.write("\n")
|
|
979
|
+
handle.write("]\n")
|
|
980
|
+
return count
|
|
981
|
+
|
|
982
|
+
|
|
983
|
+
__all__ = [
|
|
984
|
+
"iter_delimited",
|
|
985
|
+
"iter_fasta",
|
|
986
|
+
"iter_fastq",
|
|
987
|
+
"iter_json_array",
|
|
988
|
+
"iter_json_lines",
|
|
989
|
+
"write_delimited",
|
|
990
|
+
"write_fasta",
|
|
991
|
+
"write_fastq",
|
|
992
|
+
"write_json_array",
|
|
993
|
+
"write_json_lines",
|
|
994
|
+
]
|