hashfasta 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hashfasta/__init__.py +52 -0
- hashfasta/__main__.py +5 -0
- hashfasta/cli.py +50 -0
- hashfasta/core.py +418 -0
- hashfasta/py.typed +0 -0
- hashfasta-0.1.0.dist-info/METADATA +100 -0
- hashfasta-0.1.0.dist-info/RECORD +11 -0
- hashfasta-0.1.0.dist-info/WHEEL +5 -0
- hashfasta-0.1.0.dist-info/entry_points.txt +2 -0
- hashfasta-0.1.0.dist-info/licenses/LICENSE +21 -0
- hashfasta-0.1.0.dist-info/top_level.txt +1 -0
hashfasta/__init__.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""HashFASTA: FASTA records keyed by the SHA-256 of their normalized sequence.
|
|
2
|
+
|
|
3
|
+
Command line::
|
|
4
|
+
|
|
5
|
+
hashfasta input.fasta[.gz|.bz2|.xz] [-o output.hash.fasta.gz]
|
|
6
|
+
|
|
7
|
+
Library::
|
|
8
|
+
|
|
9
|
+
import hashfasta
|
|
10
|
+
hashfasta.convert("sample.fna.gz") # -> "sample.hash.fasta.gz"
|
|
11
|
+
for rec in hashfasta.read_hashfasta("sample.hash.fasta.gz"):
|
|
12
|
+
print(rec.key, rec.description, len(rec.sequence))
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from .core import (
|
|
16
|
+
HashFastaError,
|
|
17
|
+
Record,
|
|
18
|
+
convert,
|
|
19
|
+
convert_stream,
|
|
20
|
+
default_output_path,
|
|
21
|
+
escape_header,
|
|
22
|
+
format_record,
|
|
23
|
+
iter_records,
|
|
24
|
+
normalize_sequence,
|
|
25
|
+
open_input,
|
|
26
|
+
read_fasta,
|
|
27
|
+
read_hashfasta,
|
|
28
|
+
sequence_key,
|
|
29
|
+
unescape_header,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
__version__ = "0.1.0"
|
|
33
|
+
SPEC_VERSION = "0.1"
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"HashFastaError",
|
|
37
|
+
"Record",
|
|
38
|
+
"SPEC_VERSION",
|
|
39
|
+
"__version__",
|
|
40
|
+
"convert",
|
|
41
|
+
"convert_stream",
|
|
42
|
+
"default_output_path",
|
|
43
|
+
"escape_header",
|
|
44
|
+
"format_record",
|
|
45
|
+
"iter_records",
|
|
46
|
+
"normalize_sequence",
|
|
47
|
+
"open_input",
|
|
48
|
+
"read_fasta",
|
|
49
|
+
"read_hashfasta",
|
|
50
|
+
"sequence_key",
|
|
51
|
+
"unescape_header",
|
|
52
|
+
]
|
hashfasta/__main__.py
ADDED
hashfasta/cli.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""``hashfasta INPUT [-o OUTPUT]`` command-line interface."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
from . import __version__
|
|
6
|
+
from .core import HashFastaError, convert
|
|
7
|
+
|
|
8
|
+
PROG = "hashfasta"
|
|
9
|
+
USAGE = "usage: %s INPUT [-o OUTPUT] ('-' means stdin/stdout)\n" % PROG
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_args(argv):
|
|
13
|
+
"""Return (input, output) or None for a usage error. Exits for -h/--version."""
|
|
14
|
+
input_path = None
|
|
15
|
+
output_path = None
|
|
16
|
+
i = 0
|
|
17
|
+
while i < len(argv):
|
|
18
|
+
arg = argv[i]
|
|
19
|
+
if arg in ("-h", "--help"):
|
|
20
|
+
sys.stdout.write(USAGE)
|
|
21
|
+
sys.exit(0)
|
|
22
|
+
if arg == "--version":
|
|
23
|
+
sys.stdout.write("%s %s\n" % (PROG, __version__))
|
|
24
|
+
sys.exit(0)
|
|
25
|
+
if arg == "-o":
|
|
26
|
+
if i + 1 >= len(argv):
|
|
27
|
+
return None
|
|
28
|
+
output_path = argv[i + 1]
|
|
29
|
+
i += 2
|
|
30
|
+
continue
|
|
31
|
+
if (arg != "-" and arg.startswith("-")) or input_path is not None:
|
|
32
|
+
return None
|
|
33
|
+
input_path = arg
|
|
34
|
+
i += 1
|
|
35
|
+
if input_path is None:
|
|
36
|
+
return None
|
|
37
|
+
return input_path, output_path
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def main(argv=None):
|
|
41
|
+
args = parse_args(sys.argv[1:] if argv is None else argv)
|
|
42
|
+
if args is None:
|
|
43
|
+
sys.stderr.write(USAGE)
|
|
44
|
+
return 2
|
|
45
|
+
try:
|
|
46
|
+
convert(*args)
|
|
47
|
+
except HashFastaError as e:
|
|
48
|
+
sys.stderr.write("%s: error: %s\n" % (PROG, e))
|
|
49
|
+
return 1
|
|
50
|
+
return 0
|
hashfasta/core.py
ADDED
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
"""HashFASTA v0.1 reference implementation (library core).
|
|
2
|
+
|
|
3
|
+
Converts plain, gzip, bzip2 or xz FASTA into gzip-compressed HashFASTA whose
|
|
4
|
+
record IDs are SHA-256 digests of the normalized sequence, and reads
|
|
5
|
+
HashFASTA back. See spec/HASHFASTA-v0.1.md. Standard library only.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import bz2
|
|
9
|
+
import collections
|
|
10
|
+
import gzip
|
|
11
|
+
import hashlib
|
|
12
|
+
import io
|
|
13
|
+
import lzma
|
|
14
|
+
import os
|
|
15
|
+
import re
|
|
16
|
+
import sys
|
|
17
|
+
import zlib
|
|
18
|
+
|
|
19
|
+
LINE_WIDTH = 80
|
|
20
|
+
COMPRESSION_SUFFIXES = (".gz", ".bgz", ".bz2", ".xz")
|
|
21
|
+
FASTA_SUFFIXES = (".fa", ".fas", ".fasta", ".fna", ".ffn", ".faa", ".frn", ".fsa")
|
|
22
|
+
OUTPUT_SUFFIX = ".hash.fasta.gz"
|
|
23
|
+
|
|
24
|
+
GZIP_MAGIC = b"\x1f\x8b"
|
|
25
|
+
BZIP2_MAGIC = b"BZh"
|
|
26
|
+
XZ_MAGIC = b"\xfd7zXZ\x00"
|
|
27
|
+
UNSUPPORTED_MAGIC = ((b"\x28\xb5\x2f\xfd", "zstd"), (b"PK\x03\x04", "ZIP"))
|
|
28
|
+
|
|
29
|
+
# Whitespace removed before hashing: space, HT, LF, VT, FF, CR.
|
|
30
|
+
WHITESPACE = b" \t\n\x0b\x0c\r"
|
|
31
|
+
UPPER = bytes.maketrans(b"abcdefghijklmnopqrstuvwxyz", b"ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
|
32
|
+
# After whitespace removal, only printable non-space ASCII (0x21-0x7e) other
|
|
33
|
+
# than ';' is valid. A line starting with ';' is a comment (spec section 3).
|
|
34
|
+
VALID = bytes(b for b in range(0x21, 0x7F) if b != 0x3B)
|
|
35
|
+
COMMENT = b";"
|
|
36
|
+
|
|
37
|
+
_DECODE_ERRORS = (OSError, EOFError, zlib.error, lzma.LZMAError)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class HashFastaError(Exception):
|
|
41
|
+
"""Invalid input, or a failure reading or writing a file."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Record(collections.namedtuple("Record", "key header sequence")):
|
|
45
|
+
"""One HashFASTA record.
|
|
46
|
+
|
|
47
|
+
key -- ``sha256_<64 hex>`` identity (str)
|
|
48
|
+
header -- original definition line without '>' (bytes, CR removed, unescaped)
|
|
49
|
+
sequence -- normalized sequence (bytes, uppercase, no whitespace)
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
__slots__ = ()
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def description(self):
|
|
56
|
+
"""The original definition line decoded as UTF-8 (undecodable bytes replaced)."""
|
|
57
|
+
return self.header.decode("utf-8", "replace")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def escape_header(header):
|
|
61
|
+
"""Escape ``\\``, ``[`` and ``]`` for the bracketed provenance field."""
|
|
62
|
+
return (header.replace(b"\\", b"\\\\")
|
|
63
|
+
.replace(b"[", b"\\[")
|
|
64
|
+
.replace(b"]", b"\\]"))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
_UNESCAPE = re.compile(rb"\\([\\\[\]])")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def unescape_header(escaped):
|
|
71
|
+
"""Reverse :func:`escape_header`."""
|
|
72
|
+
return _UNESCAPE.sub(rb"\1", escaped)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _invalid_byte(b):
|
|
76
|
+
if b == COMMENT[0]:
|
|
77
|
+
return "invalid sequence byte ';' (allowed only at the start of a comment line)"
|
|
78
|
+
return "invalid sequence byte 0x%02x (non-ASCII or non-printable)" % b
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def normalize_sequence(data):
|
|
82
|
+
"""Remove ASCII whitespace and uppercase a-z; raise on invalid bytes."""
|
|
83
|
+
seq = data.translate(UPPER, WHITESPACE)
|
|
84
|
+
bad = seq.translate(None, VALID)
|
|
85
|
+
if bad:
|
|
86
|
+
raise HashFastaError(_invalid_byte(bad[0]))
|
|
87
|
+
return seq
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def sequence_key(normalized):
|
|
91
|
+
"""``sha256_<hex>`` key for an already-normalized sequence."""
|
|
92
|
+
return "sha256_" + hashlib.sha256(normalized).hexdigest()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def format_record(record):
|
|
96
|
+
"""HashFASTA text (bytes) for one :class:`Record`, sequence wrapped at 80."""
|
|
97
|
+
seq = record.sequence
|
|
98
|
+
return (b">" + record.key.encode("ascii") + b" [" + escape_header(record.header) + b"]\n"
|
|
99
|
+
+ b"".join(seq[i:i + LINE_WIDTH] + b"\n" for i in range(0, len(seq), LINE_WIDTH)))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def default_output_path(input_path):
|
|
103
|
+
"""Default sibling output name for a named input (spec section 5)."""
|
|
104
|
+
name = input_path
|
|
105
|
+
for suffix in COMPRESSION_SUFFIXES:
|
|
106
|
+
if name.endswith(suffix):
|
|
107
|
+
name = name[:-len(suffix)]
|
|
108
|
+
break
|
|
109
|
+
for suffix in FASTA_SUFFIXES:
|
|
110
|
+
if name.endswith(suffix):
|
|
111
|
+
name = name[:-len(suffix)]
|
|
112
|
+
break
|
|
113
|
+
return name + OUTPUT_SUFFIX
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _Prefixed(io.RawIOBase):
|
|
117
|
+
"""Raw stream that replays already-read leading bytes before the rest."""
|
|
118
|
+
|
|
119
|
+
def __init__(self, prefix, raw):
|
|
120
|
+
self._prefix = prefix
|
|
121
|
+
self._raw = raw
|
|
122
|
+
|
|
123
|
+
def readable(self):
|
|
124
|
+
return True
|
|
125
|
+
|
|
126
|
+
def readinto(self, b):
|
|
127
|
+
if self._prefix:
|
|
128
|
+
n = min(len(b), len(self._prefix))
|
|
129
|
+
b[:n] = self._prefix[:n]
|
|
130
|
+
self._prefix = self._prefix[n:]
|
|
131
|
+
return n
|
|
132
|
+
data = self._raw.read(len(b))
|
|
133
|
+
b[:len(data)] = data
|
|
134
|
+
return len(data)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
_CHUNK = 1 << 16
|
|
138
|
+
_ZLIB = type(zlib.decompressobj())
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class _Decompressed(io.RawIOBase):
|
|
142
|
+
"""Concatenated gzip, bzip2 or xz streams decoded as one input.
|
|
143
|
+
|
|
144
|
+
After each complete stream the input must end or start another stream of
|
|
145
|
+
the same format; anything else (a corrupt stream, trailing junk or zero
|
|
146
|
+
bytes) is an error. xz also allows its own stream padding: NUL bytes in
|
|
147
|
+
multiples of four. The standard library's file wrappers silently stop at
|
|
148
|
+
a corrupt trailing bzip2 or xz stream, so they are not used here.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
def __init__(self, raw, new_decoder, name, xz_padding=False):
|
|
152
|
+
self._raw = raw
|
|
153
|
+
self._new = new_decoder
|
|
154
|
+
self._name = name
|
|
155
|
+
self._xz_padding = xz_padding
|
|
156
|
+
self._dec = None
|
|
157
|
+
self._buf = b"" # input not yet given to a decoder
|
|
158
|
+
self._nul = 0
|
|
159
|
+
|
|
160
|
+
def readable(self):
|
|
161
|
+
return True
|
|
162
|
+
|
|
163
|
+
def _next_stream(self):
|
|
164
|
+
"""Start the next stream; return False at a clean end of input."""
|
|
165
|
+
while True:
|
|
166
|
+
if not self._buf:
|
|
167
|
+
self._buf = self._raw.read(_CHUNK)
|
|
168
|
+
if not self._buf:
|
|
169
|
+
if self._nul % 4:
|
|
170
|
+
raise HashFastaError("invalid xz stream padding")
|
|
171
|
+
return False
|
|
172
|
+
if self._xz_padding:
|
|
173
|
+
stripped = self._buf.lstrip(b"\0")
|
|
174
|
+
self._nul += len(self._buf) - len(stripped)
|
|
175
|
+
self._buf = stripped
|
|
176
|
+
if not stripped:
|
|
177
|
+
continue
|
|
178
|
+
if self._nul % 4:
|
|
179
|
+
raise HashFastaError("invalid xz stream padding")
|
|
180
|
+
self._nul = 0
|
|
181
|
+
self._dec = self._new()
|
|
182
|
+
return True
|
|
183
|
+
|
|
184
|
+
def readinto(self, b):
|
|
185
|
+
while True:
|
|
186
|
+
if self._dec is None and not self._next_stream():
|
|
187
|
+
return 0
|
|
188
|
+
dec = self._dec
|
|
189
|
+
zlib_dec = isinstance(dec, _ZLIB)
|
|
190
|
+
if not self._buf and (zlib_dec or dec.needs_input):
|
|
191
|
+
self._buf = self._raw.read(_CHUNK)
|
|
192
|
+
if not self._buf:
|
|
193
|
+
raise HashFastaError("%s input is truncated" % self._name)
|
|
194
|
+
out = dec.decompress(self._buf, len(b))
|
|
195
|
+
self._buf = dec.unconsumed_tail if zlib_dec else b""
|
|
196
|
+
if dec.eof:
|
|
197
|
+
self._buf = dec.unused_data
|
|
198
|
+
self._dec = None
|
|
199
|
+
if out:
|
|
200
|
+
b[:len(out)] = out
|
|
201
|
+
return len(out)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def open_input(raw):
|
|
205
|
+
"""Wrap a binary stream, decompressing gzip, bzip2 or xz detected by magic bytes."""
|
|
206
|
+
head = b""
|
|
207
|
+
while len(head) < 6:
|
|
208
|
+
chunk = raw.read(6 - len(head))
|
|
209
|
+
if not chunk:
|
|
210
|
+
break
|
|
211
|
+
head += chunk
|
|
212
|
+
for magic, name in UNSUPPORTED_MAGIC:
|
|
213
|
+
if head.startswith(magic):
|
|
214
|
+
raise HashFastaError("%s-compressed input is not supported; decompress it first" % name)
|
|
215
|
+
stream = _Prefixed(head, raw)
|
|
216
|
+
if head.startswith(GZIP_MAGIC):
|
|
217
|
+
stream = _Decompressed(stream, lambda: zlib.decompressobj(16 + zlib.MAX_WBITS), "gzip")
|
|
218
|
+
elif head.startswith(BZIP2_MAGIC):
|
|
219
|
+
stream = _Decompressed(stream, bz2.BZ2Decompressor, "bzip2")
|
|
220
|
+
elif head.startswith(XZ_MAGIC):
|
|
221
|
+
stream = _Decompressed(stream, lambda: lzma.LZMADecompressor(lzma.FORMAT_XZ), "xz",
|
|
222
|
+
xz_padding=True)
|
|
223
|
+
return io.BufferedReader(stream, _CHUNK)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _parse(inp):
|
|
227
|
+
"""Yield (header, header_line, normalized sequence) for each FASTA record."""
|
|
228
|
+
header = None
|
|
229
|
+
header_line = 0
|
|
230
|
+
seq = bytearray()
|
|
231
|
+
lineno = 0
|
|
232
|
+
try:
|
|
233
|
+
for line in inp:
|
|
234
|
+
lineno += 1
|
|
235
|
+
if line.startswith(b">"):
|
|
236
|
+
if header is not None:
|
|
237
|
+
yield header, header_line, bytes(seq)
|
|
238
|
+
header = line[1:].replace(b"\r", b"").rstrip(b"\n")
|
|
239
|
+
header_line = lineno
|
|
240
|
+
seq = bytearray()
|
|
241
|
+
continue
|
|
242
|
+
if line.startswith(COMMENT):
|
|
243
|
+
continue
|
|
244
|
+
chunk = line.translate(UPPER, WHITESPACE)
|
|
245
|
+
if not chunk:
|
|
246
|
+
continue
|
|
247
|
+
if header is None:
|
|
248
|
+
raise HashFastaError(
|
|
249
|
+
"line %d: sequence data before the first '>' header" % lineno)
|
|
250
|
+
bad = chunk.translate(None, VALID)
|
|
251
|
+
if bad:
|
|
252
|
+
raise HashFastaError("line %d: %s in record starting at line %d"
|
|
253
|
+
% (lineno, _invalid_byte(bad[0]), header_line))
|
|
254
|
+
seq += chunk
|
|
255
|
+
except _DECODE_ERRORS as e:
|
|
256
|
+
raise HashFastaError("cannot read input: %s" % e)
|
|
257
|
+
if header is None:
|
|
258
|
+
raise HashFastaError("input contains no FASTA records")
|
|
259
|
+
yield header, header_line, bytes(seq)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def iter_records(inp):
|
|
263
|
+
"""Yield a :class:`Record` per FASTA record of a decompressed binary stream.
|
|
264
|
+
|
|
265
|
+
Holds one record in memory. Raises :class:`HashFastaError` on invalid input.
|
|
266
|
+
"""
|
|
267
|
+
for header, header_line, seq in _parse(inp):
|
|
268
|
+
if not seq:
|
|
269
|
+
raise HashFastaError("record at line %d has an empty sequence" % header_line)
|
|
270
|
+
yield Record(sequence_key(seq), header, seq)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _open_source(source):
|
|
274
|
+
"""Return (binary stream, should_close) for a path, '-', or binary file object."""
|
|
275
|
+
if source == "-":
|
|
276
|
+
return sys.stdin.buffer, False
|
|
277
|
+
if isinstance(source, (str, bytes, os.PathLike)):
|
|
278
|
+
try:
|
|
279
|
+
return open(source, "rb"), True
|
|
280
|
+
except OSError as e:
|
|
281
|
+
raise HashFastaError("cannot open input %s: %s" % (os.fsdecode(source), e.strerror))
|
|
282
|
+
return source, False
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def read_fasta(source):
|
|
286
|
+
"""Yield :class:`Record` objects from FASTA at a path, '-' or binary file object.
|
|
287
|
+
|
|
288
|
+
Plain, gzip, bzip2 and xz input are all accepted.
|
|
289
|
+
"""
|
|
290
|
+
raw, close = _open_source(source)
|
|
291
|
+
try:
|
|
292
|
+
for rec in iter_records(open_input(raw)):
|
|
293
|
+
yield rec
|
|
294
|
+
finally:
|
|
295
|
+
if close:
|
|
296
|
+
raw.close()
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
_HASH_HEADER = re.compile(rb"^>(sha256_[0-9a-f]{64}) \[(.*)\]$", re.S)
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def read_hashfasta(source, verify=True):
|
|
303
|
+
"""Yield :class:`Record` objects from an existing HashFASTA file.
|
|
304
|
+
|
|
305
|
+
The header is unescaped back to the original definition line. With
|
|
306
|
+
``verify`` (the default) each key is recomputed from the sequence and a
|
|
307
|
+
mismatch raises :class:`HashFastaError`.
|
|
308
|
+
"""
|
|
309
|
+
raw, close = _open_source(source)
|
|
310
|
+
try:
|
|
311
|
+
key = header = None
|
|
312
|
+
seq = bytearray()
|
|
313
|
+
|
|
314
|
+
def finish():
|
|
315
|
+
rec = Record(key, unescape_header(header), bytes(seq))
|
|
316
|
+
if verify and sequence_key(rec.sequence) != key:
|
|
317
|
+
raise HashFastaError("key %s does not match its sequence" % key)
|
|
318
|
+
return rec
|
|
319
|
+
|
|
320
|
+
try:
|
|
321
|
+
for line in open_input(raw):
|
|
322
|
+
line = line.rstrip(b"\n")
|
|
323
|
+
if line.startswith(b">"):
|
|
324
|
+
if key is not None:
|
|
325
|
+
yield finish()
|
|
326
|
+
m = _HASH_HEADER.match(line)
|
|
327
|
+
if not m:
|
|
328
|
+
raise HashFastaError("not a HashFASTA header: %r" % line[:100])
|
|
329
|
+
key, header = m.group(1).decode("ascii"), m.group(2)
|
|
330
|
+
seq = bytearray()
|
|
331
|
+
elif key is None:
|
|
332
|
+
raise HashFastaError("sequence data before the first header")
|
|
333
|
+
else:
|
|
334
|
+
seq += line
|
|
335
|
+
except _DECODE_ERRORS as e:
|
|
336
|
+
raise HashFastaError("cannot read input: %s" % e)
|
|
337
|
+
if key is not None:
|
|
338
|
+
yield finish()
|
|
339
|
+
finally:
|
|
340
|
+
if close:
|
|
341
|
+
raw.close()
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def convert_stream(inp, out):
|
|
345
|
+
"""Convert decompressed FASTA ``inp`` to HashFASTA text on binary ``out``.
|
|
346
|
+
|
|
347
|
+
``out`` receives uncompressed text; wrap it in a gzip writer for a
|
|
348
|
+
conforming file (see :func:`convert`). Returns the number of records.
|
|
349
|
+
"""
|
|
350
|
+
n = 0
|
|
351
|
+
for rec in iter_records(inp):
|
|
352
|
+
out.write(format_record(rec))
|
|
353
|
+
n += 1
|
|
354
|
+
return n
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _gzip_writer(fileobj):
|
|
358
|
+
return gzip.GzipFile(fileobj=fileobj, mode="wb", mtime=0, filename="", compresslevel=6)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _same_file(a, b):
|
|
362
|
+
try:
|
|
363
|
+
return os.path.samefile(a, b)
|
|
364
|
+
except OSError:
|
|
365
|
+
return False
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _create_temp(output_path):
|
|
369
|
+
"""Exclusively create a temporary sibling of output_path; return (path, fd).
|
|
370
|
+
|
|
371
|
+
O_EXCL refuses any existing name, including a symlink or hard link to the
|
|
372
|
+
input, so a planted temporary path can never redirect the write.
|
|
373
|
+
"""
|
|
374
|
+
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_BINARY", 0)
|
|
375
|
+
for n in range(1000):
|
|
376
|
+
path = "%s.tmp.%d.%d" % (output_path, os.getpid(), n)
|
|
377
|
+
try:
|
|
378
|
+
return path, os.open(path, flags, 0o666)
|
|
379
|
+
except FileExistsError:
|
|
380
|
+
continue
|
|
381
|
+
raise HashFastaError("cannot create a temporary file beside %s" % output_path)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def convert(input_path, output_path=None):
|
|
385
|
+
"""Convert ``input_path`` to gzip HashFASTA; return the output path.
|
|
386
|
+
|
|
387
|
+
``'-'`` means stdin / stdout. Without ``output_path`` a named input is
|
|
388
|
+
written beside itself as ``<stem>.hash.fasta.gz`` and stdin goes to
|
|
389
|
+
stdout. A file output is written to a temporary sibling and renamed into
|
|
390
|
+
place only on success. The input is never overwritten.
|
|
391
|
+
"""
|
|
392
|
+
if output_path is None:
|
|
393
|
+
output_path = "-" if input_path == "-" else default_output_path(os.fspath(input_path))
|
|
394
|
+
if input_path != "-" and output_path != "-" and _same_file(input_path, output_path):
|
|
395
|
+
raise HashFastaError("output path %s would overwrite the input" % output_path)
|
|
396
|
+
|
|
397
|
+
raw, close = _open_source(input_path)
|
|
398
|
+
tmp_path = None
|
|
399
|
+
try:
|
|
400
|
+
inp = open_input(raw)
|
|
401
|
+
if output_path == "-":
|
|
402
|
+
with _gzip_writer(sys.stdout.buffer) as gz:
|
|
403
|
+
convert_stream(inp, gz)
|
|
404
|
+
sys.stdout.buffer.flush()
|
|
405
|
+
else:
|
|
406
|
+
tmp_path, fd = _create_temp(output_path)
|
|
407
|
+
with os.fdopen(fd, "wb") as fh, _gzip_writer(fh) as gz:
|
|
408
|
+
convert_stream(inp, gz)
|
|
409
|
+
os.replace(tmp_path, output_path)
|
|
410
|
+
tmp_path = None
|
|
411
|
+
except OSError as e:
|
|
412
|
+
raise HashFastaError("I/O error: %s" % e)
|
|
413
|
+
finally:
|
|
414
|
+
if tmp_path is not None and os.path.exists(tmp_path):
|
|
415
|
+
os.unlink(tmp_path)
|
|
416
|
+
if close:
|
|
417
|
+
raw.close()
|
|
418
|
+
return output_path
|
hashfasta/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hashfasta
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Convert FASTA to HashFASTA: records keyed by the SHA-256 of their normalized sequence, gzip output
|
|
5
|
+
Author-email: Rob Edwards <robert.edwards@flinders.edu.au>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/linsalrob/HashFASTA
|
|
8
|
+
Project-URL: Specification, https://github.com/linsalrob/HashFASTA/blob/main/spec/HASHFASTA-v0.1.md
|
|
9
|
+
Project-URL: Issues, https://github.com/linsalrob/HashFASTA/issues
|
|
10
|
+
Keywords: FASTA,bioinformatics,SHA-256,sequence,provenance
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# hashfasta (Python)
|
|
22
|
+
|
|
23
|
+
Python reference implementation of [HashFASTA](https://github.com/linsalrob/HashFASTA), both a command-line tool and a library. HashFASTA rewrites each FASTA record so that its ID is the SHA-256 of its normalized sequence and keeps the original definition line in brackets:
|
|
24
|
+
|
|
25
|
+
```text
|
|
26
|
+
>sha256_<64 hex digits> [<original definition line>]
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Two different sequences that share a source ID get different keys. Identical sequences get the same key, and both records are kept. Output is always gzip. See the [specification](https://github.com/linsalrob/HashFASTA/blob/main/spec/HASHFASTA-v0.1.md).
|
|
30
|
+
|
|
31
|
+
Standard library only. Installing needs Python ≥ 3.9; the code itself also runs on Python 3.6+ straight from a checkout through `run_hashfasta.py`.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```sh
|
|
36
|
+
pip install "git+https://github.com/linsalrob/HashFASTA#subdirectory=implementations/python"
|
|
37
|
+
# or, from a checkout
|
|
38
|
+
pip install implementations/python
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
To run from a checkout without installing:
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
python3 implementations/python/run_hashfasta.py sample.fasta.gz
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Command line
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
hashfasta sample.fna.gz # writes sample.hash.fasta.gz
|
|
51
|
+
hashfasta sample.fa.bz2 -o out.hash.fasta.gz
|
|
52
|
+
zcat x.fa.gz | hashfasta - > x.hash.fasta.gz
|
|
53
|
+
python -m hashfasta sample.fasta.xz
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Input may be plain FASTA or FASTA compressed with gzip/BGZF, bzip2 or xz. The format is detected from the content, not the file name.
|
|
57
|
+
|
|
58
|
+
## Library
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import hashfasta
|
|
62
|
+
|
|
63
|
+
out = hashfasta.convert("sample.fna.gz") # "sample.hash.fasta.gz"
|
|
64
|
+
|
|
65
|
+
for rec in hashfasta.read_hashfasta(out): # keys are re-verified by default
|
|
66
|
+
print(rec.key, rec.description, len(rec.sequence))
|
|
67
|
+
|
|
68
|
+
for rec in hashfasta.read_fasta("raw.fa.xz"): # convert in memory, record by record
|
|
69
|
+
print(rec.key, rec.header) # header is bytes, exactly as in the source
|
|
70
|
+
|
|
71
|
+
hashfasta.sequence_key(hashfasta.normalize_sequence(b"acgt\n"))
|
|
72
|
+
# 'sha256_...'
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
| Function | Purpose |
|
|
76
|
+
| --- | --- |
|
|
77
|
+
| `convert(input, output=None)` | File (or `"-"`) to gzip HashFASTA; returns the output path |
|
|
78
|
+
| `read_fasta(source)` | Yield `Record(key, header, sequence)` from plain/gz/bz2/xz FASTA |
|
|
79
|
+
| `read_hashfasta(source, verify=True)` | Read a HashFASTA file back, unescaping headers |
|
|
80
|
+
| `convert_stream(inp, out)` | Decompressed stream to uncompressed HashFASTA text |
|
|
81
|
+
| `normalize_sequence`, `sequence_key` | The normalization and key rules |
|
|
82
|
+
| `escape_header`, `unescape_header` | Bracket escaping for the provenance field |
|
|
83
|
+
| `format_record`, `iter_records`, `open_input`, `default_output_path` | Lower-level building blocks |
|
|
84
|
+
|
|
85
|
+
Every error is raised as `hashfasta.HashFastaError`.
|
|
86
|
+
|
|
87
|
+
## Development
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
python3 -m unittest discover -s implementations/python/tests # library tests
|
|
91
|
+
python3 tests/run_conformance.py python # shared conformance suite
|
|
92
|
+
python3 -m pip wheel --no-deps -w dist implementations/python # build a wheel
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Releasing to PyPI
|
|
96
|
+
|
|
97
|
+
`.github/workflows/python-package.yml` builds, checks and tests the sdist and wheel on every push. To publish:
|
|
98
|
+
|
|
99
|
+
1. One-time setup: on PyPI, add a trusted publisher for project `hashfasta`, repository `linsalrob/HashFASTA`, workflow `python-package.yml` and environment `pypi`. Then create a `pypi` environment in the GitHub repository settings.
|
|
100
|
+
2. Bump `__version__` in `src/hashfasta/__init__.py`, merge, then push a tag matching it: `git tag python-v0.1.0 && git push origin python-v0.1.0`.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
hashfasta/__init__.py,sha256=2UHfRyj-a3JYBZ_M8xt-Kkae_vsGb78ItPNvnO7OlCA,1074
|
|
2
|
+
hashfasta/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
+
hashfasta/cli.py,sha256=TJPv7Egsus1IoWUogtIousxPNo7sczrwaoU3EOtvFUY,1351
|
|
4
|
+
hashfasta/core.py,sha256=CnkQfMG93MtrZBVqcSkhE48eI9oEpTXuc96NWes5-zY,14381
|
|
5
|
+
hashfasta/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
hashfasta-0.1.0.dist-info/licenses/LICENSE,sha256=IXFGx8QPUM3XijUW6qquLjWv26rvf3vmEnsJZrBp6zg,1068
|
|
7
|
+
hashfasta-0.1.0.dist-info/METADATA,sha256=QSJiGXKeAMjdM-CYh7pYTxq7bZyU-DY5amI5mwJaIdw,4342
|
|
8
|
+
hashfasta-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
9
|
+
hashfasta-0.1.0.dist-info/entry_points.txt,sha256=qOboNb7wkwbqf9q88VoX4xpYZZosBFrz9rgpzoeo3u0,49
|
|
10
|
+
hashfasta-0.1.0.dist-info/top_level.txt,sha256=MHqWFmOP4jPbNHdTVPIOjtXRZekwmnoV29uxOej3zg4,10
|
|
11
|
+
hashfasta-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rob Edwards
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
hashfasta
|