dna-parser 0.6.1__cp315-cp315-win32.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dna_parser/__init__.py +7 -0
- dna_parser/call_functions.py +14 -0
- dna_parser/dna_parser.cp315-win32.pyd +0 -0
- dna_parser/tfidf_class.py +142 -0
- dna_parser-0.6.1.dist-info/METADATA +121 -0
- dna_parser-0.6.1.dist-info/RECORD +9 -0
- dna_parser-0.6.1.dist-info/WHEEL +4 -0
- dna_parser-0.6.1.dist-info/licenses/LICENSE +21 -0
- dna_parser-0.6.1.dist-info/sboms/dna_parser.cyclonedx.json +2615 -0
dna_parser/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from .dna_parser import *
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def make_kmers(sequences, kmer_size=3, stride=3, drop_remainder=False, n_jobs=1):
|
|
5
|
+
|
|
6
|
+
seqs = make_kmers_rust(sequences, kmer_size, stride, drop_remainder)
|
|
7
|
+
results = [seq.view(f'S{kmer_size}').reshape(-1).astype(str).tolist() for seq in seqs]
|
|
8
|
+
|
|
9
|
+
if not drop_remainder:
|
|
10
|
+
for kmer_list in results:
|
|
11
|
+
if kmer_list:
|
|
12
|
+
kmer_list[-1] = kmer_list[-1].rstrip('\x00')
|
|
13
|
+
|
|
14
|
+
return results
|
|
Binary file
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
from .dna_parser import *
|
|
2
|
+
from scipy.sparse import csr_matrix
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
class WrongTfidfConfig(Exception):
|
|
6
|
+
pass
|
|
7
|
+
|
|
8
|
+
class NotFittedError(Exception):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
class UnextendableCorpus(Exception):
|
|
12
|
+
pass
|
|
13
|
+
|
|
14
|
+
class NoReader(Exception):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
class Tfidf:
|
|
18
|
+
|
|
19
|
+
def __init__(self, corpus, kmer, vocabulary=None, smooth_idf=True, original_idf=False, n_jobs=1, reader_fn=None, format=None):
|
|
20
|
+
|
|
21
|
+
if isinstance(corpus, str) and reader_fn is None:
|
|
22
|
+
raise WrongTfidfConfig("If the corpus is a str, it should be a path to a file, and a function to read it (reader_fn) should be provided")
|
|
23
|
+
|
|
24
|
+
self.set_vocabulary(vocabulary)
|
|
25
|
+
self.corpus = corpus
|
|
26
|
+
self.kmer_size = kmer
|
|
27
|
+
|
|
28
|
+
self.idf = None
|
|
29
|
+
self.smooth_idf= smooth_idf
|
|
30
|
+
self.original_idf= original_idf
|
|
31
|
+
self.n_jobs = n_jobs
|
|
32
|
+
|
|
33
|
+
self.reader_fn = reader_fn
|
|
34
|
+
self.format = format
|
|
35
|
+
|
|
36
|
+
def set_vocabulary(self, vocabulary):
|
|
37
|
+
|
|
38
|
+
if vocabulary:
|
|
39
|
+
self.vocabulary = Vocabulary(vocabulary)
|
|
40
|
+
|
|
41
|
+
else:
|
|
42
|
+
self.vocabulary = None
|
|
43
|
+
|
|
44
|
+
self.is_idf_uptodate = False
|
|
45
|
+
|
|
46
|
+
def set_threads(self, n_jobs):
|
|
47
|
+
self.n_jobs = n_jobs
|
|
48
|
+
|
|
49
|
+
def add_to_corpus(self, new_corpus):
|
|
50
|
+
|
|
51
|
+
if isinstance(new_corpus, str):
|
|
52
|
+
raise TypeError("The new corpus cannot be a str. If you want to add a single sequence pass it as list[str]")
|
|
53
|
+
|
|
54
|
+
if hasattr(self.corpus, "extend") and callable(getattr(self.corpus, "extend")):
|
|
55
|
+
self.corpus.extend(new_corpus)
|
|
56
|
+
elif isinstance(self.corpus, list):
|
|
57
|
+
self.corpus.extend(new_corpus)
|
|
58
|
+
elif isinstance(self.corpus, set) or isinstance(self.corpus, tuple) :
|
|
59
|
+
self.corpus = list(self.corpus) + list(new_corpus)
|
|
60
|
+
else:
|
|
61
|
+
raise UnextendableCorpus(("The corpus cannot be extended."
|
|
62
|
+
"A corpus can only be extended if it is a collection of sequences (such as list[str], set[str], tuple[str])\n"
|
|
63
|
+
"Check that your initial corpus is not a path to a file."))
|
|
64
|
+
|
|
65
|
+
self.is_idf_uptodate = False
|
|
66
|
+
|
|
67
|
+
def _get_sequence_source(self, sequences):
|
|
68
|
+
"""
|
|
69
|
+
If sequences is a file path (string) and a reader_fn is provided,
|
|
70
|
+
invoke the reader to generate a fresh iterator. Otherwise, return sequences directly.
|
|
71
|
+
"""
|
|
72
|
+
if isinstance(sequences, str) and self.reader_fn is not None:
|
|
73
|
+
if self.format:
|
|
74
|
+
return self.reader_fn(sequences, self.format)
|
|
75
|
+
return self.reader_fn(sequences)
|
|
76
|
+
|
|
77
|
+
if isinstance(sequences, str) and self.reader_fn is None:
|
|
78
|
+
raise NoReader("A string (path/to/file) was passed but no reader function is specified.\nIf you want to encode a single sequence wrap it in a list -> ['agatatc'] ")
|
|
79
|
+
return sequences
|
|
80
|
+
|
|
81
|
+
def _reset_if_needed(self, sequences):
|
|
82
|
+
"""Resets native stateful readers (like Rust SequenceReader) if they support it."""
|
|
83
|
+
if hasattr(sequences, "reset") and callable(getattr(sequences, "reset")):
|
|
84
|
+
sequences.reset()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def fit(self):
|
|
88
|
+
seq_source = self._get_sequence_source(self.corpus)
|
|
89
|
+
|
|
90
|
+
if not self.vocabulary:
|
|
91
|
+
self.vocabulary = get_vocab(seq_source, self.kmer_size)
|
|
92
|
+
self._reset_if_needed(seq_source)
|
|
93
|
+
|
|
94
|
+
# If we just exhausted an iterator from reader_fn, we must spawn a fresh one for the transform step
|
|
95
|
+
seq_source = self._get_sequence_source(self.corpus)
|
|
96
|
+
|
|
97
|
+
self.idf= fit_rust(seq_source, self.vocabulary, self.kmer_size, self.smooth_idf, self.original_idf, self.n_jobs)
|
|
98
|
+
|
|
99
|
+
self.is_idf_uptodate= True
|
|
100
|
+
self._reset_if_needed(seq_source)
|
|
101
|
+
|
|
102
|
+
def transform(self, sequences=None, l2_norm=True):
|
|
103
|
+
if not self.is_idf_uptodate:
|
|
104
|
+
raise NotFittedError("This Tfidf instance is not fitted. Please use fit() or fit_transform()")
|
|
105
|
+
|
|
106
|
+
target_sequences = self.corpus if sequences is None else sequences
|
|
107
|
+
seq_source = self._get_sequence_source(target_sequences)
|
|
108
|
+
|
|
109
|
+
val, col, indptr = transform_rust(seq_source, self.idf, self.vocabulary, self.kmer_size,
|
|
110
|
+
l2_norm, self.n_jobs)
|
|
111
|
+
self._reset_if_needed(seq_source)
|
|
112
|
+
|
|
113
|
+
tfidf_csr = csr_matrix(
|
|
114
|
+
arg1=(val, col, indptr),
|
|
115
|
+
shape=(len(indptr)-1, len(self.vocabulary)),
|
|
116
|
+
dtype=np.float64
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
return tfidf_csr
|
|
120
|
+
|
|
121
|
+
def fit_transform(self, l2_norm=True):
|
|
122
|
+
seq_source = self._get_sequence_source(self.corpus)
|
|
123
|
+
|
|
124
|
+
if not self.vocabulary:
|
|
125
|
+
self.vocabulary, val, col, indptr, self.idf = fit_transform_rust(seq_source, self.kmer_size,
|
|
126
|
+
l2_norm, self.smooth_idf, self.original_idf,
|
|
127
|
+
n_jobs= self.n_jobs)
|
|
128
|
+
else:
|
|
129
|
+
val, col, indptr, self.idf = fit_transform_with_voc(seq_source, self.vocabulary, self.kmer_size,
|
|
130
|
+
l2_norm, self.smooth_idf, self.original_idf,
|
|
131
|
+
n_jobs= self.n_jobs)
|
|
132
|
+
|
|
133
|
+
self._reset_if_needed(seq_source)
|
|
134
|
+
self.is_idf_uptodate=True
|
|
135
|
+
|
|
136
|
+
tfidf_csr = csr_matrix(
|
|
137
|
+
arg1=(val, col, indptr),
|
|
138
|
+
shape=(len(indptr)-1, len(self.vocabulary)),
|
|
139
|
+
dtype=np.float64
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
return tfidf_csr
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dna_parser
|
|
3
|
+
Version: 0.6.1
|
|
4
|
+
Classifier: Programming Language :: Rust
|
|
5
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
6
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
7
|
+
Requires-Dist: numpy>=1.16.0
|
|
8
|
+
Requires-Dist: scipy>=1.3.0
|
|
9
|
+
Requires-Dist: pytest ; extra == 'dev'
|
|
10
|
+
Requires-Dist: mkdocs ; extra == 'dev'
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
15
|
+
|
|
16
|
+
<h1 style="text-align:center;"><b>dna-parser</b></h1>
|
|
17
|
+
|
|
18
|
+
<p style="text-align:center;"><b>dna-parser is a Python library written in Rust to encode (or perform feature extraction on) DNA/RNA sequences for machine learning.</b></p>
|
|
19
|
+
|
|
20
|
+
The source code is available on [GitHub](https://github.com/Mvila035/dna_parser)
|
|
21
|
+
|
|
22
|
+
The documentation is available [here](https://mvila035.github.io/dna_parser/).
|
|
23
|
+
|
|
24
|
+
## **Installation**
|
|
25
|
+
|
|
26
|
+
To install dna-parser run:
|
|
27
|
+
```sh
|
|
28
|
+
pip install dna-parser
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
If there is no Python wheel available for your OS, you can install Rust and re-install dna-parser which should now compile on your machine.
|
|
32
|
+
Run the following command on Unix-like OS to install Rust:
|
|
33
|
+
```sh
|
|
34
|
+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
|
35
|
+
```
|
|
36
|
+
or see more options at [https://www.rust-lang.org/tools/install](https://www.rust-lang.org/tools/install).
|
|
37
|
+
|
|
38
|
+
## **Quick Start**
|
|
39
|
+
|
|
40
|
+
### Loading Sequences
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import dna_parser as dps
|
|
44
|
+
|
|
45
|
+
reader= dps.SequenceReader("path/to/file")
|
|
46
|
+
reader.get_sequences()
|
|
47
|
+
# Output:
|
|
48
|
+
# [agcgcggtgcgc, agcggcgcgtata, ...]
|
|
49
|
+
|
|
50
|
+
reader.reset() # read all sequences and exhausted the reader so we reset it.
|
|
51
|
+
reader.get_ids()
|
|
52
|
+
# Output:
|
|
53
|
+
#['sequence1', 'sequence2', 'sequence3']
|
|
54
|
+
|
|
55
|
+
reader.reset()
|
|
56
|
+
for seq in reader.get_records():
|
|
57
|
+
print(seq) # seq -> SequenceRecord object
|
|
58
|
+
# with attributes .id .format .seq .qual (if Fastq file)
|
|
59
|
+
# Output:
|
|
60
|
+
#[ID: sequence1
|
|
61
|
+
# Format: Fasta
|
|
62
|
+
# Sequence: acgtatgcgtcgtc...
|
|
63
|
+
# Quality: None]
|
|
64
|
+
#
|
|
65
|
+
#[ID: sequence2
|
|
66
|
+
# ...
|
|
67
|
+
```
|
|
68
|
+
If the file does not fit in memory you can use the `batch_size` option to return a list containing the desired number of objects.
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
import dna_parser as dps
|
|
72
|
+
|
|
73
|
+
reader= dps.SequenceReader("path/to/file.fasta", batch_size=1)
|
|
74
|
+
reader.get_sequences()
|
|
75
|
+
# Output:
|
|
76
|
+
# [agcgcggtgcgc]
|
|
77
|
+
reader.get_sequences()
|
|
78
|
+
# Output:
|
|
79
|
+
# [agcggcgcgtata]
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Encoding Sequences
|
|
83
|
+
Encoding functions accept any iterables (list, set, tuple etc.) containing strings. It also accepts dna-parser `SequenceReader`, pysam `FastxFile` and Biopython `SeqIO.read` objects.
|
|
84
|
+
```python
|
|
85
|
+
import dna_parser as dps
|
|
86
|
+
|
|
87
|
+
sequences= ["agt","acc"]
|
|
88
|
+
encodings= dps.onehot_encoding(sequences)
|
|
89
|
+
print(encodings)
|
|
90
|
+
# Output: list of numpy ndarray
|
|
91
|
+
#[[[0 0 1 0]
|
|
92
|
+
# [0 1 0 0]
|
|
93
|
+
# [0 0 0 1]]
|
|
94
|
+
|
|
95
|
+
# [[0 0 1 0]
|
|
96
|
+
# [1 0 0 0]
|
|
97
|
+
# [1 0 0 0]]]
|
|
98
|
+
|
|
99
|
+
reader= dps.SequenceReader("path/to/file") #or pysam.FastxFile("path/to/file") or SeqIO.parse("path/to/file", "fasta")
|
|
100
|
+
encodings= dps.eiip_encoding(reader)
|
|
101
|
+
print(encodings)
|
|
102
|
+
# Output: list of numpy ndarray
|
|
103
|
+
#[array([0.126 , 0.134 , 0.0806, 0.1335, 0.126 , 0.1335, 0.0806, 0.134]),
|
|
104
|
+
# array([0.134 , 0.134 , 0.134 , 0.0806, 0.1335, 0.0806, 0.126 , 0. ],
|
|
105
|
+
# ... ]
|
|
106
|
+
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Most encodings provide options to pad/trim the sequence and to encode them in parallel for shorter encoding time.
|
|
110
|
+
```python
|
|
111
|
+
import dna_parser as dps
|
|
112
|
+
|
|
113
|
+
sequences= ["agt","accgt"]
|
|
114
|
+
encodings= dps.eiip_encoding(sequences, pad_type="after", pad_length=-2, n_jobs=2) #-2 pad to longest sequence, -1 trim to shortest
|
|
115
|
+
print(encodings) # 0 no padding, 1+ fixed length trim/padding
|
|
116
|
+
# Output: numpy ndarray
|
|
117
|
+
# array([[0.126 , 0.0806, 0.1335, 0. , 0. ],
|
|
118
|
+
# [0.126 , 0.134 , 0.134 , 0.0806, 0.1335]])
|
|
119
|
+
```
|
|
120
|
+
All encodings with examples are available in the [Documentation](https://mvila035.github.io/dna_parser/) section.
|
|
121
|
+
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
dna_parser/__init__.py,sha256=zDXZxxl17i-cGhOIxLt2mnfbQPdKWq_5T8AOWa3ZTqg,186
|
|
2
|
+
dna_parser/call_functions.py,sha256=CnGCGCMZzwOXc_sc-l3MVeY5fbvDzlCmlND05-bhDWo,458
|
|
3
|
+
dna_parser/dna_parser.cp315-win32.pyd,sha256=OU8O5HPPshqmUlnYlCDq-HmQa2XZ_vl-MSc_gCRWZpo,1574400
|
|
4
|
+
dna_parser/tfidf_class.py,sha256=LBK_2EJ01H06P2bAooH1J0O1KHXx0P-eIzi9cM93R7A,5601
|
|
5
|
+
dna_parser-0.6.1.dist-info/METADATA,sha256=P90auj_jw1ZS7CbsItXEMV3hxbIiV9WinrC0r6u86aQ,3945
|
|
6
|
+
dna_parser-0.6.1.dist-info/WHEEL,sha256=CJsT6FKm1obvTswo3GZhqvCqNlOFU_3ZnlijrQkujK4,93
|
|
7
|
+
dna_parser-0.6.1.dist-info/licenses/LICENSE,sha256=Vvbjeik3sJC1JpZH-bFF82bGY6aDfFMThI3CeCl0VZI,1086
|
|
8
|
+
dna_parser-0.6.1.dist-info/sboms/dna_parser.cyclonedx.json,sha256=8r0IEHWUv-EAaM5O8XX8LbF7gdNtMSuCERDrWubTDn8,81680
|
|
9
|
+
dna_parser-0.6.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Mvila035
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|