py-ard 2.4.1__tar.gz → 2.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {py_ard-2.4.1 → py_ard-2.4.2}/PKG-INFO +2 -2
- {py_ard-2.4.1 → py_ard-2.4.2}/README.md +1 -1
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/__init__.py +1 -1
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/cli/reduce_csv.py +156 -118
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/simple_table.py +5 -2
- {py_ard-2.4.1 → py_ard-2.4.2}/pyproject.toml +1 -1
- {py_ard-2.4.1 → py_ard-2.4.2}/.gitignore +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/CONTRIBUTING.rst +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/HISTORY.rst +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/LICENSE +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/ard.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/blender.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/cli/__init__.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/cli/import_db.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/cli/redux.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/cli/status.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/config.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/constants.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/data_repository.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/db.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/dna_relshp.csv +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/drbx.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/exceptions.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/__init__.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/allele_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/gl_string_processor.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/hats_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/mac_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/serology_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/shortnull_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/v2_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/handlers/xx_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/CWD2.csv +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/__init__.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/allele_list.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/cwd.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/g_group.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/mac_codes.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/p_group.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/serology.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/version.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/loader/wmda-data-model.md +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/mappings.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/misc.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/rc.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/__init__.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/base_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/default_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/exon_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/first_field_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/g_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/hats_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/lg_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/p_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/reducer_factory.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/s_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/u2_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/reducers/w_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/serology.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/pyard/smart_sort.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/api-test.hurl +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/environment.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/1f_reduction.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/Gexpand.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/Pexpand.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/Wexpand.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/XX.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/allele.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/cwd.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/drbx.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/drbx_blender.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/glstring.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/mac.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/p_g_group.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/p_group.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/serology.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/serology_broad_splits.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/serology_typings_redux.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/shortnulls.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/u2_reduction.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/valid_allele.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/version2.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/features/who.feature +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/XX.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/broad_splits.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/cwd.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/drbx_blender.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/generate_drbx.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/mac.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/redux_allele.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/steps/validate_allele.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/expected-serology.json +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/expected.json +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/__init__.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/test_allele_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/test_gl_string_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/test_mac_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/test_serology_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/test_shortnull_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/test_v2_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/handlers/test_xx_handler.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/__init__.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_all_reducers.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_base_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_default_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_exon_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_first_field_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_g_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_lg_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_p_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_reducer_factory.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_s_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_u2_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/reducers/test_w_reducer.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/simple_table/__init__.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/simple_table/test_column.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/simple_table/test_simple_table.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/simple_table/test_simple_table_failures.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/test_cli.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/test_load_allele_list.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/test_load_serology_broad_split_mapping.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/test_load_serology_mappings.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/test_pyard.py +0 -0
- {py_ard-2.4.1 → py_ard-2.4.2}/tests/unit/test_smart_sort.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: py-ard
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.2
|
|
4
4
|
Summary: ARD reduction for HLA with Python
|
|
5
5
|
Project-URL: Homepage, https://github.com/nmdp-bioinformatics/py-ard
|
|
6
6
|
Project-URL: Repository, https://github.com/nmdp-bioinformatics/py-ard
|
|
@@ -36,7 +36,7 @@ Swiss army knife of **HLA** Nomenclature
|
|
|
36
36
|
|
|
37
37
|
[](https://pypi.python.org/pypi/py-ard)
|
|
38
38
|
|
|
39
|
-

|
|
40
40
|
|
|
41
41
|
**Note:**
|
|
42
42
|
|
|
@@ -26,16 +26,21 @@
|
|
|
26
26
|
# For Excel output, openpyxl library needs to be installed.
|
|
27
27
|
# pip install openpyxl
|
|
28
28
|
#
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
29
31
|
import argparse
|
|
30
32
|
import json
|
|
33
|
+
import pprint
|
|
31
34
|
import re
|
|
32
35
|
import sys
|
|
36
|
+
from typing import Any
|
|
33
37
|
from urllib.error import HTTPError
|
|
34
38
|
|
|
35
39
|
import pandas as pd
|
|
36
40
|
|
|
37
41
|
import pyard
|
|
38
42
|
from pyard import drbx
|
|
43
|
+
from pyard.ard import ARD
|
|
39
44
|
from pyard.db import similar_alleles
|
|
40
45
|
from pyard.exceptions import InvalidAlleleError, InvalidTypingError, PyArdError
|
|
41
46
|
from pyard.misc import download_to_file, get_data_dir, get_imgt_version
|
|
@@ -45,6 +50,8 @@ from pyard.misc import download_to_file, get_data_dir, get_imgt_version
|
|
|
45
50
|
ard = None
|
|
46
51
|
ard_config = None
|
|
47
52
|
verbose = False
|
|
53
|
+
error_file = sys.stderr
|
|
54
|
+
log_file = sys.stdout
|
|
48
55
|
failed_to_reduce_alleles = []
|
|
49
56
|
white_space_regex = re.compile(r"\s+")
|
|
50
57
|
|
|
@@ -126,7 +133,7 @@ def redux(allele, locus, column_name):
|
|
|
126
133
|
else:
|
|
127
134
|
# Watch out, we may get floats when exported from Excel.
|
|
128
135
|
message = f"Failed reducing '{allele}' in column {column_name}"
|
|
129
|
-
print(message)
|
|
136
|
+
print(message, file=error_file)
|
|
130
137
|
failed_to_reduce_alleles.append((column_name, allele))
|
|
131
138
|
return allele
|
|
132
139
|
|
|
@@ -137,9 +144,10 @@ def redux(allele, locus, column_name):
|
|
|
137
144
|
reduced_allele = ard.redux(locus_allele, ard_config["redux_type"])
|
|
138
145
|
except PyArdError as e:
|
|
139
146
|
if verbose:
|
|
140
|
-
print(e)
|
|
147
|
+
print(e.message, file=error_file)
|
|
141
148
|
message = f"Failed reducing '{locus_allele}' in column {column_name}"
|
|
142
|
-
print(message)
|
|
149
|
+
print(message, file=log_file)
|
|
150
|
+
print(message, file=log_file)
|
|
143
151
|
failed_to_reduce_alleles.append((column_name, locus_allele))
|
|
144
152
|
return allele
|
|
145
153
|
# print(f"reduced to '{reduced_allele}'")
|
|
@@ -150,9 +158,9 @@ def redux(allele, locus, column_name):
|
|
|
150
158
|
allele = remove_locus_name(reduced_allele)
|
|
151
159
|
else:
|
|
152
160
|
if verbose:
|
|
153
|
-
print(f"Failed to reduce {locus_allele}")
|
|
161
|
+
print(f"Failed to reduce {locus_allele}", file=log_file)
|
|
154
162
|
if verbose:
|
|
155
|
-
print(f"\t{locus_allele} => {allele}")
|
|
163
|
+
print(f"\t{locus_allele} => {allele}", file=log_file)
|
|
156
164
|
else:
|
|
157
165
|
if ard_config.get("convert_v2_to_v3"):
|
|
158
166
|
if ard.is_v2(locus_allele):
|
|
@@ -162,7 +170,7 @@ def redux(allele, locus, column_name):
|
|
|
162
170
|
else:
|
|
163
171
|
allele = v3_allele
|
|
164
172
|
if verbose:
|
|
165
|
-
print(f"\t{locus_allele} => {allele}")
|
|
173
|
+
print(f"\t{locus_allele} => {allele}", file=log_file)
|
|
166
174
|
elif ard_config.get("keep_locus_in_allele_name"):
|
|
167
175
|
allele = locus_allele
|
|
168
176
|
|
|
@@ -193,7 +201,7 @@ def reduce_locus_columns(df, ard_config, locus_column_mapping, verbose):
|
|
|
193
201
|
locus_columns = locus_column_mapping[subject][locus]
|
|
194
202
|
for column in locus_columns:
|
|
195
203
|
if verbose:
|
|
196
|
-
print(f"Column:{column} =>")
|
|
204
|
+
print(f"Column:{column} =>", file=log_file)
|
|
197
205
|
if ard_config.get("new_column_for_redux"):
|
|
198
206
|
# insert a new column
|
|
199
207
|
new_column_name = f"{reduce_prefix}{column}"
|
|
@@ -232,28 +240,32 @@ def reduce_locus_columns(df, ard_config, locus_column_mapping, verbose):
|
|
|
232
240
|
)
|
|
233
241
|
df[f"{subject}_DRBX_1"], df[f"{subject}_DRBX_2"] = zip(*df_drbx)
|
|
234
242
|
|
|
235
|
-
if ard_config.get("generate_glstring")
|
|
243
|
+
if ard_config.get("generate_glstring", False) or ard_config.get(
|
|
244
|
+
"generate_slugs", False
|
|
245
|
+
):
|
|
236
246
|
for subject in locus_column_mapping:
|
|
247
|
+
subject_loci_columns = locus_column_mapping[subject]
|
|
237
248
|
slug_columns = []
|
|
238
|
-
for locus in
|
|
239
|
-
slug_column = locus
|
|
249
|
+
for locus in subject_loci_columns:
|
|
250
|
+
slug_column = f"{subject}_{locus}_slug"
|
|
240
251
|
slug_columns.append(slug_column)
|
|
241
|
-
locus_typ_pair =
|
|
252
|
+
locus_typ_pair = subject_loci_columns[locus]
|
|
242
253
|
if len(locus_typ_pair) > 1:
|
|
243
254
|
df[slug_column] = df[locus_typ_pair].apply(
|
|
244
255
|
create_reduced_slug, axis=1
|
|
245
256
|
)
|
|
246
257
|
else:
|
|
247
|
-
df[slug_column] = df[
|
|
258
|
+
df[slug_column] = df[subject_loci_columns[locus][0]]
|
|
248
259
|
|
|
249
260
|
if ard_config.get("suppress_reduced_locus_column"):
|
|
250
261
|
df.drop(columns=locus_typ_pair, inplace=True)
|
|
251
262
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
263
|
+
if ard_config.get("generate_glstring"):
|
|
264
|
+
df[subject + "_gl"] = df[slug_columns].agg(
|
|
265
|
+
lambda slugs: "^".join(slug for slug in slugs if slug), axis=1
|
|
266
|
+
)
|
|
267
|
+
if not ard_config.get("generate_slugs"):
|
|
268
|
+
df.drop(columns=slug_columns, inplace=True)
|
|
257
269
|
|
|
258
270
|
|
|
259
271
|
def create_reduced_slug(locus_typ1_typ2_pair):
|
|
@@ -308,6 +320,9 @@ def apply_drbx(gl_string):
|
|
|
308
320
|
|
|
309
321
|
def reduce_glstring(glstring: str) -> str:
|
|
310
322
|
try:
|
|
323
|
+
if not glstring:
|
|
324
|
+
print("Missing glstring", file=error_file)
|
|
325
|
+
return ""
|
|
311
326
|
ard_redux = ard.redux(glstring, ard_config["redux_type"])
|
|
312
327
|
if ard_config.get("map_drb345_to_drbx"):
|
|
313
328
|
glstring_drbx = apply_drbx(ard_redux)
|
|
@@ -315,7 +330,7 @@ def reduce_glstring(glstring: str) -> str:
|
|
|
315
330
|
else:
|
|
316
331
|
return ard_redux
|
|
317
332
|
except (InvalidTypingError, InvalidAlleleError) as e:
|
|
318
|
-
print(f"Error reducing {glstring} \n", e.message, file=
|
|
333
|
+
print(f"Error reducing {glstring} \n", e.message, file=error_file)
|
|
319
334
|
return "Failed"
|
|
320
335
|
|
|
321
336
|
|
|
@@ -335,8 +350,119 @@ def reduce_glstring_columns(df, ard_config, glstring_columns):
|
|
|
335
350
|
df[column] = df[column].apply(reduce_glstring)
|
|
336
351
|
|
|
337
352
|
|
|
353
|
+
def reduce_and_save(
|
|
354
|
+
ard: ARD, ard_config, in_csv_filename, out_csv_filename, verbose: bool | Any
|
|
355
|
+
):
|
|
356
|
+
global failed_to_reduce_alleles
|
|
357
|
+
# Read the Input File
|
|
358
|
+
# Read only the columns to be saved.
|
|
359
|
+
# Header is the first row
|
|
360
|
+
# Don't convert to NAs
|
|
361
|
+
columns_from_csv = ard_config["columns_from_csv"]
|
|
362
|
+
try:
|
|
363
|
+
df = pd.read_csv(
|
|
364
|
+
in_csv_filename,
|
|
365
|
+
usecols=columns_from_csv,
|
|
366
|
+
header=0,
|
|
367
|
+
dtype=str,
|
|
368
|
+
keep_default_na=False,
|
|
369
|
+
)[columns_from_csv]
|
|
370
|
+
except FileNotFoundError as e:
|
|
371
|
+
print(
|
|
372
|
+
f"File not found {in_csv_filename}. Error: {e}",
|
|
373
|
+
file=error_file,
|
|
374
|
+
)
|
|
375
|
+
sys.exit(1)
|
|
376
|
+
|
|
377
|
+
failed_to_reduce_alleles = []
|
|
378
|
+
locus_column_mapping = ard_config.get("locus_column_mapping", None)
|
|
379
|
+
if locus_column_mapping:
|
|
380
|
+
reduce_locus_columns(df, ard_config, locus_column_mapping, verbose)
|
|
381
|
+
|
|
382
|
+
glstring_columns = ard_config.get("glstring_columns", None)
|
|
383
|
+
if glstring_columns:
|
|
384
|
+
reduce_glstring_columns(df, ard_config, glstring_columns)
|
|
385
|
+
|
|
386
|
+
# Save as XLSX if specified
|
|
387
|
+
if ard_config["output_file_format"] == "xlsx":
|
|
388
|
+
out_csv_filename = f"{ard_config['out_csv_filename']}.xlsx"
|
|
389
|
+
df.to_excel(out_csv_filename, index=False)
|
|
390
|
+
else:
|
|
391
|
+
# Save as compressed CSV if specified
|
|
392
|
+
compression_type = ard_config["apply_compression"]
|
|
393
|
+
# Valid compression_type: gzip, zip, null
|
|
394
|
+
if compression_type == "gzip":
|
|
395
|
+
out_csv_filename = out_csv_filename + ".gz"
|
|
396
|
+
elif compression_type == "zip":
|
|
397
|
+
out_csv_filename = out_csv_filename + ".zip"
|
|
398
|
+
|
|
399
|
+
df.to_csv(
|
|
400
|
+
out_csv_filename,
|
|
401
|
+
index=False,
|
|
402
|
+
compression=compression_type,
|
|
403
|
+
header=True,
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
if len(failed_to_reduce_alleles) == 0:
|
|
407
|
+
print("No Errors", file=error_file)
|
|
408
|
+
else:
|
|
409
|
+
print("Summary", file=log_file)
|
|
410
|
+
print("-------", file=log_file)
|
|
411
|
+
print(
|
|
412
|
+
f"{len(failed_to_reduce_alleles)} alleles failed to reduce.",
|
|
413
|
+
file=log_file,
|
|
414
|
+
)
|
|
415
|
+
print(
|
|
416
|
+
"| Column Name | Allele | Did you mean ? ",
|
|
417
|
+
file=log_file,
|
|
418
|
+
)
|
|
419
|
+
print(
|
|
420
|
+
"| --------------- | ---------------- | ------------------------- ",
|
|
421
|
+
file=log_file,
|
|
422
|
+
)
|
|
423
|
+
for column_name, locus_allele in failed_to_reduce_alleles:
|
|
424
|
+
similar_allele_names = similar_alleles(ard.db_connection, locus_allele)
|
|
425
|
+
if similar_allele_names:
|
|
426
|
+
similar_allele_names = ",".join(
|
|
427
|
+
sorted(similar_allele_names, reverse=True)
|
|
428
|
+
)
|
|
429
|
+
else:
|
|
430
|
+
similar_allele_names = "NA"
|
|
431
|
+
print(
|
|
432
|
+
f"| {column_name:15} | {locus_allele:16} | {similar_allele_names} ",
|
|
433
|
+
file=log_file,
|
|
434
|
+
)
|
|
435
|
+
# Done
|
|
436
|
+
print(f"Saved result to file: {out_csv_filename}")
|
|
437
|
+
if log_file is not sys.stdout:
|
|
438
|
+
print(f"Saved log to file: {log_file.name}")
|
|
439
|
+
if error_file is not sys.stderr:
|
|
440
|
+
print(f"Saved errors to file: {error_file.name}")
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def get_redux_config(ard_config) -> dict[str | Any, Any]:
|
|
444
|
+
csv_redux_config = {
|
|
445
|
+
"reduce_serology": ard_config.get("reduce_serology", True),
|
|
446
|
+
"reduce_v2": ard_config.get("reduce_v2", True),
|
|
447
|
+
"reduce_3field": ard_config.get("reduce_3field", True),
|
|
448
|
+
"reduce_P": ard_config.get("reduce_P", True),
|
|
449
|
+
"reduce_XX": ard_config.get("reduce_XX", True),
|
|
450
|
+
"reduce_MAC": ard_config.get("reduce_MAC", True),
|
|
451
|
+
"reduce_shortnull": ard_config.get("reduce_shortnull", True),
|
|
452
|
+
"ping": ard_config.get("ping", True),
|
|
453
|
+
"verbose_log": ard_config.get("verbose_log", True),
|
|
454
|
+
"ARS_as_lg": ard_config.get("ARS_as_lg", False),
|
|
455
|
+
"strict": ard_config.get("strict", True),
|
|
456
|
+
"ignore_allele_with_suffixes": tuple(
|
|
457
|
+
ard_config.get("ignore_allele_with_suffixes", "").split(",")
|
|
458
|
+
),
|
|
459
|
+
}
|
|
460
|
+
pprint.pprint(csv_redux_config, indent=4, stream=log_file)
|
|
461
|
+
return csv_redux_config
|
|
462
|
+
|
|
463
|
+
|
|
338
464
|
def main():
|
|
339
|
-
global ard, ard_config, verbose
|
|
465
|
+
global ard, ard_config, verbose
|
|
340
466
|
|
|
341
467
|
# config is specified with a -c parameter
|
|
342
468
|
parser = argparse.ArgumentParser()
|
|
@@ -383,17 +509,17 @@ def main():
|
|
|
383
509
|
try:
|
|
384
510
|
url = f"https://raw.githubusercontent.com/nmdp-bioinformatics/py-ard/master/extras/{sample_file}"
|
|
385
511
|
download_to_file(url, sample_file)
|
|
386
|
-
print(f"Created {sample_file}")
|
|
512
|
+
print(f"Created {sample_file}", file=log_file)
|
|
387
513
|
except HTTPError:
|
|
388
|
-
print(f"Download failed for {sample_file}")
|
|
514
|
+
print(f"Download failed for {sample_file}", file=error_file)
|
|
389
515
|
sys.exit(0)
|
|
390
516
|
|
|
391
517
|
config_filename = args.config
|
|
392
518
|
if not config_filename:
|
|
393
|
-
print("Config file required. Specify with -c/--config")
|
|
519
|
+
print("Config file required. Specify with -c/--config", file=error_file)
|
|
394
520
|
sys.exit(1)
|
|
395
521
|
|
|
396
|
-
print("Using config file:", config_filename)
|
|
522
|
+
print("Using config file:", config_filename, file=log_file)
|
|
397
523
|
with open(config_filename) as conf_file:
|
|
398
524
|
ard_config = json.load(conf_file)
|
|
399
525
|
|
|
@@ -402,8 +528,6 @@ def main():
|
|
|
402
528
|
else:
|
|
403
529
|
verbose = False
|
|
404
530
|
|
|
405
|
-
white_space_regex = re.compile(r"\s+")
|
|
406
|
-
|
|
407
531
|
if ard_config.get("output_file_format") == "xlsx":
|
|
408
532
|
from importlib import util
|
|
409
533
|
|
|
@@ -411,31 +535,16 @@ def main():
|
|
|
411
535
|
if not excel_support_available:
|
|
412
536
|
print(
|
|
413
537
|
"For Excel output, openpyxl library needs to be installed. "
|
|
414
|
-
"Install with:"
|
|
538
|
+
"Install with:",
|
|
539
|
+
file=error_file,
|
|
415
540
|
)
|
|
416
|
-
print(" pip install openpyxl")
|
|
541
|
+
print(" pip install openpyxl", file=error_file)
|
|
417
542
|
sys.exit(1)
|
|
418
543
|
|
|
419
544
|
data_dir = get_data_dir(args.data_dir)
|
|
420
545
|
imgt_version = get_imgt_version(args.ipd_version)
|
|
421
546
|
max_cache_size = ard_config.get("redux_cache_size", pyard.DEFAULT_CACHE_SIZE)
|
|
422
|
-
csv_redux_config =
|
|
423
|
-
"reduce_serology": ard_config.get("reduce_serology", True),
|
|
424
|
-
"reduce_v2": ard_config.get("reduce_v2", True),
|
|
425
|
-
"reduce_3field": ard_config.get("reduce_3field", True),
|
|
426
|
-
"reduce_P": ard_config.get("reduce_P", True),
|
|
427
|
-
"reduce_XX": ard_config.get("reduce_XX", True),
|
|
428
|
-
"reduce_MAC": ard_config.get("reduce_MAC", True),
|
|
429
|
-
"reduce_shortnull": ard_config.get("reduce_shortnull", True),
|
|
430
|
-
"ping": ard_config.get("ping", True),
|
|
431
|
-
"verbose_log": ard_config.get("verbose_log", True),
|
|
432
|
-
"ARS_as_lg": ard_config.get("ARS_as_lg", False),
|
|
433
|
-
"strict": ard_config.get("strict", True),
|
|
434
|
-
"ignore_allele_with_suffixes": tuple(
|
|
435
|
-
ard_config.get("ignore_allele_with_suffixes", "").split(",")
|
|
436
|
-
),
|
|
437
|
-
}
|
|
438
|
-
print(csv_redux_config)
|
|
547
|
+
csv_redux_config = get_redux_config(ard_config)
|
|
439
548
|
ard = pyard.init(
|
|
440
549
|
imgt_version=imgt_version,
|
|
441
550
|
data_dir=data_dir,
|
|
@@ -443,81 +552,10 @@ def main():
|
|
|
443
552
|
config=csv_redux_config,
|
|
444
553
|
)
|
|
445
554
|
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
# Header is the first row
|
|
449
|
-
# Don't convert to NAs
|
|
450
|
-
try:
|
|
451
|
-
df = pd.read_csv(
|
|
452
|
-
ard_config["in_csv_filename"],
|
|
453
|
-
usecols=ard_config["columns_from_csv"],
|
|
454
|
-
header=0,
|
|
455
|
-
dtype=str,
|
|
456
|
-
keep_default_na=False,
|
|
457
|
-
)
|
|
458
|
-
except FileNotFoundError as e:
|
|
459
|
-
print(
|
|
460
|
-
f"File not found {ard_config.get('in_csv_filename')}. Error: {e}",
|
|
461
|
-
file=sys.stderr,
|
|
462
|
-
)
|
|
463
|
-
sys.exit(1)
|
|
464
|
-
|
|
465
|
-
failed_to_reduce_alleles = []
|
|
466
|
-
locus_column_mapping = ard_config.get("locus_column_mapping", None)
|
|
467
|
-
if locus_column_mapping:
|
|
468
|
-
reduce_locus_columns(df, ard_config, locus_column_mapping, verbose)
|
|
469
|
-
|
|
470
|
-
glstring_columns = ard_config.get("glstring_columns", None)
|
|
471
|
-
if glstring_columns:
|
|
472
|
-
reduce_glstring_columns(df, ard_config, glstring_columns)
|
|
555
|
+
in_csv_filename = ard_config["in_csv_filename"]
|
|
556
|
+
out_csv_filename = ard_config["out_csv_filename"]
|
|
473
557
|
|
|
474
|
-
|
|
475
|
-
if ard_config["output_file_format"] == "xlsx":
|
|
476
|
-
out_file_name = f"{ard_config['out_csv_filename']}.xlsx"
|
|
477
|
-
df.to_excel(out_file_name, index=False)
|
|
478
|
-
else:
|
|
479
|
-
# Save as compressed CSV if specified
|
|
480
|
-
out_file_name = ard_config["out_csv_filename"]
|
|
481
|
-
compression_type = ard_config["apply_compression"]
|
|
482
|
-
# Valid compression_type: gzip, zip, null
|
|
483
|
-
if compression_type == "gzip":
|
|
484
|
-
out_file_name = out_file_name + ".gz"
|
|
485
|
-
elif compression_type == "zip":
|
|
486
|
-
out_file_name = out_file_name + ".zip"
|
|
487
|
-
|
|
488
|
-
df.to_csv(out_file_name, index=False, compression=compression_type)
|
|
489
|
-
|
|
490
|
-
if len(failed_to_reduce_alleles) == 0:
|
|
491
|
-
print("No Errors", file=sys.stderr)
|
|
492
|
-
else:
|
|
493
|
-
print("Summary", file=sys.stderr)
|
|
494
|
-
print("-------", file=sys.stderr)
|
|
495
|
-
print(
|
|
496
|
-
f"{len(failed_to_reduce_alleles)} alleles failed to reduce.",
|
|
497
|
-
file=sys.stderr,
|
|
498
|
-
)
|
|
499
|
-
print(
|
|
500
|
-
"| Column Name | Allele | Did you mean ? ",
|
|
501
|
-
file=sys.stderr,
|
|
502
|
-
)
|
|
503
|
-
print(
|
|
504
|
-
"| --------------- | ---------------- | ------------------------- ",
|
|
505
|
-
file=sys.stderr,
|
|
506
|
-
)
|
|
507
|
-
for column_name, locus_allele in failed_to_reduce_alleles:
|
|
508
|
-
similar_allele_names = similar_alleles(ard.db_connection, locus_allele)
|
|
509
|
-
if similar_allele_names:
|
|
510
|
-
similar_allele_names = ",".join(
|
|
511
|
-
sorted(similar_allele_names, reverse=True)
|
|
512
|
-
)
|
|
513
|
-
else:
|
|
514
|
-
similar_allele_names = "NA"
|
|
515
|
-
print(
|
|
516
|
-
f"| {column_name:15} | {locus_allele:16} | {similar_allele_names} ",
|
|
517
|
-
file=sys.stderr,
|
|
518
|
-
)
|
|
519
|
-
# Done
|
|
520
|
-
print(f"Saved result to file:{out_file_name}")
|
|
558
|
+
reduce_and_save(ard, ard_config, in_csv_filename, out_csv_filename, verbose)
|
|
521
559
|
|
|
522
560
|
|
|
523
561
|
if __name__ == "__main__":
|
|
@@ -172,8 +172,11 @@ class Table:
|
|
|
172
172
|
|
|
173
173
|
def __setitem__(self, column: str, values):
|
|
174
174
|
if column in self.columns:
|
|
175
|
-
|
|
176
|
-
|
|
175
|
+
# UPDATE without where clears the whole column
|
|
176
|
+
# Fix for sqlite < 3.35.0 Issue#403
|
|
177
|
+
self._conn.execute(f"UPDATE {self._name} SET `{column}` = NULL")
|
|
178
|
+
else:
|
|
179
|
+
self._conn.execute(f"ALTER TABLE {self._name} ADD COLUMN `{column}` TEXT")
|
|
177
180
|
for i, value in enumerate(values):
|
|
178
181
|
self._conn.execute(
|
|
179
182
|
f"UPDATE {self._name} SET `{column}` = ? WHERE rowid = ?",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|