biodumpy 0.1.6__tar.gz → 0.1.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {biodumpy-0.1.6 → biodumpy-0.1.8}/PKG-INFO +1 -1
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/biodumpy.py +55 -2
- biodumpy-0.1.8/biodumpy/data/citations.json +50 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/COL.py +14 -5
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/NCBI.py +4 -3
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/ZooBank.py +9 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/utils.py +21 -8
- biodumpy-0.1.8/biodumpy/version.py +3 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy.egg-info/PKG-INFO +1 -1
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy.egg-info/SOURCES.txt +1 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/setup.cfg +6 -2
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/BOLD.py +3 -2
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/COL.py +6 -3
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/Crossref.py +2 -7
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/GBIF.py +14 -14
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/INaturalist.py +5 -5
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/IUCN.py +2 -2
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/NCBI.py +2 -2
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/OBIS.py +4 -6
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/WORMS.py +2 -2
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/ZooBank.py +2 -2
- biodumpy-0.1.6/biodumpy/version.py +0 -3
- {biodumpy-0.1.6 → biodumpy-0.1.8}/LICENSE.md +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/README.md +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/__init__.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/input.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/BOLD.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/Crossref.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/GBIF.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/INaturalist.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/IUCN.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/OBIS.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/WORMS.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy/inputs/__init__.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy.egg-info/dependency_links.txt +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy.egg-info/not-zip-safe +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy.egg-info/requires.txt +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/biodumpy.egg-info/top_level.txt +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/pyproject.toml +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/setup.py +0 -0
- {biodumpy-0.1.6 → biodumpy-0.1.8}/tests/__init__.py +0 -0
|
@@ -8,6 +8,9 @@ from .utils import dump, create_directory
|
|
|
8
8
|
from tqdm import tqdm
|
|
9
9
|
import logging
|
|
10
10
|
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
|
|
11
14
|
|
|
12
15
|
class BiodumpyException(Exception):
|
|
13
16
|
pass
|
|
@@ -27,14 +30,36 @@ class Biodumpy:
|
|
|
27
30
|
debug : bool
|
|
28
31
|
If True, enables printing of detailed information during execution.
|
|
29
32
|
Default is True
|
|
33
|
+
cit_style: str
|
|
34
|
+
Citation style used when generating references for the downloaded data.
|
|
35
|
+
Admitted values include "apa" and "bib".
|
|
36
|
+
Default is "apa".
|
|
30
37
|
"""
|
|
31
38
|
|
|
32
|
-
|
|
39
|
+
SHOW_REMINDER_CITE = True
|
|
40
|
+
|
|
41
|
+
def __init__(self, inputs: list[Input], loading_bar: bool = True, debug: bool = False, cit_style: str = "apa") -> None:
|
|
33
42
|
super().__init__()
|
|
43
|
+
|
|
44
|
+
if Biodumpy.SHOW_REMINDER_CITE:
|
|
45
|
+
print(
|
|
46
|
+
"\n 🐔 Please remember to cite biodumpy in your work."
|
|
47
|
+
"\n Consult the citation file to ensure proper attribution of all modules used. "
|
|
48
|
+
"\n Citation: Cancellario, T., Golomb Durán, T., Far, A. J., Roldán, A., & Capa, M. (2025). biodumpy: A Comprehensive Biological Data Downloader. bioRxiv, 2025-07. \n\n"
|
|
49
|
+
)
|
|
50
|
+
Biodumpy.SHOW_REMINDER_CITE = False
|
|
51
|
+
|
|
52
|
+
for input in inputs:
|
|
53
|
+
if not isinstance(input, Input):
|
|
54
|
+
raise BiodumpyException(f"Input module/s must be of type Input and got type {type(input)}")
|
|
55
|
+
|
|
34
56
|
self.inputs = inputs
|
|
35
57
|
self.debug = debug
|
|
36
58
|
self.loading_bar = loading_bar
|
|
37
|
-
|
|
59
|
+
self.cit_style = cit_style
|
|
60
|
+
|
|
61
|
+
if self.cit_style not in ("apa", "bib"):
|
|
62
|
+
raise ValueError("Please provide a valid cit_style, the parameter options are 'apa' or 'bib'")
|
|
38
63
|
|
|
39
64
|
# elements must be a flat list of strings
|
|
40
65
|
def start(self, elements, output_path="downloads/{date}/{module}/{name}"):
|
|
@@ -51,6 +76,7 @@ class Biodumpy:
|
|
|
51
76
|
last_tick = {}
|
|
52
77
|
try:
|
|
53
78
|
for el in tqdm(elements, desc="Biodumpy list", unit=" elements", disable=not self.loading_bar, smoothing=0, file=sys.stdout, colour="#FECC45"):
|
|
79
|
+
# Check whether the variable el is a string.
|
|
54
80
|
if isinstance(el, str):
|
|
55
81
|
el = {"query": el}
|
|
56
82
|
|
|
@@ -90,6 +116,15 @@ class Biodumpy:
|
|
|
90
116
|
for inp, payload in bulk_input.items():
|
|
91
117
|
dump(file_name=output_path.format(date=current_date, module=type(inp).__name__, name="bulk"), obj_list=payload, output_format=inp.output_format)
|
|
92
118
|
|
|
119
|
+
citation_path = []
|
|
120
|
+
for folder in output_path.split("/"):
|
|
121
|
+
citation_path.append(folder)
|
|
122
|
+
if "{name}" in folder:
|
|
123
|
+
break
|
|
124
|
+
|
|
125
|
+
citation_path = "/".join(citation_path)
|
|
126
|
+
self._generate_citation(citation_path.format(date=current_date, module="", name="citation"))
|
|
127
|
+
|
|
93
128
|
if log_handler.buffer:
|
|
94
129
|
print("---- Please review the dump file; errors have been detected ----")
|
|
95
130
|
down_path = str()
|
|
@@ -103,3 +138,21 @@ class Biodumpy:
|
|
|
103
138
|
for record in log_handler.buffer:
|
|
104
139
|
log_entry = f"{record.levelname}: {record.getMessage()}\n"
|
|
105
140
|
f.write(log_entry)
|
|
141
|
+
|
|
142
|
+
def _generate_citation(self, output_path):
|
|
143
|
+
# Open file with bibliography information
|
|
144
|
+
base_dir = os.path.dirname(__file__) # directory of current script
|
|
145
|
+
with open(f"{base_dir}/data/citations.json", "r") as file:
|
|
146
|
+
cit_data = json.load(file)
|
|
147
|
+
|
|
148
|
+
citations = [{"module": "biodumpy", "citation": cit_data.get("biodumpy").get(self.cit_style)}]
|
|
149
|
+
|
|
150
|
+
for inp in self.inputs:
|
|
151
|
+
module_name = type(inp).__name__
|
|
152
|
+
citations.append({"module": module_name, "citation": cit_data.get(module_name).get(self.cit_style)})
|
|
153
|
+
|
|
154
|
+
citation_text = []
|
|
155
|
+
for item in citations:
|
|
156
|
+
citation_text.append(f"---- {item.get('module')} ----\n{item.get('citation')}")
|
|
157
|
+
|
|
158
|
+
dump(output_path, "\n\n".join(citation_text), output_format="txt")
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"biodumpy": {
|
|
3
|
+
"bib": "@article{cancellario2025biodumpy, \n title={biodumpy: A Comprehensive Biological Data Downloader}, \n author={Cancellario, T and Golomb Duran, T and Far, AJ and Roldan, A and Capa, M}, \n journal={bioRxiv}, \n pages={2025--07}, \n year={2025}, \n publisher={Cold Spring Harbor Laboratory}}",
|
|
4
|
+
"apa": "Cancellario, T., Golomb Durán, T., Far, A. J., Roldán, A., & Capa, M. (2025). biodumpy: A Comprehensive Biological Data Downloader. bioRxiv, 2025-07."
|
|
5
|
+
},
|
|
6
|
+
"AVONET": {
|
|
7
|
+
"bib": "@article{tobias2022avonet,\n author = {Tobias, J. A. and Sheard, C. and Pigot, A. L. and Devenish, A. J. and Yang, J. and Sayol, F. and ... and Schleuning, M.},\n title = {AVONET: morphological, ecological and geographical data for all birds},\n journal = {Ecology Letters},\n year = {2022},\n volume = {25},\n number = {3},\n pages = {581-597}\n}",
|
|
8
|
+
"apa": "Tobias, J. A., Sheard, C., Pigot, A. L., Devenish, A. J., Yang, J., Sayol, F., ... & Schleuning, M. (2022). AVONET: morphological, ecological and geographical data for all birds. Ecology Letters, 25(3), 581-597."
|
|
9
|
+
},
|
|
10
|
+
"BOLD": {
|
|
11
|
+
"bib": "@misc{ratnasingham2024bold,\n author = {Ratnasingham, Sujeevan and Wei, C. and Chan, D. and Agda, J. and Agda, J. and Ballesteros-Mejia, L. and Ait Boutou, H. and El Bastami, Z. M. and Ma, E. and Manjunath, R. and Rea, D. and Ho, C. and Telfer, A. and McKeowan, J. and Rahulan, M. and Steinke, C. and Dorsheimer, J. and Milton, M. and Hebert, P. D. N.},\n title = {BOLD v4: A centralized bioinformatics platform for DNA-based biodiversity data},\n booktitle = {DNA Barcoding: Methods and Protocols},\n chapter = {26},\n pages = {403--441},\n year = {2024},\n publisher = {Springer US},\n address = {New York, NY}\n}",
|
|
12
|
+
"apa": "Ratnasingham, S., Wei, C., Chan, D., Agda, J., Agda, J., Ballesteros-Mejia, L., Ait Boutou, H., El Bastami, Z. M., Ma, E., Manjunath, R., Rea, D., Ho, C., Telfer, A., McKeowan, J., Rahulan, M., Steinke, C., Dorsheimer, J., Milton, M., & Hebert, P. D. N. (2024). BOLD v4: A centralized bioinformatics platform for DNA-based biodiversity data. In *DNA barcoding: Methods and protocols* (Chapter 26, pp. 403–441). Springer US."
|
|
13
|
+
},
|
|
14
|
+
"Crossref": {
|
|
15
|
+
"bib": "@misc{crossref,\n author = {{Crossref}},\n title = {Crossref},\n year = {n.d.},\n url = {https://www.crossref.org/},\n note = {Accessed: 2025-06-09}\n}",
|
|
16
|
+
"apa": "Crossref. (n.d.). *Crossref*. https://www.crossref.org/"
|
|
17
|
+
},
|
|
18
|
+
"COL": {
|
|
19
|
+
"bib": "@misc{banki2025catalogue,\n author = {Bánki, O. and Roskov, Y. and Döring, M. and Ower, G. and Hernández Robles, D. R. and Plata Corredor, C. A. and Stjernegaard Jeppesen, T. and Örn, A. and Pape, T. and Hobern, D. and Garnett, S. and Little, H. and DeWalt, R. E. and Ma, K. and Miller, J. and Orrell, T. and Aalbu, R. and Abbott, J. and Adlard, R. and others},\n title = {Catalogue of Life (Version 2025-05-13)},\n year = {2025},\n publisher = {Catalogue of Life},\n address = {Amsterdam, Netherlands},\n doi = {10.48580/dgqdn},\n url = {https://doi.org/10.48580/dgqdn}\n}",
|
|
20
|
+
"apa": "Bánki, O., Roskov, Y., Döring, M., Ower, G., Hernández Robles, D. R., Plata Corredor, C. A., Stjernegaard Jeppesen, T., Örn, A., Pape, T., Hobern, D., Garnett, S., Little, H., DeWalt, R. E., Ma, K., Miller, J., Orrell, T., Aalbu, R., Abbott, J., Adlard, R., et al. (2025). *Catalogue of Life (Version 2025-05-13)*. Catalogue of Life. https://doi.org/10.48580/dgqdn"
|
|
21
|
+
},
|
|
22
|
+
"GBIF":{
|
|
23
|
+
"bib": "@misc{gbif,\n author = {{GBIF.org}},\n title = {GBIF Home Page},\n year = {year},\n url = {https://www.gbif.org},\n note = {Accessed: DD Month YYYY}\n}",
|
|
24
|
+
"apa": "GBIF.org. (year). *GBIF Home Page*. Available from https://www.gbif.org (Accessed: DD Month YYYY)."
|
|
25
|
+
},
|
|
26
|
+
"INaturalist": {
|
|
27
|
+
"bib": "@misc{inaturalist,\n author = {{iNaturalist}},\n title = {iNaturalist},\n url = {https://www.inaturalist.org},\n note = {Accessed: DD Month YYYY}\n}",
|
|
28
|
+
"apa": "iNaturalist. (n.d.). *iNaturalist*. Available from https://www.inaturalist.org (Accessed: DD Month YYYY)."
|
|
29
|
+
},
|
|
30
|
+
"IUCN": {
|
|
31
|
+
"bib": "@misc{iucn2025,\n author = {{IUCN}},\n title = {The IUCN Red List of Threatened Species. Version year-X},\n year = {year},\n url = {https://www.iucnredlist.org},\n note = {Accessed on DD Month YYYY}\n}",
|
|
32
|
+
"apa": "IUCN. (year). *The IUCN Red List of Threatened Species. Version year-X*. https://www.iucnredlist.org (Accessed on DD Month YYYY)."
|
|
33
|
+
},
|
|
34
|
+
"NCBI": {
|
|
35
|
+
"bib": "@article{sayers2024database,\n author = {Sayers, E. W. and Beck, J. and Bolton, E. E. and Brister, J. R. and Chan, J. and Comeau, D. C. and Connor, R. and DiCuccio, M. and Farrell, C. M. and Feldgarden, M. and Fine, A. M. and Funk, K. and Hatcher, E. and Hoeppner, M. and Kane, M. and Kannan, S. and Katz, K. S. and Kelly, C. and Klimke, W. and Kim, S. and ... Sherry, S. T.},\n title = {Database resources of the National Center for Biotechnology Information},\n journal = {Nucleic Acids Research},\n volume = {52},\n number = {D1},\n pages = {D33--D43},\n year = {2024},\n doi = {10.1093/nar/gkad1044},\n url = {https://doi.org/10.1093/nar/gkad1044}\n}",
|
|
36
|
+
"apa": "Sayers, E. W., Beck, J., Bolton, E. E., Brister, J. R., Chan, J., Comeau, D. C., Connor, R., DiCuccio, M., Farrell, C. M., Feldgarden, M., Fine, A. M., Funk, K., Hatcher, E., Hoeppner, M., Kane, M., Kannan, S., Katz, K. S., Kelly, C., Klimke, W., Kim, S., … Sherry, S. T. (2024). Database resources of the National Center for Biotechnology Information. *Nucleic Acids Research, 52*(D1), D33–D43. https://doi.org/10.1093/nar/gkad1044"
|
|
37
|
+
},
|
|
38
|
+
"OBIS": {
|
|
39
|
+
"bib": "@misc{obis,\n author = {{OBIS}},\n title = {Ocean Biodiversity Information System},\n year = {YEAR},\n howpublished = {Intergovernmental Oceanographic Commission of UNESCO},\n url = {https://obis.org}\n}",
|
|
40
|
+
"apa": "OBIS. (YEAR). *Ocean Biodiversity Information System*. Intergovernmental Oceanographic Commission of UNESCO. https://obis.org"
|
|
41
|
+
},
|
|
42
|
+
"WORMS": {
|
|
43
|
+
"bib": "@misc{worms2025,\n author = {Ahyong, S. and Boyko, C.B. and Bernot, J. and Brandão, S.N. and Daly, M. and De Grave, S. and de Voogd, N.J. and Gofas, S. and Hernandez, F. and Hughes, L. and Neubauer, T.A. and Paulay, G. and van der Meij, S. and Boydens, B. and Decock, W. and Dekeyzer, S. and Goharimanesh, M. and Vandepitte, L. and Vanhoorne, B. and others},\n title = {World Register of Marine Species (WoRMS)},\n year = {2025},\n publisher = {WoRMS Editorial Board},\n url = {https://www.marinespecies.org},\n note = {Accessed: DD Month YYYY}\n}",
|
|
44
|
+
"apa": "Ahyong, S., Boyko, C. B., Bernot, J., Brandão, S. N., Daly, M., De Grave, S., de Voogd, N. J., Gofas, S., Hernandez, F., Hughes, L., Neubauer, T. A., Paulay, G., van der Meij, S., Boydens, B., Decock, W., Dekeyzer, S., Goharimanesh, M., Vandepitte, L., Vanhoorne, B., & others. (2025). World Register of Marine Species (WoRMS). WoRMS Editorial Board. https://www.marinespecies.org"
|
|
45
|
+
},
|
|
46
|
+
"ZooBank": {
|
|
47
|
+
"bib": "@misc{pyle2025,\n author = {Pyle, R.},\n title = {ZooBank},\n year = {year},\n version = {v.},\n howpublished = {International Commission on Zoological Nomenclature. Checklist dataset},\n url = {https://doi.org/10.15468/wkr0kn},\n}",
|
|
48
|
+
"apa": "Pyle, R. (year). ZooBank (Version v.) [Checklist dataset]. International Commission on Zoological Nomenclature. https://doi.org/10.15468/wkr0kn (accessed via GBIF.org on 2025-06-09)"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -39,7 +39,7 @@ class COL(Input):
|
|
|
39
39
|
|
|
40
40
|
ACCEPTED_TERMS = ["accepted", "provisionally accepted"]
|
|
41
41
|
|
|
42
|
-
def __init__(self, check_syn: bool = False, dataset_key: int =
|
|
42
|
+
def __init__(self, check_syn: bool = False, dataset_key: int = None, **kwargs):
|
|
43
43
|
super().__init__(**kwargs)
|
|
44
44
|
self.check_syn = check_syn
|
|
45
45
|
self.dataset_key = dataset_key
|
|
@@ -47,6 +47,9 @@ class COL(Input):
|
|
|
47
47
|
if self.output_format != "json":
|
|
48
48
|
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
49
49
|
|
|
50
|
+
if self.dataset_key is None:
|
|
51
|
+
raise ValueError("Please provide a valid dataset_key, or visit https://www.catalogueoflife.org/data/changelog to use the latest ChecklistBank.")
|
|
52
|
+
|
|
50
53
|
def _download(self, query, **kwargs) -> list:
|
|
51
54
|
response = requests.get(
|
|
52
55
|
f"https://api.checklistbank.org/dataset/{self.dataset_key}/nameusage/search?",
|
|
@@ -66,14 +69,20 @@ class COL(Input):
|
|
|
66
69
|
# Multiple IDs
|
|
67
70
|
if len(result) > 1:
|
|
68
71
|
ids = [item.get("id") for item in result if "id" in item]
|
|
69
|
-
ids = ", ".join(ids)
|
|
70
|
-
id_input = input(f"Please enter the correct taxon ID of {query} \n ID: {ids}; Skip \nInsert the ID:")
|
|
71
72
|
|
|
72
|
-
|
|
73
|
-
|
|
73
|
+
# Generate web links for each ID
|
|
74
|
+
web_links = "\n".join([f"https://www.checklistbank.org/dataset/{self.dataset_key}/taxon/{id_}" for id_ in ids])
|
|
75
|
+
id_input = input(f"\n Please enter the correct taxon ID of {query} \n ID: {ids}; Skip \n\nWeb links:\n{web_links}\nInsert the ID: \n")
|
|
76
|
+
|
|
77
|
+
# Check if id_input is contained in ids (if the user write a wrong id)
|
|
78
|
+
if id_input not in ids and id_input != "Skip":
|
|
79
|
+
id_input = input(f"\n Please enter the CORRECT taxon ID of {query} \n ID: {ids}; Skip \n\nWeb links:\n{web_links}\nInsert the ID: \n")
|
|
74
80
|
else:
|
|
75
81
|
result = [item for item in result if item["id"] == id_input]
|
|
76
82
|
|
|
83
|
+
if id_input == "Skip":
|
|
84
|
+
result = [{"id": None, "usage": None, "status": None, "classification": None}]
|
|
85
|
+
|
|
77
86
|
id = result[0].get("id")
|
|
78
87
|
usage = result[0].get("usage")
|
|
79
88
|
status = usage.get("status") if usage else None
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import json
|
|
2
|
+
import sys
|
|
2
3
|
import time
|
|
3
4
|
|
|
4
5
|
from biodumpy import Input
|
|
@@ -141,14 +142,14 @@ class NCBI(Input):
|
|
|
141
142
|
ids_list = self._download_ids(term=f"{query}{self.query_type}", step_id=self.step_id)
|
|
142
143
|
|
|
143
144
|
if self.summary:
|
|
144
|
-
with tqdm(total=len(ids_list), desc="
|
|
145
|
+
with tqdm(total=len(ids_list), desc="\tNCBI summary retrieve", unit=" Summary", colour="#3c91d5", file=sys.stdout, position=0) as pbar:
|
|
145
146
|
for seq_id in split_to_batches(list(ids_list), self.step_seq):
|
|
146
147
|
for sumr in self._download_summary(seq_id):
|
|
147
148
|
sumr["query"] = f"{query}{self.query_type}"
|
|
148
149
|
payload.append(json.loads(json.dumps(sumr, cls=CustomEncoder)))
|
|
149
150
|
pbar.update(len(seq_id))
|
|
150
151
|
else:
|
|
151
|
-
with tqdm(total=len(ids_list), desc="
|
|
152
|
+
with tqdm(total=len(ids_list), desc="\tNCBI sequences retrieve", unit=" Sequences", colour="#3c91d5", file=sys.stdout, position=0) as pbar:
|
|
152
153
|
for seq_id in split_to_batches(list(ids_list), self.step_seq):
|
|
153
154
|
for seq in self._download_seq(seq_id, rettype=self.rettype, db=self.db):
|
|
154
155
|
payload.append(json.loads(json.dumps(seq, cls=CustomEncoder)))
|
|
@@ -185,7 +186,7 @@ class NCBI(Input):
|
|
|
185
186
|
|
|
186
187
|
id_bp_list = set()
|
|
187
188
|
total_ids = int(record["Count"])
|
|
188
|
-
with tqdm(total=total_ids, desc="
|
|
189
|
+
with tqdm(total=total_ids, desc="\tNCBI IDs retrieve", unit=" IDs", colour="#3c91d5", file=sys.stdout, position=0) as pbar:
|
|
189
190
|
for start in range(0, total_ids, step_id):
|
|
190
191
|
try:
|
|
191
192
|
handle = Entrez.esearch(db=self.db, retstart=start, retmax=step_id, term=term)
|
|
@@ -41,12 +41,21 @@ class ZooBank(Input):
|
|
|
41
41
|
self.dataset_size = dataset_size
|
|
42
42
|
self.info = info
|
|
43
43
|
|
|
44
|
+
robot_input = input(
|
|
45
|
+
"\nBefore using the ZooBank module, please confirm that you are not a robot at: https://zoobank.org/\n"
|
|
46
|
+
"Be considerate and avoid overloading the system; download responsibly.\n\n"
|
|
47
|
+
"If you have visited the page, type 'yes' to continue with the download.\n"
|
|
48
|
+
)
|
|
49
|
+
|
|
44
50
|
if self.dataset_size not in ["small", "large"]:
|
|
45
51
|
raise ValueError("Invalid dataset_size. Expected 'small' or 'large'.")
|
|
46
52
|
|
|
47
53
|
if self.output_format != "json":
|
|
48
54
|
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
49
55
|
|
|
56
|
+
if robot_input != "yes":
|
|
57
|
+
raise ValueError("Please visit the ZooBank website and confirm that you are not a robot.")
|
|
58
|
+
|
|
50
59
|
def _download(self, query, **kwargs) -> list:
|
|
51
60
|
payload = []
|
|
52
61
|
|
|
@@ -37,7 +37,7 @@ def dump(file_name, obj_list, output_format="json"):
|
|
|
37
37
|
if output_format == "fasta":
|
|
38
38
|
for line in obj_list:
|
|
39
39
|
output_file.write(f"{line}\n")
|
|
40
|
-
elif output_format
|
|
40
|
+
elif output_format in {"pdf", "txt"}:
|
|
41
41
|
output_file.write(obj_list)
|
|
42
42
|
else:
|
|
43
43
|
json.dump(obj_list, output_file, indent=4)
|
|
@@ -77,8 +77,21 @@ def remove_tags(text: str) -> str:
|
|
|
77
77
|
|
|
78
78
|
def clean_nones(value):
|
|
79
79
|
"""
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
Recursively remove all None values from dictionaries and lists, and returns
|
|
81
|
+
the result as a new dictionary or list.
|
|
82
|
+
|
|
83
|
+
Example:
|
|
84
|
+
data = {
|
|
85
|
+
"name": "Alice",
|
|
86
|
+
"age": None,
|
|
87
|
+
"hobbies": ["reading", None, "swimming"],
|
|
88
|
+
"address": {
|
|
89
|
+
"city": "Wonderland",
|
|
90
|
+
"zip": None
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
clean_nones(data)
|
|
94
|
+
{'name': 'Alice', 'hobbies': ['reading', 'swimming'], 'address': {'city': 'Wonderland'}}
|
|
82
95
|
"""
|
|
83
96
|
if isinstance(value, list):
|
|
84
97
|
return [clean_nones(x) for x in value if x is not None]
|
|
@@ -140,7 +153,7 @@ def split_to_batches(input_list, batch_size: int):
|
|
|
140
153
|
Returns:
|
|
141
154
|
list of lists: A list containing the smaller batches.
|
|
142
155
|
|
|
143
|
-
Example
|
|
156
|
+
Example:
|
|
144
157
|
input_list = [1, 2, 3, 4, 5, 6, 7, 8, 9]
|
|
145
158
|
batch_size = 3
|
|
146
159
|
batches = divide_list_into_batches_by_size(input_list, batch_size)
|
|
@@ -256,7 +269,7 @@ def haplo_collapse(fasta_file):
|
|
|
256
269
|
|
|
257
270
|
Example
|
|
258
271
|
-------
|
|
259
|
-
|
|
272
|
+
a = haplo_collapse(fasta_file)
|
|
260
273
|
"""
|
|
261
274
|
|
|
262
275
|
if not all("id" in item and "sequence" in item for item in fasta_file):
|
|
@@ -288,9 +301,9 @@ def rm_dup(data: list) -> list:
|
|
|
288
301
|
list: A list containing only unique dictionaries, in their original order.
|
|
289
302
|
|
|
290
303
|
Example:
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
304
|
+
data = [{'a': 1}, {'a': 2}, {'a': 1}]
|
|
305
|
+
rm_dup(data)
|
|
306
|
+
[{'a': 1}, {'a': 2}]
|
|
294
307
|
"""
|
|
295
308
|
|
|
296
309
|
unique = [] # Final list of unique dictionaries to return
|
|
@@ -31,8 +31,8 @@ project_urls =
|
|
|
31
31
|
Issue Tracker = https://github.com/centrebalearbiodiversitat/cbbdb/issues
|
|
32
32
|
|
|
33
33
|
[options]
|
|
34
|
-
zip_safe =
|
|
35
|
-
include_package_data =
|
|
34
|
+
zip_safe = false
|
|
35
|
+
include_package_data = true
|
|
36
36
|
packages = find:
|
|
37
37
|
install_requires =
|
|
38
38
|
biopython
|
|
@@ -49,6 +49,10 @@ dev =
|
|
|
49
49
|
pytest-cov==5.0.0
|
|
50
50
|
pytest==8.3.3
|
|
51
51
|
|
|
52
|
+
[options.package_data]
|
|
53
|
+
biodumpy =
|
|
54
|
+
data/*
|
|
55
|
+
|
|
52
56
|
[options.packages.find]
|
|
53
57
|
where = .
|
|
54
58
|
|
|
@@ -13,7 +13,7 @@ from biodumpy.inputs import BOLD
|
|
|
13
13
|
trap = io.StringIO()
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def bold_query(query, summary, output_format):
|
|
16
|
+
def bold_query(query, summary, output_format, dir_module="BOLD"):
|
|
17
17
|
# Create temporary directory
|
|
18
18
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
19
19
|
# Construct the dynamic path using formatted strings
|
|
@@ -25,7 +25,7 @@ def bold_query(query, summary, output_format):
|
|
|
25
25
|
|
|
26
26
|
# Retrieve a file path
|
|
27
27
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
28
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
28
|
+
# dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
29
29
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
30
30
|
|
|
31
31
|
file = os.path.join(f"{dynamic_path}/downloads/{dir_date}/{dir_module}/{file_list}")
|
|
@@ -47,6 +47,7 @@ def test_bold_initialization():
|
|
|
47
47
|
assert bold.summary == False
|
|
48
48
|
assert bold.bulk == False
|
|
49
49
|
assert bold.output_format == "json"
|
|
50
|
+
assert bold.sleep == 3
|
|
50
51
|
|
|
51
52
|
# Objective: Verify that the class raises a ValueError when an invalid value is provided for the
|
|
52
53
|
# output_format parameter.
|
|
@@ -14,7 +14,7 @@ trap = io.StringIO()
|
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
# Remember to check the latest dataset_key
|
|
17
|
-
def col_query(query, check_syn, dataset_key):
|
|
17
|
+
def col_query(query, check_syn, dataset_key, dir_module="COL"):
|
|
18
18
|
# Create temporary directory
|
|
19
19
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
20
20
|
# Construct the dynamic path using formatted strings
|
|
@@ -26,7 +26,6 @@ def col_query(query, check_syn, dataset_key):
|
|
|
26
26
|
|
|
27
27
|
# Retrieve a file path
|
|
28
28
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
29
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
30
29
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
31
30
|
|
|
32
31
|
# Open file
|
|
@@ -39,15 +38,19 @@ def col_query(query, check_syn, dataset_key):
|
|
|
39
38
|
|
|
40
39
|
def test_col_initialization():
|
|
41
40
|
# Test default initialization
|
|
42
|
-
col = COL()
|
|
41
|
+
col = COL(dataset_key=309120)
|
|
43
42
|
|
|
44
43
|
assert col.output_format == "json"
|
|
44
|
+
assert col.sleep == 3
|
|
45
45
|
|
|
46
46
|
# Objective: Verify that the class raises a ValueError when an invalid value is provided for the
|
|
47
47
|
# output_format parameter.
|
|
48
48
|
with pytest.raises(ValueError, match="Invalid output_format. Expected 'json'."):
|
|
49
49
|
COL(output_format="xml")
|
|
50
50
|
|
|
51
|
+
with pytest.raises(ValueError, match="Please provide a valid dataset_key, or visit https://www.catalogueoflife.org/data/changelog to use the latest ChecklistBank."):
|
|
52
|
+
COL(dataset_key=None)
|
|
53
|
+
|
|
51
54
|
|
|
52
55
|
@pytest.mark.parametrize("query, check_syn, dataset_key", [(["Bufo roseus"], True, 309120), (["Bufo roseus"], False, 309120)])
|
|
53
56
|
def test_download(query, check_syn, dataset_key):
|
|
@@ -13,7 +13,7 @@ from biodumpy.inputs import Crossref
|
|
|
13
13
|
trap = io.StringIO()
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def crossref_query(query, summary):
|
|
16
|
+
def crossref_query(query, summary, dir_module="Crossref"):
|
|
17
17
|
# Create temporary directory
|
|
18
18
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
19
19
|
# Construct dynamic path using formatted strings
|
|
@@ -25,7 +25,6 @@ def crossref_query(query, summary):
|
|
|
25
25
|
|
|
26
26
|
# Retrieve a file path
|
|
27
27
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
28
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
29
28
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
30
29
|
|
|
31
30
|
# Open file
|
|
@@ -44,6 +43,7 @@ def test_crossref_initialization():
|
|
|
44
43
|
# correct default values.
|
|
45
44
|
assert crossref.summary == False
|
|
46
45
|
assert crossref.output_format == "json"
|
|
46
|
+
assert crossref.sleep == 3
|
|
47
47
|
|
|
48
48
|
# Objective: Verify that class raises a ValueError when an invalid value is provided for the
|
|
49
49
|
# output_format parameter.
|
|
@@ -76,7 +76,6 @@ def test_download(query, summary):
|
|
|
76
76
|
assert "language" in data, "language is not in data"
|
|
77
77
|
assert data["language"] == "en", "language is not en"
|
|
78
78
|
assert "URL" in data, "URL is not in data"
|
|
79
|
-
# assert data["URL"] == "http://dx.doi.org/10.1038/s44185-022-00001-3", "URL is not http://dx.doi.org/10.1038/s44185-022-00001-3"
|
|
80
79
|
assert "published" in data, "published is not in data"
|
|
81
80
|
assert "title" in data, "title is not in data"
|
|
82
81
|
assert data["title"] == "Climate change will redefine taxonomic, functional, and phylogenetic diversity of Odonata in space and time", (
|
|
@@ -90,12 +89,10 @@ def test_download(query, summary):
|
|
|
90
89
|
assert "reference-count" in data, "reference-count is not in data"
|
|
91
90
|
assert data["reference-count"] == 132, "reference-count is not 132"
|
|
92
91
|
assert "publisher" in data, "publisher is not in data"
|
|
93
|
-
# assert data["publisher"] == "Springer Science and Business Media LLC", "publisher is not Springer Science and Business Media LLC"
|
|
94
92
|
assert "issue" in data, "issue is not in data"
|
|
95
93
|
assert "license" in data, "license is not in data"
|
|
96
94
|
assert "content-domain" in data, "content-domain is not in data"
|
|
97
95
|
assert "short-container-title" in data, "short-container-title is not in data"
|
|
98
|
-
# assert data["short-container-title"][0] == "npj biodivers", "short-container-title is not npj biodivers"
|
|
99
96
|
assert "abstract" in data, "abstract is not in data"
|
|
100
97
|
assert "DOI" in data, "DOI is not in data"
|
|
101
98
|
assert data["DOI"] == "10.1038/s44185-022-00001-3", "DOI is not 10.1038/s44185-022-00001-3"
|
|
@@ -110,7 +107,6 @@ def test_download(query, summary):
|
|
|
110
107
|
"title is not Climate change will redefine taxonomic, functional, and phylogenetic diversity of Odonata in space and time"
|
|
111
108
|
)
|
|
112
109
|
assert "prefix" in data, "prefix is not in data"
|
|
113
|
-
# assert data["prefix"] == "10.1038", "prefix is not 10.1038"
|
|
114
110
|
assert "volume" in data, "volume is not in data"
|
|
115
111
|
assert data["volume"] == "1", "volume is not 1"
|
|
116
112
|
assert "author" in data, "author is not in data"
|
|
@@ -132,7 +128,6 @@ def test_download(query, summary):
|
|
|
132
128
|
assert "journal-issue" in data, "journal-issue is not in data"
|
|
133
129
|
assert "alternative-id" in data, "alternative-id is not in data"
|
|
134
130
|
assert "URL" in data, "URL is not in data"
|
|
135
|
-
# assert data["URL"] == "http://dx.doi.org/10.1038/s44185-022-00001-3", "URL is not http://dx.doi.org/10.1038/s44185-022-00001-3"
|
|
136
131
|
assert "relation" in data, "relation is not in data"
|
|
137
132
|
assert "ISSN" in data, "ISSN is not in data"
|
|
138
133
|
assert "issn-type" in data, "issn-type is not in data"
|
|
@@ -15,7 +15,7 @@ trap = io.StringIO()
|
|
|
15
15
|
gbif_backbone = "d7dddbf4-2cf0-4f39-9b2a-bb099caae36c"
|
|
16
16
|
|
|
17
17
|
|
|
18
|
-
def gbif_query(query, accepted_only, occ, geometry):
|
|
18
|
+
def gbif_query(query, accepted_only, occ, geometry, dir_module="GBIF"):
|
|
19
19
|
# Create temporary directory
|
|
20
20
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
21
21
|
# Construct the dynamic path using formatted strings
|
|
@@ -27,7 +27,6 @@ def gbif_query(query, accepted_only, occ, geometry):
|
|
|
27
27
|
|
|
28
28
|
# Retrieve a file path
|
|
29
29
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
30
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
31
30
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
32
31
|
|
|
33
32
|
# Open file
|
|
@@ -50,6 +49,7 @@ def test_gbif_initialization():
|
|
|
50
49
|
assert gbif.geometry is None
|
|
51
50
|
assert gbif.bulk == False
|
|
52
51
|
assert gbif.output_format == "json"
|
|
52
|
+
assert gbif.sleep == 3
|
|
53
53
|
|
|
54
54
|
# Objective: Verify that the class raises a ValueError when an invalid value is provided for the
|
|
55
55
|
# output_format parameter.
|
|
@@ -158,9 +158,9 @@ def test_download(query, accepted_only, occ, geometry):
|
|
|
158
158
|
assert "crawlId" in data, "crawlId is not in data"
|
|
159
159
|
assert "extensions" in data, "extensions is not in data"
|
|
160
160
|
assert "basisOfRecord" in data, "basisOfRecord is not in data"
|
|
161
|
-
assert "individualCount" in data, "individualCount is not in data"
|
|
161
|
+
# assert "individualCount" in data, "individualCount is not in data"
|
|
162
162
|
assert "occurrenceStatus" in data, "occurrenceStatus is not in data"
|
|
163
|
-
assert "lifeStage" in data, "lifeStage is not in data"
|
|
163
|
+
# assert "lifeStage" in data, "lifeStage is not in data"
|
|
164
164
|
assert "taxonKey" in data, "taxonKey is not in data"
|
|
165
165
|
assert "kingdomKey" in data, "kingdomKey is not in data"
|
|
166
166
|
assert "phylumKey" in data, "phylumKey is not in data"
|
|
@@ -206,7 +206,7 @@ def test_download(query, accepted_only, occ, geometry):
|
|
|
206
206
|
assert "relations" in data, "relations is not in data"
|
|
207
207
|
assert "isInCluster" in data, "isInCluster is not in data"
|
|
208
208
|
assert "recordedBy" in data, "recordedBy is not in data"
|
|
209
|
-
assert "samplingProtocol" in data, "samplingProtocol is not in data"
|
|
209
|
+
# assert "samplingProtocol" in data, "samplingProtocol is not in data"
|
|
210
210
|
assert "geodeticDatum" in data, "geodeticDatum is not in data"
|
|
211
211
|
assert "class" in data, "class is not in data"
|
|
212
212
|
assert "countryCode" in data, "countryCode is not in data"
|
|
@@ -217,12 +217,12 @@ def test_download(query, accepted_only, occ, geometry):
|
|
|
217
217
|
assert "publishedByGbifRegion" in data, "publishedByGbifRegion is not in data"
|
|
218
218
|
assert "rightsHolder" in data, "rightsHolder is not in data"
|
|
219
219
|
assert "identifier" in data, "identifier is not in data"
|
|
220
|
-
assert "vernacularName" in data, "vernacularName is not in data"
|
|
221
|
-
assert "habitat" in data, "habitat is not in data"
|
|
222
|
-
assert "dataGeneralizations" in data, "dataGeneralizations is not in data"
|
|
223
|
-
assert "eventTime" in data, "eventTime is not in data"
|
|
224
|
-
assert "locality" in data, "locality is not in data"
|
|
225
|
-
assert "identificationVerificationStatus" in data, "identificationVerificationStatus is not in data"
|
|
226
|
-
assert "gbifID" in data, "gbifID is not in data"
|
|
227
|
-
assert "eventType" in data, "eventType is not in data"
|
|
228
|
-
assert "occurrenceID" in data, "occurrenceID is not in data"
|
|
220
|
+
# assert "vernacularName" in data, "vernacularName is not in data"
|
|
221
|
+
# assert "habitat" in data, "habitat is not in data"
|
|
222
|
+
# assert "dataGeneralizations" in data, "dataGeneralizations is not in data"
|
|
223
|
+
# assert "eventTime" in data, "eventTime is not in data"
|
|
224
|
+
# assert "locality" in data, "locality is not in data"
|
|
225
|
+
# assert "identificationVerificationStatus" in data, "identificationVerificationStatus is not in data"
|
|
226
|
+
# assert "gbifID" in data, "gbifID is not in data"
|
|
227
|
+
# assert "eventType" in data, "eventType is not in data"
|
|
228
|
+
# assert "occurrenceID" in data, "occurrenceID is not in data"
|
|
@@ -13,7 +13,7 @@ from biodumpy.inputs import INaturalist
|
|
|
13
13
|
trap = io.StringIO()
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def inat_query(query):
|
|
16
|
+
def inat_query(query, dir_module="INaturalist"):
|
|
17
17
|
# Create temporary directory
|
|
18
18
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
19
19
|
# Construct the dynamic path using formatted strings
|
|
@@ -25,7 +25,6 @@ def inat_query(query):
|
|
|
25
25
|
|
|
26
26
|
# Retrieve a file path
|
|
27
27
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
28
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
29
28
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
30
29
|
|
|
31
30
|
file = os.path.join(f"{dynamic_path}/downloads/{dir_date}/{dir_module}/{file_list}")
|
|
@@ -37,11 +36,12 @@ def inat_query(query):
|
|
|
37
36
|
|
|
38
37
|
def test_inat_initialization():
|
|
39
38
|
# Test default initialization
|
|
40
|
-
|
|
39
|
+
inat = INaturalist()
|
|
41
40
|
|
|
42
41
|
# Verify default parameters
|
|
43
|
-
assert
|
|
44
|
-
assert
|
|
42
|
+
assert inat.bulk == False
|
|
43
|
+
assert inat.output_format == "json"
|
|
44
|
+
assert inat.sleep == 3
|
|
45
45
|
|
|
46
46
|
# Objective: Verify that the class raises a ValueError when an invalid value is provided for the
|
|
47
47
|
# output_format parameter.
|
|
@@ -33,7 +33,7 @@ IUCN_SCOPE = [
|
|
|
33
33
|
]
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
def iucn_query(query, authorization, assess_details, latest, scope, output_format):
|
|
36
|
+
def iucn_query(query, authorization, assess_details, latest, scope, output_format, dir_module="IUCN"):
|
|
37
37
|
# Create temporary directory
|
|
38
38
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
39
39
|
# Construct the dynamic path using formatted strings
|
|
@@ -46,7 +46,6 @@ def iucn_query(query, authorization, assess_details, latest, scope, output_forma
|
|
|
46
46
|
|
|
47
47
|
# Retrieve a file path
|
|
48
48
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
49
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
50
49
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
51
50
|
|
|
52
51
|
file = os.path.join(f"{dynamic_path}/downloads/{dir_date}/{dir_module}/{file_list}")
|
|
@@ -65,6 +64,7 @@ def test_iucn_initialization():
|
|
|
65
64
|
assert iucn.latest == False
|
|
66
65
|
assert iucn.assess_details == False
|
|
67
66
|
assert iucn.output_format == "json"
|
|
67
|
+
assert iucn.sleep == 3
|
|
68
68
|
|
|
69
69
|
# Objective: Verify that the class raises a ValueError when an invalid value is provided for the
|
|
70
70
|
# output_format parameter.
|
|
@@ -13,7 +13,7 @@ from biodumpy.inputs import NCBI
|
|
|
13
13
|
trap = io.StringIO()
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def ncbi_query(query, summary, output_format, max_bp, db, step_id, step_seq, rettype, query_type, by_id, taxonomy, taxonomy_only, mail):
|
|
16
|
+
def ncbi_query(query, summary, output_format, max_bp, db, step_id, step_seq, rettype, query_type, by_id, taxonomy, taxonomy_only, mail, dir_module="NCBI"):
|
|
17
17
|
# Create temporary directory
|
|
18
18
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
19
19
|
# Construct the dynamic path using formatted strings
|
|
@@ -43,7 +43,6 @@ def ncbi_query(query, summary, output_format, max_bp, db, step_id, step_seq, ret
|
|
|
43
43
|
|
|
44
44
|
# Retrieve a file path
|
|
45
45
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
46
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
47
46
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
48
47
|
|
|
49
48
|
file = os.path.join(f"{dynamic_path}/downloads/{dir_date}/{dir_module}/{file_list}")
|
|
@@ -74,6 +73,7 @@ def test_ncbi_initialization():
|
|
|
74
73
|
assert ncbi.taxonomy_only == False
|
|
75
74
|
assert ncbi.bulk == False
|
|
76
75
|
assert ncbi.output_format == "json"
|
|
76
|
+
assert ncbi.sleep == 3
|
|
77
77
|
|
|
78
78
|
with pytest.raises(ValueError, match="Invalid output_format or rettype. Expected fasta."):
|
|
79
79
|
NCBI(output_format="fasta", rettype="gb") # Should raise the error
|
|
@@ -13,7 +13,7 @@ from biodumpy.inputs import OBIS
|
|
|
13
13
|
trap = io.StringIO()
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def obis_query(query, occ, geometry, areaid):
|
|
16
|
+
def obis_query(query, occ, geometry, areaid, dir_module="OBIS"):
|
|
17
17
|
# Create temporary directory
|
|
18
18
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
19
19
|
# Construct the dynamic path using formatted strings
|
|
@@ -25,7 +25,6 @@ def obis_query(query, occ, geometry, areaid):
|
|
|
25
25
|
|
|
26
26
|
# Retrieve a file path
|
|
27
27
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
28
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
29
28
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
30
29
|
|
|
31
30
|
# Open file
|
|
@@ -46,6 +45,7 @@ def test_obis_initialization():
|
|
|
46
45
|
assert obis.areaid is None
|
|
47
46
|
assert obis.bulk == False
|
|
48
47
|
assert obis.output_format == "json"
|
|
48
|
+
assert obis.sleep == 3
|
|
49
49
|
|
|
50
50
|
# Objective: Verify that the class raises a ValueError when an invalid value is provided for the
|
|
51
51
|
# output_format parameter.
|
|
@@ -75,7 +75,6 @@ def test_download(query, occ, geometry, areaid):
|
|
|
75
75
|
# Check the main info in an OBIS JSON file
|
|
76
76
|
|
|
77
77
|
data = data[0]
|
|
78
|
-
|
|
79
78
|
if occ is False:
|
|
80
79
|
assert "scientificName" in data, "scientificName is not in data"
|
|
81
80
|
assert data["scientificName"] == "Pinna nobilis", "scientificName is not Pinna nobilis"
|
|
@@ -83,8 +82,8 @@ def test_download(query, occ, geometry, areaid):
|
|
|
83
82
|
assert data["scientificNameAuthorship"] == "Linnaeus, 1758", "scientificNameAuthorship is not Linnaeus, 1758"
|
|
84
83
|
assert "taxonID" in data, "taxonID is not in data"
|
|
85
84
|
assert data["taxonID"] == 140780, "taxonID is not 140780"
|
|
86
|
-
assert "bold_id" in data, "bold_id is not in data"
|
|
87
|
-
assert data["bold_id"] == 79749, "bold_id is not 79749"
|
|
85
|
+
# assert "bold_id" in data, "bold_id is not in data"
|
|
86
|
+
# assert data["bold_id"] == 79749, "bold_id is not 79749"
|
|
88
87
|
assert "ncbi_id" in data, "ncbi_id is not in data"
|
|
89
88
|
assert data["ncbi_id"] == 111169, "ncbi_id is not 111169"
|
|
90
89
|
assert "taxonRank" in data, "taxonRank is not in data"
|
|
@@ -159,7 +158,6 @@ def test_download(query, occ, geometry, areaid):
|
|
|
159
158
|
assert "eventID" in data, "eventID is not in data"
|
|
160
159
|
assert "footprintSRS" in data, "footprintSRS is not in data"
|
|
161
160
|
assert "identifiedBy" in data, "identifiedBy is not in data"
|
|
162
|
-
assert "language" in data, "language is not in data"
|
|
163
161
|
assert "maximumDepthInMeters" in data, "maximumDepthInMeters is not in data"
|
|
164
162
|
assert "minimumDepthInMeters" in data, "minimumDepthInMeters is not in data"
|
|
165
163
|
assert "modified" in data, "modified is not in data"
|
|
@@ -13,7 +13,7 @@ from biodumpy.inputs import WORMS
|
|
|
13
13
|
trap = io.StringIO()
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def worms_query(query, distribution, marine_only):
|
|
16
|
+
def worms_query(query, distribution, marine_only, dir_module="WORMS"):
|
|
17
17
|
# Create temporary directory
|
|
18
18
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
19
19
|
# Construct the dynamic path using formatted strings
|
|
@@ -25,7 +25,6 @@ def worms_query(query, distribution, marine_only):
|
|
|
25
25
|
|
|
26
26
|
# Retrieve a file path
|
|
27
27
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
28
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
29
28
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
30
29
|
|
|
31
30
|
# Open file
|
|
@@ -41,6 +40,7 @@ def test_worms_initialization():
|
|
|
41
40
|
worms = WORMS()
|
|
42
41
|
|
|
43
42
|
assert worms.output_format == "json"
|
|
43
|
+
assert worms.sleep == 3
|
|
44
44
|
|
|
45
45
|
# Objective: Verify that the class raises a ValueError when an invalid value is provided for the
|
|
46
46
|
# output_format parameter.
|
|
@@ -13,7 +13,7 @@ from biodumpy.inputs import ZooBank
|
|
|
13
13
|
trap = io.StringIO()
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def zoobank_query(query, info, dataset_size):
|
|
16
|
+
def zoobank_query(query, info, dataset_size, dir_module="ZooBank"):
|
|
17
17
|
# Create temporary directory
|
|
18
18
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
19
19
|
# Construct the dynamic path using formatted strings
|
|
@@ -25,7 +25,6 @@ def zoobank_query(query, info, dataset_size):
|
|
|
25
25
|
|
|
26
26
|
# Retrieve a file path
|
|
27
27
|
dir_date = os.listdir(f"{dynamic_path}/downloads/")[0]
|
|
28
|
-
dir_module = os.listdir(f"{dynamic_path}/downloads/{dir_date}")[0]
|
|
29
28
|
file_list = os.listdir(f"{dynamic_path}/downloads/{dir_date}/{dir_module}")[0]
|
|
30
29
|
|
|
31
30
|
# Open file
|
|
@@ -45,6 +44,7 @@ def test_zoobank_initialization():
|
|
|
45
44
|
assert zoobank.dataset_size == "small"
|
|
46
45
|
assert zoobank.info == False
|
|
47
46
|
assert zoobank.output_format == "json"
|
|
47
|
+
assert zoobank.sleep == 3
|
|
48
48
|
|
|
49
49
|
# Objective: Verify that the class correctly raises a ValueError when an invalid value is provided for the
|
|
50
50
|
# dataset_size parameter.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|