biodumpy 0.1.9__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {biodumpy-0.1.9 → biodumpy-0.2.0}/PKG-INFO +29 -30
- {biodumpy-0.1.9 → biodumpy-0.2.0}/README.md +27 -28
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/biodumpy.py +7 -2
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/data/citations.json +2 -2
- biodumpy-0.2.0/biodumpy/inputs/AVONET.py +92 -0
- biodumpy-0.2.0/biodumpy/inputs/BOLD.py +202 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/__init__.py +1 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/utils.py +13 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/pyproject.toml +3 -3
- biodumpy-0.1.9/biodumpy/data/.DS_Store +0 -0
- biodumpy-0.1.9/biodumpy/inputs/BOLD.py +0 -119
- {biodumpy-0.1.9 → biodumpy-0.2.0}/LICENSE +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/__init__.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/input.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/COL.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/Crossref.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/GBIF.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/INaturalist.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/IUCN.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/NCBI.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/OBIS.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/WORMS.py +0 -0
- {biodumpy-0.1.9 → biodumpy-0.2.0}/biodumpy/inputs/ZooBank.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: biodumpy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A Comprehensive Biological Data Downloader.
|
|
5
5
|
License-Expression: GPL-3.0-or-later
|
|
6
6
|
License-File: LICENSE
|
|
@@ -15,11 +15,13 @@ Classifier: Programming Language :: Python :: 3.13
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.14
|
|
16
16
|
Requires-Dist: beautifulsoup4 (>=4.12.3,<5.0)
|
|
17
17
|
Requires-Dist: biopython (>=1.84,<2.0)
|
|
18
|
-
Requires-Dist: requests (>=2.
|
|
18
|
+
Requires-Dist: requests (>=2.34.2,<3.0)
|
|
19
19
|
Requires-Dist: tqdm (>=4.66.4,<5.0)
|
|
20
20
|
Description-Content-Type: text/markdown
|
|
21
21
|
|
|
22
|
-
<
|
|
22
|
+
<p align="center">
|
|
23
|
+
<img src="docs/source/static/Biodumpy_logo.png" alt="Project Logo" width="200">
|
|
24
|
+
</p>
|
|
23
25
|
|
|
24
26
|
# biodumpy: A Comprehensive Biological Data Downloader
|
|
25
27
|
|
|
@@ -38,7 +40,8 @@ Description-Content-Type: text/markdown
|
|
|
38
40
|
## Key Features
|
|
39
41
|
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically designed for retrieving information from its respective source. The modules implemented so far are:
|
|
40
42
|
|
|
41
|
-
-
|
|
43
|
+
- AVONET (AVONET)
|
|
44
|
+
- Barcode of life data system (BOLD)
|
|
42
45
|
- Catalogue of life (COL)
|
|
43
46
|
- Crossref
|
|
44
47
|
- Global biodiversity information facility (GBIF)
|
|
@@ -76,45 +79,41 @@ In detail, we described:
|
|
|
76
79
|
**Example N.1**
|
|
77
80
|
|
|
78
81
|
``` python
|
|
82
|
+
# Import biodumpy package
|
|
83
|
+
from biodumpy import Biodumpy
|
|
79
84
|
|
|
80
|
-
|
|
81
|
-
|
|
85
|
+
# Import GBIF module
|
|
86
|
+
from biodumpy.inputs import GBIF
|
|
82
87
|
|
|
83
|
-
|
|
84
|
-
|
|
88
|
+
# Create a list of taxa
|
|
89
|
+
taxa = ["Alytes muletensis (Sanchíz & Adrover, 1979)", "Bufotes viridis (Laurenti, 1768)", "Hyla meridionalis Boettger, 1874", "Anax imperator Leach, 1815"]
|
|
85
90
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
'Hyla meridionalis Boettger, 1874', 'Anax imperator Leach, 1815']
|
|
91
|
+
# Set the Biodumpy function with the specific parameters
|
|
92
|
+
bdp = Biodumpy([GBIF(sleep=3, bulk=False, accepted_only=True)])
|
|
89
93
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
# Start the download
|
|
94
|
-
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
94
|
+
# Start the download
|
|
95
|
+
bdp.start(taxa, output_path="./downloads/{date}/{module}/{name}")
|
|
95
96
|
```
|
|
96
97
|
|
|
97
98
|
**Example N.2**
|
|
98
99
|
|
|
99
100
|
``` python
|
|
101
|
+
# Import biodumpy package
|
|
102
|
+
from biodumpy import Biodumpy
|
|
100
103
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
# Import GBIF and IUCN modules
|
|
105
|
-
from biodumpy.inputs import GBIF, IUCN
|
|
104
|
+
# Import GBIF and IUCN modules
|
|
105
|
+
from biodumpy.inputs import GBIF, IUCN
|
|
106
106
|
|
|
107
|
-
|
|
107
|
+
api_key = "YOUR_IUCN_API_KEY"
|
|
108
108
|
|
|
109
|
-
|
|
110
|
-
|
|
109
|
+
# Create a list of taxa
|
|
110
|
+
taxa = ["Alytes muletensis", "Bufotes viridis", "Hyla meridionalis", "Anax imperator"]
|
|
111
111
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
IUCN(api_key=api_key, bulk=True, region=['Global'])])
|
|
112
|
+
# Set the Biodumpy functions with the specific parameters
|
|
113
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True), IUCN(api_key=api_key, bulk=True, region=["Global"])])
|
|
115
114
|
|
|
116
|
-
|
|
117
|
-
|
|
115
|
+
# Start the download
|
|
116
|
+
bdp.start(taxa, output_path="./downloads/{date}/{module}/{name}")
|
|
118
117
|
```
|
|
119
118
|
|
|
120
119
|
## Documentation and Support
|
|
@@ -124,7 +123,7 @@ For detailed documentation and tutorials, please visit the ``biodumpy`` Read the
|
|
|
124
123
|
## Contribution
|
|
125
124
|
``biodumpy`` is an open-source project, and contributions are welcome!
|
|
126
125
|
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub repository or contact with the support team
|
|
127
|
-
[✉️ here](mailto:
|
|
126
|
+
[✉️ here](mailto:tommaso.cancellario@gmail.com?subject=biodumpy_support).
|
|
128
127
|
|
|
129
128
|
|
|
130
129
|
## License
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
<
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="docs/source/static/Biodumpy_logo.png" alt="Project Logo" width="200">
|
|
3
|
+
</p>
|
|
2
4
|
|
|
3
5
|
# biodumpy: A Comprehensive Biological Data Downloader
|
|
4
6
|
|
|
@@ -17,7 +19,8 @@
|
|
|
17
19
|
## Key Features
|
|
18
20
|
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically designed for retrieving information from its respective source. The modules implemented so far are:
|
|
19
21
|
|
|
20
|
-
-
|
|
22
|
+
- AVONET (AVONET)
|
|
23
|
+
- Barcode of life data system (BOLD)
|
|
21
24
|
- Catalogue of life (COL)
|
|
22
25
|
- Crossref
|
|
23
26
|
- Global biodiversity information facility (GBIF)
|
|
@@ -55,45 +58,41 @@ In detail, we described:
|
|
|
55
58
|
**Example N.1**
|
|
56
59
|
|
|
57
60
|
``` python
|
|
61
|
+
# Import biodumpy package
|
|
62
|
+
from biodumpy import Biodumpy
|
|
58
63
|
|
|
59
|
-
|
|
60
|
-
|
|
64
|
+
# Import GBIF module
|
|
65
|
+
from biodumpy.inputs import GBIF
|
|
61
66
|
|
|
62
|
-
|
|
63
|
-
|
|
67
|
+
# Create a list of taxa
|
|
68
|
+
taxa = ["Alytes muletensis (Sanchíz & Adrover, 1979)", "Bufotes viridis (Laurenti, 1768)", "Hyla meridionalis Boettger, 1874", "Anax imperator Leach, 1815"]
|
|
64
69
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
'Hyla meridionalis Boettger, 1874', 'Anax imperator Leach, 1815']
|
|
70
|
+
# Set the Biodumpy function with the specific parameters
|
|
71
|
+
bdp = Biodumpy([GBIF(sleep=3, bulk=False, accepted_only=True)])
|
|
68
72
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
# Start the download
|
|
73
|
-
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
73
|
+
# Start the download
|
|
74
|
+
bdp.start(taxa, output_path="./downloads/{date}/{module}/{name}")
|
|
74
75
|
```
|
|
75
76
|
|
|
76
77
|
**Example N.2**
|
|
77
78
|
|
|
78
79
|
``` python
|
|
80
|
+
# Import biodumpy package
|
|
81
|
+
from biodumpy import Biodumpy
|
|
79
82
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
# Import GBIF and IUCN modules
|
|
84
|
-
from biodumpy.inputs import GBIF, IUCN
|
|
83
|
+
# Import GBIF and IUCN modules
|
|
84
|
+
from biodumpy.inputs import GBIF, IUCN
|
|
85
85
|
|
|
86
|
-
|
|
86
|
+
api_key = "YOUR_IUCN_API_KEY"
|
|
87
87
|
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
# Create a list of taxa
|
|
89
|
+
taxa = ["Alytes muletensis", "Bufotes viridis", "Hyla meridionalis", "Anax imperator"]
|
|
90
90
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
IUCN(api_key=api_key, bulk=True, region=['Global'])])
|
|
91
|
+
# Set the Biodumpy functions with the specific parameters
|
|
92
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True), IUCN(api_key=api_key, bulk=True, region=["Global"])])
|
|
94
93
|
|
|
95
|
-
|
|
96
|
-
|
|
94
|
+
# Start the download
|
|
95
|
+
bdp.start(taxa, output_path="./downloads/{date}/{module}/{name}")
|
|
97
96
|
```
|
|
98
97
|
|
|
99
98
|
## Documentation and Support
|
|
@@ -103,7 +102,7 @@ For detailed documentation and tutorials, please visit the ``biodumpy`` Read the
|
|
|
103
102
|
## Contribution
|
|
104
103
|
``biodumpy`` is an open-source project, and contributions are welcome!
|
|
105
104
|
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub repository or contact with the support team
|
|
106
|
-
[✉️ here](mailto:
|
|
105
|
+
[✉️ here](mailto:tommaso.cancellario@gmail.com?subject=biodumpy_support).
|
|
107
106
|
|
|
108
107
|
|
|
109
108
|
## License
|
|
@@ -4,7 +4,7 @@ from datetime import datetime
|
|
|
4
4
|
from logging.handlers import MemoryHandler
|
|
5
5
|
|
|
6
6
|
from .input import Input
|
|
7
|
-
from .utils import dump, create_directory
|
|
7
|
+
from .utils import dump, create_directory, is_na
|
|
8
8
|
from tqdm import tqdm
|
|
9
9
|
import logging
|
|
10
10
|
|
|
@@ -45,7 +45,7 @@ class Biodumpy:
|
|
|
45
45
|
print(
|
|
46
46
|
"\n 🐔 Please remember to cite biodumpy in your work."
|
|
47
47
|
"\n Consult the citation file to ensure proper attribution of all modules used. "
|
|
48
|
-
"\n
|
|
48
|
+
"\n Cancellario, T., Golomb Durán, T., Far, A. J., Roldán, A., & Capa, M. (2026). Biodumpy: A comprehensive biological data downloader. Biodiversity Informatics, 20(1). https://doi.org/10.17161/bi.v20i1.24725 \n\n"
|
|
49
49
|
)
|
|
50
50
|
Biodumpy.SHOW_REMINDER_CITE = False
|
|
51
51
|
|
|
@@ -66,6 +66,11 @@ class Biodumpy:
|
|
|
66
66
|
if not isinstance(elements, list):
|
|
67
67
|
raise ValueError("Invalid query. Expected a list of taxa to query.")
|
|
68
68
|
|
|
69
|
+
# Check if there are NAs in the element list. If NAs are present, remove them and show a message
|
|
70
|
+
if any(is_na(e) for e in elements):
|
|
71
|
+
print("Found NA values, removing them... \n------------------------------------")
|
|
72
|
+
elements = [e for e in elements if not is_na(e)]
|
|
73
|
+
|
|
69
74
|
current_date = datetime.now().strftime("%Y-%m-%d")
|
|
70
75
|
|
|
71
76
|
log_handler = MemoryHandler(capacity=1024)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"biodumpy": {
|
|
3
|
-
"bib": "@article{
|
|
4
|
-
"apa": "Cancellario, T., Golomb Durán, T., Far, A. J., Roldán, A., & Capa, M. (
|
|
3
|
+
"bib": "@article{cancellario2026biodumpy, title={Biodumpy: A comprehensive biological data downloader}, volume={20}, url={https://journals.ku.edu/jbi/article/view/24725}, DOI={10.17161/bi.v20i1.24725}, number={1}, journal={Biodiversity Informatics}, author={Cancellario, Tommaso and Golomb Durán, Tomas and Far, Antoni Josep and Roldán, Alejandro and Capa, Maria}, year={2026}, month={Mar.} }",
|
|
4
|
+
"apa": "Cancellario, T., Golomb Durán, T., Far, A. J., Roldán, A., & Capa, M. (2026). Biodumpy: A comprehensive biological data downloader. Biodiversity Informatics, 20(1). https://doi.org/10.17161/bi.v20i1.24725"
|
|
5
5
|
},
|
|
6
6
|
"AVONET": {
|
|
7
7
|
"bib": "@article{tobias2022avonet,\n author = {Tobias, J. A. and Sheard, C. and Pigot, A. L. and Devenish, A. J. and Yang, J. and Sayol, F. and ... and Schleuning, M.},\n title = {AVONET: morphological, ecological and geographical data for all birds},\n journal = {Ecology Letters},\n year = {2022},\n volume = {25},\n number = {3},\n pages = {581-597}\n}",
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import requests
|
|
5
|
+
|
|
6
|
+
from biodumpy import Input, BiodumpyException
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AVONET(Input):
|
|
10
|
+
"""
|
|
11
|
+
AVONET plugin to access bird trait data from the AVONET dataset.
|
|
12
|
+
It allows querying species traits by species name.
|
|
13
|
+
|
|
14
|
+
Parameters
|
|
15
|
+
----------
|
|
16
|
+
data_path : str or Path, optional
|
|
17
|
+
Custom path to an AVONET JSON file. If omitted, the default plugin dataset is used.
|
|
18
|
+
output_format : str
|
|
19
|
+
Output format for the plugin data. Must be "json".
|
|
20
|
+
|
|
21
|
+
Example
|
|
22
|
+
-------
|
|
23
|
+
>>> from biodumpy import Biodumpy
|
|
24
|
+
>>> from biodumpy.inputs import AVONET
|
|
25
|
+
# Taxa list
|
|
26
|
+
>>> taxa = ['Accipiter albogularis', 'Accipiter fasciatus', 'Accipiter tachiro']
|
|
27
|
+
# Set the module and start the download
|
|
28
|
+
>>> bdp = Biodumpy([AVONET(bulk=True, output_format="json")])
|
|
29
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
DATA_URL = "https://github.com/centrebalearbiodiversitat/biodumpy/releases/download/AVONET/AVONET1_BirdLife_plugin.json"
|
|
33
|
+
|
|
34
|
+
_DATA_CACHE = None
|
|
35
|
+
|
|
36
|
+
def __init__(self, data_path=None, **kwargs):
|
|
37
|
+
super().__init__(**kwargs)
|
|
38
|
+
|
|
39
|
+
if self.output_format not in {"json"}:
|
|
40
|
+
raise ValueError('Invalid output_format. Expected "json".')
|
|
41
|
+
|
|
42
|
+
if data_path:
|
|
43
|
+
self.json_file = Path(data_path)
|
|
44
|
+
else:
|
|
45
|
+
package_dir = Path(__file__).resolve().parent.parent / "data" / "plugin"
|
|
46
|
+
try:
|
|
47
|
+
package_dir.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
self.json_file = package_dir / "AVONET1_BirdLife_plugin.json"
|
|
49
|
+
except (PermissionError, OSError):
|
|
50
|
+
cache_dir = Path.home() / ".cache" / "biodumpy" / "data" / "plugin"
|
|
51
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
52
|
+
self.json_file = cache_dir / "AVONET1_BirdLife_plugin.json"
|
|
53
|
+
|
|
54
|
+
# Download dataset if not cached locally
|
|
55
|
+
if not self.json_file.exists():
|
|
56
|
+
self._download_dataset()
|
|
57
|
+
else:
|
|
58
|
+
print("AVONET1_BirdLife_plugin... OK")
|
|
59
|
+
|
|
60
|
+
# Load dataset into memory once
|
|
61
|
+
if not data_path and AVONET._DATA_CACHE is not None:
|
|
62
|
+
self.data = AVONET._DATA_CACHE
|
|
63
|
+
else:
|
|
64
|
+
with open(self.json_file, "r", encoding="utf-8") as f:
|
|
65
|
+
self.data = json.load(f)
|
|
66
|
+
if not data_path:
|
|
67
|
+
AVONET._DATA_CACHE = self.data
|
|
68
|
+
|
|
69
|
+
def _download_dataset(self):
|
|
70
|
+
print("Downloading AVONET1_BirdLife...")
|
|
71
|
+
try:
|
|
72
|
+
response = requests.get(self.DATA_URL, timeout=30)
|
|
73
|
+
if response.status_code == 200:
|
|
74
|
+
data = response.json()
|
|
75
|
+
with open(self.json_file, "w", encoding="utf-8") as f:
|
|
76
|
+
json.dump(data, f, indent=4)
|
|
77
|
+
return
|
|
78
|
+
else:
|
|
79
|
+
raise BiodumpyException(f"Failed to download AVONET dataset from {self.DATA_URL}: HTTP {response.status_code}")
|
|
80
|
+
except requests.RequestException as e:
|
|
81
|
+
raise BiodumpyException(f"Failed to download AVONET dataset from {self.DATA_URL}: {str(e)}")
|
|
82
|
+
|
|
83
|
+
def _download(self, query, **kwargs) -> list:
|
|
84
|
+
payload = []
|
|
85
|
+
|
|
86
|
+
result = self.data.get(query, None)
|
|
87
|
+
if result is None:
|
|
88
|
+
raise ValueError(f"Taxon '{query}' not found in AVONET dataset.")
|
|
89
|
+
|
|
90
|
+
payload.append(result)
|
|
91
|
+
|
|
92
|
+
return payload
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
from urllib.parse import quote
|
|
3
|
+
from requests.adapters import HTTPAdapter
|
|
4
|
+
from urllib3.util import Retry
|
|
5
|
+
|
|
6
|
+
from biodumpy import Input, BiodumpyException
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class BOLD(Input):
|
|
10
|
+
"""
|
|
11
|
+
Query the Barcode of Life Data System (BOLD Portal) database to retrieve taxon genetic or occurrence data.
|
|
12
|
+
|
|
13
|
+
Parameters
|
|
14
|
+
----------
|
|
15
|
+
summary : bool, optional
|
|
16
|
+
If True, the function returns a summary of the downloaded metadata instead of the full records.
|
|
17
|
+
Default is False.
|
|
18
|
+
output_format : str, optional
|
|
19
|
+
Output format for the downloaded data ("json" or "fasta"). Default is "json".
|
|
20
|
+
bulk : bool, optional
|
|
21
|
+
Whether to merge results into a single file. Default is False.
|
|
22
|
+
sleep : int, optional
|
|
23
|
+
Delay between requests in seconds. Default is 3.
|
|
24
|
+
|
|
25
|
+
Details
|
|
26
|
+
-------
|
|
27
|
+
When `summary` is True, the resulting JSON will include key metadata fields:
|
|
28
|
+
- `record_id`: The unique identifier for the BOLD record.
|
|
29
|
+
- `processid`: The process ID associated with the BOLD record.
|
|
30
|
+
- `sampleid`: The sample identifier.
|
|
31
|
+
- `bin_uri`: The BIN (Barcode Index Number) URI.
|
|
32
|
+
- `taxon`: The taxonomic name (species or identification).
|
|
33
|
+
- `species`: The species name.
|
|
34
|
+
- `identification`: The identification string.
|
|
35
|
+
- `country_ocean`: The country or ocean of collection.
|
|
36
|
+
- `province_state`: The province or state of collection.
|
|
37
|
+
- `region`: The geographic region of collection.
|
|
38
|
+
- `coord`: Geographic coordinates (latitude, longitude).
|
|
39
|
+
- `marker_code`: The marker code (e.g. COI-5P).
|
|
40
|
+
- `insdc_acs`: The GenBank / INSDC accession number.
|
|
41
|
+
- `identified_by`: The identifier person or institution.
|
|
42
|
+
- `inst`: Housing institution.
|
|
43
|
+
- `collection_date_start`: Collection date.
|
|
44
|
+
- `sequence_run_site`: Sequencing facility.
|
|
45
|
+
- `sequence_upload_date`: Date uploaded to BOLD.
|
|
46
|
+
- `specimens`: Specimen information.
|
|
47
|
+
|
|
48
|
+
Example
|
|
49
|
+
-------
|
|
50
|
+
>>> from biodumpy import Biodumpy
|
|
51
|
+
>>> from biodumpy.inputs import BOLD
|
|
52
|
+
# Taxa list
|
|
53
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator']
|
|
54
|
+
# Set the module and start the download
|
|
55
|
+
>>> bdp = Biodumpy([BOLD(bulk=True, summary=False)])
|
|
56
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
HEADERS = {
|
|
60
|
+
"User-Agent": ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"),
|
|
61
|
+
"Accept": "application/json, text/plain, */*",
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
def __init__(self, summary: bool = False, headers: dict = None, **kwargs):
|
|
65
|
+
super().__init__(**kwargs)
|
|
66
|
+
self.summary = summary
|
|
67
|
+
self.headers = headers if headers is not None else self.HEADERS.copy()
|
|
68
|
+
|
|
69
|
+
if self.output_format not in {"json", "fasta"}:
|
|
70
|
+
raise ValueError('Invalid output_format. Expected "json" or "fasta".')
|
|
71
|
+
|
|
72
|
+
if self.summary and self.output_format == "fasta":
|
|
73
|
+
raise BiodumpyException("Invalid parameter combination: output_format='fasta' cannot be used when summary=True.")
|
|
74
|
+
|
|
75
|
+
def _get_session(self):
|
|
76
|
+
session = requests.Session()
|
|
77
|
+
session.headers.update(self.headers)
|
|
78
|
+
retries = Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504])
|
|
79
|
+
adapter = HTTPAdapter(max_retries=retries)
|
|
80
|
+
session.mount("https://", adapter)
|
|
81
|
+
session.mount("http://", adapter)
|
|
82
|
+
return session
|
|
83
|
+
|
|
84
|
+
def _download(self, query, **kwargs) -> list:
|
|
85
|
+
payload = []
|
|
86
|
+
session = self._get_session()
|
|
87
|
+
|
|
88
|
+
# 1. Match the taxon query string with BOLD
|
|
89
|
+
tax_response = self._querytax(query, session=session)
|
|
90
|
+
successful_terms = tax_response.get("successful_terms", [])
|
|
91
|
+
if not successful_terms:
|
|
92
|
+
return payload
|
|
93
|
+
|
|
94
|
+
matched = successful_terms[0].get("matched")
|
|
95
|
+
if not matched:
|
|
96
|
+
return payload
|
|
97
|
+
|
|
98
|
+
# 2. Check record count
|
|
99
|
+
count_response = self._querycount(matched, session=session)
|
|
100
|
+
record_count = count_response.get("records", 0)
|
|
101
|
+
if record_count == 0:
|
|
102
|
+
return payload
|
|
103
|
+
|
|
104
|
+
# 3. Generate query ID to download the records
|
|
105
|
+
query_gen = self._querygenerator(matched, session=session)
|
|
106
|
+
query_download = query_gen.get("query_id")
|
|
107
|
+
if not query_download:
|
|
108
|
+
return payload
|
|
109
|
+
|
|
110
|
+
# 4. Download data from BOLD Portal documents endpoint
|
|
111
|
+
try:
|
|
112
|
+
response = session.get(f"https://portal.boldsystems.org/api/documents/{query_download}?length={record_count}&start=0", timeout=30)
|
|
113
|
+
except requests.RequestException as e:
|
|
114
|
+
raise BiodumpyException(f"Download BOLD request failed: {str(e)}")
|
|
115
|
+
|
|
116
|
+
if response.status_code != 200:
|
|
117
|
+
raise BiodumpyException(f"Download BOLD. Error {response.status_code}")
|
|
118
|
+
|
|
119
|
+
if response.content:
|
|
120
|
+
data_json = response.json()
|
|
121
|
+
res = data_json.get("data", [])
|
|
122
|
+
payload = res
|
|
123
|
+
|
|
124
|
+
# Fasta format
|
|
125
|
+
if self.output_format == "fasta":
|
|
126
|
+
fasta_entries = []
|
|
127
|
+
for entry in res:
|
|
128
|
+
if entry.get("nuc"):
|
|
129
|
+
header = f">{entry.get('processid')}|{entry.get('identification')}|{entry.get('marker_code')}"
|
|
130
|
+
fasta_entries.append(header)
|
|
131
|
+
fasta_entries.append(entry["nuc"])
|
|
132
|
+
payload = fasta_entries
|
|
133
|
+
|
|
134
|
+
# Summary format
|
|
135
|
+
elif self.summary:
|
|
136
|
+
summary_entries = []
|
|
137
|
+
for entry in res:
|
|
138
|
+
summary_entries.append(
|
|
139
|
+
{
|
|
140
|
+
"record_id": entry.get("record_id", None),
|
|
141
|
+
"processid": entry.get("processid", None),
|
|
142
|
+
"sampleid": entry.get("sampleid", None),
|
|
143
|
+
"insdc_acs": entry.get("insdc_acs", None),
|
|
144
|
+
"bin_uri": entry.get("bin_uri", None),
|
|
145
|
+
"taxon": entry.get("species") or entry.get("identification", None),
|
|
146
|
+
"species": entry.get("species", None),
|
|
147
|
+
"identification": entry.get("identification", None),
|
|
148
|
+
"collection_date_start": entry.get("collection_date_start", None),
|
|
149
|
+
"coord": entry.get("coord", None),
|
|
150
|
+
"country_ocean": entry.get("country/ocean", None),
|
|
151
|
+
"province_state": entry.get("province/state", None),
|
|
152
|
+
"region": entry.get("region", None),
|
|
153
|
+
"identified_by": entry.get("identified_by", None),
|
|
154
|
+
"inst": entry.get("inst", None),
|
|
155
|
+
"marker_code": entry.get("marker_code", None),
|
|
156
|
+
"sequence_run_site": entry.get("sequence_run_site", None),
|
|
157
|
+
"sequence_upload_date": entry.get("sequence_upload_date", None),
|
|
158
|
+
"specimens": entry.get("specimens", None),
|
|
159
|
+
}
|
|
160
|
+
)
|
|
161
|
+
payload = summary_entries
|
|
162
|
+
|
|
163
|
+
return payload
|
|
164
|
+
|
|
165
|
+
def _querytax(self, query, session=None):
|
|
166
|
+
session = session or self._get_session()
|
|
167
|
+
encoded = quote(query, safe="")
|
|
168
|
+
try:
|
|
169
|
+
response = session.get(f"https://portal.boldsystems.org/api/query/preprocessor?query=tax:{encoded}", timeout=30)
|
|
170
|
+
except requests.RequestException as e:
|
|
171
|
+
raise BiodumpyException(f"Query tax failed: {str(e)}")
|
|
172
|
+
|
|
173
|
+
if response.status_code != 200:
|
|
174
|
+
raise BiodumpyException(f"Query tax. Error {response.status_code}")
|
|
175
|
+
|
|
176
|
+
return response.json()
|
|
177
|
+
|
|
178
|
+
def _querycount(self, query, session=None):
|
|
179
|
+
session = session or self._get_session()
|
|
180
|
+
encoded = quote(query, safe="")
|
|
181
|
+
try:
|
|
182
|
+
response = session.get(f"https://portal.boldsystems.org/api/counts?query={encoded}", timeout=30)
|
|
183
|
+
except requests.RequestException as e:
|
|
184
|
+
raise BiodumpyException(f"Query count failed: {str(e)}")
|
|
185
|
+
|
|
186
|
+
if response.status_code != 200:
|
|
187
|
+
raise BiodumpyException(f"Query count. Error {response.status_code}")
|
|
188
|
+
|
|
189
|
+
return response.json()
|
|
190
|
+
|
|
191
|
+
def _querygenerator(self, query, session=None):
|
|
192
|
+
session = session or self._get_session()
|
|
193
|
+
encoded = quote(query, safe="")
|
|
194
|
+
try:
|
|
195
|
+
response = session.get(f"https://portal.boldsystems.org/api/query?query={encoded}&extent=full", timeout=30)
|
|
196
|
+
except requests.RequestException as e:
|
|
197
|
+
raise BiodumpyException(f"Query generator failed: {str(e)}")
|
|
198
|
+
|
|
199
|
+
if response.status_code != 200:
|
|
200
|
+
raise BiodumpyException(f"Query generator. Error {response.status_code}")
|
|
201
|
+
|
|
202
|
+
return response.json()
|
|
@@ -2,6 +2,7 @@ import csv
|
|
|
2
2
|
import json
|
|
3
3
|
import os
|
|
4
4
|
import re
|
|
5
|
+
import math
|
|
5
6
|
from copy import deepcopy
|
|
6
7
|
|
|
7
8
|
|
|
@@ -101,6 +102,18 @@ def clean_nones(value):
|
|
|
101
102
|
return value
|
|
102
103
|
|
|
103
104
|
|
|
105
|
+
def is_na(x):
|
|
106
|
+
"""
|
|
107
|
+
Function to check if a value is "NA" (None or NaN)
|
|
108
|
+
|
|
109
|
+
Example:
|
|
110
|
+
elements = [1, 2, None, np.nan, 4, float('nan'), 5]
|
|
111
|
+
elements = [e for e in elements if not is_na(e)]
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
return x is None or (isinstance(x, float) and math.isnan(x))
|
|
115
|
+
|
|
116
|
+
|
|
104
117
|
def dump_to_csv(file_name, obj_list):
|
|
105
118
|
directory = os.path.dirname(file_name)
|
|
106
119
|
if not os.path.exists(directory):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "biodumpy"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "A Comprehensive Biological Data Downloader."
|
|
5
5
|
authors = [
|
|
6
6
|
{name = "Tomas Golomb",email = "tomas.golomb@uib.cat"},
|
|
@@ -15,8 +15,8 @@ requires-python = ">=3.10"
|
|
|
15
15
|
dependencies = [
|
|
16
16
|
"biopython >=1.84,<2.0",
|
|
17
17
|
"tqdm >=4.66.4,<5.0",
|
|
18
|
-
"requests >=2.
|
|
19
|
-
"beautifulsoup4 >=4.12.3,<5.0"
|
|
18
|
+
"requests >=2.34.2,<3.0",
|
|
19
|
+
"beautifulsoup4 >=4.12.3,<5.0"
|
|
20
20
|
]
|
|
21
21
|
|
|
22
22
|
[dependency-groups]
|
|
Binary file
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
import requests
|
|
2
|
-
|
|
3
|
-
from biodumpy import Input, BiodumpyException
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class BOLD(Input):
|
|
7
|
-
"""
|
|
8
|
-
Query the Barcode of Life Data System (BOLD) database to retrieve taxon genetic or occurrence data.
|
|
9
|
-
|
|
10
|
-
Parameters
|
|
11
|
-
----------
|
|
12
|
-
query : list
|
|
13
|
-
The list of taxa to query.
|
|
14
|
-
summary : bool, optional
|
|
15
|
-
If True, the function returns a summary of the downloaded metadata instead of the full records.
|
|
16
|
-
Default is False.
|
|
17
|
-
|
|
18
|
-
Details
|
|
19
|
-
-------
|
|
20
|
-
When `summary` is True, the resulting JSON will include the following information:
|
|
21
|
-
- `record_id`: The unique identifier for the BOLD record.
|
|
22
|
-
- `processid`: The process ID associated with the BOLD record.
|
|
23
|
-
- `bin_uri`: The BIN (Barcode Index Number) URI.
|
|
24
|
-
- `taxon`: The name of the lower taxon extracted from the taxonomy information.
|
|
25
|
-
- `country`: The country where the collection event took place.
|
|
26
|
-
- `province_state`: The province or state of the collection event.
|
|
27
|
-
- `region`: The region of the collection event.
|
|
28
|
-
- `lat`: The latitude of the collection event.
|
|
29
|
-
- `lon`: The longitude of the collection event.
|
|
30
|
-
- `markercode`: The marker code from the sequence data.
|
|
31
|
-
- `genbank_accession`: The GenBank accession number from the sequence data.
|
|
32
|
-
|
|
33
|
-
Example
|
|
34
|
-
-------
|
|
35
|
-
>>> from biodumpy import Biodumpy
|
|
36
|
-
>>> from biodumpy.inputs import BOLD
|
|
37
|
-
# Taxa list
|
|
38
|
-
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator']
|
|
39
|
-
# Set the module and start the download
|
|
40
|
-
>>> bdp = Biodumpy([BOLD(bulk=True, summary=False)])
|
|
41
|
-
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
42
|
-
"""
|
|
43
|
-
|
|
44
|
-
def __init__(self, summary: bool = False, **kwargs):
|
|
45
|
-
super().__init__(**kwargs)
|
|
46
|
-
self.summary = summary
|
|
47
|
-
# self.fasta = fasta
|
|
48
|
-
|
|
49
|
-
# if self.fasta and output_format != "fasta":
|
|
50
|
-
# raise ValueError("Invalid output_format. Expected fasta.")
|
|
51
|
-
|
|
52
|
-
if self.output_format not in {"json", "fasta"}:
|
|
53
|
-
raise ValueError('Invalid output_format. Expected "json" or "fasta".')
|
|
54
|
-
|
|
55
|
-
def _download(self, query, **kwargs) -> list:
|
|
56
|
-
# if self.fasta:
|
|
57
|
-
if self.output_format == "fasta":
|
|
58
|
-
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/sequence?taxon={query}")
|
|
59
|
-
|
|
60
|
-
if response.status_code != 200:
|
|
61
|
-
raise BiodumpyException(f"Fasta sequence request. Error {response.status_code}")
|
|
62
|
-
|
|
63
|
-
if response.content:
|
|
64
|
-
response = response.content
|
|
65
|
-
data_str = response.decode()
|
|
66
|
-
|
|
67
|
-
# Split the data by '>'
|
|
68
|
-
fasta_entries = [f">{entry}" for entry in data_str.split(">") if entry]
|
|
69
|
-
|
|
70
|
-
return fasta_entries
|
|
71
|
-
else:
|
|
72
|
-
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/combined?taxon={query}&format=json")
|
|
73
|
-
|
|
74
|
-
payload = []
|
|
75
|
-
|
|
76
|
-
if response.status_code != 200:
|
|
77
|
-
raise BiodumpyException(f"Combined data request. Error {response.status_code}")
|
|
78
|
-
|
|
79
|
-
if response.content:
|
|
80
|
-
results = response.json()
|
|
81
|
-
if self.summary:
|
|
82
|
-
results_summary = results.get("bold_records", {}).get("records", [])
|
|
83
|
-
for entry in results_summary:
|
|
84
|
-
entry_summary = results_summary[entry]
|
|
85
|
-
|
|
86
|
-
# Extract the necessary fields with default None values if keys are missing.
|
|
87
|
-
# Sequence could contain more than one marker code.
|
|
88
|
-
sequences = entry_summary.get("sequences", {}).get("sequence", [])
|
|
89
|
-
markercodes = [seq.get("markercode") for seq in sequences if "markercode" in seq]
|
|
90
|
-
genbank_accession = [seq.get("genbank_accession") for seq in sequences if "genbank_accession" in seq]
|
|
91
|
-
|
|
92
|
-
# Retrieve taxonomy
|
|
93
|
-
taxonomy = entry_summary.get("taxonomy", {})
|
|
94
|
-
taxon_name = taxonomy.get(list(taxonomy.keys())[-1], {}).get("taxon", {}).get("name")
|
|
95
|
-
|
|
96
|
-
# Retrieve collection event metadata
|
|
97
|
-
collection_event = entry_summary.get("collection_event", {})
|
|
98
|
-
coordinates = collection_event.get("coordinates", {})
|
|
99
|
-
|
|
100
|
-
payload.append(
|
|
101
|
-
{
|
|
102
|
-
"record_id": entry_summary.get("record_id"),
|
|
103
|
-
"processid": entry_summary.get("processid"),
|
|
104
|
-
"bin_uri": entry_summary.get("bin_uri"),
|
|
105
|
-
"taxon": taxon_name,
|
|
106
|
-
"country": collection_event.get("country"),
|
|
107
|
-
"province_state": collection_event.get("province_state"),
|
|
108
|
-
"region": collection_event.get("region"),
|
|
109
|
-
"lat": coordinates.get("lat") if coordinates else None,
|
|
110
|
-
"lon": coordinates.get("lon") if coordinates else None,
|
|
111
|
-
"markercode": "/".join(markercodes) if markercodes else None,
|
|
112
|
-
"genbank_accession": genbank_accession[0] if genbank_accession else None,
|
|
113
|
-
}
|
|
114
|
-
)
|
|
115
|
-
else:
|
|
116
|
-
res = results["bold_records"]["records"] if "bold_records" in results and "records" in results["bold_records"] else []
|
|
117
|
-
payload = list(res.values())
|
|
118
|
-
|
|
119
|
-
return payload
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|