biodumpy 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biodumpy-0.1.2/LICENSE +8 -0
- biodumpy-0.1.2/PKG-INFO +166 -0
- biodumpy-0.1.2/README.md +134 -0
- biodumpy-0.1.2/biodumpy/__init__.py +5 -0
- biodumpy-0.1.2/biodumpy/biodumpy.py +96 -0
- biodumpy-0.1.2/biodumpy/input.py +8 -0
- biodumpy-0.1.2/biodumpy/inputs/BOLD.py +131 -0
- biodumpy-0.1.2/biodumpy/inputs/COL.py +91 -0
- biodumpy-0.1.2/biodumpy/inputs/GBIF.py +126 -0
- biodumpy-0.1.2/biodumpy/inputs/INaturalist.py +115 -0
- biodumpy-0.1.2/biodumpy/inputs/IUCN.py +143 -0
- biodumpy-0.1.2/biodumpy/inputs/NCBI.py +257 -0
- biodumpy-0.1.2/biodumpy/inputs/OBIS.py +115 -0
- biodumpy-0.1.2/biodumpy/inputs/ZooBank.py +106 -0
- biodumpy-0.1.2/biodumpy/inputs/__init__.py +9 -0
- biodumpy-0.1.2/biodumpy/utils.py +169 -0
- biodumpy-0.1.2/biodumpy.egg-info/PKG-INFO +166 -0
- biodumpy-0.1.2/biodumpy.egg-info/SOURCES.txt +24 -0
- biodumpy-0.1.2/biodumpy.egg-info/dependency_links.txt +1 -0
- biodumpy-0.1.2/biodumpy.egg-info/not-zip-safe +1 -0
- biodumpy-0.1.2/biodumpy.egg-info/requires.txt +9 -0
- biodumpy-0.1.2/biodumpy.egg-info/top_level.txt +1 -0
- biodumpy-0.1.2/pyproject.toml +3 -0
- biodumpy-0.1.2/setup.cfg +54 -0
- biodumpy-0.1.2/setup.py +3 -0
biodumpy-0.1.2/LICENSE
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
Copyright 2024 CBB
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
8
|
+
|
biodumpy-0.1.2/PKG-INFO
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: biodumpy
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: A Comprehensive Biological Data Downloader from authoritative public databases like NCBI, Catalogue Of Life (COL), GBIF, BOLD, and more for any taxa.
|
|
5
|
+
Home-page: https://github.com/centrebalearbiodiversitat/biodumpy
|
|
6
|
+
Author: Cancellario, T.; Golomb, T.; Roldán, A; Far, T.
|
|
7
|
+
Author-email: t.cancellario@uib.eu
|
|
8
|
+
License: MIT License
|
|
9
|
+
Project-URL: Homepage, https://github.com/centrebalearbiodiversitat/biodumpy
|
|
10
|
+
Project-URL: Documentation, https://biodumpy.readthedocs.io/
|
|
11
|
+
Project-URL: Issue Tracker, https://github.com/centrebalearbiodiversitat/cbbdb/issues
|
|
12
|
+
Keywords: biodiversity,data,ecology,science,genetics,download,wrapper,bibliography
|
|
13
|
+
Platform: UNKNOWN
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Operating System :: OS Independent
|
|
25
|
+
Classifier: Topic :: Scientific/Engineering
|
|
26
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
27
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
|
|
31
|
+
<img src="https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/Biodumpy_logo.png" alt="Project Logo" width="200">
|
|
32
|
+
|
|
33
|
+
# biodumpy: A Comprehensive Biological Data Downloader
|
|
34
|
+
|
|
35
|
+

|
|
36
|
+

|
|
37
|
+

|
|
38
|
+

|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
## Overview
|
|
43
|
+
``biodumpy`` is a powerful and versatile Python package designed to simplify the process of retrieving biological information
|
|
44
|
+
from several public databases.
|
|
45
|
+
With ``biodumpy``, researchers can easily download and manage data from multiple sources, ensuring access to the most
|
|
46
|
+
up to date and comprehensive biological information available.
|
|
47
|
+
|
|
48
|
+
> **Note:** This package is currently under development.
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
## Key Features
|
|
52
|
+
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically
|
|
53
|
+
designed for retrieving information from its respective source. The modules implemented so far are:
|
|
54
|
+
|
|
55
|
+
- BOLD
|
|
56
|
+
- COL
|
|
57
|
+
- GBIF
|
|
58
|
+
- iNaturalist
|
|
59
|
+
- IUCN
|
|
60
|
+
- NCBI
|
|
61
|
+
- OBIS
|
|
62
|
+
- ZooBank
|
|
63
|
+
|
|
64
|
+
This list can be expanded, thus suggestions and feedback are greatly appreciated.
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
## Main functionalities and workflow
|
|
68
|
+
Before using ``biodumpy``, users need to install the package in their Python environment with the following command:
|
|
69
|
+
|
|
70
|
+
```
|
|
71
|
+
pip install biodumpy
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Usage
|
|
75
|
+
To simplify the use of ``biodumpy``, we create a general structure common among the modules:
|
|
76
|
+
|
|
77
|
+
1) **Load the package.** Import ``biodumpy`` into your Python environment.
|
|
78
|
+
2) **Load the desired modules.** Import one or more specific modules needed to retrieve the data.
|
|
79
|
+
3) **Set up the configuration of one or more modules.** Configure the ``biodumpy`` function/s with the required parameters.
|
|
80
|
+
4) **Start the download.** Execute the function to begin retrieving the data.
|
|
81
|
+
|
|
82
|
+
Here, we provide two examples illustrating the general structure of a ``biodumpy`` script:
|
|
83
|
+
|
|
84
|
+
In detail, we described:
|
|
85
|
+
- **Single Module Example**: This example demonstrates how to use a single ``biodumpy`` module (for example, GBIF).
|
|
86
|
+
- **Multiple Modules Example**: This example shows how to use multiple ``biodumpy`` modules (for example, GBIF and IUCN).
|
|
87
|
+
|
|
88
|
+
**Example N.1**
|
|
89
|
+
|
|
90
|
+
``` python
|
|
91
|
+
|
|
92
|
+
# Import biodumpy package
|
|
93
|
+
from biodumpy import Biodumpy
|
|
94
|
+
|
|
95
|
+
# Import GBIF module
|
|
96
|
+
from biodumpy.inputs import GBIF
|
|
97
|
+
|
|
98
|
+
# Create a list of taxa
|
|
99
|
+
taxa = [
|
|
100
|
+
'Alytes muletensis (Sanchíz & Adrover, 1979)',
|
|
101
|
+
'Bufotes viridis (Laurenti, 1768)',
|
|
102
|
+
'Hyla meridionalis Boettger, 1874',
|
|
103
|
+
'Anax imperator Leach, 1815'
|
|
104
|
+
]
|
|
105
|
+
|
|
106
|
+
# Set the Biodumpy function with the specific parameters
|
|
107
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True)])
|
|
108
|
+
|
|
109
|
+
# Start the download
|
|
110
|
+
bdp.start(taxa, output_path='YOUR_OUTPUT_PATH/downloads/{date}/{module}/{name}')
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
**Example N.2**
|
|
114
|
+
|
|
115
|
+
``` python
|
|
116
|
+
|
|
117
|
+
# Import biodumpy package
|
|
118
|
+
from biodumpy import Biodumpy
|
|
119
|
+
|
|
120
|
+
# Import GBIF and IUCN modules
|
|
121
|
+
from biodumpy.inputs import GBIF, IUCN
|
|
122
|
+
|
|
123
|
+
api_key = 'YOUR_IUCN_API_KEY'
|
|
124
|
+
|
|
125
|
+
# Create a list of taxa
|
|
126
|
+
taxa = [
|
|
127
|
+
'Alytes muletensis',
|
|
128
|
+
'Bufotes viridis',
|
|
129
|
+
'Hyla meridionalis',
|
|
130
|
+
'Anax imperator'
|
|
131
|
+
]
|
|
132
|
+
|
|
133
|
+
# Set the Biodumpy functions with the specific parameters
|
|
134
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True),
|
|
135
|
+
IUCN(api_key=api_key, bulk=True, region=['global'])])
|
|
136
|
+
|
|
137
|
+
# Start the download
|
|
138
|
+
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Documentation and Support
|
|
142
|
+
For detailed documentation and tutorials, please visit the ``biodumpy`` Read the Docs documentation.
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
## Contribution
|
|
146
|
+
``biodumpy`` is an open-source project, and contributions are welcome!
|
|
147
|
+
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub
|
|
148
|
+
repository or contact with the support team at [t.cancellario@uib.eu](mailto:t.cancellario@uib.eu).
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
``biodumpy`` is licensed under the GNU GENERAL PUBLIC LICENSE. See the LICENSE file for more details.
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
## Acknowledgments
|
|
156
|
+
The project was supported by MCIN with funding from the European Union—NextGenerationEU (PRTR-C17.I1) and
|
|
157
|
+
the Government of the Balearic Islands.
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
<hr>
|
|
162
|
+
<div style="display: flex; justify-content: center">
|
|
163
|
+
<img src='https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/logo_cbb.png' alt='logo_cbb' width='200'>
|
|
164
|
+
</div>
|
|
165
|
+
|
|
166
|
+
|
biodumpy-0.1.2/README.md
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
<img src="https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/Biodumpy_logo.png" alt="Project Logo" width="200">
|
|
2
|
+
|
|
3
|
+
# biodumpy: A Comprehensive Biological Data Downloader
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+

|
|
7
|
+

|
|
8
|
+

|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
## Overview
|
|
13
|
+
``biodumpy`` is a powerful and versatile Python package designed to simplify the process of retrieving biological information
|
|
14
|
+
from several public databases.
|
|
15
|
+
With ``biodumpy``, researchers can easily download and manage data from multiple sources, ensuring access to the most
|
|
16
|
+
up to date and comprehensive biological information available.
|
|
17
|
+
|
|
18
|
+
> **Note:** This package is currently under development.
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
## Key Features
|
|
22
|
+
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically
|
|
23
|
+
designed for retrieving information from its respective source. The modules implemented so far are:
|
|
24
|
+
|
|
25
|
+
- BOLD
|
|
26
|
+
- COL
|
|
27
|
+
- GBIF
|
|
28
|
+
- iNaturalist
|
|
29
|
+
- IUCN
|
|
30
|
+
- NCBI
|
|
31
|
+
- OBIS
|
|
32
|
+
- ZooBank
|
|
33
|
+
|
|
34
|
+
This list can be expanded, thus suggestions and feedback are greatly appreciated.
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
## Main functionalities and workflow
|
|
38
|
+
Before using ``biodumpy``, users need to install the package in their Python environment with the following command:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
pip install biodumpy
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
### Usage
|
|
45
|
+
To simplify the use of ``biodumpy``, we create a general structure common among the modules:
|
|
46
|
+
|
|
47
|
+
1) **Load the package.** Import ``biodumpy`` into your Python environment.
|
|
48
|
+
2) **Load the desired modules.** Import one or more specific modules needed to retrieve the data.
|
|
49
|
+
3) **Set up the configuration of one or more modules.** Configure the ``biodumpy`` function/s with the required parameters.
|
|
50
|
+
4) **Start the download.** Execute the function to begin retrieving the data.
|
|
51
|
+
|
|
52
|
+
Here, we provide two examples illustrating the general structure of a ``biodumpy`` script:
|
|
53
|
+
|
|
54
|
+
In detail, we described:
|
|
55
|
+
- **Single Module Example**: This example demonstrates how to use a single ``biodumpy`` module (for example, GBIF).
|
|
56
|
+
- **Multiple Modules Example**: This example shows how to use multiple ``biodumpy`` modules (for example, GBIF and IUCN).
|
|
57
|
+
|
|
58
|
+
**Example N.1**
|
|
59
|
+
|
|
60
|
+
``` python
|
|
61
|
+
|
|
62
|
+
# Import biodumpy package
|
|
63
|
+
from biodumpy import Biodumpy
|
|
64
|
+
|
|
65
|
+
# Import GBIF module
|
|
66
|
+
from biodumpy.inputs import GBIF
|
|
67
|
+
|
|
68
|
+
# Create a list of taxa
|
|
69
|
+
taxa = [
|
|
70
|
+
'Alytes muletensis (Sanchíz & Adrover, 1979)',
|
|
71
|
+
'Bufotes viridis (Laurenti, 1768)',
|
|
72
|
+
'Hyla meridionalis Boettger, 1874',
|
|
73
|
+
'Anax imperator Leach, 1815'
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
# Set the Biodumpy function with the specific parameters
|
|
77
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True)])
|
|
78
|
+
|
|
79
|
+
# Start the download
|
|
80
|
+
bdp.start(taxa, output_path='YOUR_OUTPUT_PATH/downloads/{date}/{module}/{name}')
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
**Example N.2**
|
|
84
|
+
|
|
85
|
+
``` python
|
|
86
|
+
|
|
87
|
+
# Import biodumpy package
|
|
88
|
+
from biodumpy import Biodumpy
|
|
89
|
+
|
|
90
|
+
# Import GBIF and IUCN modules
|
|
91
|
+
from biodumpy.inputs import GBIF, IUCN
|
|
92
|
+
|
|
93
|
+
api_key = 'YOUR_IUCN_API_KEY'
|
|
94
|
+
|
|
95
|
+
# Create a list of taxa
|
|
96
|
+
taxa = [
|
|
97
|
+
'Alytes muletensis',
|
|
98
|
+
'Bufotes viridis',
|
|
99
|
+
'Hyla meridionalis',
|
|
100
|
+
'Anax imperator'
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
# Set the Biodumpy functions with the specific parameters
|
|
104
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True),
|
|
105
|
+
IUCN(api_key=api_key, bulk=True, region=['global'])])
|
|
106
|
+
|
|
107
|
+
# Start the download
|
|
108
|
+
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Documentation and Support
|
|
112
|
+
For detailed documentation and tutorials, please visit the ``biodumpy`` Read the Docs documentation.
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
## Contribution
|
|
116
|
+
``biodumpy`` is an open-source project, and contributions are welcome!
|
|
117
|
+
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub
|
|
118
|
+
repository or contact with the support team at [t.cancellario@uib.eu](mailto:t.cancellario@uib.eu).
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
## License
|
|
122
|
+
``biodumpy`` is licensed under the GNU GENERAL PUBLIC LICENSE. See the LICENSE file for more details.
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
## Acknowledgments
|
|
126
|
+
The project was supported by MCIN with funding from the European Union—NextGenerationEU (PRTR-C17.I1) and
|
|
127
|
+
the Government of the Balearic Islands.
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
<hr>
|
|
132
|
+
<div style="display: flex; justify-content: center">
|
|
133
|
+
<img src='https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/logo_cbb.png' alt='logo_cbb' width='200'>
|
|
134
|
+
</div>
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from .input import Input
|
|
3
|
+
from .utils import dump
|
|
4
|
+
from tqdm import tqdm
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# Setup logging configuration
|
|
9
|
+
logging.basicConfig(
|
|
10
|
+
level=logging.INFO,
|
|
11
|
+
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
12
|
+
datefmt="%Y-%m-%d %H:%M:%S",
|
|
13
|
+
filename="biodumpy_downloader.log",
|
|
14
|
+
filemode="a", # is the appended mode
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class BiodumpyException(Exception):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Biodumpy:
|
|
23
|
+
"""
|
|
24
|
+
This class is designed to download biodiversity data from various sources using multiple input modules.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
inputs : list
|
|
29
|
+
A list of input modules that handle specific biodiversity data downloads.
|
|
30
|
+
loading_bar : bool
|
|
31
|
+
If True, shows a progress bar when downloading data. If False, disable the progress bar.
|
|
32
|
+
Default is False
|
|
33
|
+
debug : bool
|
|
34
|
+
If True, enables printing of detailed information during execution.
|
|
35
|
+
Default is True
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, inputs: list[Input], loading_bar: bool = False, debug: bool = True) -> None:
|
|
39
|
+
super().__init__()
|
|
40
|
+
self.inputs = inputs
|
|
41
|
+
self.debug = debug
|
|
42
|
+
self.loading_bar = loading_bar
|
|
43
|
+
|
|
44
|
+
# elements must be a flat list of strings
|
|
45
|
+
def start(self, elements, output_path="downloads/{date}/{module}/{name}"):
|
|
46
|
+
if not isinstance(elements, list):
|
|
47
|
+
raise ValueError("Invalid query. Expected a list of taxa to query.")
|
|
48
|
+
|
|
49
|
+
current_date = datetime.now().strftime("%Y-%m-%d")
|
|
50
|
+
bulk_input = {}
|
|
51
|
+
try:
|
|
52
|
+
for el in tqdm(elements, desc="Biodumpy list", unit=" elements", disable=not self.loading_bar, smoothing=0):
|
|
53
|
+
if not el:
|
|
54
|
+
continue
|
|
55
|
+
|
|
56
|
+
if isinstance(el, str):
|
|
57
|
+
el = {"query": el}
|
|
58
|
+
|
|
59
|
+
if "query" not in el:
|
|
60
|
+
logging.error(f"Missing 'query' key for {el}")
|
|
61
|
+
raise ValueError(f"Missing 'name' key for {el}")
|
|
62
|
+
|
|
63
|
+
name = el["query"]
|
|
64
|
+
clean_name = name.replace("/", "_")
|
|
65
|
+
if self.debug:
|
|
66
|
+
print(f"Downloading {name}...")
|
|
67
|
+
|
|
68
|
+
for inp in self.inputs:
|
|
69
|
+
module_name = type(inp).__name__
|
|
70
|
+
logging.info(f"biodumpy initialized with {module_name} inputs. Taxon: {name}")
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
payload = inp._download(**el)
|
|
74
|
+
logging.info(f"Download data for {module_name} was successful.\n")
|
|
75
|
+
except Exception as e:
|
|
76
|
+
logging.error(f'[{module_name}] Failed to download data for "{name}": {str(e)} \n')
|
|
77
|
+
continue
|
|
78
|
+
|
|
79
|
+
if inp.bulk:
|
|
80
|
+
if inp not in bulk_input:
|
|
81
|
+
bulk_input[inp] = []
|
|
82
|
+
bulk_input[inp].extend(payload)
|
|
83
|
+
|
|
84
|
+
else:
|
|
85
|
+
dump(
|
|
86
|
+
file_name=f"{output_path.format(date=current_date, module=module_name, name=clean_name)}",
|
|
87
|
+
obj_list=payload,
|
|
88
|
+
output_format=inp.output_format,
|
|
89
|
+
)
|
|
90
|
+
finally:
|
|
91
|
+
for inp, payload in bulk_input.items():
|
|
92
|
+
dump(
|
|
93
|
+
file_name=output_path.format(date=current_date, module=type(inp).__name__, name="bulk"),
|
|
94
|
+
obj_list=payload,
|
|
95
|
+
output_format=inp.output_format,
|
|
96
|
+
)
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input, BiodumpyException
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BOLD(Input):
|
|
7
|
+
"""
|
|
8
|
+
Query the Barcode of Life Data System (BOLD) database to retrieve taxon genetic or occurrence data.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
query : list
|
|
13
|
+
The list of taxa to query.
|
|
14
|
+
summary : bool, optional
|
|
15
|
+
If True, the function returns a summary of the downloaded metadata instead of the full records.
|
|
16
|
+
Default is False.
|
|
17
|
+
fasta : bool, optional
|
|
18
|
+
If True, the function downloads the FASTA file.
|
|
19
|
+
Default is False.
|
|
20
|
+
bulk : bool, optional
|
|
21
|
+
If True, the function creates a bulk file.
|
|
22
|
+
For further information, see the documentation of the Biodumpy package.
|
|
23
|
+
Default is False.
|
|
24
|
+
output_format : string, optional
|
|
25
|
+
The format of the output file.
|
|
26
|
+
The options available are: 'json', 'fasta', 'pdf'.
|
|
27
|
+
Default is 'json'.
|
|
28
|
+
|
|
29
|
+
Details
|
|
30
|
+
-------
|
|
31
|
+
When `summary` is True, the resulting JSON will include the following information:
|
|
32
|
+
- `record_id`: The unique identifier for the BOLD record.
|
|
33
|
+
- `processid`: The process ID associated with the BOLD record.
|
|
34
|
+
- `bin_uri`: The BIN (Barcode Index Number) URI.
|
|
35
|
+
- `taxon`: The name of the lower taxon extracted from the taxonomy information.
|
|
36
|
+
- `country`: The country where the collection event took place.
|
|
37
|
+
- `province_state`: The province or state of the collection event.
|
|
38
|
+
- `region`: The region of the collection event.
|
|
39
|
+
- `lat`: The latitude of the collection event.
|
|
40
|
+
- `lon`: The longitude of the collection event.
|
|
41
|
+
- `markercode`: The marker code from the sequence data.
|
|
42
|
+
- `genbank_accession`: The GenBank accession number from the sequence data.
|
|
43
|
+
|
|
44
|
+
Example
|
|
45
|
+
-------
|
|
46
|
+
>>> from biodumpy import Biodumpy
|
|
47
|
+
>>> from biodumpy.inputs import BOLD
|
|
48
|
+
# Taxa list
|
|
49
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator']
|
|
50
|
+
# Set the module and start the download
|
|
51
|
+
>>> bdp = Biodumpy([BOLD(bulk=True, summary=False)])
|
|
52
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(self, summary: bool = False, fasta: bool = False, output_format: str = "json", bulk: bool = False):
|
|
56
|
+
super().__init__(output_format, bulk)
|
|
57
|
+
self.summary = summary
|
|
58
|
+
self.fasta = fasta
|
|
59
|
+
|
|
60
|
+
if self.fasta and output_format != "fasta":
|
|
61
|
+
raise ValueError("Invalid output_format. Expected fasta.")
|
|
62
|
+
|
|
63
|
+
if output_format not in {"json", "fasta"}:
|
|
64
|
+
raise ValueError('Invalid output_format. Expected "json" or "fasta".')
|
|
65
|
+
|
|
66
|
+
def _download(self, query, **kwargs) -> list:
|
|
67
|
+
if self.fasta:
|
|
68
|
+
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/sequence?taxon={query}")
|
|
69
|
+
|
|
70
|
+
if response.status_code != 200:
|
|
71
|
+
raise BiodumpyException(f"Fasta sequence request. Error {response.status_code}")
|
|
72
|
+
|
|
73
|
+
if response.content:
|
|
74
|
+
response = response.content
|
|
75
|
+
data_str = response.decode()
|
|
76
|
+
|
|
77
|
+
# Split the data by '>'
|
|
78
|
+
fasta_entries = [f">{entry}" for entry in data_str.split(">") if entry]
|
|
79
|
+
|
|
80
|
+
return fasta_entries
|
|
81
|
+
|
|
82
|
+
else:
|
|
83
|
+
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/combined?taxon={query}&format=json")
|
|
84
|
+
|
|
85
|
+
payload = []
|
|
86
|
+
|
|
87
|
+
if response.status_code != 200:
|
|
88
|
+
raise BiodumpyException(f"Combined data request. Error {response.status_code}")
|
|
89
|
+
|
|
90
|
+
if response.content:
|
|
91
|
+
results = response.json()
|
|
92
|
+
if self.summary:
|
|
93
|
+
results_summary = results.get("bold_records", {}).get("records", [])
|
|
94
|
+
for entry in results_summary:
|
|
95
|
+
entry_summary = results_summary[entry]
|
|
96
|
+
|
|
97
|
+
# Extract the necessary fields with default None values if keys are missing.
|
|
98
|
+
# Sequence could contain more than one marker code.
|
|
99
|
+
sequences = entry_summary.get("sequences", {}).get("sequence", [])
|
|
100
|
+
markercodes = [seq.get("markercode") for seq in sequences if "markercode" in seq]
|
|
101
|
+
genbank_accession = [seq.get("genbank_accession") for seq in sequences if "genbank_accession" in seq]
|
|
102
|
+
|
|
103
|
+
# Retrieve taxonomy
|
|
104
|
+
taxonomy = entry_summary.get("taxonomy", {})
|
|
105
|
+
taxon_name = taxonomy.get(list(taxonomy.keys())[-1], {}).get("taxon", {}).get("name")
|
|
106
|
+
|
|
107
|
+
# Retrieve collection event metadata
|
|
108
|
+
collection_event = entry_summary.get("collection_event", {})
|
|
109
|
+
coordinates = collection_event.get("coordinates", {})
|
|
110
|
+
|
|
111
|
+
payload.append(
|
|
112
|
+
{
|
|
113
|
+
"record_id": entry_summary.get("record_id"),
|
|
114
|
+
"processid": entry_summary.get("processid"),
|
|
115
|
+
"bin_uri": entry_summary.get("bin_uri"),
|
|
116
|
+
"taxon": taxon_name,
|
|
117
|
+
"country": collection_event.get("country"),
|
|
118
|
+
"province_state": collection_event.get("province_state"),
|
|
119
|
+
"region": collection_event.get("region"),
|
|
120
|
+
"lat": coordinates.get("lat") if coordinates else None,
|
|
121
|
+
"lon": coordinates.get("lon") if coordinates else None,
|
|
122
|
+
"markercode": "/".join(markercodes) if markercodes else None,
|
|
123
|
+
"genbank_accession": genbank_accession[0] if genbank_accession else None,
|
|
124
|
+
}
|
|
125
|
+
)
|
|
126
|
+
else:
|
|
127
|
+
payload.append(
|
|
128
|
+
results["bold_records"]["records"] if "bold_records" in results and "records" in results["bold_records"] else []
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
return payload
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input, BiodumpyException
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class COL(Input):
|
|
7
|
+
"""
|
|
8
|
+
Query the Catalogue of Life (COL) database to retrieve nomenclature information of a list of taxa.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
query : list
|
|
13
|
+
The list of taxa to query.
|
|
14
|
+
check_syn : bool, optional
|
|
15
|
+
If True, the function returns only the accepted nomenclature of a taxon.
|
|
16
|
+
See Detail section for further information.
|
|
17
|
+
Default is False.
|
|
18
|
+
bulk : bool, optional
|
|
19
|
+
If True, the function creates a bulk file.
|
|
20
|
+
For further information, see the documentation of the Biodumpy package.
|
|
21
|
+
Default is False.
|
|
22
|
+
output_format : string, optional
|
|
23
|
+
The format of the output file.
|
|
24
|
+
The options available are: 'json', 'fasta', 'pdf'.
|
|
25
|
+
Default is 'json'.
|
|
26
|
+
|
|
27
|
+
Details
|
|
28
|
+
-------
|
|
29
|
+
When check_syn is set to True, the resulting JSON will include only the nomenclature of the accepted taxon.
|
|
30
|
+
For instance, if check_syn is True, the output for the species Bufo roseus will only show the nomenclature for
|
|
31
|
+
Bufotes viridis. Conversely, if check_syn is set to False, the JSON will include the nomenclature for both
|
|
32
|
+
Bufo roseus and Bufotes viridis.
|
|
33
|
+
|
|
34
|
+
Example
|
|
35
|
+
-------
|
|
36
|
+
>>> from biodumpy import Biodumpy
|
|
37
|
+
>>> from biodumpy.inputs import COL
|
|
38
|
+
# List oF taxa
|
|
39
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator', 'Bufo roseus', 'Stollia betae']
|
|
40
|
+
# Start the download
|
|
41
|
+
>>> bdp = Biodumpy([COL(bulk=True, check_syn=False)])
|
|
42
|
+
>>> bdp.start(taxa, output_path='./biodumpy/downloads/{date}/{module}/{name}')
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
ACCEPTED_TERMS = ["accepted", "provisionally accepted"]
|
|
46
|
+
|
|
47
|
+
def __init__(self, output_format: str = "json", bulk: bool = False, check_syn: bool = False):
|
|
48
|
+
super().__init__(output_format, bulk)
|
|
49
|
+
self.check_syn = check_syn
|
|
50
|
+
|
|
51
|
+
if output_format != "json":
|
|
52
|
+
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
53
|
+
|
|
54
|
+
def _download(self, query, **kwargs) -> list:
|
|
55
|
+
response = requests.get(
|
|
56
|
+
f"https://api.checklistbank.org/dataset/9923/nameusage/search?q={query}&content=SCIENTIFIC_NAME&type=EXACT&offset=0&limit=10"
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
if response.status_code != 200:
|
|
60
|
+
raise BiodumpyException(f"Taxonomy request. Error {response.status_code}")
|
|
61
|
+
|
|
62
|
+
payload = response.json()
|
|
63
|
+
|
|
64
|
+
if payload["empty"]:
|
|
65
|
+
payload = [{"origin_taxon": query, "taxon_id": None, "status": None, "usage": None, "classification": None}]
|
|
66
|
+
else:
|
|
67
|
+
result = response.json()["result"]
|
|
68
|
+
|
|
69
|
+
# Multiple IDs
|
|
70
|
+
if len(result) > 1:
|
|
71
|
+
ids = [item.get("id") for item in result if "id" in item]
|
|
72
|
+
ids = ", ".join(ids)
|
|
73
|
+
id_input = input(f"Please enter the correct taxon ID of {query} \n ID: {ids}; Skip \n" f"Insert the ID:")
|
|
74
|
+
|
|
75
|
+
if id_input == "Skip":
|
|
76
|
+
result = [{"id": None, "usage": None, "status": None, "classification": None}]
|
|
77
|
+
else:
|
|
78
|
+
result = [item for item in result if item["id"] == id_input]
|
|
79
|
+
|
|
80
|
+
id = result[0].get("id")
|
|
81
|
+
usage = result[0].get("usage")
|
|
82
|
+
status = usage.get("status") if usage else None
|
|
83
|
+
|
|
84
|
+
classification = result[0].get("classification")
|
|
85
|
+
if self.check_syn and status not in COL.ACCEPTED_TERMS:
|
|
86
|
+
synonym_id = usage.get("id") if usage else None
|
|
87
|
+
classification = [item for item in classification if item["id"] != synonym_id] if classification else None
|
|
88
|
+
|
|
89
|
+
payload = [{"origin_taxon": query, "taxon_id": id, "status": status, "usage": usage, "classification": classification}]
|
|
90
|
+
|
|
91
|
+
return payload
|