biodumpy 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biodumpy/__init__.py +5 -0
- biodumpy/biodumpy.py +96 -0
- biodumpy/input.py +8 -0
- biodumpy/inputs/BOLD.py +131 -0
- biodumpy/inputs/COL.py +91 -0
- biodumpy/inputs/GBIF.py +126 -0
- biodumpy/inputs/INaturalist.py +115 -0
- biodumpy/inputs/IUCN.py +143 -0
- biodumpy/inputs/NCBI.py +257 -0
- biodumpy/inputs/OBIS.py +115 -0
- biodumpy/inputs/ZooBank.py +106 -0
- biodumpy/inputs/__init__.py +9 -0
- biodumpy/utils.py +169 -0
- biodumpy-0.1.2.dist-info/LICENSE +8 -0
- biodumpy-0.1.2.dist-info/METADATA +175 -0
- biodumpy-0.1.2.dist-info/RECORD +18 -0
- biodumpy-0.1.2.dist-info/WHEEL +5 -0
- biodumpy-0.1.2.dist-info/top_level.txt +1 -0
biodumpy/utils.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
from Bio import Entrez
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def dump(file_name, obj_list, output_format="json"):
|
|
8
|
+
"""
|
|
9
|
+
Dump a list of objects to JSON files. Optionally split into multiple files for bulk processing.
|
|
10
|
+
|
|
11
|
+
Parameters:
|
|
12
|
+
file_name (str): Base name of the output JSON file.
|
|
13
|
+
obj_list (list): List of objects to be written to JSON.
|
|
14
|
+
output_format: output format. Default is "json". Other formats can be "fasta", "pdf".
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
directory = os.path.dirname(file_name)
|
|
18
|
+
if not os.path.exists(directory):
|
|
19
|
+
os.makedirs(directory)
|
|
20
|
+
|
|
21
|
+
with open(f"{file_name}.{output_format}", "wb+" if output_format == "pdf" else "w+") as output_file:
|
|
22
|
+
if output_format == "fasta":
|
|
23
|
+
for line in obj_list:
|
|
24
|
+
output_file.write(f"{line}\n")
|
|
25
|
+
elif output_format == "pdf":
|
|
26
|
+
output_file.write(obj_list)
|
|
27
|
+
else:
|
|
28
|
+
json.dump(obj_list, output_file, indent=4)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def clean_nones(value):
|
|
32
|
+
"""
|
|
33
|
+
Recursively remove all None values from dictionaries and lists, and returns
|
|
34
|
+
the result as a new dictionary or list.
|
|
35
|
+
"""
|
|
36
|
+
if isinstance(value, list):
|
|
37
|
+
return [clean_nones(x) for x in value if x is not None]
|
|
38
|
+
elif isinstance(value, dict):
|
|
39
|
+
return {key: clean_nones(val) for key, val in value.items() if val is not None}
|
|
40
|
+
else:
|
|
41
|
+
return value
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def dump_to_csv(file_name, obj_list):
|
|
45
|
+
directory = os.path.dirname(file_name)
|
|
46
|
+
if not os.path.exists(directory):
|
|
47
|
+
os.makedirs(directory)
|
|
48
|
+
|
|
49
|
+
headers = []
|
|
50
|
+
unroll_headers = []
|
|
51
|
+
if len(obj_list) > 0:
|
|
52
|
+
headers = []
|
|
53
|
+
unroll_headers = []
|
|
54
|
+
for obj in obj_list:
|
|
55
|
+
headers = list(obj_list[0].keys())
|
|
56
|
+
for k, v in obj_list[0].items():
|
|
57
|
+
if isinstance(v, dict):
|
|
58
|
+
for kk, vv in v.items():
|
|
59
|
+
unroll_headers.append(kk)
|
|
60
|
+
else:
|
|
61
|
+
unroll_headers.append(k)
|
|
62
|
+
|
|
63
|
+
with open(file_name, mode="w+") as csv_file:
|
|
64
|
+
writer = csv.DictWriter(csv_file, fieldnames=unroll_headers)
|
|
65
|
+
|
|
66
|
+
writer.writeheader()
|
|
67
|
+
for obj in obj_list:
|
|
68
|
+
row = {}
|
|
69
|
+
for field in headers:
|
|
70
|
+
if field in obj:
|
|
71
|
+
current = obj[field]
|
|
72
|
+
if isinstance(current, list):
|
|
73
|
+
row[field] = json.dumps(current)
|
|
74
|
+
elif isinstance(current, dict):
|
|
75
|
+
for k, v in current.items():
|
|
76
|
+
row[k] = json.dumps(v) if isinstance(v, dict) or isinstance(v, list) else v
|
|
77
|
+
else:
|
|
78
|
+
row[field] = current
|
|
79
|
+
else:
|
|
80
|
+
row[field] = None
|
|
81
|
+
|
|
82
|
+
writer.writerow(row)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def split_to_batches(input_list, batch_size: int):
|
|
86
|
+
"""
|
|
87
|
+
Divides a list into smaller batches of a specified size.
|
|
88
|
+
|
|
89
|
+
Parameters:
|
|
90
|
+
input_list (list): The list to be divided into batches.
|
|
91
|
+
batch_size (int): The size of each batch.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
list of lists: A list containing the smaller batches.
|
|
95
|
+
|
|
96
|
+
Example usage:
|
|
97
|
+
input_list = [1, 2, 3, 4, 5, 6, 7, 8, 9]
|
|
98
|
+
batch_size = 3
|
|
99
|
+
batches = divide_list_into_batches_by_size(input_list, batch_size)
|
|
100
|
+
print(batches) # Output: [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
|
|
101
|
+
"""
|
|
102
|
+
return [input_list[i : i + batch_size] for i in range(0, len(input_list), batch_size)]
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def parse_lat_lon(lat_lon: str):
|
|
106
|
+
"""
|
|
107
|
+
Parse coordinate.
|
|
108
|
+
|
|
109
|
+
Args:
|
|
110
|
+
lat_lon: String containing latitude and longitude.
|
|
111
|
+
|
|
112
|
+
Returns:
|
|
113
|
+
List of coordinates.
|
|
114
|
+
|
|
115
|
+
Example:
|
|
116
|
+
parse_lat_lon("34.0522 N 118.2437 E")
|
|
117
|
+
[34.0522, 118.2437]
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
if not lat_lon:
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
lat_lon = lat_lon.split(" ")
|
|
124
|
+
lat = float(lat_lon[0])
|
|
125
|
+
lon = float(lat_lon[2])
|
|
126
|
+
|
|
127
|
+
if lat_lon[1] == "S":
|
|
128
|
+
lat = -lat
|
|
129
|
+
if lat_lon[3] == "W":
|
|
130
|
+
lon = -lon
|
|
131
|
+
|
|
132
|
+
return [lat, lon]
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def download_taxonomy(taxon: str, mail="A.N.Other@example.com"):
|
|
136
|
+
"""
|
|
137
|
+
Download taxonomy of a taxon from NCBI Taxonomy database.
|
|
138
|
+
|
|
139
|
+
Args:
|
|
140
|
+
taxon: String containing taxon name.
|
|
141
|
+
mail: NCBI requires you to specify your email address with each request.
|
|
142
|
+
|
|
143
|
+
Returns:
|
|
144
|
+
None
|
|
145
|
+
|
|
146
|
+
Example:
|
|
147
|
+
x = download_taxonomy('Alytes muletensis')
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
Entrez.email = mail
|
|
151
|
+
|
|
152
|
+
# Retrieve taxonomy ID by taxon name
|
|
153
|
+
handle = Entrez.esearch(db="Taxonomy", term=f"{taxon}[All Names]", retmode="xml")
|
|
154
|
+
taxon_id = Entrez.read(handle) # retrieve taxon ID
|
|
155
|
+
handle.close()
|
|
156
|
+
|
|
157
|
+
if int(taxon_id["Count"]) > 0:
|
|
158
|
+
# Retrieve taxonomy by taxon ID
|
|
159
|
+
handle = Entrez.efetch(db="Taxonomy", id=taxon_id["IdList"], retmode="xml")
|
|
160
|
+
records = Entrez.read(handle)
|
|
161
|
+
handle.close()
|
|
162
|
+
|
|
163
|
+
lin = records[0]["LineageEx"]
|
|
164
|
+
lin.append = {"TaxId": records[0]["TaxId"], "ScientificName": records[0]["ScientificName"].split()[-1], "Rank": records[0]["Rank"]}
|
|
165
|
+
|
|
166
|
+
else:
|
|
167
|
+
lin = None
|
|
168
|
+
|
|
169
|
+
return lin
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
Copyright 2024 CBB
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
8
|
+
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: biodumpy
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: A Comprehensive Biological Data Downloader from authoritative public databases like NCBI, Catalogue Of Life (COL), GBIF, BOLD, and more for any taxa.
|
|
5
|
+
Home-page: https://github.com/centrebalearbiodiversitat/biodumpy
|
|
6
|
+
Author: Cancellario, T.; Golomb, T.; Roldán, A; Far, T.
|
|
7
|
+
Author-email: t.cancellario@uib.eu
|
|
8
|
+
License: MIT License
|
|
9
|
+
Project-URL: Homepage, https://github.com/centrebalearbiodiversitat/biodumpy
|
|
10
|
+
Project-URL: Documentation, https://biodumpy.readthedocs.io/
|
|
11
|
+
Project-URL: Issue Tracker, https://github.com/centrebalearbiodiversitat/cbbdb/issues
|
|
12
|
+
Keywords: biodiversity,data,ecology,science,genetics,download,wrapper,bibliography
|
|
13
|
+
Platform: UNKNOWN
|
|
14
|
+
Classifier: Development Status :: 4 - Beta
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Intended Audience :: Science/Research
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.7
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
24
|
+
Classifier: Operating System :: OS Independent
|
|
25
|
+
Classifier: Topic :: Scientific/Engineering
|
|
26
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
27
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Requires-Dist: beautifulsoup4==4.12.3
|
|
31
|
+
Requires-Dist: biopython==1.84
|
|
32
|
+
Requires-Dist: lxml==5.2.2
|
|
33
|
+
Requires-Dist: numpy==2.0.0
|
|
34
|
+
Requires-Dist: pandas==2.2.2
|
|
35
|
+
Requires-Dist: pytest==8.3.2
|
|
36
|
+
Requires-Dist: requests==2.32.2
|
|
37
|
+
Requires-Dist: selenium==4.23.0
|
|
38
|
+
Requires-Dist: tqdm==4.66.4
|
|
39
|
+
|
|
40
|
+
<img src="https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/Biodumpy_logo.png" alt="Project Logo" width="200">
|
|
41
|
+
|
|
42
|
+
# biodumpy: A Comprehensive Biological Data Downloader
|
|
43
|
+
|
|
44
|
+

|
|
45
|
+

|
|
46
|
+

|
|
47
|
+

|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
## Overview
|
|
52
|
+
``biodumpy`` is a powerful and versatile Python package designed to simplify the process of retrieving biological information
|
|
53
|
+
from several public databases.
|
|
54
|
+
With ``biodumpy``, researchers can easily download and manage data from multiple sources, ensuring access to the most
|
|
55
|
+
up to date and comprehensive biological information available.
|
|
56
|
+
|
|
57
|
+
> **Note:** This package is currently under development.
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
## Key Features
|
|
61
|
+
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically
|
|
62
|
+
designed for retrieving information from its respective source. The modules implemented so far are:
|
|
63
|
+
|
|
64
|
+
- BOLD
|
|
65
|
+
- COL
|
|
66
|
+
- GBIF
|
|
67
|
+
- iNaturalist
|
|
68
|
+
- IUCN
|
|
69
|
+
- NCBI
|
|
70
|
+
- OBIS
|
|
71
|
+
- ZooBank
|
|
72
|
+
|
|
73
|
+
This list can be expanded, thus suggestions and feedback are greatly appreciated.
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
## Main functionalities and workflow
|
|
77
|
+
Before using ``biodumpy``, users need to install the package in their Python environment with the following command:
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
pip install biodumpy
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### Usage
|
|
84
|
+
To simplify the use of ``biodumpy``, we create a general structure common among the modules:
|
|
85
|
+
|
|
86
|
+
1) **Load the package.** Import ``biodumpy`` into your Python environment.
|
|
87
|
+
2) **Load the desired modules.** Import one or more specific modules needed to retrieve the data.
|
|
88
|
+
3) **Set up the configuration of one or more modules.** Configure the ``biodumpy`` function/s with the required parameters.
|
|
89
|
+
4) **Start the download.** Execute the function to begin retrieving the data.
|
|
90
|
+
|
|
91
|
+
Here, we provide two examples illustrating the general structure of a ``biodumpy`` script:
|
|
92
|
+
|
|
93
|
+
In detail, we described:
|
|
94
|
+
- **Single Module Example**: This example demonstrates how to use a single ``biodumpy`` module (for example, GBIF).
|
|
95
|
+
- **Multiple Modules Example**: This example shows how to use multiple ``biodumpy`` modules (for example, GBIF and IUCN).
|
|
96
|
+
|
|
97
|
+
**Example N.1**
|
|
98
|
+
|
|
99
|
+
``` python
|
|
100
|
+
|
|
101
|
+
# Import biodumpy package
|
|
102
|
+
from biodumpy import Biodumpy
|
|
103
|
+
|
|
104
|
+
# Import GBIF module
|
|
105
|
+
from biodumpy.inputs import GBIF
|
|
106
|
+
|
|
107
|
+
# Create a list of taxa
|
|
108
|
+
taxa = [
|
|
109
|
+
'Alytes muletensis (Sanchíz & Adrover, 1979)',
|
|
110
|
+
'Bufotes viridis (Laurenti, 1768)',
|
|
111
|
+
'Hyla meridionalis Boettger, 1874',
|
|
112
|
+
'Anax imperator Leach, 1815'
|
|
113
|
+
]
|
|
114
|
+
|
|
115
|
+
# Set the Biodumpy function with the specific parameters
|
|
116
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True)])
|
|
117
|
+
|
|
118
|
+
# Start the download
|
|
119
|
+
bdp.start(taxa, output_path='YOUR_OUTPUT_PATH/downloads/{date}/{module}/{name}')
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Example N.2**
|
|
123
|
+
|
|
124
|
+
``` python
|
|
125
|
+
|
|
126
|
+
# Import biodumpy package
|
|
127
|
+
from biodumpy import Biodumpy
|
|
128
|
+
|
|
129
|
+
# Import GBIF and IUCN modules
|
|
130
|
+
from biodumpy.inputs import GBIF, IUCN
|
|
131
|
+
|
|
132
|
+
api_key = 'YOUR_IUCN_API_KEY'
|
|
133
|
+
|
|
134
|
+
# Create a list of taxa
|
|
135
|
+
taxa = [
|
|
136
|
+
'Alytes muletensis',
|
|
137
|
+
'Bufotes viridis',
|
|
138
|
+
'Hyla meridionalis',
|
|
139
|
+
'Anax imperator'
|
|
140
|
+
]
|
|
141
|
+
|
|
142
|
+
# Set the Biodumpy functions with the specific parameters
|
|
143
|
+
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True),
|
|
144
|
+
IUCN(api_key=api_key, bulk=True, region=['global'])])
|
|
145
|
+
|
|
146
|
+
# Start the download
|
|
147
|
+
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
## Documentation and Support
|
|
151
|
+
For detailed documentation and tutorials, please visit the ``biodumpy`` Read the Docs documentation.
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
## Contribution
|
|
155
|
+
``biodumpy`` is an open-source project, and contributions are welcome!
|
|
156
|
+
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub
|
|
157
|
+
repository or contact with the support team at [t.cancellario@uib.eu](mailto:t.cancellario@uib.eu).
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
## License
|
|
161
|
+
``biodumpy`` is licensed under the GNU GENERAL PUBLIC LICENSE. See the LICENSE file for more details.
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
## Acknowledgments
|
|
165
|
+
The project was supported by MCIN with funding from the European Union—NextGenerationEU (PRTR-C17.I1) and
|
|
166
|
+
the Government of the Balearic Islands.
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
<hr>
|
|
171
|
+
<div style="display: flex; justify-content: center">
|
|
172
|
+
<img src='https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/logo_cbb.png' alt='logo_cbb' width='200'>
|
|
173
|
+
</div>
|
|
174
|
+
|
|
175
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
biodumpy/__init__.py,sha256=dkhZ5rofUp_aAmX-Ge319rjAF9CNZODUY0yY4_7PJ5w,131
|
|
2
|
+
biodumpy/biodumpy.py,sha256=c9KvSPFIBCU8ry-3IePp733Oh_ArrHiC5d9PpSjBDK8,2770
|
|
3
|
+
biodumpy/input.py,sha256=RUwBf_opy2P41_Tbd4CcbDMuI_OwDkZ5ilT-lIySTBM,216
|
|
4
|
+
biodumpy/utils.py,sha256=qtY5ON59dw3GhJEhSEHaH1sS-V-glxys4z8WSp-ZD7Q,4240
|
|
5
|
+
biodumpy/inputs/BOLD.py,sha256=cuGaX-NZEeNsr_VBdHa2NqZEAoWlnpb4YofoEEhiJDs,4979
|
|
6
|
+
biodumpy/inputs/COL.py,sha256=rasje94XCbqsxCb24N6-Ks-MN0NkxBiTTY1vmVEIoYY,3322
|
|
7
|
+
biodumpy/inputs/GBIF.py,sha256=_ES1VoY39WexNwVpqHTF4_kPbL85VgX0kC39emb1qnI,4238
|
|
8
|
+
biodumpy/inputs/INaturalist.py,sha256=YHhjJzOit4u_fy-6vopVRx2BCULQBHiP3IyKR8jkhYo,5008
|
|
9
|
+
biodumpy/inputs/IUCN.py,sha256=w7HPU6RXpztqlIDZongivcI_RdGVO0Rm-vBuXfJ0D1g,4402
|
|
10
|
+
biodumpy/inputs/NCBI.py,sha256=QDbLqdlzEuwXpVKKmgRs1uRCZiTfOSWcwv9fSZSA2DM,9448
|
|
11
|
+
biodumpy/inputs/OBIS.py,sha256=SWpP1ATEzTK8FRVCGK55YCjOUdgrpLotta_TfP0Zcd0,3557
|
|
12
|
+
biodumpy/inputs/ZooBank.py,sha256=KRTgjeIJnNViooUS1iboDlXNsy8bRmz_9tqx0b23WzQ,3862
|
|
13
|
+
biodumpy/inputs/__init__.py,sha256=u2BpDQk_5cSer48BYWNjzpJ_xwl5x-yzaBKsFwv94zM,223
|
|
14
|
+
biodumpy-0.1.2.dist-info/LICENSE,sha256=ORhfyTkndrEturRXUBZp6Nvc_jNTEuMe4Kj6C_5j0ZE,1052
|
|
15
|
+
biodumpy-0.1.2.dist-info/METADATA,sha256=PF4TVo3aaFptdKpskxI1sDK4ztY3uc42HT3hqgYBJwk,6290
|
|
16
|
+
biodumpy-0.1.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
17
|
+
biodumpy-0.1.2.dist-info/top_level.txt,sha256=AnpMhIDQDuHix3YGhu9-dRL6ZRQSSdbduss2mOxGh3Q,9
|
|
18
|
+
biodumpy-0.1.2.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
biodumpy
|