biodumpy 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biodumpy/__init__.py +5 -0
- biodumpy/biodumpy.py +96 -0
- biodumpy/input.py +8 -0
- biodumpy/inputs/BOLD.py +131 -0
- biodumpy/inputs/COL.py +91 -0
- biodumpy/inputs/GBIF.py +126 -0
- biodumpy/inputs/INaturalist.py +115 -0
- biodumpy/inputs/IUCN.py +143 -0
- biodumpy/inputs/NCBI.py +257 -0
- biodumpy/inputs/OBIS.py +115 -0
- biodumpy/inputs/ZooBank.py +106 -0
- biodumpy/inputs/__init__.py +9 -0
- biodumpy/utils.py +169 -0
- biodumpy-0.1.2.dist-info/LICENSE +8 -0
- biodumpy-0.1.2.dist-info/METADATA +175 -0
- biodumpy-0.1.2.dist-info/RECORD +18 -0
- biodumpy-0.1.2.dist-info/WHEEL +5 -0
- biodumpy-0.1.2.dist-info/top_level.txt +1 -0
biodumpy/inputs/IUCN.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input, BiodumpyException
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class IUCN(Input):
|
|
7
|
+
"""
|
|
8
|
+
Query the IUCN Red List API for information about a specific species in a given region.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
query : list
|
|
13
|
+
The list of taxa to query.
|
|
14
|
+
api_key : str
|
|
15
|
+
Your IUCN API key.
|
|
16
|
+
habitat : str
|
|
17
|
+
The name of an IUCN habitat.
|
|
18
|
+
historical : bool, optional
|
|
19
|
+
If True, the function returns also the historical threats of a taxon. Default is False.
|
|
20
|
+
threats : bool, optional
|
|
21
|
+
If True, the function returns also the threat list of a taxon. Default is False.
|
|
22
|
+
regions : list, optional
|
|
23
|
+
The list of specific IUCN regions. For further information, see the IUCN API documentation. Default is an empty list.
|
|
24
|
+
bulk : bool, optional
|
|
25
|
+
If True, the function creates a bulk file. For further information, see the documentation of the Biodumpy package. Default is False.
|
|
26
|
+
output_format : string, optional
|
|
27
|
+
The format of the output file. The options available are: 'json', 'fasta', 'pdf'. Default is 'json'.
|
|
28
|
+
|
|
29
|
+
Example
|
|
30
|
+
-------
|
|
31
|
+
>>> from biodumpy import Biodumpy
|
|
32
|
+
>>> from biodumpy.inputs import IUCN
|
|
33
|
+
# Insert your API key
|
|
34
|
+
>>> api_key = 'YOUR_API_KEY'
|
|
35
|
+
# Taxa list
|
|
36
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis']
|
|
37
|
+
# Select your output path
|
|
38
|
+
>>> output_path = 'YOUR_OUTPUT_PATH'
|
|
39
|
+
# Select the regions
|
|
40
|
+
>>> regions = ['global', 'europe']
|
|
41
|
+
>>> bdp = Biodumpy([IUCN(api_key=api_key, bulk=True, regions=regions)])
|
|
42
|
+
>>> bdp.start(taxa, output_path=output_path)
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
api_key: str,
|
|
48
|
+
habitat: bool = False,
|
|
49
|
+
historical: bool = False,
|
|
50
|
+
threats: bool = False,
|
|
51
|
+
regions: list = None,
|
|
52
|
+
output_format: str = "json",
|
|
53
|
+
bulk: bool = False,
|
|
54
|
+
):
|
|
55
|
+
super().__init__(output_format, bulk)
|
|
56
|
+
if regions is None:
|
|
57
|
+
regions = ["global"]
|
|
58
|
+
self.api_key = api_key
|
|
59
|
+
self.habitat = habitat
|
|
60
|
+
self.regions = regions
|
|
61
|
+
self.historical = historical
|
|
62
|
+
self.threats = threats
|
|
63
|
+
|
|
64
|
+
iucn_regions = [
|
|
65
|
+
"northern_africa",
|
|
66
|
+
"global",
|
|
67
|
+
"pan-africa",
|
|
68
|
+
"central_africa",
|
|
69
|
+
"eastern_africa",
|
|
70
|
+
"northeastern_africa",
|
|
71
|
+
"western_africa",
|
|
72
|
+
"southern_africa",
|
|
73
|
+
"mediterranean",
|
|
74
|
+
"europe",
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
if output_format != "json":
|
|
78
|
+
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
79
|
+
|
|
80
|
+
for regions in self.regions:
|
|
81
|
+
if regions not in iucn_regions:
|
|
82
|
+
raise ValueError(f"Choose an IUCN region from the following options: {iucn_regions}.")
|
|
83
|
+
|
|
84
|
+
def _download(self, query, **kwargs) -> list:
|
|
85
|
+
payload = []
|
|
86
|
+
|
|
87
|
+
for region in self.regions:
|
|
88
|
+
taxon_info = self._icun_request(f"https://apiv3.iucnredlist.org/api/v3/species/{query}/region/{region}?token={self.api_key}")
|
|
89
|
+
|
|
90
|
+
if not taxon_info or taxon_info.get("taxonid") is None:
|
|
91
|
+
continue
|
|
92
|
+
|
|
93
|
+
payload.append(taxon_info)
|
|
94
|
+
taxon_id = taxon_info["taxonid"]
|
|
95
|
+
|
|
96
|
+
if taxon_info:
|
|
97
|
+
if self.habitat:
|
|
98
|
+
habitat_info = self._icun_request(
|
|
99
|
+
f"https://apiv3.iucnredlist.org/api/v3/habitats/species/id/{taxon_id}/region/{region}?token={self.api_key}"
|
|
100
|
+
)
|
|
101
|
+
taxon_info.update({"habitat": habitat_info})
|
|
102
|
+
|
|
103
|
+
if self.historical:
|
|
104
|
+
habitat_info = self._icun_request(
|
|
105
|
+
f"https://apiv3.iucnredlist.org/api/v3/species/history/id/{taxon_id}/region/{region}?token={self.api_key}"
|
|
106
|
+
)
|
|
107
|
+
taxon_info.update({"historical": habitat_info})
|
|
108
|
+
|
|
109
|
+
if self.threats:
|
|
110
|
+
habitat_info = self._icun_request(
|
|
111
|
+
f"https://apiv3.iucnredlist.org/api/v3/threats/species/id/{taxon_id}/region/{region}?token={self.api_key}"
|
|
112
|
+
)
|
|
113
|
+
taxon_info.update({"threats": habitat_info})
|
|
114
|
+
|
|
115
|
+
return payload
|
|
116
|
+
|
|
117
|
+
def _icun_request(self, query_path):
|
|
118
|
+
"""
|
|
119
|
+
Query the IUCN Red List API for information about a specific species in a given region.
|
|
120
|
+
|
|
121
|
+
Parameters:
|
|
122
|
+
query_path: str The path to the IUCN Red List API properly formatted.
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
dict: A dictionary containing information about the species in the specified region if found.
|
|
126
|
+
|
|
127
|
+
Example:
|
|
128
|
+
result = taxon_iucn(f"https://apiv3.iucnredlist.org/api/v3/habitats/species/id/{taxon_id}/region/{region}?token={self.api_key}")
|
|
129
|
+
"""
|
|
130
|
+
response = requests.get(query_path)
|
|
131
|
+
|
|
132
|
+
if response.status_code != 200:
|
|
133
|
+
raise BiodumpyException(f"Error {response.status_code}")
|
|
134
|
+
|
|
135
|
+
if response.content:
|
|
136
|
+
response = response.json()
|
|
137
|
+
|
|
138
|
+
if not (len(response.get("result", [])) == 0 or (response.get("value") == "0")):
|
|
139
|
+
result = response.get("result", [])[0]
|
|
140
|
+
if "region_identifier" in response:
|
|
141
|
+
result.update({"region": response["region_identifier"]})
|
|
142
|
+
|
|
143
|
+
return result
|
biodumpy/inputs/NCBI.py
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input
|
|
4
|
+
from biodumpy.utils import split_to_batches
|
|
5
|
+
|
|
6
|
+
from tqdm import tqdm
|
|
7
|
+
import time
|
|
8
|
+
from Bio import Entrez, SeqIO
|
|
9
|
+
from http.client import IncompleteRead
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class CustomEncoder(json.JSONEncoder):
|
|
13
|
+
def default(self, obj):
|
|
14
|
+
if hasattr(obj, "to_dict"):
|
|
15
|
+
return obj.to_dict()
|
|
16
|
+
elif hasattr(obj, "__dict__"):
|
|
17
|
+
if obj.__dict__:
|
|
18
|
+
return obj.__dict__
|
|
19
|
+
else:
|
|
20
|
+
try:
|
|
21
|
+
return str(obj) if obj else None
|
|
22
|
+
except Exception as e:
|
|
23
|
+
return str(e)
|
|
24
|
+
else:
|
|
25
|
+
return super().default(obj)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class NCBI(Input):
|
|
29
|
+
"""
|
|
30
|
+
Query the National Center for Biotechnology Information (NCBI) database to retrieve taxon retlated data.
|
|
31
|
+
|
|
32
|
+
Parameters
|
|
33
|
+
----------
|
|
34
|
+
query : list
|
|
35
|
+
A list of taxa to query in the BOLD database.
|
|
36
|
+
mail : str, optional
|
|
37
|
+
Email address used with Entrez functions to identify the user.
|
|
38
|
+
This is required by NCBI to track usage and report problems.
|
|
39
|
+
Default is None.
|
|
40
|
+
db : str, optional
|
|
41
|
+
NCBI database to search and download data from. Options include "nucleotide", "protein", "gene", and others.
|
|
42
|
+
Default is "nucleotide".
|
|
43
|
+
rettype : str, optional
|
|
44
|
+
The format for data retrieval from NCBI. Common formats include 'gb' (GenBank), 'fasta', 'xml', etc.
|
|
45
|
+
Default is 'gb'.
|
|
46
|
+
query_type : str, optional
|
|
47
|
+
Defines the type of query search, such as "[Organism]" or "[Gene]".
|
|
48
|
+
This determines how the query is interpreted by NCBI.
|
|
49
|
+
Default is "[Organism]".
|
|
50
|
+
step : int, optional
|
|
51
|
+
Number of records to download per chunk. For example, if the total data to download is 10,000 and the step is set to 100,
|
|
52
|
+
the function will download in chunks of 100 records per request.
|
|
53
|
+
Default is 100.
|
|
54
|
+
max_bp : int, optional
|
|
55
|
+
Maximum number of base pairs allowed for each sequence. Records with more base pairs than this value will be excluded.
|
|
56
|
+
If None, all records will be downloaded regardless of size.
|
|
57
|
+
Default is None.
|
|
58
|
+
summary : bool, optional
|
|
59
|
+
If True, the function returns a summary of the downloaded metadata instead of the full records.
|
|
60
|
+
Default is False.
|
|
61
|
+
by_id : bool, optional
|
|
62
|
+
If True, the function downloads the data using NCBI accession numbers as inputs.
|
|
63
|
+
Default is False.
|
|
64
|
+
bulk : bool, optional
|
|
65
|
+
If True, the function creates a bulk file for large downloads.
|
|
66
|
+
For more information, refer to the Biodumpy package documentation.
|
|
67
|
+
Default is False.
|
|
68
|
+
output_format : str, optional
|
|
69
|
+
The format of the output file. Available options are: 'json', 'fasta', 'pdf'.
|
|
70
|
+
Default is 'json'.
|
|
71
|
+
|
|
72
|
+
Details
|
|
73
|
+
-------
|
|
74
|
+
When `summary` is True, the resulting JSON will include the following information:
|
|
75
|
+
- `Id`: A numerical identifier (GI Number) that used to be assigned to each sequence version (e.g., "345678912").
|
|
76
|
+
- `Caption`: A unique identifier (accession number) assigned to a sequence when it is submitted to GenBank (e.g., "NM_001256789").
|
|
77
|
+
- `Title`: A short description or title of the sequence, often including information about the gene, organism, and type of sequence.
|
|
78
|
+
- `Length`: The length of the sequence in base pairs (for nucleotide sequences) or amino acids (for protein sequences).
|
|
79
|
+
- `query`: The original search term or query string used to retrieve this result.
|
|
80
|
+
|
|
81
|
+
Example
|
|
82
|
+
-------
|
|
83
|
+
>>> from biodumpy import Biodumpy
|
|
84
|
+
>>> from biodumpy.inputs import NCBI
|
|
85
|
+
# List of taxa
|
|
86
|
+
>>> taxa = ['Alytes muletensis', 'Hyla meridionalis', 'Anax imperator', 'Bufo roseus', 'Stollia betae']
|
|
87
|
+
# Set the module and start the download
|
|
88
|
+
>>> bdp = Biodumpy([NCBI(bulk=False, mail="hola@quetal.com", db="nucleotide", rettype="gb", query_type='[Organism]')])
|
|
89
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
def __init__(
|
|
93
|
+
self,
|
|
94
|
+
mail: str = None,
|
|
95
|
+
db: str = "nucleotide",
|
|
96
|
+
rettype: str = "gb",
|
|
97
|
+
query_type: str = "[Organism]",
|
|
98
|
+
step: int = 100,
|
|
99
|
+
max_bp: int = None,
|
|
100
|
+
summary: bool = False,
|
|
101
|
+
by_id: bool = False,
|
|
102
|
+
output_format: str = "json",
|
|
103
|
+
bulk: bool = False,
|
|
104
|
+
):
|
|
105
|
+
super().__init__(output_format, bulk)
|
|
106
|
+
self.max_bp = max_bp
|
|
107
|
+
self.db = db
|
|
108
|
+
self.step = step
|
|
109
|
+
self.rettype = rettype
|
|
110
|
+
self.query_type = query_type
|
|
111
|
+
self.summary = summary
|
|
112
|
+
self.by_id = by_id
|
|
113
|
+
Entrez.email = mail
|
|
114
|
+
|
|
115
|
+
if self.output_format == "fasta" and self.rettype != "fasta":
|
|
116
|
+
raise ValueError("Invalid output_format. Expected fasta.")
|
|
117
|
+
|
|
118
|
+
if self.by_id and self.query_type is not None:
|
|
119
|
+
raise ValueError("Invalid parameters: 'by_id' is True, so 'query_type' must be None.")
|
|
120
|
+
|
|
121
|
+
if self.summary and self.output_format == "fasta" and self.rettype == "fasta":
|
|
122
|
+
raise ValueError("Invalid parameters: 'summary' is True, so 'output_format' cannot be 'fasta'.")
|
|
123
|
+
|
|
124
|
+
if output_format not in {"json", "fasta"}:
|
|
125
|
+
raise ValueError('Invalid output_format. Expected "json" or "fasta".')
|
|
126
|
+
|
|
127
|
+
def _download(self, query, **kwargs) -> list:
|
|
128
|
+
if self.by_id:
|
|
129
|
+
ids_list = {query}
|
|
130
|
+
else:
|
|
131
|
+
ids_list = self._download_ids(term=f"{query}{self.query_type}", step=self.step)
|
|
132
|
+
|
|
133
|
+
payload = []
|
|
134
|
+
if self.summary:
|
|
135
|
+
with tqdm(total=len(ids_list), desc="NCBI summary retrieve", unit=" Summary") as pbar:
|
|
136
|
+
for seq_id in split_to_batches(list(ids_list), self.step):
|
|
137
|
+
for sumr in self._download_summary(seq_id):
|
|
138
|
+
sumr["query"] = f"{query}{self.query_type}"
|
|
139
|
+
payload.append(json.loads(json.dumps(sumr, cls=CustomEncoder)))
|
|
140
|
+
pbar.update(len(seq_id))
|
|
141
|
+
|
|
142
|
+
else:
|
|
143
|
+
with tqdm(total=len(ids_list), desc="NCBI sequences retrieve", unit=" Sequences") as pbar:
|
|
144
|
+
for seq_id in split_to_batches(list(ids_list), self.step):
|
|
145
|
+
for seq in self._download_seq(seq_id, rettype=self.rettype, db=self.db):
|
|
146
|
+
payload.append(json.loads(json.dumps(seq, cls=CustomEncoder)))
|
|
147
|
+
pbar.update(len(seq_id))
|
|
148
|
+
|
|
149
|
+
return payload
|
|
150
|
+
|
|
151
|
+
def _download_ids(self, term, step):
|
|
152
|
+
"""
|
|
153
|
+
Downloads NCBI IDs based on a search term and counts the total base pairs (bp) for the retrieved sequences.
|
|
154
|
+
|
|
155
|
+
Parameters:
|
|
156
|
+
term (str): Search term for querying the NCBI database.
|
|
157
|
+
db (str): NCBI database to search (default is 'nucleotide').
|
|
158
|
+
step (int): Number of IDs to retrieve per batch (default is 10).
|
|
159
|
+
mail (str): Email address to be used with Entrez (required by NCBI).
|
|
160
|
+
|
|
161
|
+
Returns:
|
|
162
|
+
tuple: A list of dictionaries with 'id' and 'bp' keys, and the total count of base pairs.
|
|
163
|
+
|
|
164
|
+
Example usage:
|
|
165
|
+
id_bp_list, total_bp = download_NCBI_ids_and_count_bp(term="Alytes muletensis[Organism]", db='nucleotide', step=10, mail='your-email@example.com')
|
|
166
|
+
print(id_bp_list)
|
|
167
|
+
"""
|
|
168
|
+
|
|
169
|
+
handle = Entrez.esearch(db=self.db, term=term, retmax=0)
|
|
170
|
+
record = Entrez.read(handle)
|
|
171
|
+
handle.close()
|
|
172
|
+
|
|
173
|
+
id_bp_list = set()
|
|
174
|
+
total_ids = int(record["Count"])
|
|
175
|
+
with tqdm(total=total_ids, desc="NCBI IDs retrieve", unit=" IDs") as pbar:
|
|
176
|
+
for start in range(0, total_ids, step):
|
|
177
|
+
try:
|
|
178
|
+
handle = Entrez.esearch(db=self.db, retstart=start, retmax=step, term=term)
|
|
179
|
+
record = Entrez.read(handle)
|
|
180
|
+
handle.close()
|
|
181
|
+
|
|
182
|
+
if self.max_bp:
|
|
183
|
+
# Retrieve summaries and calculate total bp
|
|
184
|
+
summary_handle = Entrez.esummary(db=self.db, id=",".join(record["IdList"]))
|
|
185
|
+
summaries = Entrez.read(summary_handle)
|
|
186
|
+
summary_handle.close()
|
|
187
|
+
|
|
188
|
+
for summary in summaries:
|
|
189
|
+
bp = int(summary["Length"])
|
|
190
|
+
if bp <= self.max_bp and summary["Id"] not in id_bp_list: # Ensure unique entries
|
|
191
|
+
id_bp_list.add(summary["Id"])
|
|
192
|
+
else:
|
|
193
|
+
id_bp_list.update(record["IdList"])
|
|
194
|
+
|
|
195
|
+
pbar.update(len(record["IdList"]))
|
|
196
|
+
except Exception as e:
|
|
197
|
+
print(f"Error retrieving IDs or summaries: {e}")
|
|
198
|
+
break
|
|
199
|
+
|
|
200
|
+
return id_bp_list
|
|
201
|
+
|
|
202
|
+
def _download_summary(self, seq_id):
|
|
203
|
+
keys_to_keep = ["Id", "Caption", "Title", "Length"]
|
|
204
|
+
summary_list = list()
|
|
205
|
+
try:
|
|
206
|
+
summary_handle = Entrez.esummary(db=self.db, id=seq_id)
|
|
207
|
+
summaries = Entrez.read(summary_handle)
|
|
208
|
+
summary_handle.close()
|
|
209
|
+
for summary in summaries:
|
|
210
|
+
summary_list.append({key: summary[key] for key in keys_to_keep})
|
|
211
|
+
except Exception as e:
|
|
212
|
+
print(f"Error retrieving IDs or summaries: {e}")
|
|
213
|
+
|
|
214
|
+
return summary_list
|
|
215
|
+
|
|
216
|
+
def _download_seq(self, seq_id, db=None, rettype=None, retmode="text", retries=3, webenv=None, query_key=None, history="y"):
|
|
217
|
+
"""
|
|
218
|
+
Downloads a full Entrez record, saves it to a file, parses it, and updates the result.
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
organism_id: NCBI database ID of the record to download.
|
|
222
|
+
rettype: Entrez return type (e.g., "gbwithparts").
|
|
223
|
+
retmode: Entrez return mode (e.g., "text").
|
|
224
|
+
retries: Number of retries in case of a connection error.
|
|
225
|
+
webenv: Web environment key for NCBI history.
|
|
226
|
+
query_key: Query key for NCBI history.
|
|
227
|
+
|
|
228
|
+
Returns:
|
|
229
|
+
A list of sequences.
|
|
230
|
+
"""
|
|
231
|
+
|
|
232
|
+
attempt = 0
|
|
233
|
+
while attempt < retries:
|
|
234
|
+
try:
|
|
235
|
+
handle = Entrez.efetch(
|
|
236
|
+
db=db, id=seq_id, rettype=rettype, retmode=retmode, usehistory=history, WebEnv=webenv, query_key=query_key
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
if self.rettype == "fasta":
|
|
240
|
+
return handle.read().split("\n\n")[:-1]
|
|
241
|
+
else:
|
|
242
|
+
parsed_records = SeqIO.parse(handle, rettype)
|
|
243
|
+
return SeqIO.to_dict(parsed_records).values()
|
|
244
|
+
except IncompleteRead as e:
|
|
245
|
+
# logging.warning(f"IncompleteRead error: {e}. Retrying {attempt + 1}/{retries}...")
|
|
246
|
+
print(f"IncompleteRead error: {e}. Retrying {attempt + 1}/{retries}...")
|
|
247
|
+
attempt += 1
|
|
248
|
+
time.sleep(2) # Wait before retrying
|
|
249
|
+
|
|
250
|
+
except Exception as e:
|
|
251
|
+
# logging.error(f"Error downloading or processing record: {e}")
|
|
252
|
+
print(f"Error downloading or processing record: {e}")
|
|
253
|
+
break
|
|
254
|
+
|
|
255
|
+
if attempt == retries:
|
|
256
|
+
# logging.error(f"Failed to download record {seq_id} after {retries} attempts.")
|
|
257
|
+
print(f"Failed to download record {seq_id} after {retries} attempts.")
|
biodumpy/inputs/OBIS.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from tqdm import tqdm
|
|
4
|
+
from biodumpy import Input, BiodumpyException
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class OBIS(Input):
|
|
8
|
+
"""
|
|
9
|
+
Query the OBIS database to retrieve taxon data.
|
|
10
|
+
|
|
11
|
+
Parameters
|
|
12
|
+
----------
|
|
13
|
+
query : list
|
|
14
|
+
The list of taxa to query.
|
|
15
|
+
occ : bool, optional
|
|
16
|
+
If True, the function also returns the occurrences of a taxon. Default is False.
|
|
17
|
+
geometry : str, optional
|
|
18
|
+
A spatial polygon to filter occurrences within a specified area. Default is an empty string.
|
|
19
|
+
area : int, optional
|
|
20
|
+
A marine area to filter occurrences. Default is an empty string.
|
|
21
|
+
bulk : bool, optional
|
|
22
|
+
If True, the function creates a bulk file. For further information, see the documentation of the Biodumpy package.
|
|
23
|
+
Default is False.
|
|
24
|
+
output_format : string, optional
|
|
25
|
+
The format of the output file. The options available is: 'json'. Default is 'json'.
|
|
26
|
+
|
|
27
|
+
Example
|
|
28
|
+
-------
|
|
29
|
+
>>> from biodumpy import Biodumpy
|
|
30
|
+
>>> from biodumpy.inputs import OBIS
|
|
31
|
+
# Taxa list
|
|
32
|
+
>>> taxa = ['Pinna nobilis', 'Delphinus delphis', 'Plerogyra sinuosa']
|
|
33
|
+
# Set the module and start the download
|
|
34
|
+
>>> bdp = Biodumpy([OBIS(bulk=False, occ=True)])
|
|
35
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}_occ/{name}')
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, occ: bool = False, geometry: str = None, areaid: int = None, output_format: str = "json", bulk: bool = False):
|
|
39
|
+
super().__init__(output_format, bulk)
|
|
40
|
+
self.occ = occ
|
|
41
|
+
self.geometry = geometry
|
|
42
|
+
self.areaid = areaid
|
|
43
|
+
|
|
44
|
+
if output_format != "json":
|
|
45
|
+
raise ValueError('Invalid output_format. Expected "json".')
|
|
46
|
+
|
|
47
|
+
# if occ is False, areaid and pylogon cannot both be True
|
|
48
|
+
if not self.occ and (self.areaid is not None or self.geometry):
|
|
49
|
+
raise ValueError('"If "occ" is False, "areaid" and "geometry" cannot be set."')
|
|
50
|
+
|
|
51
|
+
def _download(self, query, **kwargs) -> list:
|
|
52
|
+
payload = []
|
|
53
|
+
response = requests.get(f"https://api.obis.org/v3/taxon/{query}")
|
|
54
|
+
|
|
55
|
+
if response.status_code != 200:
|
|
56
|
+
raise BiodumpyException(f"Taxonomy request. Error {response.status_code}")
|
|
57
|
+
|
|
58
|
+
if response.content:
|
|
59
|
+
payload = response.json()["results"]
|
|
60
|
+
|
|
61
|
+
if self.occ and len(payload) > 0:
|
|
62
|
+
tax_key = payload[0]["taxonID"]
|
|
63
|
+
payload = self._download_obis_occ(taxon_key=tax_key, geometry=self.geometry, areaid=self.areaid)
|
|
64
|
+
|
|
65
|
+
return payload
|
|
66
|
+
|
|
67
|
+
def _download_obis_occ(self, taxon_key: int, geometry: str = None, areaid: int = None):
|
|
68
|
+
total_records = None
|
|
69
|
+
payload_occ = []
|
|
70
|
+
|
|
71
|
+
params = {
|
|
72
|
+
"taxonid": taxon_key,
|
|
73
|
+
"size": 10000, # Max size in OBIS is 10000
|
|
74
|
+
"after": None,
|
|
75
|
+
"geometry": geometry,
|
|
76
|
+
"areaid": areaid,
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
while True:
|
|
81
|
+
response = requests.get("https://api.obis.org/v3/occurrence", params=params)
|
|
82
|
+
|
|
83
|
+
if response.status_code != 200:
|
|
84
|
+
raise BiodumpyException(f"Occurrences request. Error {response.status_code}")
|
|
85
|
+
|
|
86
|
+
response_json = response.json()
|
|
87
|
+
data = response_json["results"]
|
|
88
|
+
|
|
89
|
+
if not data: # If the data list is empty, break the loop
|
|
90
|
+
break
|
|
91
|
+
|
|
92
|
+
# Initialize the progress bar on the first request
|
|
93
|
+
if total_records is None:
|
|
94
|
+
total_records = response_json["total"]
|
|
95
|
+
pbar = tqdm(total=total_records, desc="Downloading")
|
|
96
|
+
|
|
97
|
+
# Update the progress bar
|
|
98
|
+
pbar.update(len(data))
|
|
99
|
+
|
|
100
|
+
# Append the current batch of results to the list
|
|
101
|
+
payload_occ.extend(data)
|
|
102
|
+
|
|
103
|
+
# Check if we've retrieved all records
|
|
104
|
+
if len(payload_occ) >= total_records:
|
|
105
|
+
break
|
|
106
|
+
|
|
107
|
+
# Update the 'after' parameter with the last id
|
|
108
|
+
params["after"] = data[-1]["id"]
|
|
109
|
+
|
|
110
|
+
finally:
|
|
111
|
+
# Ensure the progress bar is closed properly even if an error occurs
|
|
112
|
+
if total_records is not None:
|
|
113
|
+
pbar.close()
|
|
114
|
+
|
|
115
|
+
return payload_occ
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from bs4 import BeautifulSoup
|
|
4
|
+
from tqdm import tqdm
|
|
5
|
+
from biodumpy import Input, BiodumpyException
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ZooBank(Input):
|
|
9
|
+
"""
|
|
10
|
+
Query the Official Registry of Zoological Nomenclature (ZooBank) database to retrieve scientific bibliographic
|
|
11
|
+
information.
|
|
12
|
+
|
|
13
|
+
Parameters
|
|
14
|
+
----------
|
|
15
|
+
query : list
|
|
16
|
+
The list of taxa to query.
|
|
17
|
+
dataset_size : str, optional
|
|
18
|
+
This parameter is useful for managing the download of bibliographic information based on the number of
|
|
19
|
+
scientific articles stored in ZooBank for each taxon. You can set this parameter to either 'small' or 'large'.
|
|
20
|
+
We recommend choosing 'small' if the number of articles for a given taxon is lower than 200, or 'large' if
|
|
21
|
+
it exceeds 200. Default is 'small'.
|
|
22
|
+
info : bool, optional
|
|
23
|
+
If set to True, the function will download only additional article information not included in the main research,
|
|
24
|
+
such as the DOI. Default is False.
|
|
25
|
+
bulk : bool, optional
|
|
26
|
+
If True, the function creates a bulk file. For further information, see the documentation of the Biodumpy
|
|
27
|
+
package. Default is False.
|
|
28
|
+
output_format : str, optional
|
|
29
|
+
The format of the output file. The available option is 'json'. Default is 'json'.
|
|
30
|
+
|
|
31
|
+
Example
|
|
32
|
+
-------
|
|
33
|
+
>>> from biodumpy import Biodumpy
|
|
34
|
+
>>> from biodumpy.inputs import ZooBank
|
|
35
|
+
# Taxa list
|
|
36
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator']
|
|
37
|
+
# Set the module and start the download
|
|
38
|
+
>>> bdp = Biodumpy([ZooBank(bulk=True, dataset_size='small', info=False)])
|
|
39
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(self, dataset_size: str = "small", output_format: str = "json", info: bool = False, bulk: bool = False):
|
|
43
|
+
super().__init__(output_format, bulk)
|
|
44
|
+
self.dataset_size = dataset_size
|
|
45
|
+
self.info = info
|
|
46
|
+
|
|
47
|
+
if self.dataset_size not in ["small", "large"]:
|
|
48
|
+
raise ValueError("Invalid dataset_size. Expected 'small' or 'large'.")
|
|
49
|
+
|
|
50
|
+
if output_format != "json":
|
|
51
|
+
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
52
|
+
|
|
53
|
+
def _download(self, query, **kwargs) -> list:
|
|
54
|
+
payload = []
|
|
55
|
+
|
|
56
|
+
if self.dataset_size == "small":
|
|
57
|
+
response = requests.get(f"https://zoobank.org/References.json?search_term={query}")
|
|
58
|
+
|
|
59
|
+
if response.status_code != 200:
|
|
60
|
+
raise BiodumpyException(f"Reference request. Error {response.status_code}")
|
|
61
|
+
|
|
62
|
+
payload = response.json()
|
|
63
|
+
else:
|
|
64
|
+
print("Searching in ZooBank...")
|
|
65
|
+
response_pub = requests.get(f"https://zoobank.org/Search?search_term={query}")
|
|
66
|
+
|
|
67
|
+
if response_pub.status_code != 200:
|
|
68
|
+
raise BiodumpyException(f"Term search request. Error {response_pub.status_code}")
|
|
69
|
+
|
|
70
|
+
html_content = response_pub.text
|
|
71
|
+
|
|
72
|
+
# Parsing the HTML content with BeautifulSoup
|
|
73
|
+
soup = BeautifulSoup(html_content, "html.parser")
|
|
74
|
+
referenceuuid = [
|
|
75
|
+
entry["href"].replace("/References/", "") for entry in soup.find_all(class_="biblio-entry") if "href" in entry.attrs
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
for ref in tqdm(referenceuuid, desc="Fetching paper info"):
|
|
79
|
+
response_pub = requests.get(f"https://zoobank.org/References.json/{ref}")
|
|
80
|
+
if response_pub.status_code == 200: # Check if the request was successful
|
|
81
|
+
try:
|
|
82
|
+
json_content = response_pub.json()[0]
|
|
83
|
+
payload.append(json_content)
|
|
84
|
+
except Exception as e:
|
|
85
|
+
print(f"An error occurred: {e} .... referenceuuid: {ref}")
|
|
86
|
+
else:
|
|
87
|
+
print(f"Failed to retrieve data for {ref}")
|
|
88
|
+
|
|
89
|
+
if not self.info:
|
|
90
|
+
return payload
|
|
91
|
+
|
|
92
|
+
referenceuuid = [item["referenceuuid"] for item in payload]
|
|
93
|
+
|
|
94
|
+
payload = []
|
|
95
|
+
if referenceuuid != [""]:
|
|
96
|
+
for refuid in referenceuuid:
|
|
97
|
+
response_id = requests.get(f"https://zoobank.org/Identifiers.json/{refuid}")
|
|
98
|
+
|
|
99
|
+
if response_id.status_code != 200:
|
|
100
|
+
raise BiodumpyException(f"Referenceuuid request. Error {response_id.status_code}")
|
|
101
|
+
|
|
102
|
+
payload.append(response_id.json())
|
|
103
|
+
else:
|
|
104
|
+
print("No referenceuuid found.")
|
|
105
|
+
|
|
106
|
+
return payload
|