biodumpy 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biodumpy/__init__.py +5 -0
- biodumpy/biodumpy.py +96 -0
- biodumpy/input.py +8 -0
- biodumpy/inputs/BOLD.py +131 -0
- biodumpy/inputs/COL.py +91 -0
- biodumpy/inputs/GBIF.py +126 -0
- biodumpy/inputs/INaturalist.py +115 -0
- biodumpy/inputs/IUCN.py +143 -0
- biodumpy/inputs/NCBI.py +257 -0
- biodumpy/inputs/OBIS.py +115 -0
- biodumpy/inputs/ZooBank.py +106 -0
- biodumpy/inputs/__init__.py +9 -0
- biodumpy/utils.py +169 -0
- biodumpy-0.1.2.dist-info/LICENSE +8 -0
- biodumpy-0.1.2.dist-info/METADATA +175 -0
- biodumpy-0.1.2.dist-info/RECORD +18 -0
- biodumpy-0.1.2.dist-info/WHEEL +5 -0
- biodumpy-0.1.2.dist-info/top_level.txt +1 -0
biodumpy/__init__.py
ADDED
biodumpy/biodumpy.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from .input import Input
|
|
3
|
+
from .utils import dump
|
|
4
|
+
from tqdm import tqdm
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# Setup logging configuration
|
|
9
|
+
logging.basicConfig(
|
|
10
|
+
level=logging.INFO,
|
|
11
|
+
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
12
|
+
datefmt="%Y-%m-%d %H:%M:%S",
|
|
13
|
+
filename="biodumpy_downloader.log",
|
|
14
|
+
filemode="a", # is the appended mode
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class BiodumpyException(Exception):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Biodumpy:
|
|
23
|
+
"""
|
|
24
|
+
This class is designed to download biodiversity data from various sources using multiple input modules.
|
|
25
|
+
|
|
26
|
+
Parameters
|
|
27
|
+
----------
|
|
28
|
+
inputs : list
|
|
29
|
+
A list of input modules that handle specific biodiversity data downloads.
|
|
30
|
+
loading_bar : bool
|
|
31
|
+
If True, shows a progress bar when downloading data. If False, disable the progress bar.
|
|
32
|
+
Default is False
|
|
33
|
+
debug : bool
|
|
34
|
+
If True, enables printing of detailed information during execution.
|
|
35
|
+
Default is True
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
def __init__(self, inputs: list[Input], loading_bar: bool = False, debug: bool = True) -> None:
|
|
39
|
+
super().__init__()
|
|
40
|
+
self.inputs = inputs
|
|
41
|
+
self.debug = debug
|
|
42
|
+
self.loading_bar = loading_bar
|
|
43
|
+
|
|
44
|
+
# elements must be a flat list of strings
|
|
45
|
+
def start(self, elements, output_path="downloads/{date}/{module}/{name}"):
|
|
46
|
+
if not isinstance(elements, list):
|
|
47
|
+
raise ValueError("Invalid query. Expected a list of taxa to query.")
|
|
48
|
+
|
|
49
|
+
current_date = datetime.now().strftime("%Y-%m-%d")
|
|
50
|
+
bulk_input = {}
|
|
51
|
+
try:
|
|
52
|
+
for el in tqdm(elements, desc="Biodumpy list", unit=" elements", disable=not self.loading_bar, smoothing=0):
|
|
53
|
+
if not el:
|
|
54
|
+
continue
|
|
55
|
+
|
|
56
|
+
if isinstance(el, str):
|
|
57
|
+
el = {"query": el}
|
|
58
|
+
|
|
59
|
+
if "query" not in el:
|
|
60
|
+
logging.error(f"Missing 'query' key for {el}")
|
|
61
|
+
raise ValueError(f"Missing 'name' key for {el}")
|
|
62
|
+
|
|
63
|
+
name = el["query"]
|
|
64
|
+
clean_name = name.replace("/", "_")
|
|
65
|
+
if self.debug:
|
|
66
|
+
print(f"Downloading {name}...")
|
|
67
|
+
|
|
68
|
+
for inp in self.inputs:
|
|
69
|
+
module_name = type(inp).__name__
|
|
70
|
+
logging.info(f"biodumpy initialized with {module_name} inputs. Taxon: {name}")
|
|
71
|
+
|
|
72
|
+
try:
|
|
73
|
+
payload = inp._download(**el)
|
|
74
|
+
logging.info(f"Download data for {module_name} was successful.\n")
|
|
75
|
+
except Exception as e:
|
|
76
|
+
logging.error(f'[{module_name}] Failed to download data for "{name}": {str(e)} \n')
|
|
77
|
+
continue
|
|
78
|
+
|
|
79
|
+
if inp.bulk:
|
|
80
|
+
if inp not in bulk_input:
|
|
81
|
+
bulk_input[inp] = []
|
|
82
|
+
bulk_input[inp].extend(payload)
|
|
83
|
+
|
|
84
|
+
else:
|
|
85
|
+
dump(
|
|
86
|
+
file_name=f"{output_path.format(date=current_date, module=module_name, name=clean_name)}",
|
|
87
|
+
obj_list=payload,
|
|
88
|
+
output_format=inp.output_format,
|
|
89
|
+
)
|
|
90
|
+
finally:
|
|
91
|
+
for inp, payload in bulk_input.items():
|
|
92
|
+
dump(
|
|
93
|
+
file_name=output_path.format(date=current_date, module=type(inp).__name__, name="bulk"),
|
|
94
|
+
obj_list=payload,
|
|
95
|
+
output_format=inp.output_format,
|
|
96
|
+
)
|
biodumpy/input.py
ADDED
biodumpy/inputs/BOLD.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input, BiodumpyException
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BOLD(Input):
|
|
7
|
+
"""
|
|
8
|
+
Query the Barcode of Life Data System (BOLD) database to retrieve taxon genetic or occurrence data.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
query : list
|
|
13
|
+
The list of taxa to query.
|
|
14
|
+
summary : bool, optional
|
|
15
|
+
If True, the function returns a summary of the downloaded metadata instead of the full records.
|
|
16
|
+
Default is False.
|
|
17
|
+
fasta : bool, optional
|
|
18
|
+
If True, the function downloads the FASTA file.
|
|
19
|
+
Default is False.
|
|
20
|
+
bulk : bool, optional
|
|
21
|
+
If True, the function creates a bulk file.
|
|
22
|
+
For further information, see the documentation of the Biodumpy package.
|
|
23
|
+
Default is False.
|
|
24
|
+
output_format : string, optional
|
|
25
|
+
The format of the output file.
|
|
26
|
+
The options available are: 'json', 'fasta', 'pdf'.
|
|
27
|
+
Default is 'json'.
|
|
28
|
+
|
|
29
|
+
Details
|
|
30
|
+
-------
|
|
31
|
+
When `summary` is True, the resulting JSON will include the following information:
|
|
32
|
+
- `record_id`: The unique identifier for the BOLD record.
|
|
33
|
+
- `processid`: The process ID associated with the BOLD record.
|
|
34
|
+
- `bin_uri`: The BIN (Barcode Index Number) URI.
|
|
35
|
+
- `taxon`: The name of the lower taxon extracted from the taxonomy information.
|
|
36
|
+
- `country`: The country where the collection event took place.
|
|
37
|
+
- `province_state`: The province or state of the collection event.
|
|
38
|
+
- `region`: The region of the collection event.
|
|
39
|
+
- `lat`: The latitude of the collection event.
|
|
40
|
+
- `lon`: The longitude of the collection event.
|
|
41
|
+
- `markercode`: The marker code from the sequence data.
|
|
42
|
+
- `genbank_accession`: The GenBank accession number from the sequence data.
|
|
43
|
+
|
|
44
|
+
Example
|
|
45
|
+
-------
|
|
46
|
+
>>> from biodumpy import Biodumpy
|
|
47
|
+
>>> from biodumpy.inputs import BOLD
|
|
48
|
+
# Taxa list
|
|
49
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator']
|
|
50
|
+
# Set the module and start the download
|
|
51
|
+
>>> bdp = Biodumpy([BOLD(bulk=True, summary=False)])
|
|
52
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(self, summary: bool = False, fasta: bool = False, output_format: str = "json", bulk: bool = False):
|
|
56
|
+
super().__init__(output_format, bulk)
|
|
57
|
+
self.summary = summary
|
|
58
|
+
self.fasta = fasta
|
|
59
|
+
|
|
60
|
+
if self.fasta and output_format != "fasta":
|
|
61
|
+
raise ValueError("Invalid output_format. Expected fasta.")
|
|
62
|
+
|
|
63
|
+
if output_format not in {"json", "fasta"}:
|
|
64
|
+
raise ValueError('Invalid output_format. Expected "json" or "fasta".')
|
|
65
|
+
|
|
66
|
+
def _download(self, query, **kwargs) -> list:
|
|
67
|
+
if self.fasta:
|
|
68
|
+
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/sequence?taxon={query}")
|
|
69
|
+
|
|
70
|
+
if response.status_code != 200:
|
|
71
|
+
raise BiodumpyException(f"Fasta sequence request. Error {response.status_code}")
|
|
72
|
+
|
|
73
|
+
if response.content:
|
|
74
|
+
response = response.content
|
|
75
|
+
data_str = response.decode()
|
|
76
|
+
|
|
77
|
+
# Split the data by '>'
|
|
78
|
+
fasta_entries = [f">{entry}" for entry in data_str.split(">") if entry]
|
|
79
|
+
|
|
80
|
+
return fasta_entries
|
|
81
|
+
|
|
82
|
+
else:
|
|
83
|
+
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/combined?taxon={query}&format=json")
|
|
84
|
+
|
|
85
|
+
payload = []
|
|
86
|
+
|
|
87
|
+
if response.status_code != 200:
|
|
88
|
+
raise BiodumpyException(f"Combined data request. Error {response.status_code}")
|
|
89
|
+
|
|
90
|
+
if response.content:
|
|
91
|
+
results = response.json()
|
|
92
|
+
if self.summary:
|
|
93
|
+
results_summary = results.get("bold_records", {}).get("records", [])
|
|
94
|
+
for entry in results_summary:
|
|
95
|
+
entry_summary = results_summary[entry]
|
|
96
|
+
|
|
97
|
+
# Extract the necessary fields with default None values if keys are missing.
|
|
98
|
+
# Sequence could contain more than one marker code.
|
|
99
|
+
sequences = entry_summary.get("sequences", {}).get("sequence", [])
|
|
100
|
+
markercodes = [seq.get("markercode") for seq in sequences if "markercode" in seq]
|
|
101
|
+
genbank_accession = [seq.get("genbank_accession") for seq in sequences if "genbank_accession" in seq]
|
|
102
|
+
|
|
103
|
+
# Retrieve taxonomy
|
|
104
|
+
taxonomy = entry_summary.get("taxonomy", {})
|
|
105
|
+
taxon_name = taxonomy.get(list(taxonomy.keys())[-1], {}).get("taxon", {}).get("name")
|
|
106
|
+
|
|
107
|
+
# Retrieve collection event metadata
|
|
108
|
+
collection_event = entry_summary.get("collection_event", {})
|
|
109
|
+
coordinates = collection_event.get("coordinates", {})
|
|
110
|
+
|
|
111
|
+
payload.append(
|
|
112
|
+
{
|
|
113
|
+
"record_id": entry_summary.get("record_id"),
|
|
114
|
+
"processid": entry_summary.get("processid"),
|
|
115
|
+
"bin_uri": entry_summary.get("bin_uri"),
|
|
116
|
+
"taxon": taxon_name,
|
|
117
|
+
"country": collection_event.get("country"),
|
|
118
|
+
"province_state": collection_event.get("province_state"),
|
|
119
|
+
"region": collection_event.get("region"),
|
|
120
|
+
"lat": coordinates.get("lat") if coordinates else None,
|
|
121
|
+
"lon": coordinates.get("lon") if coordinates else None,
|
|
122
|
+
"markercode": "/".join(markercodes) if markercodes else None,
|
|
123
|
+
"genbank_accession": genbank_accession[0] if genbank_accession else None,
|
|
124
|
+
}
|
|
125
|
+
)
|
|
126
|
+
else:
|
|
127
|
+
payload.append(
|
|
128
|
+
results["bold_records"]["records"] if "bold_records" in results and "records" in results["bold_records"] else []
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
return payload
|
biodumpy/inputs/COL.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input, BiodumpyException
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class COL(Input):
|
|
7
|
+
"""
|
|
8
|
+
Query the Catalogue of Life (COL) database to retrieve nomenclature information of a list of taxa.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
query : list
|
|
13
|
+
The list of taxa to query.
|
|
14
|
+
check_syn : bool, optional
|
|
15
|
+
If True, the function returns only the accepted nomenclature of a taxon.
|
|
16
|
+
See Detail section for further information.
|
|
17
|
+
Default is False.
|
|
18
|
+
bulk : bool, optional
|
|
19
|
+
If True, the function creates a bulk file.
|
|
20
|
+
For further information, see the documentation of the Biodumpy package.
|
|
21
|
+
Default is False.
|
|
22
|
+
output_format : string, optional
|
|
23
|
+
The format of the output file.
|
|
24
|
+
The options available are: 'json', 'fasta', 'pdf'.
|
|
25
|
+
Default is 'json'.
|
|
26
|
+
|
|
27
|
+
Details
|
|
28
|
+
-------
|
|
29
|
+
When check_syn is set to True, the resulting JSON will include only the nomenclature of the accepted taxon.
|
|
30
|
+
For instance, if check_syn is True, the output for the species Bufo roseus will only show the nomenclature for
|
|
31
|
+
Bufotes viridis. Conversely, if check_syn is set to False, the JSON will include the nomenclature for both
|
|
32
|
+
Bufo roseus and Bufotes viridis.
|
|
33
|
+
|
|
34
|
+
Example
|
|
35
|
+
-------
|
|
36
|
+
>>> from biodumpy import Biodumpy
|
|
37
|
+
>>> from biodumpy.inputs import COL
|
|
38
|
+
# List oF taxa
|
|
39
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator', 'Bufo roseus', 'Stollia betae']
|
|
40
|
+
# Start the download
|
|
41
|
+
>>> bdp = Biodumpy([COL(bulk=True, check_syn=False)])
|
|
42
|
+
>>> bdp.start(taxa, output_path='./biodumpy/downloads/{date}/{module}/{name}')
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
ACCEPTED_TERMS = ["accepted", "provisionally accepted"]
|
|
46
|
+
|
|
47
|
+
def __init__(self, output_format: str = "json", bulk: bool = False, check_syn: bool = False):
|
|
48
|
+
super().__init__(output_format, bulk)
|
|
49
|
+
self.check_syn = check_syn
|
|
50
|
+
|
|
51
|
+
if output_format != "json":
|
|
52
|
+
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
53
|
+
|
|
54
|
+
def _download(self, query, **kwargs) -> list:
|
|
55
|
+
response = requests.get(
|
|
56
|
+
f"https://api.checklistbank.org/dataset/9923/nameusage/search?q={query}&content=SCIENTIFIC_NAME&type=EXACT&offset=0&limit=10"
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
if response.status_code != 200:
|
|
60
|
+
raise BiodumpyException(f"Taxonomy request. Error {response.status_code}")
|
|
61
|
+
|
|
62
|
+
payload = response.json()
|
|
63
|
+
|
|
64
|
+
if payload["empty"]:
|
|
65
|
+
payload = [{"origin_taxon": query, "taxon_id": None, "status": None, "usage": None, "classification": None}]
|
|
66
|
+
else:
|
|
67
|
+
result = response.json()["result"]
|
|
68
|
+
|
|
69
|
+
# Multiple IDs
|
|
70
|
+
if len(result) > 1:
|
|
71
|
+
ids = [item.get("id") for item in result if "id" in item]
|
|
72
|
+
ids = ", ".join(ids)
|
|
73
|
+
id_input = input(f"Please enter the correct taxon ID of {query} \n ID: {ids}; Skip \n" f"Insert the ID:")
|
|
74
|
+
|
|
75
|
+
if id_input == "Skip":
|
|
76
|
+
result = [{"id": None, "usage": None, "status": None, "classification": None}]
|
|
77
|
+
else:
|
|
78
|
+
result = [item for item in result if item["id"] == id_input]
|
|
79
|
+
|
|
80
|
+
id = result[0].get("id")
|
|
81
|
+
usage = result[0].get("usage")
|
|
82
|
+
status = usage.get("status") if usage else None
|
|
83
|
+
|
|
84
|
+
classification = result[0].get("classification")
|
|
85
|
+
if self.check_syn and status not in COL.ACCEPTED_TERMS:
|
|
86
|
+
synonym_id = usage.get("id") if usage else None
|
|
87
|
+
classification = [item for item in classification if item["id"] != synonym_id] if classification else None
|
|
88
|
+
|
|
89
|
+
payload = [{"origin_taxon": query, "taxon_id": id, "status": status, "usage": usage, "classification": classification}]
|
|
90
|
+
|
|
91
|
+
return payload
|
biodumpy/inputs/GBIF.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input, BiodumpyException
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class GBIF(Input):
|
|
7
|
+
"""
|
|
8
|
+
Query the GBIF database to retrieve taxon data.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
query : list
|
|
13
|
+
The list of taxa to query.
|
|
14
|
+
dataset_key : str
|
|
15
|
+
GBIF dataset key. The default is set to the GBIF Backbone Taxonomy dataset key.
|
|
16
|
+
limit : int
|
|
17
|
+
The maximum number of names to retrieve from the taxonomy backbone for a taxon. Default is 20.
|
|
18
|
+
accepted_only : bool, optional
|
|
19
|
+
If True, the function returns only the accepted name. Default is True.
|
|
20
|
+
occ : bool, optional
|
|
21
|
+
If True, the function also returns the occurrences of a taxon. Default is False.
|
|
22
|
+
geometry : str, optional
|
|
23
|
+
A spatial polygon to filter occurrences within a specified area. Default is an empty string.
|
|
24
|
+
bulk : bool, optional
|
|
25
|
+
If True, the function creates a bulk file. For further information, see the documentation of the Biodumpy package. Default is False.
|
|
26
|
+
output_format : str, optional
|
|
27
|
+
The format of the output file. The options available are: 'json', 'fasta', 'pdf'. Default is 'json'.
|
|
28
|
+
|
|
29
|
+
Example
|
|
30
|
+
-------
|
|
31
|
+
>>> from biodumpy import Biodumpy
|
|
32
|
+
>>> from biodumpy.inputs import GBIF
|
|
33
|
+
# GBIF dataset key
|
|
34
|
+
>>> gbif_backbone = 'd7dddbf4-2cf0-4f39-9b2a-bb099caae36c'
|
|
35
|
+
# Taxa list
|
|
36
|
+
>>> taxa = ['Alytes muletensis (Sanchíz & Adrover, 1979)', 'Bufotes viridis (Laurenti, 1768)']
|
|
37
|
+
# Set the module and start the download
|
|
38
|
+
>>> bdp = Biodumpy([GBIF(dataset_key=gbif_backbone, limit=20, accepted_only=True, occ=False, bulk=False, output_format='json')])
|
|
39
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
dataset_key: str = "d7dddbf4-2cf0-4f39-9b2a-bb099caae36c",
|
|
45
|
+
limit: int = 20,
|
|
46
|
+
accepted_only: bool = True,
|
|
47
|
+
occ: bool = False,
|
|
48
|
+
geometry: str = None,
|
|
49
|
+
output_format: str = "json",
|
|
50
|
+
bulk: bool = False,
|
|
51
|
+
):
|
|
52
|
+
super().__init__(output_format, bulk)
|
|
53
|
+
self.dataset_key = dataset_key
|
|
54
|
+
self.limit = limit # Limit to find name in taxonomy backbone
|
|
55
|
+
self.accepted = accepted_only
|
|
56
|
+
self.occ = occ
|
|
57
|
+
self.geometry = geometry
|
|
58
|
+
|
|
59
|
+
if output_format != "json":
|
|
60
|
+
raise ValueError('Invalid output_format. Expected "json".')
|
|
61
|
+
|
|
62
|
+
if occ and not accepted_only:
|
|
63
|
+
raise ValueError("Invalid accepted_only. Expected True.")
|
|
64
|
+
|
|
65
|
+
def _download(self, query, **kwargs) -> list:
|
|
66
|
+
payload = []
|
|
67
|
+
response = requests.get(f"https://api.gbif.org/v1/species/search?datasetKey={self.dataset_key}&q={query}&limit={self.limit}")
|
|
68
|
+
|
|
69
|
+
if response.status_code != 200:
|
|
70
|
+
raise BiodumpyException(f"Taxonomy request. Error {response.status_code}")
|
|
71
|
+
|
|
72
|
+
if response.content:
|
|
73
|
+
payload = response.json()["results"]
|
|
74
|
+
if self.accepted:
|
|
75
|
+
# We keep the record only if the query corresponds to the scientific name in the data downloaded.
|
|
76
|
+
payload = list(
|
|
77
|
+
filter(lambda x: x.get("taxonomicStatus") == "ACCEPTED" and str(query[0]) in x.get("scientificName", ""), payload)
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
if self.occ and len(payload) > 0:
|
|
81
|
+
tax_key = payload[0]["nubKey"]
|
|
82
|
+
payload = self._download_gbif_occ(taxon_key=tax_key, geometry=self.geometry)
|
|
83
|
+
|
|
84
|
+
return payload
|
|
85
|
+
|
|
86
|
+
def _download_gbif_occ(self, taxon_key: int, geometry: str):
|
|
87
|
+
response_occ = requests.get(
|
|
88
|
+
f"https://api.gbif.org/v1/occurrence/search",
|
|
89
|
+
params={"acceptedTaxonKey": taxon_key, "occurrenceStatus": "PRESENT", "geometry": geometry, "limit": 300},
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
if response_occ.status_code != 200:
|
|
93
|
+
raise BiodumpyException(f"Occurrence request. Error {response_occ.status_code}")
|
|
94
|
+
|
|
95
|
+
if response_occ.content:
|
|
96
|
+
payload_occ = response_occ.json()
|
|
97
|
+
if payload_occ["endOfRecords"] and payload_occ["count"] > 0:
|
|
98
|
+
return payload_occ["results"]
|
|
99
|
+
elif not payload_occ["endOfRecords"]:
|
|
100
|
+
total_records = payload_occ["count"]
|
|
101
|
+
|
|
102
|
+
# Initialize variables
|
|
103
|
+
payload_occ = []
|
|
104
|
+
offset = 0
|
|
105
|
+
|
|
106
|
+
# Loop to download data
|
|
107
|
+
while offset < total_records:
|
|
108
|
+
response_occ = requests.get(
|
|
109
|
+
f"https://api.gbif.org/v1/occurrence/search",
|
|
110
|
+
params={
|
|
111
|
+
"acceptedTaxonKey": taxon_key,
|
|
112
|
+
"occurrenceStatus": "PRESENT",
|
|
113
|
+
"geometry": geometry,
|
|
114
|
+
"limit": 300,
|
|
115
|
+
"offset": offset,
|
|
116
|
+
},
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
data = response_occ.json()
|
|
120
|
+
occurrences = data["results"]
|
|
121
|
+
payload_occ.extend(occurrences)
|
|
122
|
+
offset = offset + 300
|
|
123
|
+
|
|
124
|
+
return payload_occ
|
|
125
|
+
else:
|
|
126
|
+
return []
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy import Input, BiodumpyException
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class INaturalist(Input):
|
|
7
|
+
"""
|
|
8
|
+
Query the iNaturalist database to retrieve photo links of a list of taxa.
|
|
9
|
+
|
|
10
|
+
Parameters
|
|
11
|
+
----------
|
|
12
|
+
query : list
|
|
13
|
+
The list of taxa to query.
|
|
14
|
+
bulk : bool, optional
|
|
15
|
+
If True, the function creates a bulk file.
|
|
16
|
+
For further information, see the documentation of the Biodumpy package. Default is False.
|
|
17
|
+
output_format : str, optional
|
|
18
|
+
The format of the output file. The options available is: 'json'. Default is 'json'.
|
|
19
|
+
|
|
20
|
+
Details
|
|
21
|
+
-------
|
|
22
|
+
To view the photo, append the image_id value to the end of the following link:
|
|
23
|
+
https://inaturalist-open-data.s3.amazonaws.com/photos/
|
|
24
|
+
|
|
25
|
+
Photos can be downloaded only if they have one of the valid licenses. If an iNaturalist photo has a different license than those specified, it can't be downloaded.
|
|
26
|
+
|
|
27
|
+
A detailed list of the available licenses can be found here:
|
|
28
|
+
|
|
29
|
+
- CC0 (Public Domain Dedication): No rights reserved. The creator waives all copyright, allowing anyone to use the work for any purpose without permission.
|
|
30
|
+
- CC BY (Attribution): Others can distribute, remix, adapt, and build upon the work, even commercially, as long as they give the appropriate credit to the creator.
|
|
31
|
+
- CC BY-NC (Attribution-NonCommercial): Others can remix, adapt, and build upon the work non-commercially. While new works must also acknowledge the creator, they don’t have to be licensed on the same terms.
|
|
32
|
+
- CC BY-NC-ND (Attribution-NonCommercial-NoDerivs): Others can download and share the work, but can’t change it or use it commercially, and must give credit to the creator.
|
|
33
|
+
- CC BY-SA (Attribution-ShareAlike): Others can remix, adapt, and build upon the work, even commercially, but must credit the creator and license their new creations under the same terms.
|
|
34
|
+
- CC BY-ND (Attribution-NoDerivs): Others can share the work, but they can't alter it or use it commercially, and they must credit the creator.
|
|
35
|
+
- CC BY-NC-SA (Attribution-NonCommercial-ShareAlike): Others can remix, adapt, and build upon the work non-commercially, as long as they give credit and license their new creations under the same terms.
|
|
36
|
+
|
|
37
|
+
Example
|
|
38
|
+
-------
|
|
39
|
+
>>> from biodumpy import Biodumpy
|
|
40
|
+
>>> from biodumpy.inputs import INaturalist
|
|
41
|
+
# List of taxa
|
|
42
|
+
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator', 'Bufo roseus', 'Stollia betae']
|
|
43
|
+
# Start the download
|
|
44
|
+
>>> bdp = Biodumpy([INaturalist(bulk=True)])
|
|
45
|
+
>>> bdp.start(taxa, output_path='./biodumpy/downloads/{date}/{module}/{name}')
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
output_format: str = "json",
|
|
51
|
+
bulk: bool = False,
|
|
52
|
+
):
|
|
53
|
+
super().__init__(output_format, bulk)
|
|
54
|
+
|
|
55
|
+
if output_format != "json":
|
|
56
|
+
raise ValueError('Invalid output_format. Expected "json".')
|
|
57
|
+
|
|
58
|
+
def _download(self, query, **kwargs) -> list:
|
|
59
|
+
# API request
|
|
60
|
+
response = requests.get(f"https://api.inaturalist.org/v1/taxa?q={query}&order=desc&order_by=observations_count")
|
|
61
|
+
|
|
62
|
+
if response.status_code != 200:
|
|
63
|
+
raise BiodumpyException(f"Observation request. Error {response.status_code}")
|
|
64
|
+
|
|
65
|
+
# Dictionary for empty records
|
|
66
|
+
photo_details_empty = {"taxon": query, "image_id": None, "license_code": None, "attribution": None}
|
|
67
|
+
photo_details = photo_details_empty
|
|
68
|
+
|
|
69
|
+
# Photo license values valid for the download
|
|
70
|
+
photo_license = ["cc0", "cc-by", "cc-by-nc", "cc-by-nc-nd", "cc-by-sa", "cc-by-nd", "cc-by-nc-sa"]
|
|
71
|
+
|
|
72
|
+
if response.status_code == 200:
|
|
73
|
+
results = response.json()["results"]
|
|
74
|
+
results_filtered = next(filter(lambda x: x["name"] == query, results), None)
|
|
75
|
+
|
|
76
|
+
if results_filtered:
|
|
77
|
+
taxon_id = results_filtered["id"]
|
|
78
|
+
|
|
79
|
+
if results_filtered["default_photo"] is not None:
|
|
80
|
+
photo_info = results_filtered["default_photo"] if results_filtered["default_photo"]["license_code"] else None
|
|
81
|
+
else:
|
|
82
|
+
photo_info = None
|
|
83
|
+
|
|
84
|
+
# Search the photo into the occurrences
|
|
85
|
+
if photo_info is None:
|
|
86
|
+
response_id = requests.get(f"https://api.inaturalist.org/v1/taxa/{taxon_id}")
|
|
87
|
+
|
|
88
|
+
if response_id.status_code == 200:
|
|
89
|
+
results_id = response_id.json()["results"]
|
|
90
|
+
photo_id = results_id[0]["taxon_photos"]
|
|
91
|
+
|
|
92
|
+
photo_details = next(filter(lambda x: x["photo"]["license_code"] in photo_license, photo_id), None)
|
|
93
|
+
|
|
94
|
+
if photo_details:
|
|
95
|
+
url_photo = photo_details["photo"]["url"]
|
|
96
|
+
url_photo = url_photo.split("/")[-2] + "/" + url_photo.split("/")[-1]
|
|
97
|
+
photo_details = {
|
|
98
|
+
"taxon": query,
|
|
99
|
+
"image_id": url_photo.replace("square", "medium"),
|
|
100
|
+
"license_code": photo_details["photo"]["license_code"],
|
|
101
|
+
"attribution": photo_details["photo"]["attribution"],
|
|
102
|
+
}
|
|
103
|
+
else:
|
|
104
|
+
photo_details = photo_details_empty
|
|
105
|
+
else:
|
|
106
|
+
url_photo = photo_info["url"]
|
|
107
|
+
url_photo = url_photo.split("/")[-2] + "/" + url_photo.split("/")[-1]
|
|
108
|
+
photo_details = {
|
|
109
|
+
"taxon": query,
|
|
110
|
+
"image_id": url_photo.replace("square", "medium"),
|
|
111
|
+
"license_code": photo_info["license_code"],
|
|
112
|
+
"attribution": photo_info["attribution"],
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return [photo_details]
|