biodumpy 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- biodumpy-0.1.4/LICENSE.md +28 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/PKG-INFO +26 -40
- {biodumpy-0.1.2 → biodumpy-0.1.4}/README.md +22 -36
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/__init__.py +2 -3
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/biodumpy.py +39 -30
- biodumpy-0.1.4/biodumpy/input.py +26 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/inputs/BOLD.py +11 -23
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/inputs/COL.py +12 -15
- biodumpy-0.1.4/biodumpy/inputs/Crossref.py +80 -0
- biodumpy-0.1.4/biodumpy/inputs/GBIF.py +152 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/inputs/INaturalist.py +61 -54
- biodumpy-0.1.4/biodumpy/inputs/IUCN.py +159 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/inputs/NCBI.py +99 -56
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/inputs/OBIS.py +7 -13
- biodumpy-0.1.4/biodumpy/inputs/WORMS.py +64 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/inputs/ZooBank.py +19 -28
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy/inputs/__init__.py +2 -0
- biodumpy-0.1.4/biodumpy/utils.py +304 -0
- biodumpy-0.1.4/biodumpy/version.py +3 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy.egg-info/PKG-INFO +26 -40
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy.egg-info/SOURCES.txt +16 -2
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy.egg-info/requires.txt +5 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy.egg-info/top_level.txt +1 -0
- biodumpy-0.1.4/pyproject.toml +8 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/setup.cfg +9 -3
- biodumpy-0.1.4/tests/BOLD.py +136 -0
- biodumpy-0.1.4/tests/COL.py +101 -0
- biodumpy-0.1.4/tests/Crossref.py +142 -0
- biodumpy-0.1.4/tests/GBIF.py +228 -0
- biodumpy-0.1.4/tests/INaturalist.py +71 -0
- biodumpy-0.1.4/tests/IUCN.py +165 -0
- biodumpy-0.1.4/tests/NCBI.py +212 -0
- biodumpy-0.1.4/tests/OBIS.py +215 -0
- biodumpy-0.1.4/tests/WORMS.py +138 -0
- biodumpy-0.1.4/tests/ZooBank.py +111 -0
- biodumpy-0.1.4/tests/__init__.py +1 -0
- biodumpy-0.1.2/LICENSE +0 -8
- biodumpy-0.1.2/biodumpy/input.py +0 -8
- biodumpy-0.1.2/biodumpy/inputs/GBIF.py +0 -126
- biodumpy-0.1.2/biodumpy/inputs/IUCN.py +0 -143
- biodumpy-0.1.2/biodumpy/utils.py +0 -169
- biodumpy-0.1.2/pyproject.toml +0 -3
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy.egg-info/dependency_links.txt +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/biodumpy.egg-info/not-zip-safe +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.4}/setup.py +0 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
Copyright Statement
|
|
2
|
+
Copyright © 2025 CBB
|
|
3
|
+
|
|
4
|
+
# LICENSE for Biodumpy
|
|
5
|
+
|
|
6
|
+
## MIT License
|
|
7
|
+
``biodumpy`` is licensed under the MIT License. This means you are free to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the software, subject to the following conditions:
|
|
8
|
+
|
|
9
|
+
### TERMS OF SERVICE
|
|
10
|
+
This software is provided "as is", without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages, or other liability, whether in an action of contract, tort, or otherwise, arising from, out of, or in connection with the software or the use or other dealings in the software.
|
|
11
|
+
|
|
12
|
+
### Full License Text
|
|
13
|
+
You should have received a copy of the MIT License along with this program. If not, see https://opensource.org/licenses/MIT.
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## Creative Commons Attribution (CC BY)
|
|
17
|
+
Any creative works associated with this project (e.g., documentation, visuals, promotional materials) are licensed under the Creative Commons Attribution (CC BY) license. This means you are free to:
|
|
18
|
+
|
|
19
|
+
- Share — copy and redistribute the material in any medium or format
|
|
20
|
+
- Adapt — remix, transform, and build upon the material for any purpose, even commercially.
|
|
21
|
+
|
|
22
|
+
### Requirements
|
|
23
|
+
Under the following terms:
|
|
24
|
+
|
|
25
|
+
Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.
|
|
26
|
+
|
|
27
|
+
### Full License Text
|
|
28
|
+
The full text of the CC BY license can be found at: https://creativecommons.org/licenses/by/4.0/legalcode
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: biodumpy
|
|
3
|
-
Version: 0.1.
|
|
4
|
-
Summary: A Comprehensive Biological Data Downloader
|
|
3
|
+
Version: 0.1.4
|
|
4
|
+
Summary: A Comprehensive Biological Data Downloader.
|
|
5
5
|
Home-page: https://github.com/centrebalearbiodiversitat/biodumpy
|
|
6
|
-
Author: Cancellario, T.; Golomb, T.; Roldán, A
|
|
6
|
+
Author: Cancellario, T.; Golomb, T.; Roldán, A.; Far, A.
|
|
7
7
|
Author-email: t.cancellario@uib.eu
|
|
8
8
|
License: MIT License
|
|
9
9
|
Project-URL: Homepage, https://github.com/centrebalearbiodiversitat/biodumpy
|
|
@@ -26,7 +26,8 @@ Classifier: Topic :: Scientific/Engineering
|
|
|
26
26
|
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
27
27
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
28
28
|
Description-Content-Type: text/markdown
|
|
29
|
-
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
License-File: LICENSE.md
|
|
30
31
|
|
|
31
32
|
<img src="https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/Biodumpy_logo.png" alt="Project Logo" width="200">
|
|
32
33
|
|
|
@@ -38,27 +39,24 @@ License-File: LICENSE
|
|
|
38
39
|

|
|
39
40
|
|
|
40
41
|
|
|
41
|
-
|
|
42
42
|
## Overview
|
|
43
|
-
``biodumpy`` is a powerful and versatile Python package designed to simplify the process of retrieving biological information
|
|
44
|
-
from several public databases.
|
|
45
|
-
With ``biodumpy``, researchers can easily download and manage data from multiple sources, ensuring access to the most
|
|
46
|
-
up to date and comprehensive biological information available.
|
|
43
|
+
``biodumpy`` is a powerful and versatile Python package designed to simplify the process of retrieving biological information from several public databases. With ``biodumpy``, researchers can easily download and manage data from multiple sources, ensuring access to the most up to date and comprehensive biological information available.
|
|
47
44
|
|
|
48
45
|
> **Note:** This package is currently under development.
|
|
49
46
|
|
|
50
47
|
|
|
51
48
|
## Key Features
|
|
52
|
-
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically
|
|
53
|
-
designed for retrieving information from its respective source. The modules implemented so far are:
|
|
49
|
+
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically designed for retrieving information from its respective source. The modules implemented so far are:
|
|
54
50
|
|
|
55
|
-
- BOLD
|
|
56
|
-
- COL
|
|
57
|
-
-
|
|
51
|
+
- Barcode of life data system v4 (BOLD)
|
|
52
|
+
- Catalogue of life (COL)
|
|
53
|
+
- Crossref
|
|
54
|
+
- Global biodiversity information facility (GBIF)
|
|
58
55
|
- iNaturalist
|
|
59
|
-
- IUCN
|
|
60
|
-
- NCBI
|
|
61
|
-
- OBIS
|
|
56
|
+
- International union for the conservation of nature (IUCN)
|
|
57
|
+
- National center for biotechnology information (NCBI)
|
|
58
|
+
- Ocean biodiversity information system (OBIS)
|
|
59
|
+
- World register of marine species (WoRMS)
|
|
62
60
|
- ZooBank
|
|
63
61
|
|
|
64
62
|
This list can be expanded, thus suggestions and feedback are greatly appreciated.
|
|
@@ -96,18 +94,14 @@ In detail, we described:
|
|
|
96
94
|
from biodumpy.inputs import GBIF
|
|
97
95
|
|
|
98
96
|
# Create a list of taxa
|
|
99
|
-
taxa = [
|
|
100
|
-
|
|
101
|
-
'Bufotes viridis (Laurenti, 1768)',
|
|
102
|
-
'Hyla meridionalis Boettger, 1874',
|
|
103
|
-
'Anax imperator Leach, 1815'
|
|
104
|
-
]
|
|
97
|
+
taxa = ['Alytes muletensis (Sanchíz & Adrover, 1979)', 'Bufotes viridis (Laurenti, 1768)',
|
|
98
|
+
'Hyla meridionalis Boettger, 1874', 'Anax imperator Leach, 1815']
|
|
105
99
|
|
|
106
100
|
# Set the Biodumpy function with the specific parameters
|
|
107
|
-
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True)])
|
|
101
|
+
bdp = Biodumpy([GBIF(sleep=3, bulk=False, accepted_only=True)])
|
|
108
102
|
|
|
109
103
|
# Start the download
|
|
110
|
-
bdp.start(taxa, output_path='
|
|
104
|
+
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
111
105
|
```
|
|
112
106
|
|
|
113
107
|
**Example N.2**
|
|
@@ -123,16 +117,11 @@ In detail, we described:
|
|
|
123
117
|
api_key = 'YOUR_IUCN_API_KEY'
|
|
124
118
|
|
|
125
119
|
# Create a list of taxa
|
|
126
|
-
taxa = [
|
|
127
|
-
'Alytes muletensis',
|
|
128
|
-
'Bufotes viridis',
|
|
129
|
-
'Hyla meridionalis',
|
|
130
|
-
'Anax imperator'
|
|
131
|
-
]
|
|
120
|
+
taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator']
|
|
132
121
|
|
|
133
122
|
# Set the Biodumpy functions with the specific parameters
|
|
134
123
|
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True),
|
|
135
|
-
IUCN(api_key=api_key, bulk=True, region=['
|
|
124
|
+
IUCN(api_key=api_key, bulk=True, region=['Global'])])
|
|
136
125
|
|
|
137
126
|
# Start the download
|
|
138
127
|
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
@@ -144,23 +133,20 @@ For detailed documentation and tutorials, please visit the ``biodumpy`` Read the
|
|
|
144
133
|
|
|
145
134
|
## Contribution
|
|
146
135
|
``biodumpy`` is an open-source project, and contributions are welcome!
|
|
147
|
-
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub
|
|
148
|
-
|
|
136
|
+
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub repository or contact with the support team
|
|
137
|
+
[✉️ here](mailto:t.cancellario@uib.eu?subject=biodumpy_support).
|
|
149
138
|
|
|
150
139
|
|
|
151
140
|
## License
|
|
152
|
-
``biodumpy`` is licensed under the
|
|
141
|
+
``biodumpy`` is licensed under the MIT License for its software components. Additionally, any creative works associated with this project—such as documentation, visual assets, or other non-code materials—are licensed under the Creative Commons Attribution (CC BY 4.0) license. See the [LICENSE](LICENSE.md) file for full details.
|
|
153
142
|
|
|
154
143
|
|
|
155
144
|
## Acknowledgments
|
|
156
|
-
The project
|
|
157
|
-
the Government of the Balearic Islands.
|
|
158
|
-
|
|
145
|
+
The project is developed by the "Centre Balear de Biodiversitat" (CBB) at the University of the Balearic Islands, with support from MCIN and funding from the European Union—NextGenerationEU (PRTR-C17.I1), as well as the Government of the Balearic Islands.
|
|
159
146
|
|
|
160
147
|
|
|
161
148
|
<hr>
|
|
162
149
|
<div style="display: flex; justify-content: center">
|
|
163
|
-
<img src='https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/
|
|
150
|
+
<img src='https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/founding.png' alt='logo_cbb' width='200'>
|
|
164
151
|
</div>
|
|
165
152
|
|
|
166
|
-
|
|
@@ -8,27 +8,24 @@
|
|
|
8
8
|

|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
|
|
12
11
|
## Overview
|
|
13
|
-
``biodumpy`` is a powerful and versatile Python package designed to simplify the process of retrieving biological information
|
|
14
|
-
from several public databases.
|
|
15
|
-
With ``biodumpy``, researchers can easily download and manage data from multiple sources, ensuring access to the most
|
|
16
|
-
up to date and comprehensive biological information available.
|
|
12
|
+
``biodumpy`` is a powerful and versatile Python package designed to simplify the process of retrieving biological information from several public databases. With ``biodumpy``, researchers can easily download and manage data from multiple sources, ensuring access to the most up to date and comprehensive biological information available.
|
|
17
13
|
|
|
18
14
|
> **Note:** This package is currently under development.
|
|
19
15
|
|
|
20
16
|
|
|
21
17
|
## Key Features
|
|
22
|
-
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically
|
|
23
|
-
designed for retrieving information from its respective source. The modules implemented so far are:
|
|
18
|
+
``biodumpy`` offers dedicated modules for each supported database, with each module featuring functions specifically designed for retrieving information from its respective source. The modules implemented so far are:
|
|
24
19
|
|
|
25
|
-
- BOLD
|
|
26
|
-
- COL
|
|
27
|
-
-
|
|
20
|
+
- Barcode of life data system v4 (BOLD)
|
|
21
|
+
- Catalogue of life (COL)
|
|
22
|
+
- Crossref
|
|
23
|
+
- Global biodiversity information facility (GBIF)
|
|
28
24
|
- iNaturalist
|
|
29
|
-
- IUCN
|
|
30
|
-
- NCBI
|
|
31
|
-
- OBIS
|
|
25
|
+
- International union for the conservation of nature (IUCN)
|
|
26
|
+
- National center for biotechnology information (NCBI)
|
|
27
|
+
- Ocean biodiversity information system (OBIS)
|
|
28
|
+
- World register of marine species (WoRMS)
|
|
32
29
|
- ZooBank
|
|
33
30
|
|
|
34
31
|
This list can be expanded, thus suggestions and feedback are greatly appreciated.
|
|
@@ -66,18 +63,14 @@ In detail, we described:
|
|
|
66
63
|
from biodumpy.inputs import GBIF
|
|
67
64
|
|
|
68
65
|
# Create a list of taxa
|
|
69
|
-
taxa = [
|
|
70
|
-
|
|
71
|
-
'Bufotes viridis (Laurenti, 1768)',
|
|
72
|
-
'Hyla meridionalis Boettger, 1874',
|
|
73
|
-
'Anax imperator Leach, 1815'
|
|
74
|
-
]
|
|
66
|
+
taxa = ['Alytes muletensis (Sanchíz & Adrover, 1979)', 'Bufotes viridis (Laurenti, 1768)',
|
|
67
|
+
'Hyla meridionalis Boettger, 1874', 'Anax imperator Leach, 1815']
|
|
75
68
|
|
|
76
69
|
# Set the Biodumpy function with the specific parameters
|
|
77
|
-
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True)])
|
|
70
|
+
bdp = Biodumpy([GBIF(sleep=3, bulk=False, accepted_only=True)])
|
|
78
71
|
|
|
79
72
|
# Start the download
|
|
80
|
-
bdp.start(taxa, output_path='
|
|
73
|
+
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
81
74
|
```
|
|
82
75
|
|
|
83
76
|
**Example N.2**
|
|
@@ -93,16 +86,11 @@ In detail, we described:
|
|
|
93
86
|
api_key = 'YOUR_IUCN_API_KEY'
|
|
94
87
|
|
|
95
88
|
# Create a list of taxa
|
|
96
|
-
taxa = [
|
|
97
|
-
'Alytes muletensis',
|
|
98
|
-
'Bufotes viridis',
|
|
99
|
-
'Hyla meridionalis',
|
|
100
|
-
'Anax imperator'
|
|
101
|
-
]
|
|
89
|
+
taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator']
|
|
102
90
|
|
|
103
91
|
# Set the Biodumpy functions with the specific parameters
|
|
104
92
|
bdp = Biodumpy([GBIF(bulk=False, accepted_only=True),
|
|
105
|
-
IUCN(api_key=api_key, bulk=True, region=['
|
|
93
|
+
IUCN(api_key=api_key, bulk=True, region=['Global'])])
|
|
106
94
|
|
|
107
95
|
# Start the download
|
|
108
96
|
bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
@@ -114,21 +102,19 @@ For detailed documentation and tutorials, please visit the ``biodumpy`` Read the
|
|
|
114
102
|
|
|
115
103
|
## Contribution
|
|
116
104
|
``biodumpy`` is an open-source project, and contributions are welcome!
|
|
117
|
-
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub
|
|
118
|
-
|
|
105
|
+
If you have ideas for new features, bug fixes, or improvements, please submit an issue or pull request in our GitHub repository or contact with the support team
|
|
106
|
+
[✉️ here](mailto:t.cancellario@uib.eu?subject=biodumpy_support).
|
|
119
107
|
|
|
120
108
|
|
|
121
109
|
## License
|
|
122
|
-
``biodumpy`` is licensed under the
|
|
110
|
+
``biodumpy`` is licensed under the MIT License for its software components. Additionally, any creative works associated with this project—such as documentation, visual assets, or other non-code materials—are licensed under the Creative Commons Attribution (CC BY 4.0) license. See the [LICENSE](LICENSE.md) file for full details.
|
|
123
111
|
|
|
124
112
|
|
|
125
113
|
## Acknowledgments
|
|
126
|
-
The project
|
|
127
|
-
the Government of the Balearic Islands.
|
|
128
|
-
|
|
114
|
+
The project is developed by the "Centre Balear de Biodiversitat" (CBB) at the University of the Balearic Islands, with support from MCIN and funding from the European Union—NextGenerationEU (PRTR-C17.I1), as well as the Government of the Balearic Islands.
|
|
129
115
|
|
|
130
116
|
|
|
131
117
|
<hr>
|
|
132
118
|
<div style="display: flex; justify-content: center">
|
|
133
|
-
<img src='https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/
|
|
134
|
-
</div>
|
|
119
|
+
<img src='https://raw.githubusercontent.com/centrebalearbiodiversitat/biodumpy/refs/heads/master/docs/source/static/founding.png' alt='logo_cbb' width='200'>
|
|
120
|
+
</div>
|
|
@@ -1,20 +1,14 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import time
|
|
1
3
|
from datetime import datetime
|
|
4
|
+
from logging.handlers import MemoryHandler
|
|
5
|
+
|
|
2
6
|
from .input import Input
|
|
3
|
-
from .utils import dump
|
|
7
|
+
from .utils import dump, create_directory
|
|
4
8
|
from tqdm import tqdm
|
|
5
9
|
import logging
|
|
6
10
|
|
|
7
11
|
|
|
8
|
-
# Setup logging configuration
|
|
9
|
-
logging.basicConfig(
|
|
10
|
-
level=logging.INFO,
|
|
11
|
-
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
12
|
-
datefmt="%Y-%m-%d %H:%M:%S",
|
|
13
|
-
filename="biodumpy_downloader.log",
|
|
14
|
-
filemode="a", # is the appended mode
|
|
15
|
-
)
|
|
16
|
-
|
|
17
|
-
|
|
18
12
|
class BiodumpyException(Exception):
|
|
19
13
|
pass
|
|
20
14
|
|
|
@@ -35,11 +29,12 @@ class Biodumpy:
|
|
|
35
29
|
Default is True
|
|
36
30
|
"""
|
|
37
31
|
|
|
38
|
-
def __init__(self, inputs: list[Input], loading_bar: bool =
|
|
32
|
+
def __init__(self, inputs: list[Input], loading_bar: bool = True, debug: bool = False) -> None:
|
|
39
33
|
super().__init__()
|
|
40
34
|
self.inputs = inputs
|
|
41
35
|
self.debug = debug
|
|
42
36
|
self.loading_bar = loading_bar
|
|
37
|
+
# self.loading_bar = not debug and loading_bar
|
|
43
38
|
|
|
44
39
|
# elements must be a flat list of strings
|
|
45
40
|
def start(self, elements, output_path="downloads/{date}/{module}/{name}"):
|
|
@@ -47,12 +42,15 @@ class Biodumpy:
|
|
|
47
42
|
raise ValueError("Invalid query. Expected a list of taxa to query.")
|
|
48
43
|
|
|
49
44
|
current_date = datetime.now().strftime("%Y-%m-%d")
|
|
45
|
+
|
|
46
|
+
log_handler = MemoryHandler(capacity=1024)
|
|
47
|
+
|
|
48
|
+
logging.basicConfig(level=logging.ERROR, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", handlers=[log_handler])
|
|
49
|
+
|
|
50
50
|
bulk_input = {}
|
|
51
|
+
last_tick = {}
|
|
51
52
|
try:
|
|
52
|
-
for el in tqdm(elements, desc="Biodumpy list", unit=" elements", disable=not self.loading_bar, smoothing=0):
|
|
53
|
-
if not el:
|
|
54
|
-
continue
|
|
55
|
-
|
|
53
|
+
for el in tqdm(elements, desc="Biodumpy list", unit=" elements", disable=not self.loading_bar, smoothing=0, file=sys.stdout, colour="#FECC45"):
|
|
56
54
|
if isinstance(el, str):
|
|
57
55
|
el = {"query": el}
|
|
58
56
|
|
|
@@ -62,16 +60,22 @@ class Biodumpy:
|
|
|
62
60
|
|
|
63
61
|
name = el["query"]
|
|
64
62
|
clean_name = name.replace("/", "_")
|
|
65
|
-
|
|
66
|
-
print(f"Downloading {name}...")
|
|
63
|
+
tqdm.write(f"Downloading {name}...")
|
|
67
64
|
|
|
68
65
|
for inp in self.inputs:
|
|
69
66
|
module_name = type(inp).__name__
|
|
70
67
|
logging.info(f"biodumpy initialized with {module_name} inputs. Taxon: {name}")
|
|
71
68
|
|
|
72
69
|
try:
|
|
70
|
+
if module_name in last_tick:
|
|
71
|
+
delta_last_call = time.time() - last_tick[module_name]
|
|
72
|
+
if delta_last_call < inp.sleep:
|
|
73
|
+
if self.debug:
|
|
74
|
+
tqdm.write(f"[{module_name}] Blocking for {inp.sleep - delta_last_call} seconds...")
|
|
75
|
+
time.sleep(inp.sleep - delta_last_call)
|
|
76
|
+
tqdm.write(f"[{module_name}] Downloading...")
|
|
73
77
|
payload = inp._download(**el)
|
|
74
|
-
|
|
78
|
+
last_tick[module_name] = time.time()
|
|
75
79
|
except Exception as e:
|
|
76
80
|
logging.error(f'[{module_name}] Failed to download data for "{name}": {str(e)} \n')
|
|
77
81
|
continue
|
|
@@ -80,17 +84,22 @@ class Biodumpy:
|
|
|
80
84
|
if inp not in bulk_input:
|
|
81
85
|
bulk_input[inp] = []
|
|
82
86
|
bulk_input[inp].extend(payload)
|
|
83
|
-
|
|
84
87
|
else:
|
|
85
|
-
dump(
|
|
86
|
-
file_name=f"{output_path.format(date=current_date, module=module_name, name=clean_name)}",
|
|
87
|
-
obj_list=payload,
|
|
88
|
-
output_format=inp.output_format,
|
|
89
|
-
)
|
|
88
|
+
dump(file_name=f"{output_path.format(date=current_date, module=module_name, name=clean_name)}", obj_list=payload, output_format=inp.output_format)
|
|
90
89
|
finally:
|
|
91
90
|
for inp, payload in bulk_input.items():
|
|
92
|
-
dump(
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
)
|
|
91
|
+
dump(file_name=output_path.format(date=current_date, module=type(inp).__name__, name="bulk"), obj_list=payload, output_format=inp.output_format)
|
|
92
|
+
|
|
93
|
+
if log_handler.buffer:
|
|
94
|
+
print("---- Please review the dump file; errors have been detected ----")
|
|
95
|
+
down_path = str()
|
|
96
|
+
for folder in output_path.split("/"):
|
|
97
|
+
if "{" in folder:
|
|
98
|
+
break
|
|
99
|
+
down_path = f"{down_path}{folder}/"
|
|
100
|
+
|
|
101
|
+
create_directory(down_path)
|
|
102
|
+
with open(f"{down_path}/dump_{current_date}.log", "w") as f:
|
|
103
|
+
for record in log_handler.buffer:
|
|
104
|
+
log_entry = f"{record.levelname}: {record.getMessage()}\n"
|
|
105
|
+
f.write(log_entry)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
class Input:
|
|
2
|
+
"""
|
|
3
|
+
Base class for handling input operations with customizable parameters.
|
|
4
|
+
|
|
5
|
+
Parameters
|
|
6
|
+
----------
|
|
7
|
+
sleep : float, optional
|
|
8
|
+
Time in seconds to wait between consecutive API requests.
|
|
9
|
+
Default is 0.1 seconds.
|
|
10
|
+
output_format : str, optional
|
|
11
|
+
The format of the output file. Available option is 'json'.
|
|
12
|
+
Default is 'json'.
|
|
13
|
+
bulk : bool, optional
|
|
14
|
+
If True, enables bulk processing and creates a bulk output file.
|
|
15
|
+
For more details, refer to the documentation of the biodumpy package.
|
|
16
|
+
Default is False.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, sleep: float = 3, output_format: str = "json", bulk: bool = False):
|
|
20
|
+
super().__init__()
|
|
21
|
+
self.sleep = sleep
|
|
22
|
+
self.output_format = output_format
|
|
23
|
+
self.bulk = bulk
|
|
24
|
+
|
|
25
|
+
def _download(self, **kwargs) -> list:
|
|
26
|
+
raise NotImplementedError()
|
|
@@ -13,18 +13,7 @@ class BOLD(Input):
|
|
|
13
13
|
The list of taxa to query.
|
|
14
14
|
summary : bool, optional
|
|
15
15
|
If True, the function returns a summary of the downloaded metadata instead of the full records.
|
|
16
|
-
|
|
17
|
-
fasta : bool, optional
|
|
18
|
-
If True, the function downloads the FASTA file.
|
|
19
|
-
Default is False.
|
|
20
|
-
bulk : bool, optional
|
|
21
|
-
If True, the function creates a bulk file.
|
|
22
|
-
For further information, see the documentation of the Biodumpy package.
|
|
23
|
-
Default is False.
|
|
24
|
-
output_format : string, optional
|
|
25
|
-
The format of the output file.
|
|
26
|
-
The options available are: 'json', 'fasta', 'pdf'.
|
|
27
|
-
Default is 'json'.
|
|
16
|
+
Default is False.
|
|
28
17
|
|
|
29
18
|
Details
|
|
30
19
|
-------
|
|
@@ -52,19 +41,20 @@ class BOLD(Input):
|
|
|
52
41
|
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
53
42
|
"""
|
|
54
43
|
|
|
55
|
-
def __init__(self, summary: bool = False,
|
|
56
|
-
super().__init__(
|
|
44
|
+
def __init__(self, summary: bool = False, **kwargs):
|
|
45
|
+
super().__init__(**kwargs)
|
|
57
46
|
self.summary = summary
|
|
58
|
-
self.fasta = fasta
|
|
47
|
+
# self.fasta = fasta
|
|
59
48
|
|
|
60
|
-
if self.fasta and output_format != "fasta":
|
|
61
|
-
|
|
49
|
+
# if self.fasta and output_format != "fasta":
|
|
50
|
+
# raise ValueError("Invalid output_format. Expected fasta.")
|
|
62
51
|
|
|
63
|
-
if output_format not in {"json", "fasta"}:
|
|
52
|
+
if self.output_format not in {"json", "fasta"}:
|
|
64
53
|
raise ValueError('Invalid output_format. Expected "json" or "fasta".')
|
|
65
54
|
|
|
66
55
|
def _download(self, query, **kwargs) -> list:
|
|
67
|
-
if self.fasta:
|
|
56
|
+
# if self.fasta:
|
|
57
|
+
if self.output_format == "fasta":
|
|
68
58
|
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/sequence?taxon={query}")
|
|
69
59
|
|
|
70
60
|
if response.status_code != 200:
|
|
@@ -78,7 +68,6 @@ class BOLD(Input):
|
|
|
78
68
|
fasta_entries = [f">{entry}" for entry in data_str.split(">") if entry]
|
|
79
69
|
|
|
80
70
|
return fasta_entries
|
|
81
|
-
|
|
82
71
|
else:
|
|
83
72
|
response = requests.get(f"http://v4.boldsystems.org/index.php/API_Public/combined?taxon={query}&format=json")
|
|
84
73
|
|
|
@@ -124,8 +113,7 @@ class BOLD(Input):
|
|
|
124
113
|
}
|
|
125
114
|
)
|
|
126
115
|
else:
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
)
|
|
116
|
+
res = results["bold_records"]["records"] if "bold_records" in results and "records" in results["bold_records"] else []
|
|
117
|
+
payload = list(res.values())
|
|
130
118
|
|
|
131
119
|
return payload
|
|
@@ -11,18 +11,13 @@ class COL(Input):
|
|
|
11
11
|
----------
|
|
12
12
|
query : list
|
|
13
13
|
The list of taxa to query.
|
|
14
|
+
dataset_key : int
|
|
15
|
+
The dataset key to query. Please visit https://www.catalogueoflife.org/data/metadata to check the latest ChecklistBank release.
|
|
16
|
+
Default is 9923.
|
|
14
17
|
check_syn : bool, optional
|
|
15
18
|
If True, the function returns only the accepted nomenclature of a taxon.
|
|
16
19
|
See Detail section for further information.
|
|
17
20
|
Default is False.
|
|
18
|
-
bulk : bool, optional
|
|
19
|
-
If True, the function creates a bulk file.
|
|
20
|
-
For further information, see the documentation of the Biodumpy package.
|
|
21
|
-
Default is False.
|
|
22
|
-
output_format : string, optional
|
|
23
|
-
The format of the output file.
|
|
24
|
-
The options available are: 'json', 'fasta', 'pdf'.
|
|
25
|
-
Default is 'json'.
|
|
26
21
|
|
|
27
22
|
Details
|
|
28
23
|
-------
|
|
@@ -35,25 +30,27 @@ class COL(Input):
|
|
|
35
30
|
-------
|
|
36
31
|
>>> from biodumpy import Biodumpy
|
|
37
32
|
>>> from biodumpy.inputs import COL
|
|
38
|
-
# List
|
|
33
|
+
# List of taxa
|
|
39
34
|
>>> taxa = ['Alytes muletensis', 'Bufotes viridis', 'Hyla meridionalis', 'Anax imperator', 'Bufo roseus', 'Stollia betae']
|
|
40
35
|
# Start the download
|
|
41
36
|
>>> bdp = Biodumpy([COL(bulk=True, check_syn=False)])
|
|
42
|
-
>>> bdp.start(taxa, output_path='./
|
|
37
|
+
>>> bdp.start(taxa, output_path='./downloads/{date}/{module}/{name}')
|
|
43
38
|
"""
|
|
44
39
|
|
|
45
40
|
ACCEPTED_TERMS = ["accepted", "provisionally accepted"]
|
|
46
41
|
|
|
47
|
-
def __init__(self,
|
|
48
|
-
super().__init__(
|
|
42
|
+
def __init__(self, check_syn: bool = False, dataset_key: int = 9923, **kwargs):
|
|
43
|
+
super().__init__(**kwargs)
|
|
49
44
|
self.check_syn = check_syn
|
|
45
|
+
self.dataset_key = dataset_key
|
|
50
46
|
|
|
51
|
-
if output_format != "json":
|
|
47
|
+
if self.output_format != "json":
|
|
52
48
|
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
53
49
|
|
|
54
50
|
def _download(self, query, **kwargs) -> list:
|
|
55
51
|
response = requests.get(
|
|
56
|
-
f"https://api.checklistbank.org/dataset/
|
|
52
|
+
f"https://api.checklistbank.org/dataset/{self.dataset_key}/nameusage/search?",
|
|
53
|
+
params={"q": query, "content": "SCIENTIFIC_NAME", "type": "EXACT", "offset": 0, "limit": 10},
|
|
57
54
|
)
|
|
58
55
|
|
|
59
56
|
if response.status_code != 200:
|
|
@@ -70,7 +67,7 @@ class COL(Input):
|
|
|
70
67
|
if len(result) > 1:
|
|
71
68
|
ids = [item.get("id") for item in result if "id" in item]
|
|
72
69
|
ids = ", ".join(ids)
|
|
73
|
-
id_input = input(f"Please enter the correct taxon ID of {query} \n ID: {ids}; Skip \
|
|
70
|
+
id_input = input(f"Please enter the correct taxon ID of {query} \n ID: {ids}; Skip \nInsert the ID:")
|
|
74
71
|
|
|
75
72
|
if id_input == "Skip":
|
|
76
73
|
result = [{"id": None, "usage": None, "status": None, "classification": None}]
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import requests
|
|
2
|
+
|
|
3
|
+
from biodumpy.utils import remove_tags
|
|
4
|
+
from biodumpy import Input, BiodumpyException
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Crossref(Input):
|
|
8
|
+
"""
|
|
9
|
+
Query the Crossref database to retrieve scientific bibliographic metadata information.
|
|
10
|
+
|
|
11
|
+
Parameters
|
|
12
|
+
----------
|
|
13
|
+
query : list
|
|
14
|
+
The list of DOIs to query.
|
|
15
|
+
summary : bool, optional
|
|
16
|
+
If True, the function returns a summary of the downloaded metadata instead of the full records.
|
|
17
|
+
Default is False.
|
|
18
|
+
|
|
19
|
+
Details
|
|
20
|
+
-------
|
|
21
|
+
When `summary` is True, the resulting JSON will include the following information:
|
|
22
|
+
- `publisher`: The name of the publishing entity responsible for releasing the publication.
|
|
23
|
+
- `container-title`: The title of the journal or book in which the research is published.
|
|
24
|
+
- `DOI`: The Digital Object Identifier assigned to the publication.
|
|
25
|
+
- `type`: The type of the publication, such as article, book chapter, report, etc.
|
|
26
|
+
- `language`: The language in which the publication is written.
|
|
27
|
+
- `URL`: A direct link to the publication.
|
|
28
|
+
- `published`: The date when the research was published.
|
|
29
|
+
- `title`: The title of the publication.
|
|
30
|
+
- `author`: The names of the authors who contributed to the research along with their main academic information.
|
|
31
|
+
- `abstract`: The publication abstract (if available).
|
|
32
|
+
|
|
33
|
+
Example
|
|
34
|
+
-------
|
|
35
|
+
>>> from biodumpy import Biodumpy
|
|
36
|
+
>>> from biodumpy.inputs import Crossref
|
|
37
|
+
# Create a list of DOIs
|
|
38
|
+
>>> dois = ["10.1038/s44185-022-00001-3", "10.1111/gcb.17059", "10.1016/j.ecoinf.2024.102629"]
|
|
39
|
+
# Set the Biodumpy functions with the specific parameters
|
|
40
|
+
>>> bdp = Biodumpy([Crossref(bulk=True)])
|
|
41
|
+
>>> bdp.start(dois, output_path='./downloads/{date}/{module}/{name}')
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, summary: bool = False, **kwargs):
|
|
45
|
+
super().__init__(**kwargs)
|
|
46
|
+
self.summary = summary
|
|
47
|
+
|
|
48
|
+
if self.output_format != "json":
|
|
49
|
+
raise ValueError("Invalid output_format. Expected 'json'.")
|
|
50
|
+
|
|
51
|
+
def _download(self, query, **kwargs) -> list:
|
|
52
|
+
payload = []
|
|
53
|
+
|
|
54
|
+
response = requests.get(f"https://api.crossref.org/works/{query}")
|
|
55
|
+
|
|
56
|
+
if response.status_code != 200:
|
|
57
|
+
raise BiodumpyException(f"Reference request. Error {response.status_code}")
|
|
58
|
+
|
|
59
|
+
if response.content:
|
|
60
|
+
message = response.json().get("message", {})
|
|
61
|
+
if self.summary:
|
|
62
|
+
abstract = message.get("abstract", None)
|
|
63
|
+
payload.append(
|
|
64
|
+
{
|
|
65
|
+
"publisher": message.get("publisher"),
|
|
66
|
+
"container-title": message.get("container-title")[0],
|
|
67
|
+
"DOI": message.get("DOI"),
|
|
68
|
+
"type": message.get("type"),
|
|
69
|
+
"language": message.get("language"),
|
|
70
|
+
"URL": message.get("URL"),
|
|
71
|
+
"published": message.get("published")["date-parts"],
|
|
72
|
+
"title": message.get("title")[0],
|
|
73
|
+
"author": message.get("author"),
|
|
74
|
+
"abstract": remove_tags(abstract) if abstract else None,
|
|
75
|
+
}
|
|
76
|
+
)
|
|
77
|
+
else:
|
|
78
|
+
payload = [message]
|
|
79
|
+
|
|
80
|
+
return payload
|