biodumpy 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {biodumpy-0.1.2 → biodumpy-0.1.3}/PKG-INFO +1 -1
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/__init__.py +2 -3
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/biodumpy.py +23 -11
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/utils.py +42 -39
- biodumpy-0.1.3/biodumpy/version.py +3 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy.egg-info/PKG-INFO +1 -1
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy.egg-info/SOURCES.txt +1 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/setup.cfg +1 -1
- {biodumpy-0.1.2 → biodumpy-0.1.3}/LICENSE +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/README.md +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/input.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/BOLD.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/COL.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/GBIF.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/INaturalist.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/IUCN.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/NCBI.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/OBIS.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/ZooBank.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy/inputs/__init__.py +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy.egg-info/dependency_links.txt +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy.egg-info/not-zip-safe +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy.egg-info/requires.txt +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/biodumpy.egg-info/top_level.txt +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/pyproject.toml +0 -0
- {biodumpy-0.1.2 → biodumpy-0.1.3}/setup.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: biodumpy
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: A Comprehensive Biological Data Downloader from authoritative public databases like NCBI, Catalogue Of Life (COL), GBIF, BOLD, and more for any taxa.
|
|
5
5
|
Home-page: https://github.com/centrebalearbiodiversitat/biodumpy
|
|
6
6
|
Author: Cancellario, T.; Golomb, T.; Roldán, A; Far, T.
|
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
|
+
from logging.handlers import MemoryHandler
|
|
3
|
+
|
|
2
4
|
from .input import Input
|
|
3
|
-
from .utils import dump
|
|
5
|
+
from .utils import dump, create_directory
|
|
4
6
|
from tqdm import tqdm
|
|
5
7
|
import logging
|
|
6
8
|
|
|
7
9
|
|
|
8
|
-
# Setup logging configuration
|
|
9
|
-
logging.basicConfig(
|
|
10
|
-
level=logging.INFO,
|
|
11
|
-
format="%(asctime)s - %(levelname)s - %(message)s",
|
|
12
|
-
datefmt="%Y-%m-%d %H:%M:%S",
|
|
13
|
-
filename="biodumpy_downloader.log",
|
|
14
|
-
filemode="a", # is the appended mode
|
|
15
|
-
)
|
|
16
|
-
|
|
17
|
-
|
|
18
10
|
class BiodumpyException(Exception):
|
|
19
11
|
pass
|
|
20
12
|
|
|
@@ -47,6 +39,13 @@ class Biodumpy:
|
|
|
47
39
|
raise ValueError("Invalid query. Expected a list of taxa to query.")
|
|
48
40
|
|
|
49
41
|
current_date = datetime.now().strftime("%Y-%m-%d")
|
|
42
|
+
|
|
43
|
+
log_handler = MemoryHandler(capacity=1024)
|
|
44
|
+
|
|
45
|
+
logging.basicConfig(
|
|
46
|
+
level=logging.ERROR, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", handlers=[log_handler]
|
|
47
|
+
)
|
|
48
|
+
|
|
50
49
|
bulk_input = {}
|
|
51
50
|
try:
|
|
52
51
|
for el in tqdm(elements, desc="Biodumpy list", unit=" elements", disable=not self.loading_bar, smoothing=0):
|
|
@@ -94,3 +93,16 @@ class Biodumpy:
|
|
|
94
93
|
obj_list=payload,
|
|
95
94
|
output_format=inp.output_format,
|
|
96
95
|
)
|
|
96
|
+
|
|
97
|
+
if log_handler.buffer:
|
|
98
|
+
down_path = str()
|
|
99
|
+
for folder in output_path.split("/"):
|
|
100
|
+
if "{" in folder:
|
|
101
|
+
break
|
|
102
|
+
down_path = f"{down_path}{folder}/"
|
|
103
|
+
|
|
104
|
+
create_directory(down_path)
|
|
105
|
+
with open(f"{down_path}/dump.log", "w") as f:
|
|
106
|
+
for record in log_handler.buffer:
|
|
107
|
+
log_entry = f"{record.levelname}: {record.getMessage()}\n"
|
|
108
|
+
f.write(log_entry)
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import csv
|
|
2
2
|
import json
|
|
3
3
|
import os
|
|
4
|
-
from Bio import Entrez
|
|
5
4
|
|
|
6
5
|
|
|
7
6
|
def dump(file_name, obj_list, output_format="json"):
|
|
@@ -14,9 +13,7 @@ def dump(file_name, obj_list, output_format="json"):
|
|
|
14
13
|
output_format: output format. Default is "json". Other formats can be "fasta", "pdf".
|
|
15
14
|
"""
|
|
16
15
|
|
|
17
|
-
|
|
18
|
-
if not os.path.exists(directory):
|
|
19
|
-
os.makedirs(directory)
|
|
16
|
+
create_directory(file_name)
|
|
20
17
|
|
|
21
18
|
with open(f"{file_name}.{output_format}", "wb+" if output_format == "pdf" else "w+") as output_file:
|
|
22
19
|
if output_format == "fasta":
|
|
@@ -28,6 +25,12 @@ def dump(file_name, obj_list, output_format="json"):
|
|
|
28
25
|
json.dump(obj_list, output_file, indent=4)
|
|
29
26
|
|
|
30
27
|
|
|
28
|
+
def create_directory(file_name):
|
|
29
|
+
directory = os.path.dirname(file_name)
|
|
30
|
+
if not os.path.exists(directory):
|
|
31
|
+
os.makedirs(directory)
|
|
32
|
+
|
|
33
|
+
|
|
31
34
|
def clean_nones(value):
|
|
32
35
|
"""
|
|
33
36
|
Recursively remove all None values from dictionaries and lists, and returns
|
|
@@ -132,38 +135,38 @@ def parse_lat_lon(lat_lon: str):
|
|
|
132
135
|
return [lat, lon]
|
|
133
136
|
|
|
134
137
|
|
|
135
|
-
def download_taxonomy(taxon: str, mail="A.N.Other@example.com"):
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
138
|
+
# def download_taxonomy(taxon: str, mail="A.N.Other@example.com"):
|
|
139
|
+
# """
|
|
140
|
+
# Download taxonomy of a taxon from NCBI Taxonomy database.
|
|
141
|
+
#
|
|
142
|
+
# Args:
|
|
143
|
+
# taxon: String containing taxon name.
|
|
144
|
+
# mail: NCBI requires you to specify your email address with each request.
|
|
145
|
+
#
|
|
146
|
+
# Returns:
|
|
147
|
+
# None
|
|
148
|
+
#
|
|
149
|
+
# Example:
|
|
150
|
+
# x = download_taxonomy('Alytes muletensis')
|
|
151
|
+
# """
|
|
152
|
+
#
|
|
153
|
+
# Entrez.email = mail
|
|
154
|
+
#
|
|
155
|
+
# # Retrieve taxonomy ID by taxon name
|
|
156
|
+
# handle = Entrez.esearch(db="Taxonomy", term=f"{taxon}[All Names]", retmode="xml")
|
|
157
|
+
# taxon_id = Entrez.read(handle) # retrieve taxon ID
|
|
158
|
+
# handle.close()
|
|
159
|
+
#
|
|
160
|
+
# if int(taxon_id["Count"]) > 0:
|
|
161
|
+
# # Retrieve taxonomy by taxon ID
|
|
162
|
+
# handle = Entrez.efetch(db="Taxonomy", id=taxon_id["IdList"], retmode="xml")
|
|
163
|
+
# records = Entrez.read(handle)
|
|
164
|
+
# handle.close()
|
|
165
|
+
#
|
|
166
|
+
# lin = records[0]["LineageEx"]
|
|
167
|
+
# lin.append = {"TaxId": records[0]["TaxId"], "ScientificName": records[0]["ScientificName"].split()[-1], "Rank": records[0]["Rank"]}
|
|
168
|
+
#
|
|
169
|
+
# else:
|
|
170
|
+
# lin = None
|
|
171
|
+
#
|
|
172
|
+
# return lin
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: biodumpy
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: A Comprehensive Biological Data Downloader from authoritative public databases like NCBI, Catalogue Of Life (COL), GBIF, BOLD, and more for any taxa.
|
|
5
5
|
Home-page: https://github.com/centrebalearbiodiversitat/biodumpy
|
|
6
6
|
Author: Cancellario, T.; Golomb, T.; Roldán, A; Far, T.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[metadata]
|
|
2
2
|
name = biodumpy
|
|
3
|
-
version = attr: biodumpy.__version__
|
|
3
|
+
version = attr: biodumpy.version.__version__
|
|
4
4
|
description = A Comprehensive Biological Data Downloader from authoritative public databases like NCBI, Catalogue Of Life (COL), GBIF, BOLD, and more for any taxa.
|
|
5
5
|
long_description = file: README.md
|
|
6
6
|
long_description_content_type = text/markdown
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|