apscale-gui 3.2.4__tar.gz → 3.3.0__tar.gz

{apscale_gui-3.2.4/apscale_gui.egg-info → apscale_gui-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apscale_gui
-Version: 3.2.4
+Version: 3.3.0
 Summary: Advanced Pipeline for Simple yet Comprehensive AnaLysEs of DNA metabarcoding data - Graphical User Interface
 Home-page: https://github.com/TillMacher/apscale_gui
 Author: Till-Hendrik Macher

{apscale_gui-3.2.4 → apscale_gui-3.3.0}/apscale_gui/apscale_gui.py

@@ -25,6 +25,9 @@ import importlib.metadata
 import subprocess
 import asyncio
 import sys
+import datetime, sys, getpass, requests_html, duckdb
+from tqdm import tqdm
+from stqdm import stqdm
 
 # demultiplexer2
 from demultiplexer2.create_primerset import create_primerset
@@ -51,9 +54,10 @@ from apscale_blast.__main__ import organism_filter as organism_filter
 
 # BOLDigger3
 from boldigger3 import id_engine
-from boldigger3 import metadata_download
 from boldigger3 import add_metadata
 from boldigger3 import select_top_hit
+from boldigger3 import download_database
+import requests
 
 # help texts
 n_cores_help = """
@@ -555,7 +559,117 @@ def download_seafile_zip(public_url, output_dir="downloads"):
 
     st.success(f"Finished the extraction of {len(list(output_dir.glob('*')))} databases!")
 
-def run_boldigger3(available_fasta_files, bold_modes, bold_databases):
+class DownloadProgressBar(tqdm):
+    def update_to(self, b=1, bsize=1, tsize=None):
+        if tsize is not None:
+            self.total = tsize
+        self.update(b * bsize - self.n)
+
+def run_boldigger3_db_download(output_dir, user_name, password):
+
+    """Download the latest BOLD public database and save it as a DuckDB file.
+
+    Logs into BOLD Systems, checks whether the local database is already up to
+    date, and if not, downloads the latest Parquet release and converts it to
+    DuckDB format.
+
+    Args:
+        output_file: Full path (including filename) for the output .ddb file.
+        user_name: BOLD Systems username.
+        password: BOLD Systems password.
+    """
+
+    # --- Login ---
+    data = {
+        "name": user_name,
+        "password": password,
+        "destination": "MAS_Management_UserConsole",
+        "loginType": "",
+    }
+
+    session = requests_html.HTMLSession()
+    session.post("https://bench.boldsystems.org/index.php/Login", data=data)
+
+    r = session.get("https://bench.boldsystems.org/index.php/datapackages/Latest")
+    log_out_text = r.html.find(".site-navigation > li:nth-child(4) > a:nth-child(1)")[0].text
+
+    if log_out_text != "Log out":
+        print(f"{datetime.datetime.now():%H:%M:%S}: Login failed, please check credentials.")
+        return
+
+    st.write(f"{datetime.datetime.now():%H:%M:%S}: Login successful, checking database status.")
+
+    # --- Check DB status ---
+    r = session.get("https://bench.boldsystems.org/index.php/datapackages/Latest")
+    button = r.html.find(
+        "div.row:nth-child(5) > div:nth-child(1) > table:nth-child(3) > tbody:nth-child(1) > tr:nth-child(4) > td:nth-child(2) > button:nth-child(1)"
+    )
+    package_id = button[0].attrs["data-package-id"]
+    data_url = button[0].attrs["data-url"]
+
+    parquet_path = output_dir / f"{package_id}.parquet"
+    database_path = output_dir / f"{package_id}.duckdb"
+
+    if database_path.is_file():
+        st.info(f"{datetime.datetime.now():%H:%M:%S}: Database is up to date.")
+        return
+
+    st.write(f"{datetime.datetime.now():%H:%M:%S}: Database does not exist or is outdated.")
+
+    # --- Download and convert ---
+    uid = session.get(f"https://bench.boldsystems.org{data_url}")
+    uid = uid.text.replace('"', "")
+    download_url = f"https://bench.boldsystems.org{data_url}&uid={uid}"
+
+    st.write(f"{datetime.datetime.now():%H:%M:%S}: Downloading latest database from {download_url}.")
+    print(f"{datetime.datetime.now():%H:%M:%S}: Downloading latest database from {download_url}.")
+    st.write(f"{datetime.datetime.now():%H:%M:%S}: Storing latest database as '{parquet_path}'.")
+    print(f"{datetime.datetime.now():%H:%M:%S}: Storing latest database as '{parquet_path}'.")
+
+    progress_bar = st.progress(0.0, text="Downloading public database...")
+    status = st.empty()
+    with requests.get(download_url, stream=True) as response:
+        response.raise_for_status()
+        total_size = int(response.headers.get("content-length", 0))
+        downloaded = 0
+        with open(parquet_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
+                    downloaded += len(chunk)
+                    if total_size > 0:
+                        progress = min(downloaded / total_size, 1.0)
+                        progress_bar.progress(progress)
+                        status.text(f"{downloaded / 1024 ** 2:.1f} MB / {total_size / 1024 ** 2:.1f} MB")
+    progress_bar.progress(1.0)
+    status.text("Download complete.")
+
+    if database_path.is_file():
+        database_path.unlink()
+        st.write(f"{datetime.datetime.now():%H:%M:%S}: Outdated database successfully removed.")
+
+    if not parquet_path.is_file():
+        st.write(f"{datetime.datetime.now():%H:%M:%S}: Could not find '{parquet_path}'.")
+        return
+
+    st.write(f"{datetime.datetime.now():%H:%M:%S}: Building new database.")
+
+    with duckdb.connect(database_path) as con:
+        con.execute(f"""
+            CREATE TABLE bold_public
+            AS SELECT
+                processid, sex, life_stage, inst,
+                "country/ocean", identified_by,
+                identification_method, coord, nuc, marker_code
+            FROM read_parquet('{parquet_path}')
+        """)
+
+    st.success(f"{datetime.datetime.now():%H:%M:%S}: New database saved at {database_path}.")
+
+    if parquet_path.is_file():
+        parquet_path.unlink()
+
+def run_boldigger3_identification(available_fasta_files, bold_modes, bold_databases, db_path):
 
     st.info('Starting BOLDigger3! Please refer to the terminal for live outputs!')
     print('')
@@ -575,22 +689,15 @@ def run_boldigger3(available_fasta_files, bold_modes, bold_databases):
     thresholds.append(50)
 
     # Collect input values
-    query_fasta = str(available_fasta_files[st.session_state['bold_query_fasta']])
+    query_fasta = Path(str(available_fasta_files[st.session_state['bold_query_fasta']]))
     mode = bold_modes[st.session_state['bold_mode']]
     db = bold_databases[st.session_state['bold_database']]
 
-    # download the current metadata from BOLD
-    metadata_download.main()
-
     # run the id engine
-    id_engine.main(
-        query_fasta,
-        db,
-        mode,
-    )
+    id_engine.main(query_fasta, db, mode)
 
     # add additional data via the metadata
-    add_metadata.main(query_fasta)
+    add_metadata.main(query_fasta, db_path)
 
     # select the top hit
     select_top_hit.main(query_fasta, thresholds)
@@ -680,7 +787,7 @@ def main():
                         if f.is_dir() and "_apscale" in f.name
                     ]
 
-                    if st.button(label='Remember project folder', key='remember_project_folder',use_container_width=True):
+                    if st.button(label='Remember project folder', key='remember_project_folder', use_container_width=True):
                         script_path = Path(__file__).resolve()
                         user_data_txt = script_path.parent / '_user_data' / 'user_data.txt'
                         user_data_txt.parent.mkdir(parents=True, exist_ok=True)
@@ -991,7 +1098,6 @@ def main():
                         update_settings_file(settings_xlsx, settings_dfs)
                         run_apscale(st.session_state['run_apscale_mode'], project_folder, modules_to_run)
 
-
                 ############################################################################################################
                 st.markdown("---")
                 st.header("APSCALE BLAST")
@@ -1037,25 +1143,33 @@ def main():
                 st.header("BOLDigger3")
                 st.subheader('Settings')
                 # General settings
+                available_databases = {Path(i).name:Path(i) for i in glob.glob(str(path_to_projects / 'APSCALE_databases' / '*.duckdb'))}
                 with st.expander("🗄️ Database & Operating Mode", expanded=False):
-                    bold_databases = {
-                                    'ANIMAL LIBRARY (PUBLIC)': 1,
-                                    'ANIMAL SPECIES-LEVEL LIBRARY (PUBLIC + PRIVATE)': 2,
-                                    'ANIMAL LIBRARY (PUBLIC+PRIVATE)': 3,
-                                    'VALIDATED CANADIAN ARTHROPOD LIBRARY': 4,
-                                    'PLANT LIBRARY (PUBLIC)': 5,
-                                    'FUNGI LIBRARY (PUBLIC)': 6,
-                                    'ANIMAL SECONDARY MARKERS (PUBLIC)': 7,
-                                    'VALIDATED ANIMAL RED LIST LIBRARY': 8
-                                }
-                    st.selectbox(label='BOLD database', key='bold_database', options=list(bold_databases.keys()))
-                    bold_modes = {
-                                    'Rapid Species Search': 1,
-                                    'Genus and Species Search': 2,
-                                    'Exhaustive Search': 3
-                                }
-                    st.selectbox(label='Operating mode', key='bold_mode', options=list(bold_modes.keys()))
-                    st.selectbox(label='Query FASTA', key='bold_query_fasta', options=list(available_fasta_files.keys()))
+                    col1, col2, col3 = st.columns(3)
+                    with col1:
+                        boldigger_db_file = st.selectbox(label='Local BOLD database file', key='boldigger_db_file', options=list(available_databases.keys()))
+                        st.selectbox(label='Query FASTA', key='bold_query_fasta', options=list(available_fasta_files.keys()))
+                    with col2:
+                        bold_modes = {
+                                        'Rapid Species Search': 1,
+                                        'Genus and Species Search': 2,
+                                        'Exhaustive Search': 3
+                                    }
+                        st.selectbox(label='Operating mode', key='bold_mode', options=list(bold_modes.keys()))
+                        bold_databases = {
+                                        'ANIMAL LIBRARY (PUBLIC)': 1,
+                                        'ANIMAL SPECIES-LEVEL LIBRARY (PUBLIC + PRIVATE)': 2,
+                                        'ANIMAL LIBRARY (PUBLIC+PRIVATE)': 3,
+                                        'VALIDATED CANADIAN ARTHROPOD LIBRARY': 4,
+                                        'PLANT LIBRARY (PUBLIC)': 5,
+                                        'FUNGI LIBRARY (PUBLIC)': 6,
+                                        'ANIMAL SECONDARY MARKERS (PUBLIC)': 7,
+                                        'VALIDATED ANIMAL RED LIST LIBRARY': 8
+                                    }
+                        st.selectbox(label='BOLD database', key='bold_database', options=list(bold_databases.keys()))
+                    with col3:
+                        st.text_input("Enter BOLD user name", key="bold_user")
+                        st.text_input("Enter BOLD password", type="password", key="bold_password")
                 # Thresholds
                 with st.expander("⚖️ Thresholds", expanded=False):
                     col1, col2 = st.columns(2)
@@ -1068,8 +1182,21 @@ def main():
                         st.text_input(label='Order (%)', key='bold_order', value=85)
 
                 st.subheader('Run BOLDigger3')
-                if st.button(f'Start taxonomic assignment against BOLD'):
-                    run_boldigger3(available_fasta_files, bold_modes, bold_databases)
+                if not st.session_state['boldigger_db_file']:
+                    st.info('No local BOLD database file found: Latest Database file will be downloaded.')
+
+                if st.button(f'Download latest BOLD database', use_container_width=True):
+                    if st.session_state['bold_user'] and st.session_state['bold_password']:
+                        bold_download_folder = path_to_projects / 'APSCALE_databases'
+                        run_boldigger3_db_download(bold_download_folder, st.session_state['bold_user'], st.session_state['bold_password'])
+                    else:
+                        st.error('Please provide a BOLD user name and password.')
+                if st.session_state['boldigger_db_file']:
+                    if st.button(f'Start taxonomic assignment against BOLD', use_container_width=True):
+                        db_path = available_databases[st.session_state['boldigger_db_file']]
+                        run_boldigger3_identification(available_fasta_files, bold_modes, bold_databases, db_path)
+                else:
+                    st.info('Please download the latest BOLD database first.')
 
                 ############################################################################################################
                 st.markdown("---")

{apscale_gui-3.2.4 → apscale_gui-3.3.0/apscale_gui.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apscale_gui
-Version: 3.2.4
+Version: 3.3.0
 Summary: Advanced Pipeline for Simple yet Comprehensive AnaLysEs of DNA metabarcoding data - Graphical User Interface
 Home-page: https://github.com/TillMacher/apscale_gui
 Author: Till-Hendrik Macher

{apscale_gui-3.2.4 → apscale_gui-3.3.0}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="apscale_gui",
-    version="3.2.4",
+    version="3.3.0",
     author="Till-Hendrik Macher",
     author_email="macher@uni-trier.de",
     description="Advanced Pipeline for Simple yet Comprehensive AnaLysEs of DNA metabarcoding data - Graphical User Interface",