pyadps 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

pyadps/pages/03_Download_Raw_File.py

@@ -34,7 +34,7 @@ if "add_attributes_DRW" not in st.session_state:
 
 if "file_prefix" not in st.session_state:
     raw_basename = os.path.basename(st.session_state.fname)
-    st.session_state.filename = os.path.splitext(raw_basename)[0] 
+    st.session_state.filename = os.path.splitext(raw_basename)[0]
     st.session_state.file_prefix = st.session_state.filename
 
 
@@ -45,7 +45,6 @@ if "filename" not in st.session_state:
     st.session_state.filename = ""  # <-- Default file name if not passed
 
 
-
 ################ Functions #######################
 @st.cache_data()
 def file_access(uploaded_file):
@@ -63,6 +62,7 @@ def read_file(filepath):
         ds.fixensemble()
     st.session_state.ds = ds
 
+
 @st.cache_data
 def get_prefixed_filename(base_name):
     """Generates the file name with the optional prefix."""
@@ -74,57 +74,55 @@ def get_prefixed_filename(base_name):
 @st.cache_data
 def file_write(path, axis_option, add_attributes=True):
     tempdirname = tempfile.TemporaryDirectory(delete=False)
-    st.session_state.rawfilename = os.path.join(tempdirname.name, get_prefixed_filename("RAW_DAT.nc"))
+    st.session_state.rawfilename = os.path.join(
+        tempdirname.name, get_prefixed_filename("RAW_DAT.nc")
+    )
 
     if add_attributes:
         wr.rawnc(
             path,
             st.session_state.rawfilename,
-            st.session_state.date1,
-            axis_option,
+            axis_option=axis_option,
             attributes=st.session_state.attributes,
         )
     else:
-        wr.rawnc(
-            path, st.session_state.rawfilename, st.session_state.date1, axis_option
-        )
+        wr.rawnc(path, st.session_state.rawfilename, axis_option)
+
 
 @st.cache_data
 def file_write_flead(path, axis_option, add_attributes=True):
     tempvardirname = tempfile.TemporaryDirectory(delete=False)
-    st.session_state.fleadfilename = os.path.join(tempvardirname.name, get_prefixed_filename("RAW_FIX.nc"))
+    st.session_state.fleadfilename = os.path.join(
+        tempvardirname.name, get_prefixed_filename("RAW_FIX.nc")
+    )
 
     if add_attributes:
         wr.flead_nc(
             path,
             st.session_state.fleadfilename,
-            st.session_state.date2,
-            axis_option,
+            axis_option=axis_option,
             attributes=st.session_state.attributes,
         )
     else:
-        wr.flead_nc(
-            path, st.session_state.fleadfilename, st.session_state.date2, axis_option
-        )
+        wr.flead_nc(path, st.session_state.fleadfilename, axis_option)
+
 
 @st.cache_data
 def file_write_vlead(path, axis_option, add_attributes=True):
     tempvardirname = tempfile.TemporaryDirectory(delete=False)
-    st.session_state.vleadfilename = os.path.join(tempvardirname.name, get_prefixed_filename("RAW_VAR.nc"))
+    st.session_state.vleadfilename = os.path.join(
+        tempvardirname.name, get_prefixed_filename("RAW_VAR.nc")
+    )
 
     if add_attributes:
         wr.vlead_nc(
             path,
             st.session_state.vleadfilename,
-            st.session_state.date3,
-            axis_option,
+            axis_option=axis_option,
             attributes=st.session_state.attributes,
         )
     else:
-        wr.vlead_nc(
-            path, st.session_state.vleadfilename, st.session_state.date3, axis_option
-        )
-
+        wr.vlead_nc(path, st.session_state.vleadfilename, axis_option)
 
 
 if "axis_option" not in st.session_state:

pyadps/pages/05_QC_Test.py

@@ -92,11 +92,12 @@ def qc_submit():
     evt = st.session_state.evt_QCT
     ft = st.session_state.ft_QCT
     is3beam = st.session_state.is3beam_QCT
+    beam_ignore = st.session_state.beam_to_ignore
     mask = pg_check(ds, mask, pgt, threebeam=is3beam)
-    mask = correlation_check(ds, mask, ct)
-    mask = echo_check(ds, mask, et)
+    mask = correlation_check(ds, mask, ct,is3beam,beam_ignore=beam_ignore)
+    mask = echo_check(ds, mask, et,is3beam,beam_ignore=beam_ignore)
     mask = ev_check(ds, mask, evt)
-    mask = false_target(ds, mask, ft, threebeam=True)
+    mask = false_target(ds, mask, ft, threebeam=is3beam, beam_ignore=beam_ignore)
     # Store the processed mask in a temporary mask
     st.session_state.qc_mask_temp = mask
 
@@ -337,6 +338,22 @@ with tab2:
             "Would you like to use a three-beam solution?", (True, False)
         )
 
+        if st.session_state.is3beam_QCT:
+            beam_label_to_value = {
+                "None": None,
+                "Beam 1": 0,
+                "Beam 2": 1,
+                "Beam 3": 2,
+                "Beam 4": 3
+            }
+
+            selected_beam = st.selectbox(
+                "Select Beam to Ignore",
+                options=list(beam_label_to_value.keys()),
+                index=0  # Default is "None"
+            )
+            st.session_state.beam_to_ignore =  beam_label_to_value[selected_beam]
+
         st.session_state.pgt_QCT = st.number_input(
             "Select Percent Good Threshold",
             0,

pyadps/pages/08_Write_File.py

@@ -1,6 +1,6 @@
 import configparser
 import tempfile
-
+import os
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
@@ -506,6 +506,8 @@ if generate_config_radio == "Yes":
     config["QCTest"]["error_velocity"] = str(st.session_state.evt_QCT)
     config["QCTest"]["false_target"] = str(st.session_state.ft_QCT)
     config["QCTest"]["three_beam"] = str(st.session_state.is3beam_QCT)
+    if st.session_state.is3beam_QCT:
+        config["QCTest"]["beam_ignore"] = str(st.session_state.beam_to_ignore)
     config["QCTest"]["percent_good"] = str(st.session_state.pgt_QCT)
 
     # Tab 4

pyadps/pages/09_Add-Ons.py

@@ -0,0 +1,168 @@
+import os
+import tempfile
+from pathlib import Path
+
+import re
+import io
+import contextlib
+
+import configparser
+import streamlit as st
+from utils.autoprocess import autoprocess
+from utils.multifile import ADCPBinFileCombiner
+
+# To make the page wider if the user presses the reload button.
+st.set_page_config(layout="wide")
+
+
+def ansi_to_html(text):
+    """
+    Function to convert ANSI (console color) to HTML.
+    To display the text, map the output to st.markdown
+    """
+    text = re.sub(r"\x1b\[31m", "<span style='color:red'><br>", text)  # red
+    text = re.sub(r"\x1b\[32m", "<span style='color:green'><br>", text)  # red
+    text = re.sub(r"\x1b\[33m", "<span style='color:orange'><br>", text)  # green
+    text = re.sub(r"\x1b\[0m", "</span>", text)  # reset
+    return text
+
+
+@st.cache_data
+def file_access(uploaded_file):
+    """
+    Function creates temporary directory to store the uploaded file.
+    The path of the file is returned
+
+    Args:
+        uploaded_file (string): Name of the uploaded file
+
+    Returns:
+        path (string): Path of the uploaded file
+    """
+    temp_dir = tempfile.mkdtemp()
+    path = os.path.join(temp_dir, uploaded_file.name)
+    with open(path, "wb") as f:
+        f.write(uploaded_file.getvalue())
+    return path
+
+
+def display_config_as_json(config_file):
+    config = configparser.ConfigParser()
+    config.read_string(config_file.getvalue().decode("utf-8"))
+    st.json({section: dict(config[section]) for section in config.sections()})
+
+
+def main():
+    st.title("🧰 Add-Ons")
+    st.header("🔧 Auto Processing Tool", divider=True)
+    st.write(
+        "You can use a configuration file from `pyadps` to re-process ADCP data by simply adjusting threshold values within the file. "
+        "This allows you to fine-tune the output without repeating the full processing workflow in the software."
+    )
+    st.write(
+        "To begin, upload both a binary ADCP file and a `config.ini` file for processing."
+    )
+
+    # File Upload Section
+    uploaded_binary_file = st.file_uploader(
+        "Upload ADCP Binary File", type=["000", "bin"]
+    )
+    uploaded_config_file = st.file_uploader(
+        "Upload Config File (config.ini)", type=["ini"]
+    )
+
+    if uploaded_binary_file and uploaded_config_file:
+        st.success("Files uploaded successfully!")
+
+        # Display config.ini file content as JSON
+        display_config_as_json(uploaded_config_file)
+
+        fpath = file_access(uploaded_binary_file)
+        # Process files
+        with st.spinner("Processing files. Please wait..."):
+            autoprocess(uploaded_config_file, binary_file_path=fpath)
+            st.success("Processing completed successfully!")
+            st.write("Processed file written.")
+
+    st.header("🔗 Binary File Combiner", divider=True)
+    st.write(
+        "ADCPs  may produce multiple binary segments instead of a single continuous file. "
+        "This tool scans each uploaded binary file for the `7f7f` header, removes any broken ensembles at the beginning or the end, and combines all valid segments into a single file. "
+        "To ensure correct order during concatenation, please rename the files using sequential numbering. "
+        "For example: `KKS_000.000`, `KKS_001.000`, `KKS_002.000`."
+    )
+    output_cat_filename = "merged_000.000"
+    st.info(f"Current file name: **{output_cat_filename}**")
+    output_cat_filename_radio = st.radio(
+        "Would you like to edit the output filename?",
+        ["No", "Yes"],
+        horizontal=True,
+    )
+    if output_cat_filename_radio == "Yes":
+        output_cat_filename = st.text_input(
+            "Enter file name (e.g., GD10A000)",
+            value=output_cat_filename,
+        )
+
+    display_log = st.radio(
+        "Display log from console:",
+        ["No", "Yes"],
+        horizontal=True,
+    )
+
+    uploaded_files = st.file_uploader(
+        "Upload multiple binary files", type=["bin", "000"], accept_multiple_files=True
+    )
+
+    if uploaded_files:
+        st.info("Saving uploaded files to temporary disk files...")
+
+        # Save files to temporary path
+        temp_file_paths = []
+        for uploaded_file in uploaded_files:
+            suffix = Path(uploaded_file.name).suffix
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+                tmp.write(uploaded_file.read())
+                temp_path = Path(tmp.name)
+                temp_file_paths.append(temp_path)
+
+        st.divider()
+        st.subheader("🛠 Processing and Combining...")
+
+        if display_log == "Yes":
+            # The `buffer` is used to display console output to streamlit
+            buffer = io.StringIO()
+            with contextlib.redirect_stdout(buffer):
+                adcpcat = ADCPBinFileCombiner()
+                combined_data = adcpcat.combine_files(temp_file_paths)
+            st.markdown(ansi_to_html(buffer.getvalue()), unsafe_allow_html=True)
+        else:
+            adcpcat = ADCPBinFileCombiner()
+            combined_data = adcpcat.combine_files(temp_file_paths)
+
+        if combined_data:
+            st.success("✅ Valid binary data has been combined successfully.")
+            st.warning(
+                "⚠️ Note: The time axis in the final file may be irregular due to missing ensembles during concatenation."
+            )
+            st.download_button(
+                label="📥 Download Combined Binary File",
+                data=bytes(combined_data),
+                file_name=output_cat_filename,
+                mime="application/octet-stream",
+            )
+        else:
+            st.warning("⚠️ No valid data found to combine.")
+
+        # Optional: Clean up temporary files
+        for path in temp_file_paths:
+            try:
+                os.remove(path)
+            except Exception as e:
+                st.warning(f"Failed to delete temp file {path}: {e}")
+    else:
+        st.info("Please upload binary files to begin.")
+
+
+if __name__ == "__main__":
+    main()

pyadps/utils/__init__.py

@@ -9,4 +9,6 @@ from pyadps.utils.signal_quality import *
 from pyadps.utils.velocity_test import *
 from pyadps.utils.writenc import *
 from pyadps.utils.autoprocess import *
+from pyadps.utils.logging_utils import *
+from pyadps.utils.multifile import *
 from pyadps.utils.script import *

pyadps/utils/autoprocess.py

@@ -192,15 +192,20 @@ def autoprocess(config_file, binary_file_path=None):
             evt = config.getint("QCTest", "error_velocity")
             et = config.getint("QCTest", "echo_intensity")
             ft = config.getint("QCTest", "false_target")
-            is3Beam = config.getboolean("QCTest", "three_beam")
+            is3beam = config.getboolean("QCTest", "three_beam")
+            if is3beam != None:
+                is3beam = int(is3beam)
+            beam_ignore = config.get("QCTest","beam_ignore")
             pgt = config.getint("QCTest", "percent_good")
             orientation = config.get("QCTest", "orientation")
+            beam_ignore = config.getboolean("QCTest",)
 
-            mask = pg_check(ds, mask, pgt, threebeam=is3Beam)
-            mask = correlation_check(ds, mask, ct)
-            mask = echo_check(ds, mask, et)
+            mask = pg_check(ds, mask, pgt, threebeam=is3beam)
+            mask = correlation_check(ds, mask, ct,is3beam,beam_ignore=beam_ignore)
+            mask = echo_check(ds, mask, et,is3beam,beam_ignore=beam_ignore)
             mask = ev_check(ds, mask, evt)
-            mask = false_target(ds, mask, ft, threebeam=True)
+            mask = false_target(ds, mask, ft, threebeam=is3beam, beam_ignore=beam_ignore)
+        
 
             print("QC Check Complete.")
 

pyadps/utils/logging_utils.py

@@ -0,0 +1,269 @@
+"""
+Reusable Logging Utilities
+A clean, configurable logging module that can be used across multiple projects.
+"""
+
+import logging
+import sys
+from enum import Enum
+from typing import Optional, Union
+from pathlib import Path
+
+
+class LogLevel(Enum):
+    """Log level enumeration"""
+
+    DEBUG = logging.DEBUG
+    INFO = logging.INFO
+    WARNING = logging.WARNING
+    ERROR = logging.ERROR
+    CRITICAL = logging.CRITICAL
+
+
+class CustomFormatter(logging.Formatter):
+    """Custom colored formatter for console logging"""
+
+    COLORS = {
+        logging.DEBUG: "\x1b[36m",  # Cyan
+        logging.INFO: "\x1b[32m",  # Green
+        logging.WARNING: "\x1b[33m",  # Yellow
+        logging.ERROR: "\x1b[31m",  # Red
+        logging.CRITICAL: "\x1b[31;1m",  # Bold Red
+    }
+    RESET = "\x1b[0m"
+
+    def __init__(self, include_timestamp: bool = True, include_module: bool = False):
+        """
+        Initialize formatter with optional components
+
+        Args:
+            include_timestamp: Whether to include timestamp in log format
+            include_module: Whether to include module name in log format
+        """
+        self.include_timestamp = include_timestamp
+        self.include_module = include_module
+        super().__init__()
+
+    def format(self, record):
+        """Format log record with colors and optional components"""
+        # Build format string based on options
+        format_parts = []
+
+        if self.include_timestamp:
+            format_parts.append("%(asctime)s")
+
+        format_parts.append("%(levelname)s")
+
+        if self.include_module:
+            format_parts.append("%(name)s")
+
+        format_parts.append("%(message)s")
+
+        log_format = " - ".join(format_parts)
+
+        # Apply color
+        color = self.COLORS.get(record.levelno, "")
+        colored_format = color + log_format + self.RESET
+
+        formatter = logging.Formatter(
+            colored_format,
+            datefmt="%Y-%m-%d %H:%M:%S" if self.include_timestamp else None,
+        )
+        return formatter.format(record)
+
+
+class LoggerConfig:
+    """Configuration class for logger setup"""
+
+    def __init__(
+        self,
+        level: LogLevel = LogLevel.INFO,
+        include_timestamp: bool = True,
+        include_module: bool = False,
+        log_to_file: bool = False,
+        log_file_path: Optional[Union[str, Path]] = None,
+        file_log_level: Optional[LogLevel] = None,
+        max_file_size: int = 10 * 1024 * 1024,  # 10MB
+        backup_count: int = 5,
+    ):
+        """
+        Initialize logger configuration
+
+        Args:
+            level: Console logging level
+            include_timestamp: Include timestamp in console output
+            include_module: Include module name in output
+            log_to_file: Whether to also log to file
+            log_file_path: Path for log file (if log_to_file is True)
+            file_log_level: File logging level (defaults to console level)
+            max_file_size: Maximum size of log file before rotation
+            backup_count: Number of backup files to keep
+        """
+        self.level = level
+        self.include_timestamp = include_timestamp
+        self.include_module = include_module
+        self.log_to_file = log_to_file
+        self.log_file_path = Path(log_file_path) if log_file_path else None
+        self.file_log_level = file_log_level or level
+        self.max_file_size = max_file_size
+        self.backup_count = backup_count
+
+
+class LoggerManager:
+    """Manages logger configuration and setup"""
+
+    _loggers = {}  # Cache for created loggers
+
+    @classmethod
+    def setup_logger(
+        self, name: str = "app", config: Optional[LoggerConfig] = None
+    ) -> logging.Logger:
+        """
+        Set up and configure logger with given configuration
+
+        Args:
+            name: Logger name
+            config: Logger configuration (uses defaults if None)
+
+        Returns:
+            Configured logger instance
+        """
+        # Use default config if none provided
+        if config is None:
+            config = LoggerConfig()
+
+        # Return cached logger if it exists
+        cache_key = f"{name}_{id(config)}"
+        if cache_key in self._loggers:
+            return self._loggers[cache_key]
+
+        logger = logging.getLogger(name)
+        logger.setLevel(config.level.value)
+
+        # Remove existing handlers to avoid duplicates
+        logger.handlers.clear()
+
+        # Create console handler
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(config.level.value)
+        console_formatter = CustomFormatter(
+            include_timestamp=config.include_timestamp,
+            include_module=config.include_module,
+        )
+        console_handler.setFormatter(console_formatter)
+        logger.addHandler(console_handler)
+
+        # Add file handler if requested
+        if config.log_to_file and config.log_file_path:
+            self._add_file_handler(logger, config)
+
+        # Prevent propagation to root logger
+        logger.propagate = False
+
+        # Cache the logger
+        self._loggers[cache_key] = logger
+
+        return logger
+
+    @classmethod
+    def _add_file_handler(self, logger: logging.Logger, config: LoggerConfig):
+        """Add rotating file handler to logger"""
+        from logging.handlers import RotatingFileHandler
+
+        # Ensure log directory exists
+        config.log_file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        file_handler = RotatingFileHandler(
+            config.log_file_path,
+            maxBytes=config.max_file_size,
+            backupCount=config.backup_count,
+        )
+        file_handler.setLevel(config.file_log_level.value)
+
+        # File logs typically include more detail
+        file_formatter = logging.Formatter(
+            "%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+        file_handler.setFormatter(file_formatter)
+        logger.addHandler(file_handler)
+
+    @classmethod
+    def get_logger(self, name: str = "app") -> logging.Logger:
+        """Get existing logger or create with default config"""
+        return logging.getLogger(name) or self.setup_logger(name)
+
+    @classmethod
+    def clear_cache(self):
+        """Clear logger cache (useful for testing)"""
+        self._loggers.clear()
+
+
+# Convenience functions for quick setup
+def get_console_logger(
+    name: str = "app", level: LogLevel = LogLevel.INFO, include_timestamp: bool = True
+) -> logging.Logger:
+    """Quick setup for console-only logger"""
+    config = LoggerConfig(
+        level=level, include_timestamp=include_timestamp, include_module=False
+    )
+    return LoggerManager.setup_logger(name, config)
+
+
+def get_file_logger(
+    name: str = "app",
+    log_file: Union[str, Path] = "app.log",
+    level: LogLevel = LogLevel.INFO,
+    file_level: Optional[LogLevel] = None,
+) -> logging.Logger:
+    """Quick setup for file + console logger"""
+    config = LoggerConfig(
+        level=level,
+        log_to_file=True,
+        log_file_path=log_file,
+        file_log_level=file_level or LogLevel.DEBUG,
+    )
+    return LoggerManager.setup_logger(name, config)
+
+
+def get_detailed_logger(
+    name: str = "app",
+    log_file: Union[str, Path] = "app.log",
+    console_level: LogLevel = LogLevel.INFO,
+    file_level: LogLevel = LogLevel.DEBUG,
+) -> logging.Logger:
+    """Setup logger with detailed configuration"""
+    config = LoggerConfig(
+        level=console_level,
+        include_timestamp=True,
+        include_module=True,
+        log_to_file=True,
+        log_file_path=log_file,
+        file_log_level=file_level,
+    )
+    return LoggerManager.setup_logger(name, config)
+
+
+# Example usage
+if __name__ == "__main__":
+    # Test different logger configurations
+
+    # Simple console logger
+    simple_logger = get_console_logger("simple", LogLevel.DEBUG)
+    simple_logger.debug("Debug message")
+    simple_logger.info("Info message")
+    simple_logger.warning("Warning message")
+    simple_logger.error("Error message")
+
+    print("\n" + "=" * 50 + "\n")
+
+    # File + console logger
+    file_logger = get_file_logger("file_test", "test.log", LogLevel.INFO)
+    file_logger.info("This goes to both console and file")
+    file_logger.debug("This only goes to file")
+
+    print("\n" + "=" * 50 + "\n")
+
+    # Detailed logger
+    detailed_logger = get_detailed_logger("detailed", "detailed.log")
+    detailed_logger.info("Detailed logging with module names")