SafeLLM4SE 0.1.1__py3-none-any.whl

safellm4se/__init__.py

@@ -0,0 +1,13 @@
+"""Public package interface for SafeLLM4SE."""
+
+from safellm4se.sampling.models import SamplerSettings, SamplingObservation
+from safellm4se.sampling.sampler import AdaptiveSampler
+
+__version__: str = "0.1.0"
+
+__all__: list[str] = [
+    "AdaptiveSampler",
+    "SamplerSettings",
+    "SamplingObservation",
+    "__version__",
+]

safellm4se/compare.py

@@ -0,0 +1,57 @@
+"""Command-line entry point for SafeLLM4SE two-sample CSV comparisons."""
+
+import argparse
+from pathlib import Path
+
+from loguru import logger
+
+from safellm4se.comparing.cli import parse_args, settings_from_args
+from safellm4se.comparing.comparator import generate_comparison_report
+from safellm4se.comparing.models import ComparisonSettings
+from safellm4se.sampling.config.logger import create_log_file_path, setup_logger
+
+
+def _console_log_level_from_args(args: argparse.Namespace) -> str | None:
+    """Return the console log level requested by command-line arguments.
+
+    Args:
+        args: Parsed command-line arguments.
+
+    Returns:
+        A log level for console output, or None when console logging is disabled.
+    """
+    if args.log_level:
+        return args.log_level
+    if args.verbose:
+        return "INFO"
+    return None
+
+
+def main() -> None:
+    """Run the comparison report command-line entry point.
+
+    Raises:
+        Exception: Re-raises uncaught exceptions from comparison generation.
+    """
+    args: argparse.Namespace = parse_args()
+    settings: ComparisonSettings = settings_from_args(args)
+    log_directory: Path = settings.output_path.parent
+    log_file: Path | None = setup_logger(
+        create_log_file_path(log_directory),
+        console_level=_console_log_level_from_args(args),
+    )
+    logger.info("Running SafeLLM4SE comparison report generation...")
+    if log_file is not None:
+        logger.info(f"Execution log written to {log_file}.")
+    try:
+        generate_comparison_report(settings)
+        final_message: str = f"Comparison report written in {settings.output_path}."
+        logger.info(final_message)
+        print(final_message)
+    except Exception:
+        logger.exception("SafeLLM4SE comparison report generation failed.")
+        raise
+
+
+if __name__ == "__main__":
+    main()

safellm4se/comparing/cli.py

@@ -0,0 +1,176 @@
+"""Command-line parsing for SafeLLM4SE comparison reports."""
+
+import argparse
+from pathlib import Path
+
+from safellm4se.sampling.config.logger import LOG_LEVELS
+
+from safellm4se.reporting.cli import MandatoryAwareDefaultsHelpFormatter
+from safellm4se.comparing.models import ComparisonSettings
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the command-line parser used by comparing.py.
+
+    Returns:
+        A configured argument parser for two-sample comparison reports.
+    """
+    parser: argparse.ArgumentParser = argparse.ArgumentParser(
+        description="SafeLLM4SE two-sample CSV comparison reporting.",
+        formatter_class=MandatoryAwareDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--h",
+        action="help",
+        help="Show this help message and exit.",
+    )
+    parser.add_argument(
+        "--input",
+        required=True,
+        metavar="FILE_NAME",
+        help="Input CSV file generated by sample.py.",
+    )
+    parser.add_argument(
+        "--output",
+        default="comparing.csv",
+        metavar="FILE_NAME",
+        help="Output CSV comparison report file.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Show INFO and higher log messages on screen.",
+    )
+    parser.add_argument(
+        "--log",
+        dest="log_level",
+        choices=LOG_LEVELS,
+        type=str.upper,
+        metavar="LEVEL",
+        default=None,
+        help=(
+            "Show log messages on screen from the selected level. Accepted "
+            f"values: {', '.join(LOG_LEVELS)}."
+        ),
+    )
+    parser.add_argument(
+        "--task-id-1",
+        required=True,
+        metavar="TASK_ID",
+        help="First task identifier to include in the comparison report.",
+    )
+    parser.add_argument(
+        "--task-name-1",
+        metavar="NAME",
+        default=None,
+        help="Optional first task name to display in generated plots.",
+    )
+    parser.add_argument(
+        "--task-id-2",
+        required=True,
+        metavar="TASK_ID",
+        help="Second task identifier to include in the comparison report.",
+    )
+    parser.add_argument(
+        "--task-name-2",
+        metavar="NAME",
+        default=None,
+        help="Optional second task name to display in generated plots.",
+    )
+    parser.add_argument(
+        "--test-type",
+        required=True,
+        choices=["paired", "independent"],
+        help="Experimental design used to comparing both distributions.",
+    )
+    parser.add_argument(
+        "--confidence-level",
+        default=0.95,
+        type=float,
+        metavar="CONFIDENCE_LEVEL",
+        help="Confidence level used for confidence intervals.",
+    )
+    parser.add_argument(
+        "--boxplot",
+        metavar="FILE_NAME",
+        default=None,
+        help="Optional SVG output file for a two-sample boxplot.",
+    )
+    parser.add_argument(
+        "--violin",
+        metavar="FILE_NAME",
+        default=None,
+        help="Optional SVG output file for a two-sample violin plot.",
+    )
+    parser.add_argument(
+        "--ECDF",
+        "--ecdf",
+        dest="ecdf",
+        metavar="FILE_NAME",
+        default=None,
+        help="Optional SVG output file for a two-sample ECDF plot.",
+    )
+    parser.add_argument(
+        "--raincloud",
+        metavar="FILE_NAME",
+        default=None,
+        help="Optional SVG output file for a two-sample raincloud plot.",
+    )
+    parser.add_argument(
+        "--KDE",
+        "--kde",
+        dest="kde",
+        metavar="FILE_NAME",
+        default=None,
+        help="Optional SVG output file for a two-sample KDE plot.",
+    )
+    return parser
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command-line arguments for a comparison report generation run.
+
+    Returns:
+        The parsed command-line namespace.
+    """
+    return build_parser().parse_args()
+
+
+def settings_from_args(args: argparse.Namespace) -> ComparisonSettings:
+    """Create comparison settings from parsed command-line arguments.
+
+    Args:
+        args: Parsed command-line arguments.
+
+    Returns:
+        Runtime settings for comparison report generation.
+    """
+    return ComparisonSettings(
+        input_path=Path(args.input),
+        output_path=Path(args.output),
+        boxplot_path=_optional_path(args.boxplot),
+        violin_path=_optional_path(args.violin),
+        ecdf_path=_optional_path(args.ecdf),
+        raincloud_path=_optional_path(args.raincloud),
+        kde_path=_optional_path(args.kde),
+        task_id_1=args.task_id_1,
+        task_id_2=args.task_id_2,
+        task_name_1=args.task_name_1,
+        task_name_2=args.task_name_2,
+        test_type=args.test_type,
+        confidence_level=args.confidence_level,
+    )
+
+
+def _optional_path(file_name: str | None) -> Path | None:
+    """Return a Path for an optional file name.
+
+    Args:
+        file_name: Optional command-line file name.
+
+    Returns:
+        A path when a file name was provided; otherwise, None.
+    """
+    if file_name is None:
+        return None
+    return Path(file_name)

safellm4se/comparing/comparator.py

@@ -0,0 +1,58 @@
+"""High-level comparison workflow for SafeLLM4SE sampling outputs."""
+
+from typing import Any
+
+from loguru import logger
+
+from safellm4se.comparing.metrics import build_comparison_report_row
+from safellm4se.comparing.models import ComparisonSettings
+from safellm4se.comparing.plots import generate_requested_comparison_plots
+from safellm4se.comparing.writer import write_comparison_report
+from safellm4se.reporting.metrics import parse_theta_values
+from safellm4se.reporting.reader import read_task_rows
+
+
+def generate_comparison_report(settings: ComparisonSettings) -> dict[str, Any]:
+    """Generate a two-sample comparison report CSV.
+
+    Args:
+        settings: Runtime settings for comparison report generation.
+
+    Returns:
+        The generated comparison report row.
+
+    Raises:
+        FileNotFoundError: If the input CSV file does not exist.
+        OSError: If the output CSV file or plot files cannot be written.
+        ValueError: If the input data is invalid or inconsistent.
+    """
+    logger.info("Reading sampling measurements from {}.", settings.input_path)
+    rows_1: list[dict[str, str]] = read_task_rows(
+        settings.input_path,
+        settings.task_id_1,
+    )
+    rows_2: list[dict[str, str]] = read_task_rows(
+        settings.input_path,
+        settings.task_id_2,
+    )
+    logger.info(
+        "Generating comparison report from {} and {} filtered rows.",
+        len(rows_1),
+        len(rows_2),
+    )
+    report_row: dict[str, Any] = build_comparison_report_row(
+        rows_1,
+        rows_2,
+        settings.test_type,
+        settings.confidence_level,
+    )
+    write_comparison_report(settings.output_path, report_row)
+    logger.info("Comparison report written to {}.", settings.output_path)
+    generated_plot_paths = generate_requested_comparison_plots(
+        parse_theta_values(rows_1),
+        parse_theta_values(rows_2),
+        settings,
+    )
+    for generated_plot_path in generated_plot_paths:
+        logger.info("Comparison plot written to {}.", generated_plot_path)
+    return report_row