nm-tool-forge 0.1.0__py3-none-any.whl

loganalysis/__init__.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from .analysis import analyze_file, run_analysis
+from .converters import convert_report_md_to_html_pdf
+from .normalization import normalize_message
+from .parsing import iter_logical_entries
+
+__all__ = [
+    "analyze_file",
+    "convert_report_md_to_html_pdf",
+    "iter_logical_entries",
+    "normalize_message",
+    "run_analysis",
+]
+
+__version__ = "0.1.0"

loganalysis/__main__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .cli import main
+
+raise SystemExit(main())

loganalysis/analysis.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from pathlib import Path
+
+from .constants import DEFAULT_OUT_DIR, DEFAULT_TOP_EXAMPLES, EXIT_NO_LOG_FILES
+from .csv_export import write_csv
+from .encoding import count_physical_lines, detect_encoding
+from .filesystem import backup_file, ensure_dir
+from .models import AnalysisConfig, AnalysisRunResult, AnalysisSummary, FileAnalysis, MessageKey
+from .normalization import normalize_message
+from .parsing import iter_logical_entries, parse_entry
+from .report_markdown import build_markdown_report
+
+
+class NoLogFilesError(FileNotFoundError):
+    """Raised when no matching log files are found for an analysis run."""
+
+    exit_code = EXIT_NO_LOG_FILES
+
+
+def analyze_file(file_path: Path) -> FileAnalysis:
+    """Analyze one log file and aggregate raw and normalized message counts."""
+
+    raw_counts: Counter[MessageKey] = Counter()
+    norm_counts: Counter[MessageKey] = Counter()
+    norm_examples: dict[MessageKey, Counter[str]] = defaultdict(Counter)
+
+    unknown_lines = 0
+    total_entries = 0
+
+    encoding = detect_encoding(file_path)
+    total_lines = count_physical_lines(file_path, encoding=encoding)
+
+    for entry in iter_logical_entries(file_path, encoding=encoding):
+        total_entries += 1
+        parsed = parse_entry(entry)
+        if not parsed:
+            continue
+
+        severity = parsed.severity
+        message = parsed.message
+
+        if severity == "UNKNOWN":
+            unknown_lines += 1
+
+        raw_counts[(severity, message)] += 1
+
+        normalized_message = normalize_message(message)
+        norm_key = (severity, normalized_message)
+        norm_counts[norm_key] += 1
+        norm_examples[norm_key][message] += 1
+
+    return FileAnalysis(
+        file=file_path,
+        total_lines=total_lines,
+        total_entries=total_entries,
+        unknown_lines=unknown_lines,
+        raw_counts=raw_counts,
+        norm_counts=norm_counts,
+        norm_examples=dict(norm_examples),
+    )
+
+
+def sorted_rows(counter: Counter[MessageKey]) -> list[tuple[str, str, int]]:
+    """Return aggregated rows in a stable severity/count/message order."""
+
+    return [
+        (severity, message, count)
+        for (severity, message), count in sorted(
+            counter.items(),
+            key=lambda item: (item[0][0], -item[1], item[0][1]),
+        )
+    ]
+
+
+def find_log_files(logs_dir: Path) -> list[Path]:
+    """Return all supported log files from the configured logs directory."""
+
+    return sorted(logs_dir.glob("*.txt"))
+
+
+def build_default_config(
+    *,
+    logs_dir: Path,
+    out_dir: Path | None = None,
+    backup_dir: Path | None = None,
+    top_examples: int = DEFAULT_TOP_EXAMPLES,
+    convert: bool = False,
+) -> AnalysisConfig:
+    """Build an analysis configuration with default output locations."""
+
+    resolved_out_dir = out_dir or Path(DEFAULT_OUT_DIR)
+    resolved_backup_dir = backup_dir or (resolved_out_dir / "backup")
+    return AnalysisConfig(
+        logs_dir=logs_dir,
+        out_dir=resolved_out_dir,
+        backup_dir=resolved_backup_dir,
+        top_examples=top_examples,
+        convert=convert,
+    )
+
+
+def run_analysis(config: AnalysisConfig) -> AnalysisRunResult:
+    """Run the full analysis workflow and write CSV and report outputs."""
+
+    logs_dir = config.logs_dir
+    out_dir = config.out_dir
+    backup_dir = config.backup_dir or (out_dir / "backup")
+
+    ensure_dir(out_dir)
+    ensure_dir(backup_dir)
+
+    log_files = find_log_files(logs_dir)
+    if not log_files:
+        raise NoLogFilesError(f"No *.txt files found in: {logs_dir.resolve()}")
+
+    summary = AnalysisSummary(
+        analyses=[],
+        global_raw=Counter(),
+        global_norm=Counter(),
+        global_norm_examples={},
+    )
+
+    for log_file in log_files:
+        backup_path = backup_file(log_file, backup_dir)
+        analysis = analyze_file(log_file)
+        analysis.backup_path = backup_path
+        summary.analyses.append(analysis)
+
+        summary.global_raw.update(analysis.raw_counts)
+        summary.global_norm.update(analysis.norm_counts)
+        for key, counter in analysis.norm_examples.items():
+            summary.global_norm_examples.setdefault(key, Counter()).update(counter)
+
+        write_csv(
+            out_dir / f"{log_file.stem}.aggregated.csv",
+            sorted_rows(analysis.raw_counts),
+            headers=["SEVERITY", "MESSAGE", "COUNT"],
+        )
+        write_csv(
+            out_dir / f"{log_file.stem}.aggregated.normalized.csv",
+            sorted_rows(analysis.norm_counts),
+            headers=["SEVERITY", "MESSAGE_NORMALIZED", "COUNT"],
+        )
+
+    write_csv(
+        out_dir / "summary.all_files.csv",
+        sorted_rows(summary.global_raw),
+        headers=["SEVERITY", "MESSAGE", "COUNT"],
+    )
+    write_csv(
+        out_dir / "summary.all_files.normalized.csv",
+        sorted_rows(summary.global_norm),
+        headers=["SEVERITY", "MESSAGE_NORMALIZED", "COUNT"],
+    )
+
+    report_path = out_dir / "report.md"
+    report_path.write_text(build_markdown_report(summary, config), encoding="utf-8", newline="\n")
+
+    result = AnalysisRunResult(
+        out_dir=out_dir,
+        backup_dir=backup_dir,
+        report_path=report_path,
+        summary=summary,
+    )
+
+    if config.convert:
+        from .converters import convert_report_md_to_html_pdf
+
+        result.html_path = out_dir / "report.html"
+        result.pdf_path = out_dir / "report.pdf"
+        result.convert_status = convert_report_md_to_html_pdf(result.report_path, result.html_path, result.pdf_path)
+
+    return result

loganalysis/cli.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+from typing import Sequence
+
+from .analysis import NoLogFilesError, run_analysis
+from .constants import DEFAULT_LOGS_DIR, DEFAULT_OUT_DIR, DEFAULT_TOP_EXAMPLES, EXIT_SUCCESS
+from .models import AnalysisConfig
+from .selftest import run_self_tests
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the command-line parser for the log analysis tool."""
+
+    parser = argparse.ArgumentParser(
+        description="Aggregated analysis of log files (INFO/ERROR/WARNING) in logs/*.txt",
+    )
+    parser.add_argument("--logs-dir", default=DEFAULT_LOGS_DIR, help=f"Subdirectory with log files (Default: {DEFAULT_LOGS_DIR})")
+    parser.add_argument("--out-dir", default=DEFAULT_OUT_DIR, help=f"Output directory (Default: {DEFAULT_OUT_DIR})")
+    parser.add_argument("--backup-dir", default=None, help="Backup directory (Default: <out-dir>/backup)")
+    parser.add_argument(
+        "--top-examples",
+        type=int,
+        default=DEFAULT_TOP_EXAMPLES,
+        help=f"Number of example variants per normalized message in the report (Default: {DEFAULT_TOP_EXAMPLES})",
+    )
+    parser.add_argument(
+        "--convert",
+        action="store_true",
+        help="Convert report.md to report.html and report.pdf when supported after analysis.",
+    )
+    parser.add_argument("--self-test", action="store_true", help="Run built-in self-tests and exit.")
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """CLI entry point for `python -m loganalysis` and the console script."""
+
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.self_test:
+        run_self_tests()
+        print("Self-tests passed.")
+        return EXIT_SUCCESS
+
+    config = AnalysisConfig(
+        logs_dir=Path(args.logs_dir),
+        out_dir=Path(args.out_dir),
+        backup_dir=Path(args.backup_dir) if args.backup_dir else (Path(args.out_dir) / "backup"),
+        top_examples=args.top_examples,
+        convert=args.convert,
+    )
+
+    try:
+        result = run_analysis(config)
+    except NoLogFilesError as exc:
+        print(str(exc))
+        return exc.exit_code
+
+    if config.convert:
+        print(f"Converting {result.report_path} to HTML and PDF...")
+        if result.convert_status.get("html_created"):
+            print(f"- HTML: {result.html_path.resolve() if result.html_path else 'created'}")
+        else:
+            print("- HTML: failed")
+
+        if result.convert_status.get("pdf_created"):
+            print(f"- PDF: {result.pdf_path.resolve() if result.pdf_path else 'created'}")
+        else:
+            pdf_reason = result.convert_status.get("pdf_reason")
+            if pdf_reason == "pandoc_missing":
+                print("- PDF: skipped (pandoc not available)")
+            elif pdf_reason == "no_pdf_engine":
+                print("- PDF: skipped (no PDF engine found)")
+            elif pdf_reason == "html_failed":
+                print("- PDF: skipped (HTML conversion failed)")
+            elif pdf_reason == "pdf_not_created":
+                print("- PDF: not created")
+            else:
+                print("- PDF: failed")
+
+    print("Analysis completed.")
+    print(f"- Backups: {result.backup_dir.resolve()}")
+    print(f"- Outputs: {result.out_dir.resolve()}")
+    print(f"- Report: {result.report_path.resolve()}")
+    return EXIT_SUCCESS

loganalysis/constants.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import re
+
+DEFAULT_LOGS_DIR = "logs"
+DEFAULT_OUT_DIR = "log_analyse_out"
+DEFAULT_TOP_EXAMPLES = 3
+DEFAULT_REPORT_TITLE = "Log Analysis Report"
+REPORT_TOP_PER_FILE = 20
+REPORT_TOP_GLOBAL = 50
+
+EXIT_SUCCESS = 0
+EXIT_NO_LOG_FILES = 2
+
+SEVERITY_ALIASES = {
+    "INFO": "INFO",
+    "INFORMATION": "INFO",
+    "I": "INFO",
+    "ERROR": "ERROR",
+    "ERR": "ERROR",
+    "E": "ERROR",
+    "WARNING": "WARNING",
+    "WARN": "WARNING",
+    "W": "WARNING",
+}
+
+RE_GUID = re.compile(r"\b[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}\b")
+RE_INT = re.compile(r"\b\d+\b")
+RE_WHITESPACE = re.compile(r"\s+")
+RE_WINDOWS_PATH = re.compile(r"\b[a-zA-Z]:\\[^;\n\r,]*")
+RE_UNIX_PATH = re.compile(r"(?<![A-Za-z0-9])(?:/[^/;\s,:]+){2,}")
+RE_ENTRY_START = re.compile(
+    r"^\s*(?P<severity>Info|Information|I|Error|Err|E|Warning|Warn|W)\b(?:\s*[;,]|\t)",
+    re.IGNORECASE,
+)
+RE_LINE_PREFIX = re.compile(r"^\s*Line\s+\d+\s*:\s*", re.IGNORECASE)
+RE_TRAILING_DATASET = re.compile(r"\s*;(?:[^\n;]*;){2,}[^\n;]*\s*$")
+RE_SEMANTIC_VALUE_CHAR = re.compile(r"[0-9A-Za-zÄÖÜäöüß]")
+RE_LOOKUP_ASSIGNMENT = re.compile(
+    r'(?P<head>\b(?:Mandatory Field Invalid|Conversion)\b.*?=)'
+    r'(?P<value>.*?)'
+    r'\s*[.,;:]*\s*(?:\d+\s+)?'
+    r'The record was not found in table\s+"(?P<table>[^"]+)"\.?',
+    re.IGNORECASE,
+)
+RE_VALIDATE_QUOTED_KEY = re.compile(
+    r"(?P<head>\bValidate\b.*?\{[^}]+\}\s*')"
+    r"(?P<value>[^']*)"
+    r"(?P<tail>':)",
+    re.IGNORECASE,
+)
+RE_QUOTED_VALUE = re.compile(r"\s*'[^']*'\s*")
+RE_MARKDOWN_TABLE_SEPARATOR = re.compile(r"^\s*\|?(?:\s*:?-+:?\s*\|)+\s*$")
+RE_HTML_BREAK = re.compile(r"<br\s*/?>", re.IGNORECASE)
+RE_INLINE_CODE_SPAN = re.compile(r"(`[^`]*`)")
+RE_LEADING_TIMESTAMP_PLACEHOLDER = re.compile(r"^\s*<N>\.<N>\.<N>/<N>:<N>:<N>\s*(?:-\s*)?")
+
+LATEX_SPECIAL_CHAR_REPLACEMENTS = {
+    "\\": r"\textbackslash{}",
+    "{": r"\{",
+    "}": r"\}",
+    "_": r"\_",
+    "%": r"\%",
+    "&": r"\&",
+    "#": r"\#",
+    "$": r"\$",
+    "~": r"\textasciitilde{}",
+    "^": r"\textasciicircum{}",
+}
+LATEX_PDF_ENGINES = {"xelatex", "pdflatex"}
+COMMON_MOJIBAKE_TOKENS = ("ÔÇ£", "ÔÇ¥", "–", "—", "„", "“", "â€")
+
+NORMALIZATION_SELF_TEST_CASES: tuple[tuple[str, str], ...] = (
+    (
+        'Mandatory Field Invalid: X =. The record was not found in table "Teile".',
+        'Mandatory Field Invalid: X =<EMPTY> The record was not found in table "Teile".',
+    ),
+    (
+        'Mandatory Field Invalid: X =., The record was not found in table "Teile".',
+        'Mandatory Field Invalid: X =<EMPTY> The record was not found in table "Teile".',
+    ),
+    (
+        'Mandatory Field Invalid: X = ; The record was not found in table "Teile".',
+        'Mandatory Field Invalid: X =<EMPTY> The record was not found in table "Teile".',
+    ),
+    (
+        'Mandatory Field Invalid: X =3000613.40 138 The record was not found in table "Teile".',
+        'Mandatory Field Invalid: X =<VALUE> The record was not found in table "Teile".',
+    ),
+    (
+        'Conversion: X =3100110. 138 The record was not found in table "Teile".',
+        'Conversion: X =<VALUE> The record was not found in table "Teile".',
+    ),
+    (
+        "Validate gbS_EAN: s_trg00021 {Teil} '7006563.1,6RS':",
+        "Validate gbS_EAN: s_trg00021 {Teil} '<VALUE>':",
+    ),
+    (
+        "Validate gbS_EAN: s_trg00021 {Teil} '., ':",
+        "Validate gbS_EAN: s_trg00021 {Teil} '<EMPTY>':",
+    ),
+    (
+        "See /var/log/app/output.txt for details",
+        "See <PATH> for details",
+    ),
+    (
+        "Error x40 exclusion criteria: s_art00002 {Teil} '.': {Der Datensatz ist nicht angelegt.}",
+        "Error x40 exclusion criteria: s_art00002 {Teil}: {Der Datensatz ist nicht angelegt.}",
+    ),
+    (
+        "Error x40 exclusion criteria: s_art00002 {Teil} '.,': {Der Datensatz ist nicht angelegt.}",
+        "Error x40 exclusion criteria: s_art00002 {Teil}: {Der Datensatz ist nicht angelegt.}",
+    ),
+    (
+        "Error x40 exclusion criteria: s_art00002 {Teil} '2053052.35': {Der Datensatz ist nicht angelegt.}",
+        "Error x40 exclusion criteria: s_art00002 {Teil}: {Der Datensatz ist nicht angelegt.}",
+    ),
+    (
+        "Error x40 exclusion criteria: s_art00002 {Teil} '179020.6,3': {Der Datensatz ist nicht angelegt.}",
+        "Error x40 exclusion criteria: s_art00002 {Teil}: {Der Datensatz ist nicht angelegt.}",
+    ),
+    (
+        "Error x40 exclusion criteria: s_art00002 {Teil} '9008001.': {Der Datensatz ist nicht angelegt.}",
+        "Error x40 exclusion criteria: s_art00002 {Teil}: {Der Datensatz ist nicht angelegt.}",
+    ),
+)

loganalysis/converters.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import subprocess
+import tempfile
+from pathlib import Path
+
+from .filesystem import ensure_dir
+from .report_html import render_report_html_document, resolve_report_date
+from .report_markdown import parse_report_markdown
+from .report_pdf import build_pdf_safe_markdown, contains_common_mojibake, select_pdf_engine
+
+
+def convert_report_md_to_html_pdf(md_path: Path, html_path: Path, pdf_path: Path) -> dict[str, object]:
+    """Convert the generated markdown report to HTML and optionally to PDF."""
+
+    status: dict[str, object] = {
+        "html_created": False,
+        "pdf_created": False,
+        "pdf_reason": None,
+        "pdf_engine": None,
+    }
+
+    try:
+        ensure_dir(html_path.parent)
+        ensure_dir(pdf_path.parent)
+
+        markdown = md_path.read_text(encoding="utf-8")
+        report = parse_report_markdown(markdown)
+        report_date = resolve_report_date(report, md_path)
+        html_document = render_report_html_document(report, md_path.name, report_date)
+        html_path.write_text(html_document, encoding="utf-8")
+    except Exception as exc:
+        print(f"[ERROR] HTML generation failed: {exc}")
+        status["pdf_reason"] = "html_failed"
+        return status
+
+    html_created = html_path.exists() and html_path.stat().st_size > 0
+    status["html_created"] = html_created
+    if not html_created:
+        print("[ERROR] HTML generation failed: HTML file was not created.")
+        status["pdf_reason"] = "html_failed"
+        return status
+
+    pdf_engine = select_pdf_engine()
+    if pdf_engine is None:
+        print("[WARN] No supported PDF engine found. HTML was generated, PDF skipped.")
+        status["pdf_reason"] = "no_pdf_engine"
+        return status
+
+    status["pdf_engine"] = pdf_engine
+
+    pdf_source_path = md_path
+    temp_md_path: Path | None = None
+    temp_pdf_path: Path | None = None
+
+    try:
+        with tempfile.NamedTemporaryFile(
+            prefix=f"{pdf_path.stem}.",
+            suffix=pdf_path.suffix,
+            dir=str(pdf_path.parent),
+            delete=False,
+        ) as handle:
+            temp_pdf_path = Path(handle.name)
+
+        if pdf_engine == "weasyprint":
+            from weasyprint import HTML
+
+            HTML(filename=str(html_path), base_url=str(html_path.parent)).write_pdf(str(temp_pdf_path))
+        elif pdf_engine == "wkhtmltopdf":
+            import shutil
+
+            wkhtmltopdf_path = shutil.which("wkhtmltopdf")
+            if wkhtmltopdf_path is None:
+                print("[WARN] wkhtmltopdf is not available anymore. PDF skipped.")
+                status["pdf_reason"] = "no_pdf_engine"
+                return status
+
+            subprocess.run(
+                [
+                    wkhtmltopdf_path,
+                    "--enable-local-file-access",
+                    "--page-size",
+                    "A4",
+                    "--orientation",
+                    "Portrait",
+                    str(html_path),
+                    str(temp_pdf_path),
+                ],
+                check=True,
+            )
+        else:
+            import shutil
+
+            pandoc_path = shutil.which("pandoc")
+            if pandoc_path is None:
+                print("[WARN] pandoc is not installed or not in PATH. HTML generated, PDF skipped.")
+                status["pdf_reason"] = "pandoc_missing"
+                return status
+
+            pdf_safe_markdown = build_pdf_safe_markdown(markdown)
+            if pdf_safe_markdown != markdown:
+                with tempfile.NamedTemporaryFile(
+                    prefix=f"{md_path.stem}.pdfsafe.",
+                    suffix=md_path.suffix,
+                    dir=str(md_path.parent),
+                    mode="w",
+                    encoding="utf-8",
+                    newline="\n",
+                    delete=False,
+                ) as handle:
+                    handle.write(pdf_safe_markdown)
+                    pdf_source_path = temp_md_path = Path(handle.name)
+
+            if contains_common_mojibake(markdown):
+                print(
+                    "[WARN] Report contains suspicious mojibake sequences. PDF escaping was applied, "
+                    "but the source encoding should still be checked."
+                )
+
+            subprocess.run(
+                [
+                    pandoc_path,
+                    str(pdf_source_path),
+                    "-o",
+                    str(temp_pdf_path),
+                    "--from",
+                    "markdown",
+                    f"--pdf-engine={pdf_engine}",
+                ],
+                check=True,
+            )
+
+        if temp_pdf_path.exists() and temp_pdf_path.stat().st_size > 0:
+            temp_pdf_path.replace(pdf_path)
+            status["pdf_created"] = True
+            status["pdf_reason"] = None
+            print(f"PDF generated via {pdf_engine}: {pdf_path}")
+        else:
+            print("[ERROR] PDF generation finished without creating a PDF file.")
+            status["pdf_reason"] = "pdf_not_created"
+    except Exception as exc:
+        print(f"[ERROR] PDF generation failed via {pdf_engine}: {exc}")
+        status["pdf_reason"] = "pdf_failed"
+    finally:
+        if temp_pdf_path and temp_pdf_path.exists():
+            temp_pdf_path.unlink()
+        if temp_md_path and temp_md_path.exists():
+            temp_md_path.unlink()
+
+    return status

loganalysis/csv_export.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+from typing import Iterable
+
+from .filesystem import ensure_dir
+
+
+def write_csv(path: Path, rows: Iterable[tuple[str, str, int]], headers: list[str]) -> None:
+    """Write semicolon-separated CSV output."""
+
+    ensure_dir(path.parent)
+    with path.open("w", encoding="utf-8", newline="") as file_handle:
+        writer = csv.writer(file_handle, delimiter=";")
+        writer.writerow(headers)
+        for row in rows:
+            writer.writerow(row)

loganalysis/encoding.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import chardet
+
+
+def detect_encoding(file_path: Path) -> str:
+    """Best-effort encoding detection based on file header bytes."""
+
+    with file_path.open("rb") as file_handle:
+        raw = file_handle.read(4096)
+    result = chardet.detect(raw)
+    return result["encoding"] or "utf-8"
+
+
+def count_physical_lines(file_path: Path, encoding: str | None = None) -> int:
+    """Count physical lines in a text file using a detected or provided encoding."""
+
+    file_encoding = encoding or detect_encoding(file_path)
+    with file_path.open("r", encoding=file_encoding, errors="replace") as file_handle:
+        return sum(1 for _ in file_handle)

loganalysis/filesystem.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import datetime as dt
+import shutil
+from pathlib import Path
+
+
+def now_stamp() -> str:
+    """Return a filesystem-safe timestamp."""
+
+    return dt.datetime.now().strftime("%Y%m%d_%H%M%S")
+
+
+def ensure_dir(path: Path) -> None:
+    """Create a directory and all missing parents."""
+
+    path.mkdir(parents=True, exist_ok=True)
+
+
+def backup_file(src: Path, backup_root: Path) -> Path:
+    """Create a timestamped backup copy of a source file."""
+
+    ensure_dir(backup_root)
+    dst = backup_root / f"{src.name}.{now_stamp()}.bak"
+    shutil.copy2(src, dst)
+    return dst