nm-tool-forge 0.2.5__tar.gz → 0.3.0__tar.gz

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nm-tool-forge
-Version: 0.2.5
+Version: 0.3.0
 Summary: Analyze MigMan log files and generate aggregated CSV, Markdown, HTML, and optional PDF reports.
 Author-email: Stefan Ewald <s.ew@outlook.de>
 License-Expression: MIT

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nm-tool-forge"
-version = "0.2.5"
+version = "0.3.0"
 description = "Analyze MigMan log files and generate aggregated CSV, Markdown, HTML, and optional PDF reports."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/src/csvchunking/__init__.py

@@ -1,4 +1,4 @@
 from .chunker import ChunkResult, split_csv
 
 __all__ = ["ChunkResult", "split_csv"]
-__version__ = "0.2.5"
+__version__ = "0.3.0"

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/src/loganalysis/__init__.py

@@ -13,4 +13,4 @@ __all__ = [
     "run_analysis",
 ]
 
-__version__ = "0.2.5"
+__version__ = "0.3.0"

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/src/loganalysis/analysis.py

@@ -3,13 +3,13 @@ from __future__ import annotations
 from collections import Counter, defaultdict
 from pathlib import Path
 
-from .constants import DEFAULT_OUT_DIR, DEFAULT_TOP_EXAMPLES, EXIT_NO_LOG_FILES
-from .csv_export import write_csv
-from .encoding import count_physical_lines, detect_encoding
-from .filesystem import backup_file, ensure_dir
-from .models import AnalysisConfig, AnalysisRunResult, AnalysisSummary, FileAnalysis, MessageKey
-from .normalization import normalize_message
-from .parsing import iter_logical_entries, parse_entry
+from .constants import DEFAULT_OUT_DIR, DEFAULT_TOP_EXAMPLES, EXIT_NO_LOG_FILES
+from .csv_export import write_csv
+from .encoding import detect_encoding
+from .filesystem import backup_file, ensure_dir
+from .models import AnalysisConfig, AnalysisRunResult, AnalysisSummary, FileAnalysis, MessageKey
+from .normalization import normalize_message
+from .parsing import extract_last_import_block, iter_logical_entries_from_lines, parse_entry
 from .report_markdown import build_markdown_report
 
 
@@ -27,16 +27,22 @@ def analyze_file(file_path: Path) -> FileAnalysis:
     norm_examples: dict[MessageKey, Counter[str]] = defaultdict(Counter)
 
     unknown_lines = 0
-    total_entries = 0
-
-    encoding = detect_encoding(file_path)
-    total_lines = count_physical_lines(file_path, encoding=encoding)
-
-    for entry in iter_logical_entries(file_path, encoding=encoding):
-        total_entries += 1
-        parsed = parse_entry(entry)
-        if not parsed:
-            continue
+    total_entries = 0
+
+    encoding = detect_encoding(file_path)
+    try:
+        all_lines = file_path.read_text(encoding=encoding, errors="strict").splitlines(keepends=True)
+    except UnicodeDecodeError:
+        all_lines = file_path.read_text(encoding=encoding, errors="replace").splitlines(keepends=True)
+
+    import_block = extract_last_import_block(all_lines)
+    total_lines = len(import_block.lines)
+
+    for entry in iter_logical_entries_from_lines(import_block.lines):
+        total_entries += 1
+        parsed = parse_entry(entry)
+        if not parsed:
+            continue
 
         severity = parsed.severity
         message = parsed.message
@@ -58,8 +64,13 @@ def analyze_file(file_path: Path) -> FileAnalysis:
         unknown_lines=unknown_lines,
         raw_counts=raw_counts,
         norm_counts=norm_counts,
-        norm_examples=dict(norm_examples),
-    )
+        norm_examples=dict(norm_examples),
+        file_total_lines=len(all_lines),
+        analyzed_start_line_number=import_block.start_line_number,
+        analyzed_end_line_number=import_block.end_line_number,
+        import_start=import_block.start_metadata,
+        import_end=import_block.end_stats,
+    )
 
 
 def sorted_rows(counter: Counter[MessageKey]) -> list[tuple[str, str, int]]:

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/src/loganalysis/cli.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
-import argparse
-from collections.abc import Sequence
-from pathlib import Path
+import argparse
+from collections.abc import Sequence
+from pathlib import Path
 
 from .analysis import NoLogFilesError, run_analysis
 from .constants import DEFAULT_LOGS_DIR, DEFAULT_OUT_DIR, DEFAULT_TOP_EXAMPLES, EXIT_SUCCESS
@@ -16,11 +16,11 @@ def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         description="Aggregated analysis of log files (INFO/ERROR/WARNING) in logs/*.txt",
     )
-    parser.add_argument(
-        "--logs-dir",
-        default=DEFAULT_LOGS_DIR,
-        help=f"Subdirectory with log files (Default: {DEFAULT_LOGS_DIR})",
-    )
+    parser.add_argument(
+        "--logs-dir",
+        default=DEFAULT_LOGS_DIR,
+        help=f"Subdirectory with log files (Default: {DEFAULT_LOGS_DIR})",
+    )
     parser.add_argument("--out-dir", default=DEFAULT_OUT_DIR, help=f"Output directory (Default: {DEFAULT_OUT_DIR})")
     parser.add_argument("--backup-dir", default=None, help="Backup directory (Default: <out-dir>/backup)")
     parser.add_argument(
@@ -74,14 +74,12 @@ def main(argv: Sequence[str] | None = None) -> int:
             print(f"- PDF: {result.pdf_path.resolve() if result.pdf_path else 'created'}")
         else:
             pdf_reason = result.convert_status.get("pdf_reason")
-            if pdf_reason == "pandoc_missing":
-                print("- PDF: skipped (pandoc not available)")
-            elif pdf_reason == "no_pdf_engine":
-                print("- PDF: skipped (no PDF engine found)")
+            if pdf_reason == "weasyprint_unavailable":
+                print("- PDF: skipped (WeasyPrint is required for the formatted PDF report)")
             elif pdf_reason == "html_failed":
                 print("- PDF: skipped (HTML conversion failed)")
-            elif pdf_reason == "pdf_not_created":
-                print("- PDF: not created")
+            elif pdf_reason == "weasyprint_failed":
+                print("- PDF: failed (WeasyPrint could not create the formatted PDF report)")
             else:
                 print("- PDF: failed")
 

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/src/loganalysis/constants.py

@@ -29,11 +29,32 @@ RE_INT = re.compile(r"\b\d+\b")
 RE_WHITESPACE = re.compile(r"\s+")
 RE_WINDOWS_PATH = re.compile(r"\b[a-zA-Z]:\\[^;\n\r,]*")
 RE_UNIX_PATH = re.compile(r"(?<![A-Za-z0-9])(?:/[^/;\s,:]+){2,}")
-RE_ENTRY_START = re.compile(
-    r"^\s*(?P<severity>Info|Information|I|Error|Err|E|Warning|Warn|W)\b(?:\s*[;,]|\t)",
-    re.IGNORECASE,
-)
-RE_LINE_PREFIX = re.compile(r"^\s*Line\s+\d+\s*:\s*", re.IGNORECASE)
+RE_ENTRY_START = re.compile(
+    r"^\s*(?P<severity>Info|Information|I|Error|Err|E|Warning|Warn|W)\b(?:\s*[;,]|\t)",
+    re.IGNORECASE,
+)
+RE_IMPORT_START = re.compile(
+    r"^Info;(?P<timestamp>\d{2}\.\d{2}\.\d{4}/\d{2}:\d{2}:\d{2})\s+"
+    r"Starting import:\s*(?P<body>.*)$"
+)
+RE_IMPORT_START_KV = re.compile(
+    r"(?P<key>company|environment|language|Proalpha version|user)\s+'(?P<value>[^']*)'"
+)
+RE_IMPORT_END = re.compile(
+    r'^Info;(?P<timestamp>\d{2}\.\d{2}\.\d{4}/\d{2}:\d{2}:\d{2})\s+-\s+'
+    r"End of import:\s+"
+    r'(?P<records_to_be_edited>[\d.]*)\s*records to be edited \(incl\. the "header record"\),\s*'
+    r"(?P<records_edited>[\d.]*)\s*records edited,\s*thereof\s*"
+    r"(?P<successful>[\d.]*)\s*successful,\s*"
+    r"(?P<errors>[\d.]*)\s*errors,\s*"
+    r"(?P<warnings>[\d.]*)\s*warnings,\s*"
+    r"(?P<info_messages>[\d.]*)\s*info messages\s*$"
+)
+RE_SUCCESS_RATE_SPAN = re.compile(
+    r'^<span class="success-rate success-rate--(?P<status>red|yellow|green)">'
+    r"(?P<value>\d{1,3},\d{2}%)</span>$"
+)
+RE_LINE_PREFIX = re.compile(r"^\s*Line\s+\d+\s*:\s*", re.IGNORECASE)
 RE_TRAILING_DATASET = re.compile(r"\s*;(?:[^\n;]*;){2,}[^\n;]*\s*$")
 RE_SEMANTIC_VALUE_CHAR = re.compile(r"[0-9A-Za-zÄÖÜäöüß]")
 RE_LOOKUP_ASSIGNMENT = re.compile(

nm_tool_forge-0.3.0/src/loganalysis/converters.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+
+from .filesystem import ensure_dir
+from .report_html import render_report_html_document, resolve_report_date
+from .report_markdown import parse_report_markdown
+from .report_pdf import select_pdf_engine
+
+
+def convert_report_md_to_html_pdf(md_path: Path, html_path: Path, pdf_path: Path) -> dict[str, object]:
+    """Convert the generated markdown report to HTML and optionally to PDF."""
+
+    status: dict[str, object] = {
+        "html_created": False,
+        "pdf_created": False,
+        "pdf_reason": None,
+        "pdf_engine": None,
+    }
+
+    try:
+        ensure_dir(html_path.parent)
+        ensure_dir(pdf_path.parent)
+
+        markdown = md_path.read_text(encoding="utf-8")
+        report = parse_report_markdown(markdown)
+        report_date = resolve_report_date(report, md_path)
+        html_document = render_report_html_document(report, md_path.name, report_date)
+        html_path.write_text(html_document, encoding="utf-8")
+    except Exception as exc:
+        print(f"[ERROR] HTML generation failed: {exc}")
+        status["pdf_reason"] = "html_failed"
+        return status
+
+    html_created = html_path.exists() and html_path.stat().st_size > 0
+    status["html_created"] = html_created
+    if not html_created:
+        print("[ERROR] HTML generation failed: HTML file was not created.")
+        status["pdf_reason"] = "html_failed"
+        return status
+
+    pdf_engine = select_pdf_engine()
+    if pdf_engine is None:
+        print("[WARN] WeasyPrint is required for the formatted PDF report. HTML was generated, PDF skipped.")
+        status["pdf_reason"] = "weasyprint_unavailable"
+        return status
+
+    status["pdf_engine"] = pdf_engine
+
+    temp_pdf_path: Path | None = None
+
+    try:
+        with tempfile.NamedTemporaryFile(
+            prefix=f"{pdf_path.stem}.",
+            suffix=pdf_path.suffix,
+            dir=str(pdf_path.parent),
+            delete=False,
+        ) as handle:
+            temp_pdf_path = Path(handle.name)
+
+        from weasyprint import HTML
+
+        HTML(filename=str(html_path), base_url=str(html_path.parent)).write_pdf(str(temp_pdf_path))
+
+        if temp_pdf_path.exists() and temp_pdf_path.stat().st_size > 0:
+            temp_pdf_path.replace(pdf_path)
+            status["pdf_created"] = True
+            status["pdf_reason"] = None
+            print(f"PDF generated via {pdf_engine}: {pdf_path}")
+        else:
+            print("[ERROR] PDF generation finished without creating a PDF file.")
+            status["pdf_reason"] = "weasyprint_failed"
+    except Exception as exc:
+        print(f"[ERROR] PDF generation failed via {pdf_engine}: {exc}")
+        status["pdf_reason"] = "weasyprint_failed"
+    finally:
+        if temp_pdf_path and temp_pdf_path.exists():
+            temp_pdf_path.unlink()
+
+    return status

nm_tool_forge-0.3.0/src/loganalysis/encoding.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import chardet
+
+
+def detect_encoding(file_path: Path) -> str:
+    """Detect text encoding, preferring strict UTF-8 over heuristic guesses."""
+
+    raw = file_path.read_bytes()
+
+    for encoding in ("utf-8-sig", "utf-8"):
+        try:
+            raw.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+        return encoding
+
+    result = chardet.detect(raw)
+    detected = result.get("encoding") or "utf-8"
+    normalized = detected.lower().replace("_", "-")
+    if normalized in {"ascii", "us-ascii"} and any(byte >= 0x80 for byte in raw):
+        return "utf-8"
+    return detected
+
+
+def count_physical_lines(file_path: Path, encoding: str | None = None) -> int:
+    """Count physical lines in a text file using a detected or provided encoding."""
+
+    file_encoding = encoding or detect_encoding(file_path)
+    try:
+        with file_path.open("r", encoding=file_encoding, errors="strict") as file_handle:
+            return sum(1 for _ in file_handle)
+    except UnicodeDecodeError:
+        with file_path.open("r", encoding=file_encoding, errors="replace") as file_handle:
+            return sum(1 for _ in file_handle)

{nm_tool_forge-0.2.5 → nm_tool_forge-0.3.0}/src/loganalysis/models.py

@@ -1,32 +1,77 @@
 from __future__ import annotations
 
-from collections import Counter
-from dataclasses import dataclass, field
-from pathlib import Path
-
-MessageKey = tuple[str, str]
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+MessageKey = tuple[str, str]
+SuccessRateStatus = Literal["red", "yellow", "green", "none"]
 
 
 @dataclass(frozen=True)
-class ParsedLine:
-    """Structured representation of a parsed logical log entry."""
-
-    severity: str
-    message: str
+class ParsedLine:
+    """Structured representation of a parsed logical log entry."""
+
+    severity: str
+    message: str
+
+
+@dataclass(frozen=True)
+class ImportStartMetadata:
+    """Selected metadata from a MigMan `Starting import` line."""
+
+    timestamp: str
+    company: str | None = None
+    environment: str | None = None
+    language: str | None = None
+    proalpha_version: str | None = None
+    user: str | None = None
+
+
+@dataclass(frozen=True)
+class ImportEndStats:
+    """Counters and calculated success status from an `End of import` line."""
+
+    timestamp: str
+    records_to_be_edited: int
+    records_edited: int
+    successful: int
+    errors: int
+    warnings: int
+    info_messages: int
+    success_rate_percent: float | None
+    success_rate_status: SuccessRateStatus
+
+
+@dataclass(frozen=True)
+class ImportBlock:
+    """The physical lines and metadata for the selected import block."""
+
+    start_line_number: int | None
+    end_line_number: int | None
+    lines: tuple[str, ...]
+    start_metadata: ImportStartMetadata | None
+    end_stats: ImportEndStats | None
 
 
 @dataclass
-class FileAnalysis:
-    """Aggregated analysis results for a single log file."""
+class FileAnalysis:
+    """Aggregated analysis results for a single log file."""
 
     file: Path
     total_lines: int
     total_entries: int
     unknown_lines: int
     raw_counts: Counter[MessageKey]
-    norm_counts: Counter[MessageKey]
-    norm_examples: dict[MessageKey, Counter[str]]
-    backup_path: Path | None = None
+    norm_counts: Counter[MessageKey]
+    norm_examples: dict[MessageKey, Counter[str]]
+    backup_path: Path | None = None
+    file_total_lines: int = 0
+    analyzed_start_line_number: int | None = None
+    analyzed_end_line_number: int | None = None
+    import_start: ImportStartMetadata | None = None
+    import_end: ImportEndStats | None = None
 
 
 @dataclass

nm_tool_forge-0.3.0/src/loganalysis/parsing.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+
+from .constants import (
+    RE_ENTRY_START,
+    RE_IMPORT_END,
+    RE_IMPORT_START,
+    RE_IMPORT_START_KV,
+    RE_LINE_PREFIX,
+    RE_TRAILING_DATASET,
+    RE_WHITESPACE,
+    SEVERITY_ALIASES,
+)
+from .encoding import detect_encoding
+from .models import ImportBlock, ImportEndStats, ImportStartMetadata, ParsedLine, SuccessRateStatus
+
+
+def canonical_severity(raw: str) -> str:
+    """Map a raw severity token to the canonical output severity."""
+
+    severity = (raw or "").strip().upper()
+    return SEVERITY_ALIASES.get(severity, severity if severity else "UNKNOWN")
+
+
+def is_entry_start(line: str) -> bool:
+    """Return whether a physical line starts a new logical log entry."""
+
+    return bool(RE_ENTRY_START.match(line))
+
+
+def iter_logical_entries_from_lines(lines: Iterable[str]) -> Iterable[str]:
+    """Yield logical log entries assembled from a sequence of physical lines."""
+
+    current_lines: list[str] = []
+
+    for line in lines:
+        if is_entry_start(line):
+            if current_lines:
+                yield "".join(current_lines)
+            current_lines = [line]
+            continue
+
+        if current_lines:
+            current_lines.append(line)
+
+    if current_lines:
+        yield "".join(current_lines)
+
+
+def iter_logical_entries(file_path: Path, encoding: str | None = None) -> Iterable[str]:
+    """Yield logical log entries assembled from one or more physical lines."""
+
+    file_encoding = encoding or detect_encoding(file_path)
+
+    with file_path.open("r", encoding=file_encoding, errors="replace") as file_handle:
+        yield from iter_logical_entries_from_lines(file_handle)
+
+
+def parse_de_int(raw: str | None) -> int:
+    """Parse integers with German thousands separators. Empty values become 0."""
+
+    value = (raw or "").strip()
+    if not value:
+        return 0
+    return int(value.replace(".", ""))
+
+
+def format_de_int(value: int) -> str:
+    """Format 17705 as 17.705."""
+
+    return f"{value:,}".replace(",", ".")
+
+
+def calculate_success_rate(successful: int, records_edited: int) -> tuple[float | None, SuccessRateStatus]:
+    """Return percentage and status according to the report thresholds."""
+
+    if records_edited == 0:
+        return None, "none"
+
+    percent = successful / records_edited * 100
+    if percent < 95.0:
+        return percent, "red"
+    if percent < 98.5:
+        return percent, "yellow"
+    return percent, "green"
+
+
+def parse_import_start_line(line: str) -> ImportStartMetadata | None:
+    """Parse timestamp and selected Starting-import key/value pairs."""
+
+    match = RE_IMPORT_START.match(line.rstrip("\r\n"))
+    if not match:
+        return None
+
+    values = {
+        kv_match.group("key"): kv_match.group("value")
+        for kv_match in RE_IMPORT_START_KV.finditer(match.group("body"))
+    }
+    return ImportStartMetadata(
+        timestamp=match.group("timestamp"),
+        company=values.get("company"),
+        environment=values.get("environment"),
+        language=values.get("language"),
+        proalpha_version=values.get("Proalpha version"),
+        user=values.get("user"),
+    )
+
+
+def parse_import_end_line(line: str) -> ImportEndStats | None:
+    """Parse End-of-import counters and calculated success rate."""
+
+    match = RE_IMPORT_END.match(line.rstrip("\r\n"))
+    if not match:
+        return None
+
+    records_to_be_edited = parse_de_int(match.group("records_to_be_edited"))
+    records_edited = parse_de_int(match.group("records_edited"))
+    successful = parse_de_int(match.group("successful"))
+    errors = parse_de_int(match.group("errors"))
+    warnings = parse_de_int(match.group("warnings"))
+    info_messages = parse_de_int(match.group("info_messages"))
+    success_rate_percent, success_rate_status = calculate_success_rate(successful, records_edited)
+
+    return ImportEndStats(
+        timestamp=match.group("timestamp"),
+        records_to_be_edited=records_to_be_edited,
+        records_edited=records_edited,
+        successful=successful,
+        errors=errors,
+        warnings=warnings,
+        info_messages=info_messages,
+        success_rate_percent=success_rate_percent,
+        success_rate_status=success_rate_status,
+    )
+
+
+def extract_last_import_block(lines: Sequence[str]) -> ImportBlock:
+    """Return only the last complete import block, scanning from EOF backwards."""
+
+    last_end_idx: int | None = None
+    end_stats: ImportEndStats | None = None
+    for idx in range(len(lines) - 1, -1, -1):
+        parsed_end = parse_import_end_line(lines[idx])
+        if parsed_end is not None:
+            last_end_idx = idx
+            end_stats = parsed_end
+            break
+
+    if last_end_idx is None:
+        return ImportBlock(
+            start_line_number=None,
+            end_line_number=None,
+            lines=tuple(lines),
+            start_metadata=None,
+            end_stats=None,
+        )
+
+    start_idx: int | None = None
+    start_metadata: ImportStartMetadata | None = None
+    for idx in range(last_end_idx, -1, -1):
+        parsed_start = parse_import_start_line(lines[idx])
+        if parsed_start is not None:
+            start_idx = idx
+            start_metadata = parsed_start
+            break
+
+    if start_idx is None:
+        return ImportBlock(
+            start_line_number=None,
+            end_line_number=last_end_idx + 1,
+            lines=tuple(lines),
+            start_metadata=None,
+            end_stats=end_stats,
+        )
+
+    return ImportBlock(
+        start_line_number=start_idx + 1,
+        end_line_number=last_end_idx + 1,
+        lines=tuple(lines[start_idx : last_end_idx + 1]),
+        start_metadata=start_metadata,
+        end_stats=end_stats,
+    )
+
+
+def parse_entry(entry: str) -> ParsedLine | None:
+    """Parse one logical entry into severity and normalized message text."""
+
+    stripped = entry.strip()
+    if not stripped:
+        return None
+
+    match = RE_ENTRY_START.match(stripped)
+    if not match:
+        message = RE_WHITESPACE.sub(" ", stripped).strip()
+        return ParsedLine(severity="UNKNOWN", message=message or "(no message)")
+
+    severity = canonical_severity(match.group("severity") or "")
+    message = stripped[match.end():].strip()
+    message = RE_LINE_PREFIX.sub("", message, count=1)
+    message = RE_WHITESPACE.sub(" ", message).strip()
+    message = RE_TRAILING_DATASET.sub("", message).strip()
+    message = RE_WHITESPACE.sub(" ", message).strip()
+    return ParsedLine(severity=severity, message=message or "(no message)")
+
+
+def parse_line(line: str) -> ParsedLine | None:
+    """Backward-compatible alias for parsing a logical entry."""
+
+    return parse_entry(line)