nm-tool-forge 0.1.0__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/PKG-INFO +18 -1
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/README.md +17 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/pyproject.toml +67 -66
- nm_tool_forge-0.2.3/src/csvchunking/__init__.py +4 -0
- nm_tool_forge-0.2.3/src/csvchunking/__main__.py +4 -0
- nm_tool_forge-0.2.3/src/csvchunking/chunker.py +76 -0
- nm_tool_forge-0.2.3/src/csvchunking/cli.py +31 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/__init__.py +16 -16
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/cli.py +8 -4
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/csv_export.py +7 -7
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/parsing.py +6 -6
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/report_markdown.py +16 -12
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/report_pdf.py +9 -4
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/selftest.py +17 -11
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/nm_tool_forge.egg-info/PKG-INFO +18 -1
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/nm_tool_forge.egg-info/SOURCES.txt +5 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/nm_tool_forge.egg-info/entry_points.txt +1 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/nm_tool_forge.egg-info/top_level.txt +1 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/tests/test_analysis.py +6 -5
- nm_tool_forge-0.2.3/tests/test_csvchunking.py +63 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/tests/test_report_markdown.py +3 -1
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/LICENSE +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/setup.cfg +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/__main__.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/analysis.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/constants.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/converters.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/encoding.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/filesystem.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/models.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/normalization.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/report_html.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/loganalysis/report_models.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/nm_tool_forge.egg-info/dependency_links.txt +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/src/nm_tool_forge.egg-info/requires.txt +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/tests/test_normalization.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/tests/test_parsing.py +0 -0
- {nm_tool_forge-0.1.0 → nm_tool_forge-0.2.3}/tests/test_report_html.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nm-tool-forge
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Analyze MigMan log files and generate aggregated CSV, Markdown, HTML, and optional PDF reports.
|
|
5
5
|
Author-email: Stefan Ewald <s.ew@outlook.de>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -89,6 +89,23 @@ python -m loganalysis --self-test
|
|
|
89
89
|
|
|
90
90
|
Legacy compatibility call:
|
|
91
91
|
|
|
92
|
+
|
|
93
|
+
## Release process
|
|
94
|
+
|
|
95
|
+
To publish a new release, always test on TestPyPI first, then upload to PyPI only after successful Conda/Smoke-Tests:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
export TWINE_USERNAME="__token__"
|
|
99
|
+
export TWINE_PASSWORD="pypi-..."
|
|
100
|
+
|
|
101
|
+
bash scripts/release_testpypi.sh --bump patch
|
|
102
|
+
bash scripts/release_pypi.sh --yes
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
**Hinweise:**
|
|
106
|
+
- Erst TestPyPI ausführen und testen, dann final nach PyPI hochladen.
|
|
107
|
+
- Versionen auf PyPI können nicht überschrieben oder erneut verwendet werden.
|
|
108
|
+
|
|
92
109
|
```powershell
|
|
93
110
|
python .\log_analysis.py --convert
|
|
94
111
|
```
|
|
@@ -58,6 +58,23 @@ python -m loganalysis --self-test
|
|
|
58
58
|
|
|
59
59
|
Legacy compatibility call:
|
|
60
60
|
|
|
61
|
+
|
|
62
|
+
## Release process
|
|
63
|
+
|
|
64
|
+
To publish a new release, always test on TestPyPI first, then upload to PyPI only after successful Conda/Smoke-Tests:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
export TWINE_USERNAME="__token__"
|
|
68
|
+
export TWINE_PASSWORD="pypi-..."
|
|
69
|
+
|
|
70
|
+
bash scripts/release_testpypi.sh --bump patch
|
|
71
|
+
bash scripts/release_pypi.sh --yes
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Hinweise:**
|
|
75
|
+
- Erst TestPyPI ausführen und testen, dann final nach PyPI hochladen.
|
|
76
|
+
- Versionen auf PyPI können nicht überschrieben oder erneut verwendet werden.
|
|
77
|
+
|
|
61
78
|
```powershell
|
|
62
79
|
python .\log_analysis.py --convert
|
|
63
80
|
```
|
|
@@ -1,67 +1,68 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=69", "wheel"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "nm-tool-forge"
|
|
7
|
-
version = "0.
|
|
8
|
-
description = "Analyze MigMan log files and generate aggregated CSV, Markdown, HTML, and optional PDF reports."
|
|
9
|
-
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
|
-
requires-python = ">=3.10"
|
|
11
|
-
license = "MIT"
|
|
12
|
-
license-files = ["LICENSE"]
|
|
13
|
-
authors = [
|
|
14
|
-
{ name = "Stefan Ewald", email = "s.ew@outlook.de" }
|
|
15
|
-
]
|
|
16
|
-
keywords = ["migman", "logs", "analysis", "reporting", "csv", "markdown", "pdf"]
|
|
17
|
-
classifiers = [
|
|
18
|
-
"Development Status :: 4 - Beta",
|
|
19
|
-
"Intended Audience :: Developers",
|
|
20
|
-
"Programming Language :: Python :: 3",
|
|
21
|
-
"Programming Language :: Python :: 3.10",
|
|
22
|
-
"Programming Language :: Python :: 3.11",
|
|
23
|
-
"Programming Language :: Python :: 3.12",
|
|
24
|
-
"Programming Language :: Python :: 3.13",
|
|
25
|
-
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
26
|
-
"Topic :: Utilities",
|
|
27
|
-
]
|
|
28
|
-
|
|
29
|
-
dependencies = [
|
|
30
|
-
"chardet>=5.0",
|
|
31
|
-
]
|
|
32
|
-
|
|
33
|
-
[project.optional-dependencies]
|
|
34
|
-
pdf = [
|
|
35
|
-
"weasyprint>=62",
|
|
36
|
-
]
|
|
37
|
-
dev = [
|
|
38
|
-
"pytest>=8.0",
|
|
39
|
-
"build>=1.2",
|
|
40
|
-
"twine>=5.0",
|
|
41
|
-
"ruff>=0.11",
|
|
42
|
-
]
|
|
43
|
-
|
|
44
|
-
[project.urls]
|
|
45
|
-
Homepage = "https://github.com/Jack736-ui/migman_log"
|
|
46
|
-
Issues = "https://github.com/Jack736-ui/migman_log/issues"
|
|
47
|
-
|
|
48
|
-
[project.scripts]
|
|
49
|
-
nm-tool-forge = "loganalysis.cli:main"
|
|
50
|
-
loganalysis = "loganalysis.cli:main"
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "nm-tool-forge"
|
|
7
|
+
version = "0.2.3"
|
|
8
|
+
description = "Analyze MigMan log files and generate aggregated CSV, Markdown, HTML, and optional PDF reports."
|
|
9
|
+
readme = { file = "README.md", content-type = "text/markdown" }
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [
|
|
14
|
+
{ name = "Stefan Ewald", email = "s.ew@outlook.de" }
|
|
15
|
+
]
|
|
16
|
+
keywords = ["migman", "logs", "analysis", "reporting", "csv", "markdown", "pdf"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Programming Language :: Python :: 3.13",
|
|
25
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
26
|
+
"Topic :: Utilities",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
dependencies = [
|
|
30
|
+
"chardet>=5.0",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
pdf = [
|
|
35
|
+
"weasyprint>=62",
|
|
36
|
+
]
|
|
37
|
+
dev = [
|
|
38
|
+
"pytest>=8.0",
|
|
39
|
+
"build>=1.2",
|
|
40
|
+
"twine>=5.0",
|
|
41
|
+
"ruff>=0.11",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://github.com/Jack736-ui/migman_log"
|
|
46
|
+
Issues = "https://github.com/Jack736-ui/migman_log/issues"
|
|
47
|
+
|
|
48
|
+
[project.scripts]
|
|
49
|
+
nm-tool-forge = "loganalysis.cli:main"
|
|
50
|
+
loganalysis = "loganalysis.cli:main"
|
|
51
|
+
csvchunking = "csvchunking.cli:main"
|
|
52
|
+
|
|
53
|
+
[tool.setuptools]
|
|
54
|
+
package-dir = { "" = "src" }
|
|
55
|
+
|
|
56
|
+
[tool.setuptools.packages.find]
|
|
57
|
+
where = ["src"]
|
|
58
|
+
|
|
59
|
+
[tool.pytest.ini_options]
|
|
60
|
+
testpaths = ["tests"]
|
|
61
|
+
addopts = "--basetemp=tests_tmp"
|
|
62
|
+
|
|
63
|
+
[tool.ruff]
|
|
64
|
+
line-length = 120
|
|
65
|
+
target-version = "py310"
|
|
66
|
+
|
|
67
|
+
[tool.ruff.lint]
|
|
67
68
|
select = ["E", "F", "I", "B", "UP"]
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import csv
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen=True)
|
|
7
|
+
class ChunkResult:
|
|
8
|
+
input_file: Path
|
|
9
|
+
output_dir: Path
|
|
10
|
+
chunk_size: int
|
|
11
|
+
data_rows_processed: int
|
|
12
|
+
files_created: int
|
|
13
|
+
output_files: tuple[Path, ...]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def split_csv(
|
|
17
|
+
input_file: Path,
|
|
18
|
+
chunk_size: int,
|
|
19
|
+
encoding: str = "utf-8-sig",
|
|
20
|
+
) -> ChunkResult:
|
|
21
|
+
if not Path(input_file).is_file():
|
|
22
|
+
raise FileNotFoundError(f"Eingabedatei nicht gefunden: {input_file}")
|
|
23
|
+
if chunk_size <= 0:
|
|
24
|
+
raise ValueError("chunk_size muss > 0 sein")
|
|
25
|
+
|
|
26
|
+
input_file = Path(input_file)
|
|
27
|
+
output_dir = input_file.parent / input_file.stem
|
|
28
|
+
output_dir.mkdir(exist_ok=True)
|
|
29
|
+
|
|
30
|
+
# Delimiter automatisch erkennen
|
|
31
|
+
with open(input_file, encoding=encoding, newline="") as f:
|
|
32
|
+
sample = f.read(4096)
|
|
33
|
+
f.seek(0)
|
|
34
|
+
sniffer = csv.Sniffer()
|
|
35
|
+
try:
|
|
36
|
+
dialect = sniffer.sniff(sample)
|
|
37
|
+
except Exception:
|
|
38
|
+
dialect = csv.excel
|
|
39
|
+
dialect.delimiter = ";"
|
|
40
|
+
reader = csv.reader(f, dialect)
|
|
41
|
+
try:
|
|
42
|
+
header = next(reader)
|
|
43
|
+
except StopIteration as exc:
|
|
44
|
+
raise ValueError("Eingabedatei ist leer.") from exc
|
|
45
|
+
chunk = []
|
|
46
|
+
file_count = 0
|
|
47
|
+
data_rows = 0
|
|
48
|
+
output_files = []
|
|
49
|
+
for row in reader:
|
|
50
|
+
chunk.append(row)
|
|
51
|
+
data_rows += 1
|
|
52
|
+
if len(chunk) == chunk_size:
|
|
53
|
+
file_count += 1
|
|
54
|
+
out_path = output_dir / f"{input_file.stem}_{file_count:02d}{input_file.suffix}"
|
|
55
|
+
with open(out_path, "w", encoding=encoding, newline="") as out:
|
|
56
|
+
writer = csv.writer(out, dialect)
|
|
57
|
+
writer.writerow(header)
|
|
58
|
+
writer.writerows(chunk)
|
|
59
|
+
output_files.append(out_path)
|
|
60
|
+
chunk = []
|
|
61
|
+
if chunk:
|
|
62
|
+
file_count += 1
|
|
63
|
+
out_path = output_dir / f"{input_file.stem}_{file_count:02d}{input_file.suffix}"
|
|
64
|
+
with open(out_path, "w", encoding=encoding, newline="") as out:
|
|
65
|
+
writer = csv.writer(out, dialect)
|
|
66
|
+
writer.writerow(header)
|
|
67
|
+
writer.writerows(chunk)
|
|
68
|
+
output_files.append(out_path)
|
|
69
|
+
return ChunkResult(
|
|
70
|
+
input_file=input_file,
|
|
71
|
+
output_dir=output_dir,
|
|
72
|
+
chunk_size=chunk_size,
|
|
73
|
+
data_rows_processed=data_rows,
|
|
74
|
+
files_created=file_count,
|
|
75
|
+
output_files=tuple(output_files),
|
|
76
|
+
)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from .chunker import split_csv
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def main() -> None:
|
|
9
|
+
parser = argparse.ArgumentParser(
|
|
10
|
+
description="Teilt eine große CSV-Datei in kleinere Chunks mit Header.",
|
|
11
|
+
)
|
|
12
|
+
parser.add_argument("input_file", help="Pfad zur CSV-Datei")
|
|
13
|
+
parser.add_argument(
|
|
14
|
+
"--chunk-size",
|
|
15
|
+
type=int,
|
|
16
|
+
required=True,
|
|
17
|
+
help="Anzahl Datenzeilen pro Ausgabedatei, muss > 0 sein",
|
|
18
|
+
)
|
|
19
|
+
parser.add_argument("--encoding", default="utf-8-sig", help="Encoding für Ein- und Ausgabe (Standard: utf-8-sig)")
|
|
20
|
+
args = parser.parse_args()
|
|
21
|
+
try:
|
|
22
|
+
result = split_csv(Path(args.input_file), args.chunk_size, encoding=args.encoding)
|
|
23
|
+
except Exception as e:
|
|
24
|
+
print(f"Fehler: {e}", file=sys.stderr)
|
|
25
|
+
sys.exit(1)
|
|
26
|
+
print("CSV chunking completed.")
|
|
27
|
+
print(f"- Input: {result.input_file}")
|
|
28
|
+
print(f"- Output directory: {result.output_dir}")
|
|
29
|
+
print(f"- Chunk size: {result.chunk_size}")
|
|
30
|
+
print(f"- Data rows processed: {result.data_rows_processed}")
|
|
31
|
+
print(f"- Files created: {result.files_created}")
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from .analysis import analyze_file, run_analysis
|
|
4
|
-
from .converters import convert_report_md_to_html_pdf
|
|
5
|
-
from .normalization import normalize_message
|
|
6
|
-
from .parsing import iter_logical_entries
|
|
7
|
-
|
|
8
|
-
__all__ = [
|
|
9
|
-
"analyze_file",
|
|
10
|
-
"convert_report_md_to_html_pdf",
|
|
11
|
-
"iter_logical_entries",
|
|
12
|
-
"normalize_message",
|
|
13
|
-
"run_analysis",
|
|
14
|
-
]
|
|
15
|
-
|
|
16
|
-
__version__ = "0.
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from .analysis import analyze_file, run_analysis
|
|
4
|
+
from .converters import convert_report_md_to_html_pdf
|
|
5
|
+
from .normalization import normalize_message
|
|
6
|
+
from .parsing import iter_logical_entries
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"analyze_file",
|
|
10
|
+
"convert_report_md_to_html_pdf",
|
|
11
|
+
"iter_logical_entries",
|
|
12
|
+
"normalize_message",
|
|
13
|
+
"run_analysis",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
__version__ = "0.2.3"
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import argparse
|
|
4
|
-
from
|
|
5
|
-
from
|
|
3
|
+
import argparse
|
|
4
|
+
from collections.abc import Sequence
|
|
5
|
+
from pathlib import Path
|
|
6
6
|
|
|
7
7
|
from .analysis import NoLogFilesError, run_analysis
|
|
8
8
|
from .constants import DEFAULT_LOGS_DIR, DEFAULT_OUT_DIR, DEFAULT_TOP_EXAMPLES, EXIT_SUCCESS
|
|
@@ -16,7 +16,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
16
16
|
parser = argparse.ArgumentParser(
|
|
17
17
|
description="Aggregated analysis of log files (INFO/ERROR/WARNING) in logs/*.txt",
|
|
18
18
|
)
|
|
19
|
-
parser.add_argument(
|
|
19
|
+
parser.add_argument(
|
|
20
|
+
"--logs-dir",
|
|
21
|
+
default=DEFAULT_LOGS_DIR,
|
|
22
|
+
help=f"Subdirectory with log files (Default: {DEFAULT_LOGS_DIR})",
|
|
23
|
+
)
|
|
20
24
|
parser.add_argument("--out-dir", default=DEFAULT_OUT_DIR, help=f"Output directory (Default: {DEFAULT_OUT_DIR})")
|
|
21
25
|
parser.add_argument("--backup-dir", default=None, help="Backup directory (Default: <out-dir>/backup)")
|
|
22
26
|
parser.add_argument(
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import csv
|
|
4
|
-
from
|
|
5
|
-
from
|
|
6
|
-
|
|
7
|
-
from .filesystem import ensure_dir
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .filesystem import ensure_dir
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def write_csv(path: Path, rows: Iterable[tuple[str, str, int]], headers: list[str]) -> None:
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from
|
|
4
|
-
from
|
|
5
|
-
|
|
6
|
-
from .constants import RE_ENTRY_START, RE_LINE_PREFIX, RE_TRAILING_DATASET, RE_WHITESPACE, SEVERITY_ALIASES
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .constants import RE_ENTRY_START, RE_LINE_PREFIX, RE_TRAILING_DATASET, RE_WHITESPACE, SEVERITY_ALIASES
|
|
7
7
|
from .encoding import detect_encoding
|
|
8
8
|
from .models import ParsedLine
|
|
9
9
|
|
|
@@ -52,12 +52,14 @@ def build_markdown_report(
|
|
|
52
52
|
continue
|
|
53
53
|
|
|
54
54
|
lines.append("| Severity | Count | Normalized message | Examples |")
|
|
55
|
-
lines.append("|---|---:|---|---|")
|
|
56
|
-
for (severity, normalized_message), count in top_norm:
|
|
57
|
-
examples_counter = analysis.norm_examples[(severity, normalized_message)]
|
|
58
|
-
examples = [
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
lines.append("|---|---:|---|---|")
|
|
56
|
+
for (severity, normalized_message), count in top_norm:
|
|
57
|
+
examples_counter = analysis.norm_examples[(severity, normalized_message)]
|
|
58
|
+
examples = [
|
|
59
|
+
f"{message} ({amount})" for message, amount in examples_counter.most_common(config.top_examples)
|
|
60
|
+
]
|
|
61
|
+
examples_text = "<br>".join(examples) if examples else ""
|
|
62
|
+
lines.append(f"| {severity} | {count} | {normalized_message} | {examples_text} |")
|
|
61
63
|
lines.append("")
|
|
62
64
|
|
|
63
65
|
lines.append("## Overall summary (all files)")
|
|
@@ -65,12 +67,14 @@ def build_markdown_report(
|
|
|
65
67
|
top_global = _top_counter_items(summary.global_norm, REPORT_TOP_GLOBAL)
|
|
66
68
|
if top_global:
|
|
67
69
|
lines.append("| Severity | Count | Normalized message | Examples |")
|
|
68
|
-
lines.append("|---|---:|---|---|")
|
|
69
|
-
for (severity, normalized_message), count in top_global:
|
|
70
|
-
examples_counter = summary.global_norm_examples[(severity, normalized_message)]
|
|
71
|
-
examples = [
|
|
72
|
-
|
|
73
|
-
|
|
70
|
+
lines.append("|---|---:|---|---|")
|
|
71
|
+
for (severity, normalized_message), count in top_global:
|
|
72
|
+
examples_counter = summary.global_norm_examples[(severity, normalized_message)]
|
|
73
|
+
examples = [
|
|
74
|
+
f"{message} ({amount})" for message, amount in examples_counter.most_common(config.top_examples)
|
|
75
|
+
]
|
|
76
|
+
examples_text = "<br>".join(examples) if examples else ""
|
|
77
|
+
lines.append(f"| {severity} | {count} | {normalized_message} | {examples_text} |")
|
|
74
78
|
lines.append("")
|
|
75
79
|
else:
|
|
76
80
|
lines.append("_No messages found._")
|
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import shutil
|
|
4
|
-
from contextlib import redirect_stderr, redirect_stdout
|
|
5
|
-
from io import StringIO
|
|
6
|
-
|
|
7
|
-
from .constants import
|
|
4
|
+
from contextlib import redirect_stderr, redirect_stdout
|
|
5
|
+
from io import StringIO
|
|
6
|
+
|
|
7
|
+
from .constants import (
|
|
8
|
+
COMMON_MOJIBAKE_TOKENS,
|
|
9
|
+
LATEX_PDF_ENGINES,
|
|
10
|
+
LATEX_SPECIAL_CHAR_REPLACEMENTS,
|
|
11
|
+
RE_MARKDOWN_TABLE_SEPARATOR,
|
|
12
|
+
)
|
|
8
13
|
|
|
9
14
|
|
|
10
15
|
def select_pdf_engine() -> str | None:
|
|
@@ -9,17 +9,23 @@ from .report_pdf import build_pdf_safe_markdown, escape_latex_text, make_markdow
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
def run_self_tests() -> None:
|
|
12
|
-
"""Run deterministic built-in assertions for quick local verification."""
|
|
13
|
-
|
|
14
|
-
for raw_message, expected in NORMALIZATION_SELF_TEST_CASES:
|
|
15
|
-
actual = normalize_message(raw_message)
|
|
16
|
-
assert actual == expected, f"normalize_message({raw_message!r}) -> {actual!r}, expected {expected!r}"
|
|
17
|
-
|
|
18
|
-
assert is_entry_start("ERROR\tLine 1: tab-separated severity")
|
|
19
|
-
assert
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
12
|
+
"""Run deterministic built-in assertions for quick local verification."""
|
|
13
|
+
|
|
14
|
+
for raw_message, expected in NORMALIZATION_SELF_TEST_CASES:
|
|
15
|
+
actual = normalize_message(raw_message)
|
|
16
|
+
assert actual == expected, f"normalize_message({raw_message!r}) -> {actual!r}, expected {expected!r}"
|
|
17
|
+
|
|
18
|
+
assert is_entry_start("ERROR\tLine 1: tab-separated severity")
|
|
19
|
+
assert (
|
|
20
|
+
escape_latex_text(r"D:\DATEN_UEBERNAHME\A&B")
|
|
21
|
+
== r"D:\textbackslash{}DATEN\_UEBERNAHME\textbackslash{}A\&B"
|
|
22
|
+
)
|
|
23
|
+
assert make_markdown_table_line_pdf_safe("|---|---:|---|") == "|---|---:|---|"
|
|
24
|
+
assert (
|
|
25
|
+
make_markdown_table_line_pdf_safe(r"| ERROR | D:\DATEN_1<br>foo |")
|
|
26
|
+
== r"| ERROR | D:\textbackslash{}DATEN\_1 ; foo |"
|
|
27
|
+
)
|
|
28
|
+
assert build_pdf_safe_markdown("plain\n| A | B |\n").endswith("\n")
|
|
23
29
|
|
|
24
30
|
sample_report_markdown = """# Log Analysis Report
|
|
25
31
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nm-tool-forge
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: Analyze MigMan log files and generate aggregated CSV, Markdown, HTML, and optional PDF reports.
|
|
5
5
|
Author-email: Stefan Ewald <s.ew@outlook.de>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -89,6 +89,23 @@ python -m loganalysis --self-test
|
|
|
89
89
|
|
|
90
90
|
Legacy compatibility call:
|
|
91
91
|
|
|
92
|
+
|
|
93
|
+
## Release process
|
|
94
|
+
|
|
95
|
+
To publish a new release, always test on TestPyPI first, then upload to PyPI only after successful Conda/Smoke-Tests:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
export TWINE_USERNAME="__token__"
|
|
99
|
+
export TWINE_PASSWORD="pypi-..."
|
|
100
|
+
|
|
101
|
+
bash scripts/release_testpypi.sh --bump patch
|
|
102
|
+
bash scripts/release_pypi.sh --yes
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
**Hinweise:**
|
|
106
|
+
- Erst TestPyPI ausführen und testen, dann final nach PyPI hochladen.
|
|
107
|
+
- Versionen auf PyPI können nicht überschrieben oder erneut verwendet werden.
|
|
108
|
+
|
|
92
109
|
```powershell
|
|
93
110
|
python .\log_analysis.py --convert
|
|
94
111
|
```
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
LICENSE
|
|
2
2
|
README.md
|
|
3
3
|
pyproject.toml
|
|
4
|
+
src/csvchunking/__init__.py
|
|
5
|
+
src/csvchunking/__main__.py
|
|
6
|
+
src/csvchunking/chunker.py
|
|
7
|
+
src/csvchunking/cli.py
|
|
4
8
|
src/loganalysis/__init__.py
|
|
5
9
|
src/loganalysis/__main__.py
|
|
6
10
|
src/loganalysis/analysis.py
|
|
@@ -25,6 +29,7 @@ src/nm_tool_forge.egg-info/entry_points.txt
|
|
|
25
29
|
src/nm_tool_forge.egg-info/requires.txt
|
|
26
30
|
src/nm_tool_forge.egg-info/top_level.txt
|
|
27
31
|
tests/test_analysis.py
|
|
32
|
+
tests/test_csvchunking.py
|
|
28
33
|
tests/test_normalization.py
|
|
29
34
|
tests/test_parsing.py
|
|
30
35
|
tests/test_report_html.py
|
|
@@ -18,11 +18,12 @@ def test_analyze_file_aggregates_raw_and_normalized_counts(tmp_path: Path) -> No
|
|
|
18
18
|
result = analyze_file(log_path)
|
|
19
19
|
|
|
20
20
|
assert result.total_lines == 3
|
|
21
|
-
assert result.total_entries == 3
|
|
22
|
-
assert result.unknown_lines == 0
|
|
23
|
-
assert result.raw_counts[("WARNING", "Different issue")] == 1
|
|
24
|
-
|
|
25
|
-
assert
|
|
21
|
+
assert result.total_entries == 3
|
|
22
|
+
assert result.unknown_lines == 0
|
|
23
|
+
assert result.raw_counts[("WARNING", "Different issue")] == 1
|
|
24
|
+
normalized_key = ("ERROR", 'Conversion: X =<VALUE> The record was not found in table "Teile".')
|
|
25
|
+
assert result.norm_counts[normalized_key] == 2
|
|
26
|
+
assert len(result.norm_examples[normalized_key]) == 2
|
|
26
27
|
|
|
27
28
|
|
|
28
29
|
def test_run_analysis_writes_outputs_and_report(tmp_path: Path) -> None:
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from csvchunking.chunker import split_csv
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def make_csv(tmp_path, name, header, rows, encoding="utf-8-sig", delimiter=";"):
|
|
7
|
+
file = tmp_path / name
|
|
8
|
+
with open(file, "w", encoding=encoding, newline="") as f:
|
|
9
|
+
f.write(delimiter.join(header) + "\n")
|
|
10
|
+
for row in rows:
|
|
11
|
+
f.write(delimiter.join(row) + "\n")
|
|
12
|
+
return file
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_normale_aufteilung(tmp_path):
|
|
16
|
+
header = ["col1", "col2"]
|
|
17
|
+
rows = [["A", "1"], ["B", "2"], ["C", "3"], ["D", "4"], ["E", "5"]]
|
|
18
|
+
file = make_csv(tmp_path, "sample.csv", header, rows)
|
|
19
|
+
result = split_csv(file, chunk_size=2)
|
|
20
|
+
assert result.files_created == 3
|
|
21
|
+
for out in result.output_files:
|
|
22
|
+
with open(out, encoding="utf-8-sig") as f:
|
|
23
|
+
lines = f.read().splitlines()
|
|
24
|
+
assert lines[0] == "col1;col2"
|
|
25
|
+
assert (result.output_dir / "sample_01.csv").exists()
|
|
26
|
+
assert (result.output_dir / "sample_02.csv").exists()
|
|
27
|
+
assert (result.output_dir / "sample_03.csv").exists()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_header_in_jeder_datei(tmp_path):
|
|
31
|
+
header = ["foo", "bar"]
|
|
32
|
+
rows = [["x", "1"], ["y", "2"], ["z", "3"]]
|
|
33
|
+
file = make_csv(tmp_path, "test.csv", header, rows)
|
|
34
|
+
result = split_csv(file, chunk_size=1)
|
|
35
|
+
for out in result.output_files:
|
|
36
|
+
with open(out, encoding="utf-8-sig") as f:
|
|
37
|
+
assert f.readline().strip() == "foo;bar"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_dateiname_mit_leerzeichen(tmp_path):
|
|
41
|
+
header = ["a", "b"]
|
|
42
|
+
rows = [["1", "2"]]
|
|
43
|
+
file = make_csv(tmp_path, "Part-Storage Areas Relationships.csv", header, rows)
|
|
44
|
+
result = split_csv(file, chunk_size=1)
|
|
45
|
+
assert result.output_dir.name == "Part-Storage Areas Relationships"
|
|
46
|
+
assert (result.output_dir / "Part-Storage Areas Relationships_01.csv").exists()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_ungueltige_chunkgroesse(tmp_path):
|
|
50
|
+
header = ["a", "b"]
|
|
51
|
+
rows = [["1", "2"]]
|
|
52
|
+
file = make_csv(tmp_path, "fail.csv", header, rows)
|
|
53
|
+
with pytest.raises(ValueError):
|
|
54
|
+
split_csv(file, chunk_size=0)
|
|
55
|
+
with pytest.raises(ValueError):
|
|
56
|
+
split_csv(file, chunk_size=-1)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_leere_datei(tmp_path):
|
|
60
|
+
file = tmp_path / "empty.csv"
|
|
61
|
+
file.write_text("")
|
|
62
|
+
with pytest.raises(ValueError):
|
|
63
|
+
split_csv(file, chunk_size=1)
|
|
@@ -16,7 +16,9 @@ def test_build_and_parse_markdown_report_roundtrip(tmp_path: Path) -> None:
|
|
|
16
16
|
unknown_lines=0,
|
|
17
17
|
raw_counts=Counter({("ERROR", 'Conversion: X =3100110. 138 The record was not found in table "Teile".'): 2}),
|
|
18
18
|
norm_counts=Counter({normalized_key: 2}),
|
|
19
|
-
norm_examples={
|
|
19
|
+
norm_examples={
|
|
20
|
+
normalized_key: Counter({'Conversion: X =3100110. 138 The record was not found in table "Teile".': 2})
|
|
21
|
+
},
|
|
20
22
|
backup_path=tmp_path / "backup" / "demo.txt.bak",
|
|
21
23
|
)
|
|
22
24
|
summary = AnalysisSummary(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|