logslop 0.1.0__tar.gz

logslop-0.1.0/.github/workflows/release-trusted-publisher.yml

@@ -0,0 +1,41 @@
+name: Release (Trusted Publisher)
+
+permissions:
+  contents: write
+  id-token: write
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build package
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Upload to PyPI using Trusted Publisher
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/
+
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release create "${GITHUB_REF_NAME}" \
+            --title "${GITHUB_REF_NAME}" \
+            --generate-notes \
+            dist/*

logslop-0.1.0/LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2026 Advanced Micro Devices, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

logslop-0.1.0/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: logslop
+Version: 0.1.0
+Summary: Log deduplication: strip repeated lines so readers and agents can focus on key events.
+Author: Advanced Micro Devices, Inc.
+License: MIT
+Project-URL: homepage, https://github.com/amd/logslop
+Project-URL: repository, https://github.com/amd/logslop
+Project-URL: issues, https://github.com/amd/logslop/issues
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# LogsLop - Log Deduplication Tool
+
+LogsLop is a log summarization tool that removes repeated messages from log files, enabling readers and agents to home in quickly on key message events.
+
+In progress for public open source release.

logslop-0.1.0/README.md

@@ -0,0 +1,5 @@
+# LogsLop - Log Deduplication Tool
+
+LogsLop is a log summarization tool that removes repeated messages from log files, enabling readers and agents to home in quickly on key message events.
+
+In progress for public open source release.

logslop-0.1.0/logslop.egg-info/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.4
+Name: logslop
+Version: 0.1.0
+Summary: Log deduplication: strip repeated lines so readers and agents can focus on key events.
+Author: Advanced Micro Devices, Inc.
+License: MIT
+Project-URL: homepage, https://github.com/amd/logslop
+Project-URL: repository, https://github.com/amd/logslop
+Project-URL: issues, https://github.com/amd/logslop/issues
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# LogsLop - Log Deduplication Tool
+
+LogsLop is a log summarization tool that removes repeated messages from log files, enabling readers and agents to home in quickly on key message events.
+
+In progress for public open source release.

logslop-0.1.0/logslop.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+LICENSE
+README.md
+logslop.py
+logslop_test.py
+pyproject.toml
+.github/workflows/release-trusted-publisher.yml
+logslop.egg-info/PKG-INFO
+logslop.egg-info/SOURCES.txt
+logslop.egg-info/dependency_links.txt
+logslop.egg-info/entry_points.txt
+logslop.egg-info/top_level.txt

logslop-0.1.0/logslop.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

logslop-0.1.0/logslop.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+logslop = logslop:main

logslop-0.1.0/logslop.egg-info/top_level.txt

@@ -0,0 +1 @@
+logslop

logslop-0.1.0/logslop.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# Copyright (C) 2025 Advanced Micro Devices, Inc.
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file or at https://opensource.org/licenses/MIT.
+"""LogsLop Standalone - Log Deduplication Script
+
+Large log files are often full of repeated or near-duplicate lines.
+LogsLop strips that redundancy, so you get one representative per message type.
+Particularly useful when logs are too big to send or process as-is (e.g. into LLM context windows).
+
+Usage:
+  python3 logslop.py   (reads stdin)
+  your-command 2>&1 | python3 logslop.py   (pipes stderr to stdout and both through logslop)
+
+Examples:
+  journalctl --no-pager | python3 logslop.py
+  python3 logslop.py < your_log.txt
+
+To add to PATH: copy to e.g. ~/.local/bin, then export PATH="$HOME/.local/bin:$PATH" (add to ~/.bashrc to persist).
+
+Options (argparse, all optional):
+  -n 5000              max clusters to track
+  -t 0.6               Jaccard similarity threshold (0-1, higher = more aggressive deduping)
+  --no-normalize-digits  disable digit/hex normalization (rare)
+"""
+
+import argparse
+import re
+import sys
+
+
+def tokenize(line: str, normalize_digits: bool) -> list[str]:
+    """Tokenize with optional digit and hex normalization."""
+    placeholder = '\u0001'
+    if normalize_digits:
+        line = re.sub(r'\d+', placeholder, line)
+    word_chars = r'a-zA-Z' + (placeholder if normalize_digits else '0-9') + r'_'
+    tokens = re.findall(rf'[{word_chars}]+|[^\w\s]', line)
+    if normalize_digits:
+        hex_pattern = re.compile(r'^[' + placeholder + r'a-fA-FxX]+$')
+        tokens = [placeholder if placeholder in t and hex_pattern.match(t) else t for t in tokens]
+    return [t for t in tokens if t]
+
+
+def jaccard(a: list[str], b: list[str]) -> float:
+    """Token-counting Jaccard similarity."""
+    sa, sb = set(a), set(b)
+    inter = len(sa & sb)
+    union = len(sa | sb)
+    return inter / union if union else 1.0
+
+
+def process(lines, *, max_clusters=5000, threshold=0.6, normalize_digits=True):
+    """Yield lines that pass through (first occurrence of each cluster).
+    Emits ``  ...N...M`` markers (1-based omitted line ranges) before the next kept line; same at EOF for trailing skips."""
+    clusters = []  # [(exemplar_tokens, line_text), ...] most recent first; exemplar fixed, not updated on match
+    skipped = 0
+    lineno = 0
+    for line in lines:
+        lineno += 1
+        line = line.rstrip('\n\r')
+        tokens = tokenize(line, normalize_digits)
+        matched = False
+        for i, (ex_tokens, ex_line) in enumerate(clusters):
+            if jaccard(tokens, ex_tokens) >= threshold:
+                matched = True
+                clusters.pop(i)
+                clusters.insert(0, (ex_tokens, ex_line))  # Keep exemplar; replacing causes drift
+                break
+        if not matched:
+            if skipped > 0:
+                skip_start = lineno - skipped
+                skip_end = lineno - 1
+                yield f"  ...{skip_start}...{skip_end}"
+            yield line
+            skipped = 0
+            if max_clusters and len(clusters) >= max_clusters:
+                clusters.pop()
+            clusters.insert(0, (tokens, line))
+        else:
+            skipped += 1
+    if skipped > 0:
+        skip_start = lineno - skipped + 1
+        skip_end = lineno
+        yield f"  ...{skip_start}...{skip_end}"
+
+
+def main():
+    p = argparse.ArgumentParser(description='Reduce log redundancy: print first occurrence of each pattern.')
+    p.add_argument('-n', '--max-clusters', type=int, default=5000, metavar='N', help='Max clusters to track (default: 5000)')
+    p.add_argument('-t', '--threshold', type=float, default=0.6, help='Jaccard threshold for match (default: 0.6)')
+    p.add_argument('--no-normalize-digits', action='store_true', help='Disable digit normalization')
+    args = p.parse_args()
+    c0, c1 = ("\033[36m", "\033[0m") if sys.stdout.isatty() else ("", "")
+    print(f"{c0}# LogsLop: near-duplicate input lines removed.{c1}", flush=True)
+    print(f"{c0}#   ...N...M  = omitted lines N through M in the original input (1-based).{c1}", flush=True)
+    for line in process(sys.stdin, max_clusters=args.max_clusters, threshold=args.threshold, normalize_digits=not args.no_normalize_digits):
+        print(line, flush=True)
+
+
+if __name__ == '__main__':
+    main()

logslop-0.1.0/logslop_test.py

@@ -0,0 +1,81 @@
+# Copyright (C) 2025 Advanced Micro Devices, Inc.
+# Use of this source code is governed by an MIT-style license that can be
+# found in the LICENSE file or at https://opensource.org/licenses/MIT.
+"""Unit tests for standalone logslop."""
+
+import importlib.util
+import unittest
+from pathlib import Path
+
+_spec = importlib.util.spec_from_file_location("standalone_logslop", Path(__file__).parent / "logslop.py")
+_standalone = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_standalone)
+tokenize = _standalone.tokenize
+jaccard = _standalone.jaccard
+process = _standalone.process
+
+
+class TestTokenize(unittest.TestCase):
+    def test_normalize(self):
+        self.assertEqual(tokenize("foo 123 bar", True), tokenize("foo 456 bar", True))
+        self.assertNotEqual(tokenize("foo 123 bar", True), tokenize("foo 123 bar", False))
+
+    def test_no_normalize(self):
+        self.assertEqual(tokenize("foo 123 bar", False), ["foo", "123", "bar"])
+
+    def test_hex_normalization(self):
+        """Hex sequences (0x...) collapse to single placeholder like digit sequences."""
+        t1 = tokenize("amd_diag_apply_pgprot: 0x413d12000", True)
+        t2 = tokenize("amd_diag_apply_pgprot: 0x413d22000", True)
+        self.assertEqual(t1, t2, "hex addresses should normalize to same tokens")
+
+    def test_hex_and_digits_same_structure(self):
+        """Different hex values produce identical token sequences."""
+        self.assertEqual(
+            tokenize("pre_set_pgprot: phy_base -- 0x413d12000 and phy_type -- 0x4", True),
+            tokenize("pre_set_pgprot: phy_base -- 0x413d22000 and phy_type -- 0x5", True),
+        )
+
+
+class TestJaccard(unittest.TestCase):
+    def test_identical(self):
+        self.assertEqual(jaccard(["a", "b"], ["a", "b"]), 1.0)
+
+    def test_partial(self):
+        self.assertEqual(jaccard(["a", "b"], ["a", "c"]), 1 / 3)
+
+    def test_empty(self):
+        self.assertEqual(jaccard([], []), 1.0)
+
+
+class TestProcess(unittest.TestCase):
+    def test_dedupes_similar(self):
+        lines = ["foo 123 bar", "foo 456 bar", "foo 789 bar", "baz qux", "foo 999 bar"]
+        self.assertEqual(list(process(lines)), ["foo 123 bar", "  ...2...3", "baz qux", "  ...5...5"])
+
+    def test_no_normalize_keeps_all(self):
+        lines = ["line 1", "line 2", "line 3"]
+        self.assertEqual(list(process(lines, normalize_digits=False)), lines)
+
+    def test_threshold_stricter(self):
+        lines = ["foo bar baz", "foo bar qux", "foo bar"]
+        self.assertEqual(list(process(lines, threshold=0.9)), lines)
+        self.assertEqual(list(process(lines, threshold=0.5)), ["foo bar baz", "  ...2...3"])
+
+    def test_max_clusters(self):
+        lines = [f"unique_{i}" for i in range(5)]
+        self.assertEqual(len(list(process(lines, max_clusters=2, normalize_digits=False))), 5)
+        self.assertEqual(list(process([f"x {i}" for i in range(5)], max_clusters=2)), ["x 0", "  ...2...5"])
+
+    def test_hex_lines_deduped(self):
+        """Lines differing only in hex addresses are deduped."""
+        lines = [
+            "amd_diag_apply_pgprot: 0x413d12000",
+            "amd_diag_apply_pgprot: 0x413d22000",
+            "amd_diag_apply_pgprot: 0x413d72000",
+        ]
+        self.assertEqual(list(process(lines)), ["amd_diag_apply_pgprot: 0x413d12000", "  ...2...3"])
+
+
+if __name__ == "__main__":
+    unittest.main()

logslop-0.1.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["setuptools>=61.0", "setuptools-scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "logslop"
+dynamic = ["version"]
+description = "Log deduplication: strip repeated lines so readers and agents can focus on key events."
+readme = "README.md"
+requires-python = ">=3.8"
+license = { text = "MIT" }
+authors = [{ name = "Advanced Micro Devices, Inc." }]
+
+[project.urls]
+homepage = "https://github.com/amd/logslop"
+repository = "https://github.com/amd/logslop"
+issues = "https://github.com/amd/logslop/issues"
+
+[project.scripts]
+logslop = "logslop:main"
+
+[tool.setuptools]
+py-modules = ["logslop"]
+
+[tool.setuptools_scm]
+version_scheme = "post-release"

logslop-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+