codetool-shell 0.1.1__tar.gz

codetool_shell-0.1.1/.gitignore

@@ -0,0 +1,24 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+.pytest_cache/
+
+# Virtual environments
+.venv
+
+# Test-generated executable shims
+.pytest-executables/
+
+# Staged Rust helper binaries
+src/codetool_shell/bin/
+
+# Rust build artifacts
+rust/target/
+
+# Research file
+
+research/

codetool_shell-0.1.1/PKG-INFO

@@ -0,0 +1,152 @@
+Metadata-Version: 2.4
+Name: codetool-shell
+Version: 0.1.1
+Summary: Compress shell output text with Python and optional Rust backends
+Project-URL: Homepage, https://github.com/pbi-agent/codetool-shell
+Project-URL: Repository, https://github.com/pbi-agent/codetool-shell
+Project-URL: Issues, https://github.com/pbi-agent/codetool-shell/issues
+Project-URL: Changelog, https://github.com/pbi-agent/codetool-shell/releases
+Author-email: drod <naceur.bs@gmail.com>
+Maintainer-email: drod <naceur.bs@gmail.com>
+Keywords: agent,compression,developer-tools,llm,rust,shell,terminal
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Rust
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: System :: Shells
+Classifier: Topic :: Text Processing
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# codetool-shell
+
+Python interface library for compressing plain shell-output text before it is
+returned to an LLM/tool consumer.
+
+```python
+from codetool_shell import compress_text
+
+compressed = compress_text("one  \n\n\n two\t\n")
+print(compressed)
+```
+
+The core pipeline is intentionally conservative:
+
+- normalize CRLF/CR line endings to LF
+- trim trailing whitespace on each line
+- run dedicated output filters only when they make the text smaller
+- collapse repeated blank lines to `max_blank_lines` (default: `1`)
+
+Initial dedicated filters cover:
+
+- git output: short and long status, tabbed name-status, diff stats, and patch
+  hunks/binary-patch payloads with verbose unchanged or encoded context
+  summarized
+- test-runner output: conservative ANSI-aware pytest (including quiet
+  progress/summary output), unittest-like, vitest-like, Playwright, and
+  cargo-test summaries that preserve failure identifiers, assertion/error text,
+  source locations, artifacts/traces, and final totals
+- CI/job logs: conservative GitHub Actions / `gh run`-style summaries that
+  collapse repeated timestamped step noise while preserving step names,
+  annotations, warnings/errors, file locations, artifact/URL/path references,
+  and job/workflow conclusions
+- standalone Python tracebacks: conservative traceback summaries that preserve
+  exception type/message, file stack frames, chained-exception separators,
+  source/caret context, and small command noise while collapsing repeated or
+  internal stack frames
+- structured tables: conservative psql, SQLite/column, MySQL boxed, and long
+  Markdown pipe table summaries that preserve headers, footers, first/last
+  rows, and warning/error/URL/path/ID rows while counting omitted middle rows
+- file/tree/listing output: conservative path-only, `tree`, and safe `ls -R`
+  reducers that render repeated path prefixes as compact trees while preserving
+  visible summary totals and leaving metadata-rich listings unchanged
+- ripgrep output: direct `path:line[:column]:text`, heading-mode, context-mode
+  `path-line-context` with `--` separators, and ANSI-colored matches grouped
+  into a compact path tree when repeated paths or prefixes would otherwise be
+  duplicated, while diagnostic/table/prose-looking output is left alone
+- generic application logs: conservative long timestamp/level-prefixed log
+  summaries that collapse repeated INFO/DEBUG/TRACE patterns while preserving
+  first/last context, warnings/errors/fatal/panic/exception lines,
+  file paths/URLs, failure summaries, and unusual levels
+- package-manager logs: conservative uv/pip, npm, bun, and cargo package
+  install/update summaries that collapse progress/download/resolve/install
+  noise while preserving warnings/errors, package summaries, versions,
+  lock/frozen conflicts, audit/vulnerability summaries, hints, paths, and URLs
+- build/compiler diagnostics: conservative cargo/rustc, tsc, ESLint, and
+  mypy-like reducers that collapse build progress while preserving
+  warnings/errors, codes, file spans, source/caret snippets, notes/help, rule
+  IDs, and final summaries
+
+## Backends
+
+- `backend="python"` uses the pure-Python implementation.
+- `backend="rust"` requires the optional Rust CLI backend.
+- `backend="auto"` prefers Rust when a compatible binary is discoverable and
+  falls back to Python otherwise.
+
+Rust discovery order:
+
+1. `CODETOOL_SHELL_RUST_BINARY`
+2. packaged wheel binaries under `codetool_shell/bin/<platform>/`
+3. development binaries under `rust/target/{release,debug}/`
+4. `PATH`
+
+Build and stage the Rust CLI before creating a target wheel with:
+
+```bash
+python scripts/package_rust_binary.py --target-runtime linux-x86_64
+```
+
+Then build the Python distributions with:
+
+```bash
+uv build
+```
+
+During local development you can also rely on the development build directly:
+
+```bash
+cd rust
+cargo build
+```
+
+## Packaging and release
+
+`codetool-shell` publishes target-specific wheels that include one matching
+Rust helper binary under `codetool_shell/bin/<runtime>/`. Supported runtime keys
+are:
+
+- `linux-x86_64`
+- `linux-aarch64`
+- `macos-x86_64`
+- `macos-arm64`
+- `windows-x86_64`
+- `windows-arm64`
+
+To build a Linux x86_64 wheel locally (replace the runtime key for other
+targets):
+
+```bash
+uv run python scripts/package_rust_binary.py --target-runtime linux-x86_64
+CODETOOL_SHELL_TARGET_RUNTIME=linux-x86_64 uv build --wheel
+```
+
+The release workflow builds an sdist, manylinux wheels, native macOS/Windows
+wheels, checks the distributions, and publishes to PyPI via trusted publishing
+for tagged releases (`v*`) or manual dispatch with `publish=true`.
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest
+uv run python benchmarks/benchmark_compress_text.py
+uv run python scripts/benchmark_filters.py --iterations 1000
+```

codetool_shell-0.1.1/README.md

@@ -0,0 +1,125 @@
+# codetool-shell
+
+Python interface library for compressing plain shell-output text before it is
+returned to an LLM/tool consumer.
+
+```python
+from codetool_shell import compress_text
+
+compressed = compress_text("one  \n\n\n two\t\n")
+print(compressed)
+```
+
+The core pipeline is intentionally conservative:
+
+- normalize CRLF/CR line endings to LF
+- trim trailing whitespace on each line
+- run dedicated output filters only when they make the text smaller
+- collapse repeated blank lines to `max_blank_lines` (default: `1`)
+
+Initial dedicated filters cover:
+
+- git output: short and long status, tabbed name-status, diff stats, and patch
+  hunks/binary-patch payloads with verbose unchanged or encoded context
+  summarized
+- test-runner output: conservative ANSI-aware pytest (including quiet
+  progress/summary output), unittest-like, vitest-like, Playwright, and
+  cargo-test summaries that preserve failure identifiers, assertion/error text,
+  source locations, artifacts/traces, and final totals
+- CI/job logs: conservative GitHub Actions / `gh run`-style summaries that
+  collapse repeated timestamped step noise while preserving step names,
+  annotations, warnings/errors, file locations, artifact/URL/path references,
+  and job/workflow conclusions
+- standalone Python tracebacks: conservative traceback summaries that preserve
+  exception type/message, file stack frames, chained-exception separators,
+  source/caret context, and small command noise while collapsing repeated or
+  internal stack frames
+- structured tables: conservative psql, SQLite/column, MySQL boxed, and long
+  Markdown pipe table summaries that preserve headers, footers, first/last
+  rows, and warning/error/URL/path/ID rows while counting omitted middle rows
+- file/tree/listing output: conservative path-only, `tree`, and safe `ls -R`
+  reducers that render repeated path prefixes as compact trees while preserving
+  visible summary totals and leaving metadata-rich listings unchanged
+- ripgrep output: direct `path:line[:column]:text`, heading-mode, context-mode
+  `path-line-context` with `--` separators, and ANSI-colored matches grouped
+  into a compact path tree when repeated paths or prefixes would otherwise be
+  duplicated, while diagnostic/table/prose-looking output is left alone
+- generic application logs: conservative long timestamp/level-prefixed log
+  summaries that collapse repeated INFO/DEBUG/TRACE patterns while preserving
+  first/last context, warnings/errors/fatal/panic/exception lines,
+  file paths/URLs, failure summaries, and unusual levels
+- package-manager logs: conservative uv/pip, npm, bun, and cargo package
+  install/update summaries that collapse progress/download/resolve/install
+  noise while preserving warnings/errors, package summaries, versions,
+  lock/frozen conflicts, audit/vulnerability summaries, hints, paths, and URLs
+- build/compiler diagnostics: conservative cargo/rustc, tsc, ESLint, and
+  mypy-like reducers that collapse build progress while preserving
+  warnings/errors, codes, file spans, source/caret snippets, notes/help, rule
+  IDs, and final summaries
+
+## Backends
+
+- `backend="python"` uses the pure-Python implementation.
+- `backend="rust"` requires the optional Rust CLI backend.
+- `backend="auto"` prefers Rust when a compatible binary is discoverable and
+  falls back to Python otherwise.
+
+Rust discovery order:
+
+1. `CODETOOL_SHELL_RUST_BINARY`
+2. packaged wheel binaries under `codetool_shell/bin/<platform>/`
+3. development binaries under `rust/target/{release,debug}/`
+4. `PATH`
+
+Build and stage the Rust CLI before creating a target wheel with:
+
+```bash
+python scripts/package_rust_binary.py --target-runtime linux-x86_64
+```
+
+Then build the Python distributions with:
+
+```bash
+uv build
+```
+
+During local development you can also rely on the development build directly:
+
+```bash
+cd rust
+cargo build
+```
+
+## Packaging and release
+
+`codetool-shell` publishes target-specific wheels that include one matching
+Rust helper binary under `codetool_shell/bin/<runtime>/`. Supported runtime keys
+are:
+
+- `linux-x86_64`
+- `linux-aarch64`
+- `macos-x86_64`
+- `macos-arm64`
+- `windows-x86_64`
+- `windows-arm64`
+
+To build a Linux x86_64 wheel locally (replace the runtime key for other
+targets):
+
+```bash
+uv run python scripts/package_rust_binary.py --target-runtime linux-x86_64
+CODETOOL_SHELL_TARGET_RUNTIME=linux-x86_64 uv build --wheel
+```
+
+The release workflow builds an sdist, manylinux wheels, native macOS/Windows
+wheels, checks the distributions, and publishes to PyPI via trusted publishing
+for tagged releases (`v*`) or manual dispatch with `publish=true`.
+
+## Development
+
+```bash
+uv sync --dev
+uv run pytest
+uv run python benchmarks/benchmark_compress_text.py
+uv run python scripts/benchmark_filters.py --iterations 1000
+```

codetool_shell-0.1.1/benchmarks/benchmark_compress_text.py

@@ -0,0 +1,79 @@
+"""Tiny benchmark for compression backends."""
+
+from __future__ import annotations
+
+import argparse
+import statistics
+import time
+
+from codetool_shell import compress_text
+
+SAMPLE_OUTPUT = "\n".join(
+    [
+        "collecting workspace metadata   ",
+        "",
+        "",
+        "src/package/module.py:10: info: ok\t",
+        "",
+        "",
+        "",
+        "tests/test_module.py::test_case PASSED   ",
+        "",
+    ]
+)
+
+
+def benchmark_backend(
+    backend: str,
+    text: str,
+    max_blank_lines: int,
+    iterations: int,
+) -> dict:
+    compress_text(text, max_blank_lines=max_blank_lines, backend=backend)
+
+    durations = []
+    output = ""
+    for _ in range(iterations):
+        started = time.perf_counter()
+        output = compress_text(text, max_blank_lines=max_blank_lines, backend=backend)
+        durations.append((time.perf_counter() - started) * 1000)
+
+    verify_result(backend, text, output)
+    return {
+        "backend": backend,
+        "min_ms": min(durations),
+        "median_ms": statistics.median(durations),
+        "max_ms": max(durations),
+        "compressed_chars": len(output),
+    }
+
+
+def verify_result(backend: str, original: str, compressed: str) -> None:
+    if not isinstance(compressed, str):
+        raise RuntimeError(f"{backend} benchmark did not return text")
+    if len(compressed) > len(original):
+        raise RuntimeError(f"{backend} benchmark expanded the sample")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--backend", choices=["auto", "python", "rust"], default="auto")
+    parser.add_argument("--iterations", type=int, default=1000)
+    parser.add_argument("--max-blank-lines", type=int, default=1)
+    args = parser.parse_args()
+
+    row = benchmark_backend(
+        args.backend,
+        SAMPLE_OUTPUT,
+        args.max_blank_lines,
+        args.iterations,
+    )
+    print(
+        "{backend}: min={min_ms:.4f}ms median={median_ms:.4f}ms "
+        "max={max_ms:.4f}ms chars={compressed_chars}".format(**row)
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

codetool_shell-0.1.1/hatch_build.py

@@ -0,0 +1,203 @@
+"""Target-aware Hatch wheel hook for prebuilt Rust helper wheels."""
+
+from __future__ import annotations
+
+import os
+import sysconfig
+from pathlib import Path
+
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+
+PACKAGE = "codetool_shell"
+RUST_BINARY_NAME = "codetool-shell-rust"
+TARGET_RUNTIME_ENVS = (
+    "CODETOOL_SHELL_TARGET_RUNTIME",
+    "CODETOOL_TARGET_RUNTIME",
+)
+TARGET_WHEEL_TAG_ENVS = (
+    "CODETOOL_SHELL_TARGET_WHEEL_TAG",
+    "CODETOOL_TARGET_WHEEL_TAG",
+)
+
+SUPPORTED_RUNTIME_KEYS = (
+    "linux-x86_64",
+    "linux-aarch64",
+    "macos-x86_64",
+    "macos-arm64",
+    "windows-x86_64",
+    "windows-arm64",
+)
+
+RUNTIME_ALIASES = {
+    "linux-amd64": "linux-x86_64",
+    "linux-x64": "linux-x86_64",
+    "darwin-x86_64": "macos-x86_64",
+    "darwin-amd64": "macos-x86_64",
+    "darwin-arm64": "macos-arm64",
+    "darwin-aarch64": "macos-arm64",
+    "macos-aarch64": "macos-arm64",
+    "macosx-x86_64": "macos-x86_64",
+    "macosx-arm64": "macos-arm64",
+    "linux-arm64": "linux-aarch64",
+    "windows-amd64": "windows-x86_64",
+    "windows-x64": "windows-x86_64",
+    "windows-aarch64": "windows-arm64",
+    "win-x86_64": "windows-x86_64",
+    "win-amd64": "windows-x86_64",
+    "win-arm64": "windows-arm64",
+    "linux_x86_64": "linux-x86_64",
+    "linux_aarch64": "linux-aarch64",
+    "macos_x86_64": "macos-x86_64",
+    "macos_arm64": "macos-arm64",
+    "windows_x86_64": "windows-x86_64",
+    "windows_arm64": "windows-arm64",
+}
+
+DEFAULT_WHEEL_TAGS = {
+    "linux-x86_64": "manylinux2014_x86_64",
+    "linux-aarch64": "manylinux2014_aarch64",
+    "macos-x86_64": "macosx_10_12_x86_64",
+    "macos-arm64": "macosx_11_0_arm64",
+    "windows-x86_64": "win_amd64",
+    "windows-arm64": "win_arm64",
+}
+
+
+class CustomBuildHook(BuildHookInterface):
+    """Include exactly one target runtime binary and tag the wheel for it."""
+
+    def initialize(self, version: str, build_data: dict) -> None:
+        if self.target_name != "wheel":
+            return
+
+        runtime_key = target_runtime_key()
+        binary = staged_binary(Path(self.root), runtime_key)
+        if binary is None:
+            if target_runtime_env_is_set():
+                raise FileNotFoundError(
+                    f"no staged Rust helper for {runtime_key}; run "
+                    f"scripts/package_rust_binary.py --target-runtime {runtime_key} first"
+                )
+            return
+
+        build_data["pure_python"] = False
+        build_data["tag"] = f"py3-none-{target_wheel_tag(runtime_key)}"
+
+        force_include = build_data.setdefault("force_include", {})
+        src_root = Path(self.root) / "src"
+        force_include[binary.relative_to(self.root).as_posix()] = binary.relative_to(src_root).as_posix()
+
+
+def staged_binary(root: Path, runtime_key: str) -> Path | None:
+    runtime_key = canonical_runtime_key(runtime_key)
+    package_bin = root / "src" / PACKAGE / "bin"
+    binary_name = binary_name_for_runtime(runtime_key)
+    for key in runtime_key_candidates(runtime_key):
+        candidate = package_bin / key / binary_name
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def target_runtime_env_is_set() -> bool:
+    return _first_env(TARGET_RUNTIME_ENVS) is not None
+
+
+def target_runtime_key() -> str:
+    override = _first_env(TARGET_RUNTIME_ENVS)
+    if override:
+        return canonical_runtime_key(override)
+    return host_runtime_key()
+
+
+def target_wheel_tag(runtime_key: str) -> str:
+    override = _first_env(TARGET_WHEEL_TAG_ENVS)
+    if override:
+        return override.strip().replace("-", "_").replace(".", "_")
+    return DEFAULT_WHEEL_TAGS[canonical_runtime_key(runtime_key)]
+
+
+def binary_name_for_runtime(runtime_key: str) -> str:
+    if canonical_runtime_key(runtime_key).startswith("windows-"):
+        return f"{RUST_BINARY_NAME}.exe"
+    return RUST_BINARY_NAME
+
+
+def runtime_key_candidates(runtime_key: str) -> list[str]:
+    canonical = canonical_runtime_key(runtime_key)
+    candidates = [canonical]
+    for alias, target in RUNTIME_ALIASES.items():
+        if target == canonical and alias not in candidates:
+            candidates.append(alias)
+    return candidates
+
+
+def canonical_runtime_key(key: str) -> str:
+    normalized = key.strip().lower().replace("_", "-")
+    normalized = normalized.replace("linux-x86-64", "linux-x86_64")
+    normalized = normalized.replace("macos-x86-64", "macos-x86_64")
+    normalized = normalized.replace("windows-x86-64", "windows-x86_64")
+    normalized = normalized.replace("darwin-x86-64", "darwin-x86_64")
+    normalized = RUNTIME_ALIASES.get(normalized, normalized)
+    if normalized not in SUPPORTED_RUNTIME_KEYS:
+        raise ValueError(
+            f"unsupported runtime key {key!r}; expected one of: "
+            + ", ".join(SUPPORTED_RUNTIME_KEYS)
+        )
+    return normalized
+
+
+def _first_env(names: tuple[str, ...]) -> str | None:
+    for name in names:
+        value = os.environ.get(name)
+        if value:
+            return value
+    return None
+
+
+def host_runtime_key() -> str:
+    platform_tag = sysconfig.get_platform().replace(".", "_").replace("-", "_").lower()
+    if platform_tag.startswith("linux"):
+        if "aarch64" in platform_tag or "arm64" in platform_tag:
+            return "linux-aarch64"
+        return "linux-x86_64"
+    if platform_tag.startswith("macos"):
+        if "arm64" in platform_tag or "aarch64" in platform_tag:
+            return "macos-arm64"
+        return "macos-x86_64"
+    if platform_tag.startswith("win") or platform_tag.startswith("windows"):
+        if "arm64" in platform_tag or "aarch64" in platform_tag:
+            return "windows-arm64"
+        return "windows-x86_64"
+    return canonical_runtime_key(platform_tag.replace("_", "-"))
+
+
+def runtime_key_for_system(system: str, machine: str) -> str:
+    """Compatibility helper for older tests/importers."""
+
+    normalized_system = (system or "").strip().lower()
+    normalized_machine = (machine or "").strip().lower().replace("_", "-")
+    os_key = {
+        "linux": "linux",
+        "darwin": "macos",
+        "macos": "macos",
+        "windows": "windows",
+        "mingw": "windows",
+        "msys": "windows",
+        "cygwin": "windows",
+    }.get(normalized_system, normalized_system)
+    arch_key = {
+        "amd64": "x86_64",
+        "x64": "x86_64",
+        "x86-64": "x86_64",
+        "x86_64": "x86_64",
+        "arm64": "arm64",
+        "aarch64": "aarch64" if os_key == "linux" else "arm64",
+    }.get(normalized_machine, normalized_machine)
+    return canonical_runtime_key(f"{os_key}-{arch_key}")
+
+
+def _platform_tag() -> str:
+    """Compatibility helper for earlier tests/importers."""
+
+    return target_runtime_key()

codetool_shell-0.1.1/pyproject.toml

@@ -0,0 +1,85 @@
+[project]
+name = "codetool-shell"
+version = "0.1.1"
+description = "Compress shell output text with Python and optional Rust backends"
+readme = "README.md"
+authors = [
+    { name = "drod", email = "naceur.bs@gmail.com" }
+]
+maintainers = [
+    { name = "drod", email = "naceur.bs@gmail.com" }
+]
+requires-python = ">=3.9"
+keywords = [
+    "agent",
+    "compression",
+    "developer-tools",
+    "llm",
+    "rust",
+    "shell",
+    "terminal",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: MacOS",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Rust",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: System :: Shells",
+    "Topic :: Text Processing",
+    "Typing :: Typed",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/pbi-agent/codetool-shell"
+Repository = "https://github.com/pbi-agent/codetool-shell"
+Issues = "https://github.com/pbi-agent/codetool-shell/issues"
+Changelog = "https://github.com/pbi-agent/codetool-shell/releases"
+
+[dependency-groups]
+dev = [
+    "hatchling>=1.27",
+    "pytest>=8,<9",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/codetool_shell"]
+exclude = ["src/codetool_shell/bin/**"]
+
+[tool.hatch.build.targets.wheel.hooks.custom]
+path = "hatch_build.py"
+
+[tool.hatch.build.targets.sdist]
+include = [
+    "src/**",
+    "tests/**",
+    "benchmarks/**",
+    "scripts/**",
+    "rust/Cargo.toml",
+    "rust/Cargo.lock",
+    "rust/src/**",
+    "hatch_build.py",
+    "README.md",
+    "pyproject.toml",
+    "uv.lock",
+]
+exclude = [
+    "src/codetool_shell/bin/**",
+    "**/__pycache__/**",
+    "**/*.py[cod]",
+    "rust/target/**",
+    "dist/**",
+]
+
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"