pythinker-code 2.0.0__py3-none-any.whl

pythinker_code/utils/file_filter.py

@@ -0,0 +1,375 @@
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+from pathlib import Path
+
+_IGNORED_NAMES: frozenset[str] = frozenset(
+    (
+        # vcs metadata
+        ".DS_Store",
+        ".bzr",
+        ".git",
+        ".hg",
+        ".svn",
+        # tooling caches
+        ".build",
+        ".cache",
+        ".coverage",
+        ".fleet",
+        ".gradle",
+        ".idea",
+        ".ipynb_checkpoints",
+        ".pnpm-store",
+        ".pytest_cache",
+        ".pub-cache",
+        ".ruff_cache",
+        ".swiftpm",
+        ".tox",
+        ".venv",
+        ".vs",
+        ".vscode",
+        ".yarn",
+        ".yarn-cache",
+        # js / frontend
+        ".next",
+        ".nuxt",
+        ".parcel-cache",
+        ".svelte-kit",
+        ".turbo",
+        ".vercel",
+        "node_modules",
+        # python packaging
+        "__pycache__",
+        "build",
+        "coverage",
+        "dist",
+        "htmlcov",
+        "pip-wheel-metadata",
+        "venv",
+        # java / jvm
+        ".mvn",
+        "out",
+        "target",
+        # dotnet / native
+        "bin",
+        "cmake-build-debug",
+        "cmake-build-release",
+        "obj",
+        # bazel / buck
+        "bazel-bin",
+        "bazel-out",
+        "bazel-testlogs",
+        "buck-out",
+        # misc artifacts
+        ".dart_tool",
+        ".serverless",
+        ".stack-work",
+        ".terraform",
+        ".terragrunt-cache",
+        "DerivedData",
+        "Pods",
+        "deps",
+        "tmp",
+        "vendor",
+    )
+)
+
+_IGNORED_PATTERNS: re.Pattern[str] = re.compile(
+    r"|".join(
+        (
+            r".*_cache$",
+            r".*-cache$",
+            r".*\.egg-info$",
+            r".*\.dist-info$",
+            r".*\.py[co]$",
+            r".*\.class$",
+            r".*\.sw[po]$",
+            r".*~$",
+            r".*\.(?:tmp|bak)$",
+        )
+    ),
+    re.IGNORECASE,
+)
+
+_GIT_LS_FILES_TIMEOUT = 5
+
+
+def _scope_args(scope: str | None) -> list[str]:
+    """Return ``["--", "<scope>/"]`` if *scope* is given, else ``[]``."""
+    return ["--", scope + "/"] if scope else []
+
+
+def is_ignored(name: str) -> bool:
+    """Return *True* if *name* should be excluded from file mention results."""
+    if not name:
+        return True
+    if name in _IGNORED_NAMES:
+        return True
+    return bool(_IGNORED_PATTERNS.fullmatch(name))
+
+
+def detect_git(root: Path) -> bool:
+    """Return *True* if *root* is inside a git work tree."""
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--git-dir"],
+            cwd=root,
+            capture_output=True,
+            timeout=2,
+        )
+        return result.returncode == 0
+    except Exception:
+        return False
+
+
+def git_index_mtime(root: Path) -> float | None:
+    """Return the mtime of ``.git/index``, or *None* if unavailable."""
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", "--git-dir"],
+            cwd=root,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=2,
+        )
+        if result.returncode != 0:
+            return None
+        git_dir = Path(result.stdout.strip())
+        if not git_dir.is_absolute():
+            git_dir = root / git_dir
+        index = git_dir / "index"
+        return index.stat().st_mtime
+    except Exception:
+        return None
+
+
+def _parse_ls_files_output(stdout: str, *, filter_ignored: bool = True) -> list[str]:
+    """Parse NUL-delimited ``git ls-files -z`` output into paths with synthesised dirs.
+
+    When *filter_ignored* is *True*, paths whose segments match
+    ``is_ignored()`` are excluded so that tracked ``node_modules/``,
+    ``vendor/``, etc. do not pollute completion candidates.
+    """
+    paths: list[str] = []
+    seen_dirs: set[str] = set()
+    ignored_prefixes: set[str] = set()
+    for entry in stdout.split("\0"):
+        if not entry:
+            continue
+
+        parts = entry.split("/")
+
+        if filter_ignored:
+            skip = False
+            for i, part in enumerate(parts):
+                prefix = "/".join(parts[: i + 1]) + "/"
+                if prefix in ignored_prefixes:
+                    skip = True
+                    break
+                if is_ignored(part):
+                    ignored_prefixes.add(prefix)
+                    skip = True
+                    break
+            if skip:
+                continue
+
+        for i in range(1, len(parts)):
+            dir_path = "/".join(parts[:i]) + "/"
+            if dir_path not in seen_dirs:
+                seen_dirs.add(dir_path)
+                paths.append(dir_path)
+        paths.append(entry)
+    return paths
+
+
+def _git_deleted_files(root: Path, scope: str | None = None) -> set[str]:
+    """Return the set of tracked files deleted from the working tree."""
+    cmd = ["git", "-c", "core.quotepath=false", "ls-files", "-z", "--deleted", *_scope_args(scope)]
+    try:
+        result = subprocess.run(
+            cmd,
+            cwd=root,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=_GIT_LS_FILES_TIMEOUT,
+        )
+        if result.returncode == 0:
+            return {e for e in result.stdout.split("\0") if e}
+    except Exception:
+        pass
+    return set()
+
+
+def list_files_git(
+    root: Path,
+    scope: str | None = None,
+    *,
+    include_untracked: bool = True,
+) -> list[str] | None:
+    """List workspace paths via ``git ls-files``, or *None* on failure.
+
+    When *scope* is given (e.g. ``"src/utils"``), only files under that
+    subtree are returned.  When *include_untracked* is *True*, untracked
+    files (respecting ``.gitignore``) are appended via
+    ``--others --exclude-standard``.
+
+    Deleted working-tree files (``git ls-files --deleted``) are excluded
+    so that renamed / removed files do not appear as stale candidates.
+    """
+    if scope and ".." in scope.split("/"):
+        return None
+
+    cmd = [
+        "git",
+        "-c",
+        "core.quotepath=false",
+        "ls-files",
+        "-z",
+        "--recurse-submodules",
+        *_scope_args(scope),
+    ]
+    try:
+        result = subprocess.run(
+            cmd,
+            cwd=root,
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=_GIT_LS_FILES_TIMEOUT,
+        )
+        if result.returncode != 0:
+            return None
+    except Exception:
+        return None
+
+    deleted = _git_deleted_files(root, scope)
+    paths = _parse_ls_files_output(result.stdout)
+    if deleted:
+        paths = [p for p in paths if p.endswith("/") or p not in deleted]
+
+    if include_untracked:
+        others_cmd = [
+            "git",
+            "-c",
+            "core.quotepath=false",
+            "ls-files",
+            "-z",
+            "--others",
+            "--exclude-standard",
+            *_scope_args(scope),
+        ]
+        try:
+            others = subprocess.run(
+                others_cmd,
+                cwd=root,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=_GIT_LS_FILES_TIMEOUT,
+            )
+            if others.returncode == 0:
+                tracked = set(paths)
+                for p in _parse_ls_files_output(others.stdout):
+                    if p not in tracked:
+                        paths.append(p)
+        except Exception:
+            pass
+
+    # Prune directory entries that have no surviving file children.
+    if deleted:
+        live_dirs: set[str] = set()
+        for p in paths:
+            if not p.endswith("/"):
+                parts = p.split("/")
+                for i in range(1, len(parts)):
+                    live_dirs.add("/".join(parts[:i]) + "/")
+        paths = [p for p in paths if not p.endswith("/") or p in live_dirs]
+
+    return paths
+
+
+def list_files_walk(
+    root: Path,
+    scope: str | None = None,
+    *,
+    limit: int = 1000,
+) -> list[str]:
+    """List workspace paths via ``os.walk`` (fallback for non-git repos).
+
+    When *scope* is given, the walk starts from that subdirectory.
+    """
+    resolved_root = root.resolve()
+    walk_root = (root / scope).resolve() if scope else resolved_root
+
+    # Prevent path traversal outside the workspace (e.g. scope="../").
+    try:
+        if not walk_root.is_relative_to(resolved_root):
+            return []
+    except (OSError, ValueError):
+        return []
+
+    paths: list[str] = []
+    try:
+        for current_root, dirs, files in os.walk(walk_root):
+            relative_root = Path(current_root).resolve().relative_to(resolved_root)
+
+            dirs[:] = sorted(d for d in dirs if not is_ignored(d))
+
+            if relative_root.parts and any(is_ignored(part) for part in relative_root.parts):
+                dirs[:] = []
+                continue
+
+            if relative_root.parts:
+                paths.append(relative_root.as_posix() + "/")
+                if len(paths) >= limit:
+                    break
+
+            for file_name in sorted(files):
+                if is_ignored(file_name):
+                    continue
+                relative = (relative_root / file_name).as_posix()
+                if not relative:
+                    continue
+                paths.append(relative)
+                if len(paths) >= limit:
+                    break
+
+            if len(paths) >= limit:
+                break
+    except OSError:
+        pass
+
+    return paths
+
+
+def list_directory_filtered(directory: Path) -> list[dict[str, str | int]]:
+    """List immediate children of *directory*, filtering ignored entries.
+
+    Returns dicts with ``name``, ``type`` (``"file"``/``"directory"``), and
+    optionally ``size``.  Suitable for the web API response.
+    """
+    result: list[dict[str, str | int]] = []
+    try:
+        for subpath in directory.iterdir():
+            if is_ignored(subpath.name):
+                continue
+            if subpath.is_dir():
+                result.append({"name": subpath.name, "type": "directory"})
+            else:
+                try:
+                    size = subpath.stat().st_size
+                except OSError:
+                    size = 0
+                result.append({"name": subpath.name, "type": "file", "size": size})
+    except OSError:
+        pass
+    result.sort(key=lambda x: (str(x["type"]), str(x["name"])))
+    return result

pythinker_code/utils/frontmatter.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, cast
+
+import yaml
+
+
+def parse_frontmatter(text: str) -> dict[str, Any] | None:
+    """
+    Parse YAML frontmatter from a text blob.
+
+    Raises:
+        ValueError: If the frontmatter YAML is invalid.
+    """
+    lines = text.splitlines()
+    if not lines or lines[0].strip() != "---":
+        return None
+
+    frontmatter_lines: list[str] = []
+    for line in lines[1:]:
+        if line.strip() == "---":
+            break
+        frontmatter_lines.append(line)
+    else:
+        return None
+
+    frontmatter = "\n".join(frontmatter_lines).strip()
+    if not frontmatter:
+        return None
+
+    try:
+        raw_data: Any = yaml.safe_load(frontmatter)
+    except yaml.YAMLError as exc:
+        raise ValueError("Invalid frontmatter YAML.") from exc
+
+    if not isinstance(raw_data, dict):
+        raise ValueError("Frontmatter YAML must be a mapping.")
+
+    return cast(dict[str, Any], raw_data)
+
+
+def read_frontmatter(path: Path) -> dict[str, Any] | None:
+    """
+    Read the YAML frontmatter at the start of a file.
+
+    Args:
+        path: Path to an existing file that may contain frontmatter.
+    """
+    return parse_frontmatter(path.read_text(encoding="utf-8", errors="replace"))
+
+
+def strip_frontmatter(text: str) -> str:
+    """Return *text* with any leading YAML frontmatter block removed.
+
+    Mirrors the detection rule used by :func:`parse_frontmatter`: a frontmatter
+    block starts with ``---`` on the first line and ends at the next ``---``
+    line. Returns the original text unchanged when no valid frontmatter is
+    found. Sharing this helper lets callers skip the frontmatter in exactly the
+    same way :func:`parse_frontmatter` does, avoiding duplicate logic.
+    """
+    lines = text.splitlines(keepends=True)
+    if not lines or lines[0].strip() != "---":
+        return text
+    for i in range(1, len(lines)):
+        if lines[i].strip() == "---":
+            return "".join(lines[i + 1 :])
+    # Malformed frontmatter with no closing ``---``; parse_frontmatter returns
+    # None in that case, so we treat the whole input as body.
+    return text

pythinker_code/utils/io.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+def atomic_json_write(data: Any, path: Path) -> None:
+    """Write JSON data to a file atomically using tmp-file + os.replace.
+
+    This prevents data corruption if the process crashes mid-write: either the
+    old file is kept intact or the new file is fully committed.
+    """
+    fd, tmp_path = tempfile.mkstemp(dir=path.parent, suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp_path, path)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            os.unlink(tmp_path)
+        raise

pythinker_code/utils/logging.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import codecs
+import contextlib
+import locale
+import os
+import sys
+import threading
+from collections.abc import Iterator
+from typing import IO, Any, cast
+
+
+class _LazyLogger:
+    """Import loguru only when logging is actually used."""
+
+    def __init__(self) -> None:
+        self._logger: Any | None = None
+
+    def _get(self) -> Any:
+        if self._logger is None:
+            from loguru import logger as real_logger
+
+            # Disable logging by default for library usage.
+            # Application entry points should call logger.enable("pythinker_code")
+            # to enable logging.
+            real_logger.disable("pythinker_code")
+            self._logger = real_logger
+        return self._logger
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._get(), name)
+
+
+logger = cast(Any, _LazyLogger())
+
+
+class StderrRedirector:
+    def __init__(self, level: str = "ERROR") -> None:
+        self._level = level
+        self._encoding: str | None = None
+        self._installed = False
+        self._lock = threading.Lock()
+        self._original_fd: int | None = None
+        self._read_fd: int | None = None
+        self._thread: threading.Thread | None = None
+
+    def install(self) -> None:
+        with self._lock:
+            if self._installed:
+                return
+            with contextlib.suppress(Exception):
+                sys.stderr.flush()
+            if self._original_fd is None:
+                with contextlib.suppress(OSError):
+                    self._original_fd = os.dup(2)
+            if self._encoding is None:
+                self._encoding = (
+                    sys.stderr.encoding or locale.getpreferredencoding(False) or "utf-8"
+                )
+            read_fd, write_fd = os.pipe()
+            os.dup2(write_fd, 2)
+            os.close(write_fd)
+            self._read_fd = read_fd
+            self._thread = threading.Thread(
+                target=self._drain, name="pythinker-stderr-redirect", daemon=True
+            )
+            self._thread.start()
+            self._installed = True
+
+    def uninstall(self) -> None:
+        with self._lock:
+            if not self._installed:
+                return
+            if self._original_fd is not None:
+                os.dup2(self._original_fd, 2)
+            self._installed = False
+        if self._thread is not None:
+            self._thread.join(timeout=2.0)
+            self._thread = None
+
+    def _drain(self) -> None:
+        buffer = ""
+        read_fd = self._read_fd
+        if read_fd is None:
+            return
+        encoding = self._encoding or "utf-8"
+        decoder = codecs.getincrementaldecoder(encoding)(errors="replace")
+        try:
+            while True:
+                chunk = os.read(read_fd, 4096)
+                if not chunk:
+                    break
+                buffer += decoder.decode(chunk)
+                while "\n" in buffer:
+                    line, buffer = buffer.split("\n", 1)
+                    self._log_line(line)
+        except Exception:
+            logger.exception("Failed to read redirected stderr")
+        finally:
+            buffer += decoder.decode(b"", final=True)
+            if buffer:
+                self._log_line(buffer)
+            with contextlib.suppress(OSError):
+                os.close(read_fd)
+
+    def _log_line(self, line: str) -> None:
+        text = line.rstrip("\r")
+        if not text:
+            return
+        logger.opt(depth=2).log(self._level, text)
+
+    def open_original_stderr_handle(self) -> IO[bytes] | None:
+        if self._original_fd is None:
+            return None
+        dup_fd = os.dup(self._original_fd)
+        os.set_inheritable(dup_fd, True)
+        return os.fdopen(dup_fd, "wb", closefd=True)
+
+
+_stderr_redirector: StderrRedirector | None = None
+
+
+def redirect_stderr_to_logger(level: str = "ERROR") -> None:
+    global _stderr_redirector
+    if _stderr_redirector is None:
+        _stderr_redirector = StderrRedirector(level=level)
+    _stderr_redirector.install()
+
+
+def restore_stderr() -> None:
+    if _stderr_redirector is not None:
+        _stderr_redirector.uninstall()
+
+
+@contextlib.contextmanager
+def open_original_stderr() -> Iterator[IO[bytes] | None]:
+    redirector = _stderr_redirector
+    if redirector is None:
+        yield None
+        return
+    stream = redirector.open_original_stderr_handle()
+    try:
+        yield stream
+    finally:
+        if stream is not None:
+            stream.close()

pythinker_code/utils/media_tags.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from html import escape
+
+from pythinker_code.wire.types import ContentPart, TextPart
+
+
+def _format_tag(tag: str, attrs: Mapping[str, str | None] | None = None) -> str:
+    if not attrs:
+        return f"<{tag}>"
+    rendered: list[str] = []
+    for key, value in sorted(attrs.items()):
+        if not value:
+            continue
+        rendered.append(f'{key}="{escape(str(value), quote=True)}"')
+    if not rendered:
+        return f"<{tag}>"
+    return f"<{tag} " + " ".join(rendered) + ">"
+
+
+def wrap_media_part(
+    part: ContentPart, *, tag: str, attrs: Mapping[str, str | None] | None = None
+) -> list[ContentPart]:
+    return [
+        TextPart(text=_format_tag(tag, attrs)),
+        part,
+        TextPart(text=f"</{tag}>"),
+    ]

pythinker_code/utils/message.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pythinker_core.message import Message
+
+from pythinker_code.wire.types import AudioURLPart, ImageURLPart, TextPart, VideoURLPart
+
+
+def message_stringify(message: Message) -> str:
+    """Get a string representation of a message."""
+    # TODO: this should be merged into `pythinker_core.message.Message.extract_text`
+    parts: list[str] = []
+    for part in message.content:
+        if isinstance(part, TextPart):
+            parts.append(part.text)
+        elif isinstance(part, ImageURLPart):
+            parts.append("[image]")
+        elif isinstance(part, AudioURLPart):
+            suffix = f":{part.audio_url.id}" if part.audio_url.id else ""
+            parts.append(f"[audio{suffix}]")
+        elif isinstance(part, VideoURLPart):
+            parts.append("[video]")
+        else:
+            parts.append(f"[{part.type}]")
+    return "".join(parts)