wcgw 5.5.4__py3-none-any.whl

wcgw/client/bash_state/parser/__init__.py

@@ -0,0 +1,7 @@
+"""
+Parser for bash statements using tree-sitter.
+
+This module provides functionality to parse and identify individual bash statements.
+"""
+
+from .bash_statement_parser import BashStatementParser, Statement

wcgw/client/bash_state/parser/bash_statement_parser.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""
+Bash Statement Parser
+
+This script parses bash scripts and identifies individual statements using tree-sitter.
+It correctly handles multi-line strings, command chains with && and ||, and semicolon-separated statements.
+"""
+
+import sys
+from dataclasses import dataclass
+from typing import Any, List, Optional
+
+import tree_sitter_bash
+from tree_sitter import Language, Parser
+
+
+@dataclass
+class Statement:
+    """A bash statement with its source code and position information."""
+
+    text: str
+    start_line: int
+    end_line: int
+    start_byte: int
+    end_byte: int
+    node_type: str
+    parent_type: Optional[str] = None
+
+    def __str__(self) -> str:
+        return self.text.strip()
+
+
+class BashStatementParser:
+    def __init__(self) -> None:
+        # Use the precompiled bash language
+        self.language = Language(tree_sitter_bash.language())
+        self.parser = Parser(self.language)
+
+    def parse_file(self, file_path: str) -> List[Statement]:
+        """Parse a bash script file and return a list of statements."""
+        with open(file_path, "r", encoding="utf-8") as f:
+            content = f.read()
+        return self.parse_string(content)
+
+    def parse_string(self, content: str) -> List[Statement]:
+        """Parse a string containing bash script and return a list of statements."""
+        tree = self.parser.parse(bytes(content, "utf-8"))
+        root_node = tree.root_node
+
+        # For debugging: Uncomment to print the tree structure
+        # self._print_tree(root_node, content)
+
+        statements: List[Statement] = []
+        self._extract_statements(root_node, content, statements, None)
+
+        # Post-process statements to handle multi-line statements correctly
+        return self._post_process_statements(statements, content)
+
+    def _print_tree(self, node: Any, content: str, indent: str = "") -> None:
+        """Debug helper to print the entire syntax tree."""
+        node_text = content[node.start_byte : node.end_byte]
+        if len(node_text) > 40:
+            node_text = node_text[:37] + "..."
+        print(f"{indent}{node.type}: {repr(node_text)}")
+        for child in node.children:
+            self._print_tree(child, content, indent + "  ")
+
+    def _extract_statements(
+        self,
+        node: Any,
+        content: str,
+        statements: List[Statement],
+        parent_type: Optional[str],
+    ) -> None:
+        """Recursively extract statements from the syntax tree."""
+        # Node types that represent bash statements
+        statement_node_types = {
+            # Basic statements
+            "command",
+            "variable_assignment",
+            "declaration_command",
+            "unset_command",
+            # Control flow statements
+            "for_statement",
+            "c_style_for_statement",
+            "while_statement",
+            "if_statement",
+            "case_statement",
+            # Function definition
+            "function_definition",
+            # Command chains and groups
+            "pipeline",  # For command chains with | and |&
+            "list",  # For command chains with && and ||
+            "compound_statement",
+            "subshell",
+            "redirected_statement",
+        }
+
+        # Create a Statement object for this node if it's a recognized statement type
+        if node.type in statement_node_types:
+            # Get the text of this statement
+            start_byte = node.start_byte
+            end_byte = node.end_byte
+            statement_text = content[start_byte:end_byte]
+
+            # Get line numbers
+            start_line = (
+                node.start_point[0] + 1
+            )  # tree-sitter uses 0-indexed line numbers
+            end_line = node.end_point[0] + 1
+
+            statements.append(
+                Statement(
+                    text=statement_text,
+                    start_line=start_line,
+                    end_line=end_line,
+                    start_byte=start_byte,
+                    end_byte=end_byte,
+                    node_type=node.type,
+                    parent_type=parent_type,
+                )
+            )
+
+            # Update parent type for children
+            parent_type = node.type
+
+        # Recursively process all children
+        for child in node.children:
+            self._extract_statements(child, content, statements, parent_type)
+
+    def _post_process_statements(
+        self, statements: List[Statement], content: str
+    ) -> List[Statement]:
+        if not statements:
+            return []
+
+        # Filter out list statements that have been split
+        top_statements = []
+        for stmt in statements:
+            # Skip statements that are contained within others
+            is_contained = False
+            for other in statements:
+                if other is stmt:
+                    continue
+
+                # Check if completely contained (except for lists we've split)
+                if other.node_type != "list" or ";" not in other.text:
+                    if (
+                        other.start_line <= stmt.start_line
+                        and other.end_line >= stmt.end_line
+                        and len(other.text) > len(stmt.text)
+                        and stmt.text in other.text
+                    ):
+                        is_contained = True
+                        break
+
+            if not is_contained:
+                top_statements.append(stmt)
+
+        # Sort by position in file for consistent output
+        top_statements.sort(key=lambda s: (s.start_line, s.text))
+
+        return top_statements
+
+
+def main() -> None:
+    if len(sys.argv) < 2:
+        print("Usage: python bash_statement_parser.py <bash_script_file>")
+        sys.exit(1)
+
+    parser = BashStatementParser()
+    statements = parser.parse_file(sys.argv[1])
+
+    print(f"Found {len(statements)} statements:")
+    for i, stmt in enumerate(statements, 1):
+        print(f"\n--- Statement {i} (Lines {stmt.start_line}-{stmt.end_line}) ---")
+        print(stmt)
+
+
+if __name__ == "__main__":
+    main()

wcgw/client/common.py

@@ -0,0 +1,51 @@
+import select
+import sys
+import termios
+import tty
+from typing import Literal
+from pydantic import BaseModel
+
+
+class CostData(BaseModel):
+    cost_per_1m_input_tokens: float
+    cost_per_1m_output_tokens: float
+
+
+from openai.types.chat import (
+    ChatCompletionMessageParam,
+    ChatCompletionAssistantMessageParam,
+    ChatCompletionMessage,
+    ParsedChatCompletionMessage,
+)
+
+History = list[ChatCompletionMessageParam]
+Models = Literal["gpt-4o-2024-08-06", "gpt-4o-mini"]
+
+
+def discard_input() -> None:
+    try:
+        # Get the file descriptor for stdin
+        fd = sys.stdin.fileno()
+
+        # Save current terminal settings
+        old_settings = termios.tcgetattr(fd)
+
+        try:
+            # Switch terminal to non-canonical mode where input is read immediately
+            tty.setcbreak(fd)
+
+            # Discard all input
+            while True:
+                # Check if there is input to be read
+                if sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
+                    sys.stdin.read(
+                        1
+                    )  # Read one character at a time to flush the input buffer
+                else:
+                    break
+        finally:
+            # Restore old terminal settings
+            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
+    except (termios.error, ValueError) as e:
+        # Handle the error gracefully
+        print(f"Warning: Unable to discard input. Error: {e}")

wcgw/client/diff-instructions.txt

@@ -0,0 +1,73 @@
+
+Instructions for editing files.
+# Example
+## Input file
+```
+import numpy as np
+from impls import impl1, impl2
+
+def hello():
+    "print a greeting"
+
+    print("hello")
+
+def call_hello():
+    "call hello"
+
+    hello()
+    print("Called")
+    impl1()
+    hello()
+    impl2()
+
+```
+## Edit format on the input file
+```
+<<<<<<< SEARCH
+from impls import impl1, impl2
+=======
+from impls import impl1, impl2
+from hello import hello as hello_renamed
+>>>>>>> REPLACE
+<<<<<<< SEARCH
+def hello():
+    "print a greeting"
+
+    print("hello")
+=======
+>>>>>>> REPLACE
+<<<<<<< SEARCH
+def call_hello():
+    "call hello"
+
+    hello()
+=======
+def call_hello_renamed():
+    "call hello renamed"
+
+    hello_renamed()
+>>>>>>> REPLACE
+<<<<<<< SEARCH
+    impl1()
+    hello()
+    impl2()
+=======
+    impl1()
+    hello_renamed()
+    impl2()
+>>>>>>> REPLACE
+```
+
+# *SEARCH/REPLACE block* Rules:
+Every "<<<<<<< SEARCH" section must *EXACTLY MATCH* the existing file content, character for character, including all comments, docstrings, whitespaces, etc.
+
+Including multiple unique *SEARCH/REPLACE* blocks if needed.
+Include enough and only enough lines in each SEARCH section to uniquely match each set of lines that need to change.
+
+Keep *SEARCH/REPLACE* blocks concise.
+Break large *SEARCH/REPLACE* blocks into a series of smaller blocks that each change a small portion of the file.
+Include just the changing lines, and a few surrounding lines (0-3 lines) if needed for uniqueness.
+Other than for uniqueness, avoid including those lines which do not change in search (and replace) blocks. Target 0-3 non trivial extra lines per block.
+
+Preserve leading spaces and indentations in both SEARCH and REPLACE blocks.
+

wcgw/client/encoder/__init__.py

@@ -0,0 +1,47 @@
+import threading
+from typing import Callable, Protocol, TypeVar, cast
+
+import tokenizers  # type: ignore[import-untyped]
+
+T = TypeVar("T")
+
+
+class EncoderDecoder(Protocol[T]):
+    def encoder(self, text: str) -> list[T]: ...
+
+    def decoder(self, tokens: list[T]) -> str: ...
+
+
+class LazyEncoder:
+    def __init__(self) -> None:
+        self._tokenizer: tokenizers.Tokenizer | None = None
+        self._init_lock = threading.Lock()
+        self._init_thread = threading.Thread(target=self._initialize, daemon=True)
+        self._init_thread.start()
+
+    def _initialize(self) -> None:
+        with self._init_lock:
+            if self._tokenizer is None:
+                self._tokenizer = tokenizers.Tokenizer.from_pretrained(
+                    "Xenova/claude-tokenizer"
+                )
+
+    def _ensure_initialized(self) -> None:
+        if self._tokenizer is None:
+            with self._init_lock:
+                if self._tokenizer is None:
+                    self._init_thread.join()
+
+    def encoder(self, text: str) -> list[int]:
+        self._ensure_initialized()
+        assert self._tokenizer is not None, "Couldn't initialize tokenizer"
+        return cast(list[int], self._tokenizer.encode(text).ids)
+
+    def decoder(self, tokens: list[int]) -> str:
+        self._ensure_initialized()
+        assert self._tokenizer is not None, "Couldn't initialize tokenizer"
+        return cast(str, self._tokenizer.decode(tokens))
+
+
+def get_default_encoder() -> EncoderDecoder[int]:
+    return LazyEncoder()