code-puppy 0.0.91__py3-none-any.whl → 0.0.93__py3-none-any.whl

code_puppy/main.py

@@ -38,7 +38,10 @@ def get_secret_file_path():
 
 async def main():
     # Ensure the config directory and puppy.cfg with name info exist (prompt user if needed)
-    logfire.configure(token="pylf_v1_us_8G5nLznQtHMRsL4hsNG5v3fPWKjyXbysrMgrQ1bV1wRP")
+    logfire.configure(
+        token="pylf_v1_us_8G5nLznQtHMRsL4hsNG5v3fPWKjyXbysrMgrQ1bV1wRP",
+        console=False
+    )
     logfire.instrument_pydantic_ai()
     ensure_config_exists()
 

code_puppy/message_history_processor.py

@@ -4,7 +4,6 @@ import os
 from pathlib import Path
 
 import pydantic
-import tiktoken
 from pydantic_ai.messages import (
     ModelMessage,
     TextPart,
@@ -16,6 +15,7 @@ from pydantic_ai.messages import (
 from code_puppy.tools.common import console
 from code_puppy.model_factory import ModelFactory
 from code_puppy.config import get_model_name
+from code_puppy.token_utils import estimate_tokens
 
 # Import the status display to get token rate info
 try:
@@ -46,12 +46,12 @@ except ImportError:
         return None
 
 
+# Dummy function for backward compatibility
 def get_tokenizer_for_model(model_name: str):
     """
-    Always use cl100k_base tokenizer regardless of model type.
-    This is a simple approach that works reasonably well for most models.
+    Dummy function that returns None since we're now using len/4 heuristic.
     """
-    return tiktoken.get_encoding("cl100k_base")
+    return None
 
 
 def stringify_message_part(part) -> str:
@@ -96,17 +96,15 @@ def stringify_message_part(part) -> str:
 
 def estimate_tokens_for_message(message: ModelMessage) -> int:
     """
-    Estimate the number of tokens in a message using tiktoken with cl100k_base encoding.
-    This is more accurate than character-based estimation.
+    Estimate the number of tokens in a message using the len/4 heuristic.
+    This is a simple approximation that works reasonably well for most text.
     """
-    tokenizer = get_tokenizer_for_model(get_model_name())
     total_tokens = 0
 
     for part in message.parts:
         part_str = stringify_message_part(part)
         if part_str:
-            tokens = tokenizer.encode(part_str)
-            total_tokens += len(tokens)
+            total_tokens += estimate_tokens(part_str)
 
     return max(1, total_tokens)
 

code_puppy/status_display.py

@@ -104,9 +104,13 @@ class StatusDisplay:
 
     def update_token_count(self, tokens: int) -> None:
         """Update the token count and recalculate the rate"""
+        # Reset timing if this is the first update of a new task
         if self.start_time is None:
             self.start_time = time.time()
             self.last_update_time = self.start_time
+            # Reset token counters for new task
+            self.last_token_count = 0
+            self.current_rate = 0.0
         
         # Allow for incremental updates (common for streaming) or absolute updates
         if tokens > self.token_count or tokens < 0:
@@ -204,6 +208,13 @@ class StatusDisplay:
             avg_rate = self.token_count / elapsed if elapsed > 0 else 0
             self.console.print(f"[dim]Completed: {self.token_count} tokens in {elapsed:.1f}s ({avg_rate:.1f} t/s avg)[/dim]")
             
-            # Reset
+            # Reset state
             self.start_time = None
             self.token_count = 0
+            self.last_update_time = None
+            self.last_token_count = 0
+            self.current_rate = 0
+            
+            # Reset global rate to 0 to avoid affecting subsequent tasks
+            global CURRENT_TOKEN_RATE
+            CURRENT_TOKEN_RATE = 0.0

code_puppy/token_utils.py

@@ -1,16 +1,17 @@
 import json
-import tiktoken
 
 import pydantic
 from pydantic_ai.messages import ModelMessage
 
 
-def get_tokenizer():
+def estimate_tokens(text: str) -> int:
     """
-    Always use cl100k_base tokenizer regardless of model type.
-    This is a simple approach that works reasonably well for most models.
+    Estimate the number of tokens using the len/4 heuristic.
+    This is a simple approximation that works reasonably well for most text.
     """
-    return tiktoken.get_encoding("cl100k_base")
+    if not text:
+        return 0
+    return max(1, len(text) // 4)
 
 
 def stringify_message_part(part) -> str:
@@ -55,16 +56,14 @@ def stringify_message_part(part) -> str:
 
 def estimate_tokens_for_message(message: ModelMessage) -> int:
     """
-    Estimate the number of tokens in a message using tiktoken with cl100k_base encoding.
-    This is more accurate than character-based estimation.
+    Estimate the number of tokens in a message using the len/4 heuristic.
+    This is a simple approximation that works reasonably well for most text.
     """
-    tokenizer = get_tokenizer()
     total_tokens = 0
 
     for part in message.parts:
         part_str = stringify_message_part(part)
         if part_str:
-            tokens = tokenizer.encode(part_str)
-            total_tokens += len(tokens)
+            total_tokens += estimate_tokens(part_str)
 
     return max(1, total_tokens)

code_puppy/tools/common.py

@@ -2,7 +2,6 @@ import os
 import fnmatch
 
 from typing import Optional, Tuple
-import tiktoken
 from rapidfuzz.distance import JaroWinkler
 from rich.console import Console
 
@@ -44,48 +43,314 @@ def get_model_context_length() -> int:
 # Shared ignore patterns/helpers
 # -------------------
 IGNORE_PATTERNS = [
-    "**/node_modules/**",
-    "**/node_modules/**/*.js",
-    "node_modules/**",
-    "node_modules",
+    # Version control
     "**/.git/**",
     "**/.git",
     ".git/**",
     ".git",
+    "**/.svn/**",
+    "**/.hg/**",
+    "**/.bzr/**",
+    # Node.js / JavaScript / TypeScript
+    "**/node_modules/**",
+    "**/node_modules/**/*.js",
+    "node_modules/**",
+    "node_modules",
+    "**/npm-debug.log*",
+    "**/yarn-debug.log*",
+    "**/yarn-error.log*",
+    "**/pnpm-debug.log*",
+    "**/.npm/**",
+    "**/.yarn/**",
+    "**/.pnpm-store/**",
+    "**/coverage/**",
+    "**/.nyc_output/**",
+    "**/dist/**",
+    "**/dist",
+    "**/build/**",
+    "**/build",
+    "**/.next/**",
+    "**/.nuxt/**",
+    "**/out/**",
+    "**/.cache/**",
+    "**/.parcel-cache/**",
+    "**/.vite/**",
+    "**/storybook-static/**",
+    # Python
     "**/__pycache__/**",
     "**/__pycache__",
     "__pycache__/**",
     "__pycache__",
-    "**/.DS_Store",
-    ".DS_Store",
-    "**/.env",
-    ".env",
+    "**/*.pyc",
+    "**/*.pyo",
+    "**/*.pyd",
+    "**/.pytest_cache/**",
+    "**/.mypy_cache/**",
+    "**/.coverage",
+    "**/htmlcov/**",
+    "**/.tox/**",
+    "**/.nox/**",
+    "**/site-packages/**",
     "**/.venv/**",
     "**/.venv",
     "**/venv/**",
     "**/venv",
-    "**/.idea/**",
-    "**/.idea",
-    "**/.vscode/**",
-    "**/.vscode",
+    "**/env/**",
+    "**/ENV/**",
+    "**/.env",
+    "**/pip-wheel-metadata/**",
+    "**/*.egg-info/**",
     "**/dist/**",
-    "**/dist",
+    "**/wheels/**",
+    # Java (Maven, Gradle, SBT)
+    "**/target/**",
+    "**/target",
     "**/build/**",
     "**/build",
-    "**/*.pyc",
-    "**/*.pyo",
-    "**/*.pyd",
+    "**/.gradle/**",
+    "**/gradle-app.setting",
+    "**/*.class",
+    "**/*.jar",
+    "**/*.war",
+    "**/*.ear",
+    "**/*.nar",
+    "**/hs_err_pid*",
+    "**/.classpath",
+    "**/.project",
+    "**/.settings/**",
+    "**/bin/**",
+    "**/project/target/**",
+    "**/project/project/**",
+    # Go
+    "**/vendor/**",
+    "**/*.exe",
+    "**/*.exe~",
+    "**/*.dll",
     "**/*.so",
+    "**/*.dylib",
+    "**/*.test",
+    "**/*.out",
+    "**/go.work",
+    "**/go.work.sum",
+    # Rust
+    "**/target/**",
+    "**/Cargo.lock",
+    "**/*.pdb",
+    # Ruby
+    "**/vendor/**",
+    "**/.bundle/**",
+    "**/Gemfile.lock",
+    "**/*.gem",
+    "**/.rvm/**",
+    "**/.rbenv/**",
+    "**/coverage/**",
+    "**/.yardoc/**",
+    "**/doc/**",
+    "**/rdoc/**",
+    "**/.sass-cache/**",
+    "**/.jekyll-cache/**",
+    "**/_site/**",
+    # PHP
+    "**/vendor/**",
+    "**/composer.lock",
+    "**/.phpunit.result.cache",
+    "**/storage/logs/**",
+    "**/storage/framework/cache/**",
+    "**/storage/framework/sessions/**",
+    "**/storage/framework/testing/**",
+    "**/storage/framework/views/**",
+    "**/bootstrap/cache/**",
+    # .NET / C#
+    "**/bin/**",
+    "**/obj/**",
+    "**/packages/**",
+    "**/*.cache",
     "**/*.dll",
-    "**/.*",
+    "**/*.exe",
+    "**/*.pdb",
+    "**/*.user",
+    "**/*.suo",
+    "**/.vs/**",
+    "**/TestResults/**",
+    "**/BenchmarkDotNet.Artifacts/**",
+    # C/C++
+    "**/*.o",
+    "**/*.obj",
+    "**/*.so",
+    "**/*.dll",
+    "**/*.a",
+    "**/*.lib",
+    "**/*.dylib",
+    "**/*.exe",
+    "**/CMakeFiles/**",
+    "**/CMakeCache.txt",
+    "**/cmake_install.cmake",
+    "**/Makefile",
+    "**/compile_commands.json",
+    "**/.deps/**",
+    "**/.libs/**",
+    "**/autom4te.cache/**",
+    # Perl
+    "**/blib/**",
+    "**/_build/**",
+    "**/Build",
+    "**/Build.bat",
+    "**/*.tmp",
+    "**/*.bak",
+    "**/*.old",
+    "**/Makefile.old",
+    "**/MANIFEST.bak",
+    "**/META.yml",
+    "**/META.json",
+    "**/MYMETA.*",
+    "**/.prove",
+    # Scala
+    "**/target/**",
+    "**/project/target/**",
+    "**/project/project/**",
+    "**/.bloop/**",
+    "**/.metals/**",
+    "**/.ammonite/**",
+    "**/*.class",
+    # Elixir
+    "**/_build/**",
+    "**/deps/**",
+    "**/*.beam",
+    "**/.fetch",
+    "**/erl_crash.dump",
+    "**/*.ez",
+    "**/doc/**",
+    "**/.elixir_ls/**",
+    # Swift
+    "**/.build/**",
+    "**/Packages/**",
+    "**/*.xcodeproj/**",
+    "**/*.xcworkspace/**",
+    "**/DerivedData/**",
+    "**/xcuserdata/**",
+    "**/*.dSYM/**",
+    # Kotlin
+    "**/build/**",
+    "**/.gradle/**",
+    "**/*.class",
+    "**/*.jar",
+    "**/*.kotlin_module",
+    # Clojure
+    "**/target/**",
+    "**/.lein-**",
+    "**/.nrepl-port",
+    "**/pom.xml.asc",
+    "**/*.jar",
+    "**/*.class",
+    # Dart/Flutter
+    "**/.dart_tool/**",
+    "**/build/**",
+    "**/.packages",
+    "**/pubspec.lock",
+    "**/*.g.dart",
+    "**/*.freezed.dart",
+    "**/*.gr.dart",
+    # Haskell
+    "**/dist/**",
+    "**/dist-newstyle/**",
+    "**/.stack-work/**",
+    "**/*.hi",
+    "**/*.o",
+    "**/*.prof",
+    "**/*.aux",
+    "**/*.hp",
+    "**/*.eventlog",
+    "**/*.tix",
+    # Erlang
+    "**/ebin/**",
+    "**/rel/**",
+    "**/deps/**",
+    "**/*.beam",
+    "**/*.boot",
+    "**/*.plt",
+    "**/erl_crash.dump",
+    # Common cache and temp directories
+    "**/.cache/**",
+    "**/cache/**",
+    "**/tmp/**",
+    "**/temp/**",
+    "**/.tmp/**",
+    "**/.temp/**",
+    "**/logs/**",
+    "**/*.log",
+    "**/*.log.*",
+    # IDE and editor files
+    "**/.idea/**",
+    "**/.idea",
+    "**/.vscode/**",
+    "**/.vscode",
+    "**/*.swp",
+    "**/*.swo",
+    "**/*~",
+    "**/.#*",
+    "**/#*#",
+    "**/.emacs.d/auto-save-list/**",
+    "**/.vim/**",
+    "**/.netrwhist",
+    "**/Session.vim",
+    "**/.sublime-project",
+    "**/.sublime-workspace",
+    # OS-specific files
+    "**/.DS_Store",
+    ".DS_Store",
+    "**/Thumbs.db",
+    "**/Desktop.ini",
+    "**/.directory",
+    "**/*.lnk",
+    # Common artifacts
+    "**/*.orig",
+    "**/*.rej",
+    "**/*.patch",
+    "**/*.diff",
+    "**/.*.orig",
+    "**/.*.rej",
+    # Backup files
+    "**/*~",
+    "**/*.bak",
+    "**/*.backup",
+    "**/*.old",
+    "**/*.save",
+    # Hidden files (but be careful with this one)
+    # "**/.*",  # Commented out as it might be too aggressive
 ]
 
 
 def should_ignore_path(path: str) -> bool:
     """Return True if *path* matches any pattern in IGNORE_PATTERNS."""
+    # Convert path to Path object for better pattern matching
+    path_obj = Path(path)
+
     for pattern in IGNORE_PATTERNS:
-        if fnmatch.fnmatch(path, pattern):
-            return True
+        # Try pathlib's match method which handles ** patterns properly
+        try:
+            if path_obj.match(pattern):
+                return True
+        except ValueError:
+            # If pathlib can't handle the pattern, fall back to fnmatch
+            if fnmatch.fnmatch(path, pattern):
+                return True
+
+        # Additional check: if pattern contains **, try matching against
+        # different parts of the path to handle edge cases
+        if "**" in pattern:
+            # Convert pattern to handle different path representations
+            simplified_pattern = pattern.replace("**/", "").replace("/**", "")
+
+            # Check if any part of the path matches the simplified pattern
+            path_parts = path_obj.parts
+            for i in range(len(path_parts)):
+                subpath = Path(*path_parts[i:])
+                if fnmatch.fnmatch(str(subpath), simplified_pattern):
+                    return True
+                # Also check individual parts
+                if fnmatch.fnmatch(path_parts[i], simplified_pattern):
+                    return True
+
     return False
 
 

code_puppy/tools/file_operations.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, conint
 from pydantic_ai import RunContext
 
 from code_puppy.tools.common import console
-from code_puppy.token_utils import get_tokenizer
+from code_puppy.token_utils import estimate_tokens
 from code_puppy.tools.token_check import token_guard
 # ---------------------------------------------------------------------------
 # Module-level helper functions (exposed for unit tests _and_ used as tools)
@@ -218,8 +218,7 @@ def _read_file(context: RunContext, file_path: str, start_line: int | None = Non
                 # Read the entire file
                 content = f.read()
 
-            tokenizer = get_tokenizer()
-            num_tokens = len(tokenizer.encode(content))
+            num_tokens = estimate_tokens(content)
             if num_tokens > 10000:
                 raise ValueError("The file is massive, greater than 10,000 tokens which is dangerous to read entirely. Please read this file in chunks.")
             token_guard(num_tokens)
@@ -313,8 +312,7 @@ def list_files(
     context: RunContext, directory: str = ".", recursive: bool = True
 ) -> ListFileOutput:
     list_files_output = _list_files(context, directory, recursive)
-    tokenizer = get_tokenizer()
-    num_tokens = len(tokenizer.encode(list_files_output.model_dump_json()))
+    num_tokens = estimate_tokens(list_files_output.model_dump_json())
     if num_tokens > 10000:
         return ListFileOutput(
             files=[],

{code_puppy-0.0.91.dist-info → code_puppy-0.0.93.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: code-puppy
-Version: 0.0.91
+Version: 0.0.93
 Summary: Code generation agent
 Author: Michael Pfaffenberger
 License: MIT
@@ -27,7 +27,6 @@ Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: rapidfuzz>=3.13.0
 Requires-Dist: rich>=13.4.2
 Requires-Dist: ruff>=0.11.11
-Requires-Dist: tiktoken>=0.11.0
 Requires-Dist: tree-sitter-language-pack>=0.8.0
 Requires-Dist: tree-sitter-typescript>=0.23.2
 Description-Content-Type: text/markdown

{code_puppy-0.0.91.dist-info → code_puppy-0.0.93.dist-info}/RECORD

@@ -2,14 +2,14 @@ code_puppy/__init__.py,sha256=CWH46ZAmJRmHAbOiAhG07OrWYEcEt4yvDTkZU341Wag,169
 code_puppy/agent.py,sha256=Hwlblm9Om4qZPOur28auxBkdJRAA9Kz-36YPQg7i3SI,3321
 code_puppy/agent_prompts.py,sha256=t3-lqDKrDxCKxFa_va4Suze9BT-JOu1dh9iGiAVNFO4,6828
 code_puppy/config.py,sha256=r5nw5ChOP8xd_K5yo8U5OtO2gy2bFhARiyNtDp1JrwQ,5013
-code_puppy/main.py,sha256=_9PWsk1GLVvfH9w8KbcoMXvUCwhzxu6WRqguTIYVccE,21183
-code_puppy/message_history_processor.py,sha256=cL9DgSqpwWk0dSamc1I7NR1n1CNefvKlIMSCSA4YkM8,10034
+code_puppy/main.py,sha256=4ru_POVn6_QylecnJ12aNi95mi0bZ4A6P1Ujzn_YD4k,21220
+code_puppy/message_history_processor.py,sha256=-bdtufnxMNrX9JLY6dgmxGxjgy4sutTjA_SCKGnEaaA,9931
 code_puppy/model_factory.py,sha256=HXuFHNkVjkCcorAd3ScFmSvBILO932UTq6OmNAqisT8,10898
 code_puppy/models.json,sha256=jr0-LW87aJS79GosVwoZdHeeq5eflPzgdPoMbcqpVA8,2728
 code_puppy/state_management.py,sha256=JkTkmq6f9rl_RHPDoBqJvbAzgaMsIkJf-k38ragItIo,1692
-code_puppy/status_display.py,sha256=tzllkO0ssmRZSn20mFXnEt1BZXH4oP9eHSSN-7Jyrok,7946
+code_puppy/status_display.py,sha256=P4FVtbzdGQs11UDz4ZlWKOpuNYLQ8ILANi2NlMyjKp8,8413
 code_puppy/summarization_agent.py,sha256=jHUQe6iYJsMT0ywEwO7CrhUIKEamO5imhAsDwvNuvow,2684
-code_puppy/token_utils.py,sha256=g7Jj6NAy_a2ab7BXpwyhktruR-QlUV670H_mCPZV1N4,2110
+code_puppy/token_utils.py,sha256=c3mD-7VSvkybP9J4kptyrq_jushtzaKUTj6qj2ddqZA,2070
 code_puppy/version_checker.py,sha256=aRGulzuY4C4CdFvU1rITduyL-1xTFsn4GiD1uSfOl_Y,396
 code_puppy/command_line/__init__.py,sha256=y7WeRemfYppk8KVbCGeAIiTuiOszIURCDjOMZv_YRmU,45
 code_puppy/command_line/file_path_completion.py,sha256=gw8NpIxa6GOpczUJRyh7VNZwoXKKn-yvCqit7h2y6Gg,2931
@@ -20,13 +20,13 @@ code_puppy/command_line/prompt_toolkit_completion.py,sha256=De_grHDPOvCRph-HDOGC
 code_puppy/command_line/utils.py,sha256=7eyxDHjPjPB9wGDJQQcXV_zOsGdYsFgI0SGCetVmTqE,1251
 code_puppy/tools/__init__.py,sha256=WTHYIfRk2KMmk6o45TELpbB3GIiAm8s7GmfJ7Zy_tww,503
 code_puppy/tools/command_runner.py,sha256=9UWCSPpuEndaPx8Ecc8TRsn3rMHNd2AqerirvYPGRIw,14358
-code_puppy/tools/common.py,sha256=UkhnfLG1bmd4f9nZCcmno088AtKtAnEES1tydxUN-Fk,3265
+code_puppy/tools/common.py,sha256=xSNBpF1m6GMGNGnVK9PXly7dvfieIQgX0IegtQd6nWA,9309
 code_puppy/tools/file_modifications.py,sha256=BzQrGEacS2NZr2ru9N30x_Qd70JDudBKOAPO1XjBohg,13861
-code_puppy/tools/file_operations.py,sha256=5oJSJCKPLSgJ9YcD1CV9D-SJ0hnzVzmuAs-eiriLPLo,13615
+code_puppy/tools/file_operations.py,sha256=htrdURR1mBqBUja1mAHLlySwFN6xNHeGdYbg5eLiOug,13533
 code_puppy/tools/token_check.py,sha256=F3eygdI8fgb6dfCrSkGw_OLI7cb_Kpa5ILft4BQ7hvY,525
-code_puppy-0.0.91.data/data/code_puppy/models.json,sha256=jr0-LW87aJS79GosVwoZdHeeq5eflPzgdPoMbcqpVA8,2728
-code_puppy-0.0.91.dist-info/METADATA,sha256=PaIPBOhLU92HD2QkF1lZVCgFBi8KpTV46pzzAIf620M,6351
-code_puppy-0.0.91.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-code_puppy-0.0.91.dist-info/entry_points.txt,sha256=d8YkBvIUxF-dHNJAj-x4fPEqizbY5d_TwvYpc01U5kw,58
-code_puppy-0.0.91.dist-info/licenses/LICENSE,sha256=31u8x0SPgdOq3izJX41kgFazWsM43zPEF9eskzqbJMY,1075
-code_puppy-0.0.91.dist-info/RECORD,,
+code_puppy-0.0.93.data/data/code_puppy/models.json,sha256=jr0-LW87aJS79GosVwoZdHeeq5eflPzgdPoMbcqpVA8,2728
+code_puppy-0.0.93.dist-info/METADATA,sha256=E7tB-HnhBSsH2kaISnyyftHeX8Ippz9I39UthAVcdNk,6319
+code_puppy-0.0.93.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+code_puppy-0.0.93.dist-info/entry_points.txt,sha256=d8YkBvIUxF-dHNJAj-x4fPEqizbY5d_TwvYpc01U5kw,58
+code_puppy-0.0.93.dist-info/licenses/LICENSE,sha256=31u8x0SPgdOq3izJX41kgFazWsM43zPEF9eskzqbJMY,1075
+code_puppy-0.0.93.dist-info/RECORD,,