aider-ce 0.88.20__py3-none-any.whl

aider/help_pats.py

@@ -0,0 +1,19 @@
+# This needs to sync with MANIFEST.in
+
+exclude_website_pats = [
+    "**/.DS_Store",
+    "examples/**",
+    "_posts/**",
+    "HISTORY.md",
+    "docs/benchmarks*md",
+    "docs/ctags.md",
+    "docs/unified-diffs.md",
+    "docs/leaderboards/index.md",
+    "assets/**",
+    ".jekyll-metadata",
+    "Gemfile.lock",
+    "Gemfile",
+    "_config.yml",
+    "**/OLD/**",
+    "OLD/**",
+]

aider/helpers/__init__.py

@@ -0,0 +1,9 @@
+"""Utility functions for aider."""
+
+from .similarity import cosine_similarity, create_bigram_vector, normalize_vector
+
+__all__ = [
+    "cosine_similarity",
+    "create_bigram_vector",
+    "normalize_vector",
+]

aider/helpers/similarity.py

@@ -0,0 +1,98 @@
+import numpy as np
+
+
+def normalize_vector(vector):
+    """Normalize a vector to unit length (L2 norm).
+
+    Args:
+        vector (np.ndarray or list): Input vector
+
+    Returns:
+        np.ndarray: Normalized vector with length 1
+    """
+    vector = np.asarray(vector, dtype=np.float64)
+    magnitude = np.linalg.norm(vector)
+    if magnitude == 0:
+        return vector  # Return original if zero vector
+    return vector / magnitude
+
+
+def cosine_similarity(vector1, vector2):
+    """Calculate cosine similarity between two vectors.
+
+    Args:
+        vector1 (np.ndarray or list): First vector
+        vector2 (np.ndarray or list): Second vector
+
+    Returns:
+        float: Cosine similarity between the vectors (range: -1 to 1)
+    """
+    vector1 = np.asarray(vector1, dtype=np.float64)
+    vector2 = np.asarray(vector2, dtype=np.float64)
+
+    if len(vector1) != len(vector2):
+        raise ValueError("Vectors must have the same length")
+
+    # Use NumPy's optimized dot product and norm functions
+    dot_product = np.dot(vector1, vector2)
+    magnitude1 = np.linalg.norm(vector1)
+    magnitude2 = np.linalg.norm(vector2)
+
+    if magnitude1 == 0 or magnitude2 == 0:
+        return 0.0  # Return 0 if either vector is zero
+
+    return dot_product / (magnitude1 * magnitude2)
+
+
+def create_bigram_vector(texts):
+    """Create a bigram frequency vector using optimized NumPy operations.
+
+    This version uses pre-computed bigram indices and NumPy's bincount
+    for maximum performance on large datasets.
+
+    Args:
+        texts (tuple): Tuple of strings to process
+
+    Returns:
+        np.ndarray: Vector of bigram frequencies
+    """
+    # Pre-compute bigram indices (0 for 'aa', 1 for 'ab', ..., 675 for 'zz')
+    bigram_indices = {}
+    idx = 0
+    for i in range(ord("a"), ord("z") + 1):
+        for j in range(ord("a"), ord("z") + 1):
+            bigram = chr(i) + chr(j)
+            bigram_indices[bigram] = idx
+            idx += 1
+
+    # Initialize frequency vector
+    vector = np.zeros(26 * 26, dtype=np.int32)
+
+    # Process all texts
+    for text in texts:
+        text_lower = text.lower()
+        if len(text_lower) < 2:
+            continue
+
+        # Extract bigrams using NumPy sliding window view
+        # Convert string to character array for efficient slicing
+        chars = np.array(list(text_lower))
+
+        # Create bigrams by combining consecutive characters
+        bigrams = np.char.add(chars[:-1], chars[1:])
+
+        # Filter only alphabetic bigrams
+        mask = np.array([bg.isalpha() for bg in bigrams])
+        valid_bigrams = bigrams[mask]
+
+        # Count bigrams using bincount with pre-computed indices
+        indices = []
+        for bg in valid_bigrams:
+            if bg in bigram_indices:
+                indices.append(bigram_indices[bg])
+
+        if indices:
+            counts = np.bincount(indices, minlength=26 * 26)
+            vector += counts
+
+    return vector

aider/history.py

@@ -0,0 +1,180 @@
+import argparse
+
+from aider import models, prompts
+from aider.dump import dump  # noqa: F401
+
+
+class ChatSummary:
+    def __init__(self, models=None, max_tokens=1024):
+        if not models:
+            raise ValueError("At least one model must be provided")
+        self.models = models if isinstance(models, list) else [models]
+        self.max_tokens = max_tokens
+        self.token_count = self.models[0].token_count
+
+    def check_max_tokens(self, messages, max_tokens=None):
+        if max_tokens is None:
+            max_tokens = self.max_tokens
+
+        if not max_tokens:
+            return False
+
+        sized = self.tokenize(messages)
+        total = sum(tokens for tokens, _msg in sized)
+        return total > max_tokens
+
+    def tokenize(self, messages):
+        sized = []
+        for msg in messages:
+            tokens = self.token_count(msg)
+            sized.append((tokens, msg))
+        return sized
+
+    async def summarize(self, messages, depth=0):
+        messages = await self.summarize_real(messages)
+        if messages and messages[-1]["role"] != "assistant":
+            messages.append(dict(role="assistant", content="Ok."))
+        return messages
+
+    async def summarize_real(self, messages, depth=0):
+        if not self.models:
+            raise ValueError("No models available for summarization")
+
+        sized = self.tokenize(messages)
+        total = sum(tokens for tokens, _msg in sized)
+
+        if total <= self.max_tokens:
+            if depth == 0:
+                # All fit, no summarization needed
+                return messages
+            # This is a chunk that's small enough to summarize in one go
+            return await self.summarize_all(messages)
+
+        min_split = 4
+        if len(messages) <= min_split or depth > 4:
+            return await self.summarize_all(messages)
+
+        tail_tokens = 0
+        split_index = len(messages)
+        half_max_tokens = self.max_tokens // 2
+
+        # Iterate over the messages in reverse order
+        for i in range(len(sized) - 1, -1, -1):
+            tokens, _msg = sized[i]
+            if tail_tokens + tokens < half_max_tokens:
+                tail_tokens += tokens
+                split_index = i
+            else:
+                break
+
+        # If we couldn't find a split point from the end, it's because the
+        # last message was too big. So just split off the last message and
+        # summarize the rest. This prevents infinite recursion.
+        if split_index == len(messages):
+            split_index = len(messages) - 1
+
+        # Ensure the head ends with an assistant message
+        while messages[split_index - 1]["role"] != "assistant" and split_index > 1:
+            split_index -= 1
+
+        if split_index <= min_split:
+            return await self.summarize_all(messages)
+
+        # Split head and tail
+        head = messages[:split_index]
+        tail = messages[split_index:]
+
+        summary = await self.summarize_real(head, depth + 1)
+
+        # If the combined summary and tail still fits, return directly
+        new_messages = summary + tail
+
+        sized_new = self.tokenize(new_messages)
+        total_new = sum(tokens for tokens, _msg in sized_new)
+
+        if total_new < self.max_tokens:
+            return new_messages
+
+        # Otherwise recurse with increased depth
+        return await self.summarize_real(new_messages, depth + 1)
+
+    async def summarize_all(self, messages):
+        content = ""
+        for msg in messages:
+            role = msg["role"].upper()
+            if role not in ("USER", "ASSISTANT"):
+                continue
+            if not msg.get("content"):
+                continue
+            content += f"# {role}\n"
+            content += msg["content"]
+            if not content.endswith("\n"):
+                content += "\n"
+
+        summarize_messages = [
+            dict(role="system", content=prompts.summarize),
+            dict(role="user", content=content),
+        ]
+
+        for model in self.models:
+            try:
+                summary = await model.simple_send_with_retries(summarize_messages)
+                if summary is not None:
+                    summary = prompts.summary_prefix + summary
+                    return [dict(role="user", content=summary)]
+            except Exception as e:
+                print(f"Summarization failed for model {model.name}: {str(e)}")
+
+        err = "summarizer unexpectedly failed for all models"
+        print(err)
+        raise ValueError(err)
+
+    async def summarize_all_as_text(self, messages, prompt, max_tokens=None):
+        content = ""
+        for msg in messages:
+            role = msg["role"].upper()
+            if role not in ("USER", "ASSISTANT"):
+                continue
+            if not msg.get("content"):
+                continue
+            content += f"# {role}\n"
+            content += msg["content"]
+            if not content.endswith("\n"):
+                content += "\n"
+
+        summarize_messages = [
+            dict(role="system", content=prompt),
+            dict(role="user", content=content),
+        ]
+
+        for model in self.models:
+            try:
+                summary = await model.simple_send_with_retries(
+                    summarize_messages, max_tokens=max_tokens
+                )
+                if summary is not None:
+                    return summary
+            except Exception as e:
+                print(f"Summarization failed for model {model.name}: {str(e)}")
+
+        raise ValueError("summarizer unexpectedly failed for all models")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filename", help="Markdown file to parse")
+    args = parser.parse_args()
+
+    model_names = ["gpt-3.5-turbo", "gpt-4"]  # Add more model names as needed
+    model_list = [models.Model(name) for name in model_names]
+    summarizer = ChatSummary(model_list)
+
+    with open(args.filename, "r") as f:
+        text = f.read()
+
+    summary = summarizer.summarize_chat_history_markdown(text)
+    dump(summary)
+
+
+if __name__ == "__main__":
+    main()