zerottmm 0.1.0__py3-none-any.whl

zerottmm/gitutils.py

@@ -0,0 +1,117 @@
+"""Utilities for computing git churn.
+
+The hotspot score used by ttmm combines cyclomatic complexity and recent
+development activity.  ``compute_churn`` uses ``git log --numstat`` to
+approximate how much each file has changed recently.  A half‑life decay
+is applied so that more recent commits contribute more to the churn.
+
+If ``git`` is not installed or the repository is not a git repository,
+``compute_churn`` will return an empty dictionary, in which case
+complexity alone determines the hotspot score.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import time
+import os
+from typing import Dict, Optional
+
+
+def _run_git(args, cwd: str) -> Optional[str]:
+    try:
+        result = subprocess.run(
+            ["git"] + args,
+            cwd=cwd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=True,
+        )
+        return result.stdout
+    except Exception:
+        return None
+
+
+def compute_churn(
+    repo_path: str,
+    half_life_days: float = 90.0,
+    since_days: float = 365.0,
+) -> Dict[str, float]:
+    """Compute a recency‑weighted churn score for each file in a git repository.
+
+    Parameters
+    ----------
+    repo_path: str
+        Path to the repository root.  Must contain a ``.git`` directory.
+    half_life_days: float
+        The half‑life for decay in days.  Commits older than this
+        contribute half as much churn as very recent commits.
+    since_days: float
+        Limit churn calculation to this many days in the past.  Setting
+        a finite window improves performance on large repositories.
+
+    Returns
+    -------
+    Dict[str, float]
+        Mapping from relative file path to a churn value.  Paths use
+        forward slashes regardless of platform.  If ``git`` is not
+        available or no repository exists, returns an empty dict.
+    """
+    # Verify .git exists
+    if not os.path.isdir(os.path.join(repo_path, ".git")):
+        return {}
+    # Build git log command
+    now = time.time()
+    since_timestamp = now - since_days * 86400
+    since_date = time.strftime("%Y-%m-%d", time.gmtime(since_timestamp))
+    output = _run_git([
+        "-c", "log.showSignature=false",  # suppress GPG signature noise
+        "log",
+        "--numstat",
+        f"--since={since_date}",
+        "--pretty=format:%ct",
+    ], cwd=repo_path)
+    if output is None:
+        return {}
+    churn: Dict[str, float] = {}
+    lines = output.splitlines()
+    commit_time: Optional[int] = None
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        if line.isdigit():
+            # Start of a commit: epoch time
+            try:
+                commit_time = int(line)
+            except Exception:
+                commit_time = None
+        else:
+            if commit_time is None:
+                continue
+            parts = line.split("\t")
+            if len(parts) != 3:
+                continue
+            adds_str, dels_str, path = parts
+            # Convert additions/deletions; "-" means binary or untracked
+            try:
+                adds = int(adds_str) if adds_str != "-" else 0
+            except Exception:
+                adds = 0
+            try:
+                dels = int(dels_str) if dels_str != "-" else 0
+            except Exception:
+                dels = 0
+            total = adds + dels
+            # Weight by recency
+            age_days = (now - commit_time) / 86400.0
+            weight = 0.0
+            if half_life_days > 0:
+                # Exponential decay: half‑life h -> weight = 0.5^(age/h)
+                weight = 0.5 ** (age_days / half_life_days)
+            else:
+                weight = 1.0
+            churn.setdefault(path, 0.0)
+            churn[path] += total * weight
+    return churn

zerottmm/index.py

@@ -0,0 +1,160 @@
+"""Repository indexer for ttmm.
+
+The indexer walks a Python repository, parses functions and methods with
+``ast`` and stores them in the ttmm database via :mod:`ttmm.store`.
+It computes cyclomatic complexity and lines of code for each symbol via
+:mod:`ttmm.metrics`, and merges in git churn information via
+:mod:`ttmm.gitutils` to produce a hotspot score later on.
+
+Only top‑level functions and class methods are indexed.  Nested
+functions are ignored to keep the mental model focused on API surfaces.
+Calls made within functions are collected, but attribute calls that
+cannot be resolved statically are marked as unresolved.
+
+Example usage:
+
+```
+from ttmm.index import index_repo
+index_repo("/path/to/repo")
+```
+"""
+
+from __future__ import annotations
+
+import ast
+import os
+from typing import Dict, List, Tuple
+
+from . import metrics, gitutils, store
+
+
+def _iter_python_files(repo_path: str) -> List[str]:
+    """Recursively find all Python files in a repository.
+
+    Ignores the ``.ttmm`` directory.  Returns relative paths with
+    forward slashes.
+    """
+    py_files: List[str] = []
+    for root, dirs, files in os.walk(repo_path):
+        # Skip .ttmm directory
+        rel_root = os.path.relpath(root, repo_path)
+        if rel_root.startswith(os.path.join(".ttmm")):
+            continue
+        for fname in files:
+            if fname.endswith(".py") and not fname.startswith("."):
+                full_path = os.path.join(root, fname)
+                rel_path = os.path.relpath(full_path, repo_path).replace(os.sep, "/")
+                py_files.append(rel_path)
+    return py_files
+
+
+def index_repo(repo_path: str) -> None:
+    """Index a Python repository into the ttmm database.
+
+    This will parse all ``.py`` files under ``repo_path``, compute
+    symbol definitions, call edges and metrics, and persist them in
+    ``.ttmm/ttmm.db``.  Any existing static data in the database is
+    replaced.  Dynamic trace data is preserved.
+    """
+    repo_path = os.path.abspath(repo_path)
+    # Discover python files
+    py_files = _iter_python_files(repo_path)
+    # Compute git churn for all files
+    churn_by_file = gitutils.compute_churn(repo_path)
+    files_data: List[Tuple[str, float]] = []
+    symbols_data: List[Dict[str, object]] = []
+    calls_data: List[Dict[str, object]] = []
+    metrics_data: Dict[str, Tuple[float, int, float]] = {}
+    for rel_path in py_files:
+        abs_path = os.path.join(repo_path, rel_path)
+        try:
+            with open(abs_path, "r", encoding="utf-8") as f:
+                source = f.read()
+        except Exception:
+            # Skip unreadable files
+            continue
+        try:
+            tree = ast.parse(source, filename=abs_path)
+        except SyntaxError:
+            # Skip files with syntax errors
+            continue
+        mtime = os.path.getmtime(abs_path)
+        files_data.append((rel_path, mtime))
+        module_name = rel_path[:-3].replace("/", ".")  # strip .py
+
+        class IndexVisitor(ast.NodeVisitor):
+            """Visitor to collect top‑level functions and methods and their calls."""
+
+            def __init__(self) -> None:
+                self.class_stack: List[str] = []
+                self.func_depth = 0  # track nesting of functions
+
+            def visit_ClassDef(self, node: ast.ClassDef) -> None:
+                self.class_stack.append(node.name)
+                self.generic_visit(node)
+                self.class_stack.pop()
+
+            def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+                # Index only functions not nested inside another function (func_depth == 0)
+                if self.func_depth == 0:
+                    qualname = module_name + ":" + (
+                        f"{self.class_stack[-1]}." if self.class_stack else ""
+                    ) + node.name
+                    sym_type = "method" if self.class_stack else "function"
+                    doc = ast.get_docstring(node)
+                    symbols_data.append(
+                        {
+                            "qualname": qualname,
+                            "path": rel_path,
+                            "lineno": node.lineno,
+                            "endlineno": getattr(node, "end_lineno", node.lineno),
+                            "type": sym_type,
+                            "doc": doc.strip() if isinstance(doc, str) else None,
+                        }
+                    )
+                    # Metrics
+                    comp = metrics.compute_complexity(node)
+                    loc = metrics.compute_loc(node)
+                    churn = churn_by_file.get(rel_path, 0.0)
+                    metrics_data[qualname] = (comp, loc, churn)
+                    # Collect calls
+
+                    class CallVisitor(ast.NodeVisitor):
+                        def visit_Call(self, call: ast.Call) -> None:
+                            callee_name: str | None = None
+                            unresolved = False
+                            func = call.func
+                            if isinstance(func, ast.Name):
+                                callee_name = func.id
+                            elif isinstance(func, ast.Attribute):
+                                callee_name = func.attr
+                                unresolved = True
+                            if callee_name:
+                                calls_data.append(
+                                    {
+                                        "caller_qualname": qualname,
+                                        "callee_name": callee_name,
+                                        "unresolved": unresolved,
+                                    }
+                                )
+                            # Continue into nested calls
+                            self.generic_visit(call)
+                    CallVisitor().visit(node)
+                # Recurse into the function to handle nested functions' bodies but not index them
+                self.func_depth += 1
+                self.generic_visit(node)
+                self.func_depth -= 1
+
+            # Also handle async functions similarly
+            def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
+                # Treat async functions as normal for indexing
+                self.visit_FunctionDef(node)  # type: ignore[arg-type]
+
+        IndexVisitor().visit(tree)
+    # Insert into DB
+    conn = store.connect(repo_path)
+    try:
+        store.reset_static_tables(conn)
+        store.insert_static_data(conn, files_data, symbols_data, calls_data, metrics_data)
+    finally:
+        store.close(conn)

zerottmm/metrics.py

@@ -0,0 +1,66 @@
+"""Metrics computation for ttmm.
+
+This module provides functions to compute cyclomatic complexity and lines
+of code for Python functions and methods.  Complexity is a simple
+approximation based on counting branching constructs; it is not as
+sophisticated as tools like radon or mccabe but suffices for guiding
+reading order.  It does not require any third party dependencies.
+"""
+
+from __future__ import annotations
+
+import ast
+
+
+class ComplexityVisitor(ast.NodeVisitor):
+    """AST visitor that accumulates cyclomatic complexity.
+
+    The initial complexity is 1.  Each branching or boolean operator
+    increments the count.  Attribute calls do not contribute to
+    complexity here.
+    """
+
+    def __init__(self) -> None:
+        self.complexity = 1
+
+    def generic_visit(self, node: ast.AST) -> None:
+        # Branching constructs increase complexity by 1
+        if isinstance(
+            node,
+            (
+                ast.If,
+                ast.For,
+                ast.While,
+                ast.AsyncFor,
+                ast.With,
+                ast.AsyncWith,
+                ast.Try,
+                ast.ExceptHandler,
+            ),
+        ):
+            self.complexity += 1
+        elif isinstance(node, ast.BoolOp):
+            # bool operations like ``a and b and c`` count len(values) - 1
+            self.complexity += max(len(node.values) - 1, 0)
+        # Continue traversing
+        super().generic_visit(node)
+
+
+def compute_complexity(node: ast.AST) -> int:
+    """Compute a simple cyclomatic complexity for a function/method AST node."""
+    visitor = ComplexityVisitor()
+    visitor.visit(node)
+    return visitor.complexity
+
+
+def compute_loc(node: ast.AST) -> int:
+    """Compute the number of lines of code covered by a node.
+
+    Uses ``lineno`` and ``end_lineno`` attributes available on Python 3.8+
+    AST nodes.  Returns at least 1 even if these attributes are missing.
+    """
+    start = getattr(node, "lineno", None)
+    end = getattr(node, "end_lineno", None)
+    if start is not None and end is not None and end >= start:
+        return end - start + 1
+    return 1

zerottmm/search.py

@@ -0,0 +1,121 @@
+"""Simple TF‑IDF search over ttmm symbols.
+
+This module provides a function to answer a natural language question
+about a codebase.  It ranks functions and methods by combining
+keyword similarity with their hotspot score.  The goal is to return
+a small set of entry points that the developer should read first to
+understand the behaviour described by the query.
+"""
+
+from __future__ import annotations
+
+import re
+import math
+from typing import Dict, List, Tuple
+
+from . import store
+
+
+def _tokenize(text: str) -> List[str]:
+    """Split a string into lowercase alphanumeric tokens."""
+    return [t.lower() for t in re.findall(r"[A-Za-z0-9]+", text)]
+
+
+def answer_question(
+    repo_path: str,
+    question: str,
+    top: int = 5,
+    include_scores: bool = False,
+) -> List[Tuple[str, str, int, float]]:
+    """Answer a question by returning a minimal set of relevant symbols.
+
+    Parameters
+    ----------
+    repo_path: str
+        Path to the repository root.  The repository must have been
+        indexed previously.
+    question: str
+        Natural language query or keywords.
+    top: int
+        Number of symbols to return.
+    include_scores: bool
+        If True, return the computed score alongside each result.
+
+    Returns
+    -------
+    List[Tuple[qualname, file_path, line_no, score]]
+        Sorted list of symbol descriptors.  Each tuple contains the
+        fully qualified name, the relative file path, the starting line
+        number and the ranking score.  The score is omitted if
+        ``include_scores`` is False.
+    """
+    question_tokens = _tokenize(question)
+    if not question_tokens:
+        return []
+    conn = store.connect(repo_path)
+    try:
+        cur = conn.cursor()
+        # Load all symbols with metrics
+        cur.execute(
+            """
+            SELECT symbols.id AS id,
+                   symbols.qualname AS qualname,
+                   files.path AS file_path,
+                   symbols.lineno AS lineno,
+                   metrics.complexity AS complexity,
+                   metrics.churn AS churn,
+                   symbols.doc AS doc
+            FROM symbols
+            JOIN metrics ON metrics.symbol_id = symbols.id
+            JOIN files ON files.id = symbols.file_id
+            """
+        )
+        symbols_list = cur.fetchall()
+        # Build inverted index: token -> {symbol_index: tf}
+        token_df: Dict[str, int] = {}
+        token_tf: Dict[str, Dict[int, int]] = {}
+        docs: List[Dict[str, int]] = []
+        for idx, row in enumerate(symbols_list):
+            text_parts = [row["qualname"], row["doc"] or ""]
+            tokens = _tokenize(" ".join(text_parts))
+            tf: Dict[str, int] = {}
+            for t in tokens:
+                tf[t] = tf.get(t, 0) + 1
+            docs.append(tf)
+            for t in tf:
+                token_df[t] = token_df.get(t, 0) + 1
+                token_tf.setdefault(t, {})[idx] = tf[t]
+        n_docs = len(symbols_list)
+        # Precompute idf for query tokens only to speed up
+        idf: Dict[str, float] = {}
+        for t in question_tokens:
+            df = token_df.get(t, 0)
+            # Add 1 to denominator to avoid division by zero
+            idf[t] = math.log((n_docs + 1) / (df + 1)) + 1.0
+        # Compute similarity for each symbol
+        scores: List[Tuple[int, float]] = []
+        for idx, row in enumerate(symbols_list):
+            # Compute TF‑IDF dot product for query and document
+            score = 0.0
+            for t in question_tokens:
+                tf = token_tf.get(t, {}).get(idx, 0)
+                score += tf * idf.get(t, 0.0)
+            if score > 0.0:
+                # Multiply by hotspot score (complexity * sqrt(churn + 1))
+                complexity = row["complexity"]
+                churn = row["churn"]
+                hotspot = complexity * (1.0 + math.sqrt(churn))
+                score *= (hotspot + 1e-6)
+                scores.append((idx, score))
+        # Sort descending
+        scores.sort(key=lambda x: x[1], reverse=True)
+        results = []
+        for idx, s in scores[:top]:
+            row = symbols_list[idx]
+            if include_scores:
+                results.append((row["qualname"], row["file_path"], row["lineno"], s))
+            else:
+                results.append((row["qualname"], row["file_path"], row["lineno"],))
+        return results
+    finally:
+        store.close(conn)