pi-context-map 0.1.4 → 0.3.0

package/extensions/token-counter.ts

@@ -0,0 +1,149 @@
+/**
+ * TokenCounter
+ * Code-aware token estimation. Avoids heavy external tokenizers (e.g., tiktoken)
+ * by applying multipliers based on content structure.
+ *
+ * Heuristic base: 4 characters per token (the standard for English text).
+ * Adjustments:
+ *  - Code blocks (```...```) are denser in tokens (identifiers, symbols).
+ *  - JSON payloads have more structural overhead.
+ *  - Pure whitespace is under-weighted.
+ */
+export class TokenCounter {
+	/** Base heuristic: average English is ~4 characters per token. */
+	private static BASE_CHARS_PER_TOKEN = 4;
+
+	/** Multiplier for fenced code blocks (```` ``` ````). */
+	private static CODE_MULTIPLIER = 1.3;
+
+	/** Multiplier for JSON-like structures. */
+	private static JSON_MULTIPLIER = 1.5;
+
+	/**
+	 * Count estimated tokens for a raw string of text.
+	 */
+	public static count(text: string): number {
+		if (!text) return 0;
+
+		let total = 0;
+		let cursor = 0;
+		const len = text.length;
+
+		while (cursor < len) {
+			// Detect fenced code blocks
+			const fenceStart = text.indexOf("```", cursor);
+			if (fenceStart !== -1) {
+				// Count everything up to the fence as regular text
+				total += TokenCounter.regularChunk(text.substring(cursor, fenceStart));
+				// Find the closing fence
+				const fenceEnd = text.indexOf("```", fenceStart + 3);
+				if (fenceEnd === -1) {
+					// Unclosed fence — treat the rest as code
+					total += TokenCounter.codeChunk(text.substring(fenceStart));
+					cursor = len;
+				} else {
+					total += TokenCounter.codeChunk(
+						text.substring(fenceStart, fenceEnd + 3),
+					);
+					cursor = fenceEnd + 3;
+				}
+				continue;
+			}
+
+			// Detect JSON-like content (starts with { or [ and has balanced structure)
+			const trimmed = text.substring(cursor).trimStart();
+			const firstChar = trimmed.charAt(0);
+			if (
+				(firstChar === "{" || firstChar === "[") &&
+				TokenCounter.looksLikeJson(trimmed)
+			) {
+				const jsonLen = TokenCounter.extractJsonLength(trimmed);
+				if (jsonLen > 0) {
+					total += TokenCounter.jsonChunk(trimmed.substring(0, jsonLen));
+					cursor += text.substring(cursor).indexOf(trimmed) + jsonLen;
+					continue;
+				}
+			}
+
+			// Default: regular text
+			total += TokenCounter.regularChunk(text.substring(cursor));
+			cursor = len;
+		}
+
+		return Math.ceil(total);
+	}
+
+	/**
+	 * Convenience: count tokens for any message shape Pi uses.
+	 */
+	public static countMessage(msg: any): number {
+		if (!msg) return 0;
+		if (typeof msg.content === "string") {
+			return TokenCounter.count(msg.content);
+		}
+		if (Array.isArray(msg.content)) {
+			return msg.content.reduce((sum: number, block: any) => {
+				if (typeof block === "string") return sum + TokenCounter.count(block);
+				if (block.type === "text" && typeof block.text === "string") {
+					return sum + TokenCounter.count(block.text);
+				}
+				if (block.type === "tool_use" || block.type === "tool_result") {
+					return sum + TokenCounter.count(JSON.stringify(block));
+				}
+				return sum + TokenCounter.count(JSON.stringify(block));
+			}, 0);
+		}
+		return TokenCounter.count(JSON.stringify(msg));
+	}
+
+	private static regularChunk(text: string): number {
+		return text.length / TokenCounter.BASE_CHARS_PER_TOKEN;
+	}
+
+	private static codeChunk(text: string): number {
+		return (
+			(text.length / TokenCounter.BASE_CHARS_PER_TOKEN) *
+			TokenCounter.CODE_MULTIPLIER
+		);
+	}
+
+	private static jsonChunk(text: string): number {
+		return (
+			(text.length / TokenCounter.BASE_CHARS_PER_TOKEN) *
+			TokenCounter.JSON_MULTIPLIER
+		);
+	}
+
+	private static looksLikeJson(text: string): boolean {
+		// Quick heuristic: contains quoted keys and structural punctuation
+		return /"[^"]+"\s*:/i.test(text) && /[{}[\],]/.test(text);
+	}
+
+	private static extractJsonLength(text: string): number {
+		let depth = 0;
+		let inString = false;
+		let escape = false;
+		for (let i = 0; i < text.length; i++) {
+			const ch = text[i];
+			if (escape) {
+				escape = false;
+				continue;
+			}
+			if (ch === "\\") {
+				escape = true;
+				continue;
+			}
+			if (ch === '"') {
+				inString = !inString;
+				continue;
+			}
+			if (inString) continue;
+			if (ch === "{" || ch === "[") depth++;
+			else if (ch === "}" || ch === "]") {
+				depth--;
+				if (depth === 0) return i + 1;
+			}
+		}
+		return 0;
+	}
+}

package/extensions/types/pi-coding-agent.d.ts

@@ -0,0 +1,3 @@
+declare module "@earendil-works/pi-coding-agent" {
+	export type ExtensionAPI = any;
+}

package/package.json

@@ -1,33 +1,37 @@
 {
   "name": "pi-context-map",
-  "version": "0.1.4",
-  "description": "A Pi extension that visualizes the current session context window and token distribution.",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "scripts": {
-    "build": "tsc",
-    "dev": "tsc -w"
-  },
+  "version": "0.3.0",
+  "description": "Professional context profiler for Pi that visualizes the session context window, token distribution, and integrates with Nexus packages for actionable insights.",
   "keywords": [
-    "pi",
     "pi-package",
-    "extension",
     "context",
     "visualization",
-    "tokens"
+    "tokens",
+    "profiler",
+    "nexus"
   ],
-  "author": "ZachDreamZ",
-  "license": "MIT",
   "pi": {
-    "extensions": ["dist/index.js"]
+    "extensions": ["./extensions"]
+  },
+  "files": [
+    "extensions",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "jest",
+    "prepublishOnly": "npm run build"
   },
-  "dependencies": {},
   "peerDependencies": {
-    "@earendil-works/pi-ai": "*",
-    "@earendil-works/pi-coding-agent": "*"
+    "pi-coding-agent": "*"
   },
   "devDependencies": {
     "typescript": "^5.0.0",
+    "jest": "^29.0.0",
+    "ts-jest": "^29.0.0",
+    "@types/jest": "^29.0.0",
     "@types/node": "^20.0.0"
   }
 }

package/dist/analyzer.d.ts

@@ -1,40 +0,0 @@
-/**
- * ContextAnalyzer
- * Responsible for parsing Pi session messages to identify the active working set of files,
- * their token weights, and their temporal status.
- */
-export interface FileOp {
-    type: "read" | "write" | "edit" | "delete";
-    turn: number;
-    timestamp: number;
-}
-export interface FileContext {
-    path: string;
-    weight: number;
-    lastOp: FileOp;
-    status: "active" | "stale" | "legacy";
-}
-export interface ContextMap {
-    files: FileContext[];
-    totalTokens: number;
-    systemTokens: number;
-    historyTokens: number;
-    fileTokens: number;
-    toolTokens: number;
-}
-export declare class ContextAnalyzer {
-    /**
-     * Heuristic for token estimation: approx 4 chars per token.
-     */
-    private static TOKEN_HEURISTIC;
-    /**
-     * Analyze session messages to produce a context map.
-     * @param messages The full session conversation history.
-     * @param currentTurn The current turn number.
-     */
-    analyze(messages: any[], currentTurn: number): ContextMap;
-    private extractPath;
-    private getOpType;
-    private calculateStatus;
-    private findToolResult;
-}

package/dist/analyzer.js

@@ -1,117 +0,0 @@
-"use strict";
-/**
- * ContextAnalyzer
- * Responsible for parsing Pi session messages to identify the active working set of files,
- * their token weights, and their temporal status.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ContextAnalyzer = void 0;
-class ContextAnalyzer {
-    /**
-     * Heuristic for token estimation: approx 4 chars per token.
-     */
-    static TOKEN_HEURISTIC = 4;
-    /**
-     * Analyze session messages to produce a context map.
-     * @param messages The full session conversation history.
-     * @param currentTurn The current turn number.
-     */
-    analyze(messages, currentTurn) {
-        const fileRegistry = new Map();
-        let totalTokens = 0;
-        let fileTokens = 0;
-        let toolTokens = 0;
-        messages.forEach((msg, index) => {
-            const turn = index + 1;
-            // Basic token estimation for the message
-            const msgText = typeof msg.content === "string"
-                ? msg.content
-                : JSON.stringify(msg.content);
-            const msgTokens = Math.ceil(msgText.length / ContextAnalyzer.TOKEN_HEURISTIC);
-            totalTokens += msgTokens;
-            if (msg.role === "assistant" && Array.isArray(msg.content)) {
-                for (const block of msg.content) {
-                    if (block.type === "tool_use") {
-                        const input = block.input;
-                        const path = this.extractPath(block.name, input);
-                        if (path) {
-                            const opType = this.getOpType(block.name);
-                            // If the file is already tracked, update it
-                            // Find the tool result for this tool use to get actual content length
-                            const result = this.findToolResult(messages, index, block.id);
-                            const content = result?.content || "";
-                            const weight = Math.ceil(String(content).length / ContextAnalyzer.TOKEN_HEURISTIC);
-                            fileRegistry.set(path, {
-                                path,
-                                weight,
-                                lastOp: {
-                                    type: opType,
-                                    turn,
-                                    timestamp: msg.timestamp || Date.now(),
-                                },
-                                status: this.calculateStatus(turn, currentTurn),
-                            });
-                        }
-                    }
-                }
-            }
-            if (msg.role === "tool") {
-                toolTokens += Math.ceil(String(msg.content).length / ContextAnalyzer.TOKEN_HEURISTIC);
-            }
-        });
-        const files = Array.from(fileRegistry.values());
-        fileTokens = files.reduce((acc, f) => acc + f.weight, 0);
-        return {
-            files: files.sort((a, b) => b.weight - a.weight).slice(0, 100),
-            totalTokens,
-            systemTokens: 0, // Pi provides this via ctx, not messages
-            historyTokens: totalTokens - fileTokens - toolTokens,
-            fileTokens,
-            toolTokens,
-        };
-    }
-    extractPath(toolName, input) {
-        if (toolName === "read" || toolName === "write" || toolName === "edit") {
-            return typeof input.path === "string" ? input.path : null;
-        }
-        if (toolName === "bash") {
-            // Simple regex for paths in bash commands (e.g., cat path/to/file)
-            const match = input.command?.match(/(?:cat|ls|rm|mv|cp|vi|nano)\s+([^\s;]+)/);
-            return match ? match[1] : null;
-        }
-        return null;
-    }
-    getOpType(toolName) {
-        switch (toolName) {
-            case "write":
-                return "write";
-            case "edit":
-                return "edit";
-            case "bash":
-                return "delete"; // Simplified; usually bash implies modification or deletion
-            default:
-                return "read";
-        }
-    }
-    calculateStatus(turn, currentTurn) {
-        const diff = currentTurn - turn;
-        if (diff <= 3)
-            return "active";
-        if (diff <= 10)
-            return "stale";
-        return "legacy";
-    }
-    findToolResult(messages, toolTurnIndex, toolId) {
-        // Look for the tool result immediately following the tool use
-        for (let i = toolTurnIndex + 1; i < messages.length; i++) {
-            if (messages[i].role === "tool" && messages[i].tool_call_id === toolId) {
-                return messages[i];
-            }
-            // If we hit another assistant turn, the result for this specific call is likely gone/compacted
-            if (messages[i].role === "assistant")
-                break;
-        }
-        return null;
-    }
-}
-exports.ContextAnalyzer = ContextAnalyzer;

package/dist/generator.d.ts

@@ -1,11 +0,0 @@
-/**
- * ReportGenerator
- * Generates a visual HTML dashboard based on the ContextMap.
- */
-import type { ContextMap } from "./analyzer";
-export declare class ReportGenerator {
-    static generateHTML(map: ContextMap): string;
-    static writeReport(html: string): string;
-    private static getOpIcon;
-    private static escapeHtml;
-}

package/dist/generator.js

@@ -1,258 +0,0 @@
-"use strict";
-/**
- * ReportGenerator
- * Generates a visual HTML dashboard based on the ContextMap.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ReportGenerator = void 0;
-const node_fs_1 = require("node:fs");
-const node_path_1 = require("node:path");
-const node_os_1 = require("node:os");
-class ReportGenerator {
-    static generateHTML(map) {
-        const fileCards = map.files
-            .map((file) => `
-			<div class="file-card ${file.status}">
-				<div class="file-header">
-					<span class="file-path">${ReportGenerator.escapeHtml(file.path)}</span>
-					<span class="file-weight">${file.weight.toLocaleString()} tokens</span>
-				</div>
-				<div class="file-footer">
-					<span class="op-badge">${ReportGenerator.getOpIcon(file.lastOp.type)} ${file.lastOp.type}</span>
-					<span class="turn-badge">Turn ${file.lastOp.turn}</span>
-					<span class="status-text">${file.status.toUpperCase()}</span>
-				</div>
-				<div class="weight-bar">
-					<div class="weight-fill" style="width: ${Math.min(100, (file.weight / 1000) * 100)}%"></div>
-				</div>
-			</div>
-		`)
-            .join("");
-        const budgetPercent = (map.fileTokens / map.totalTokens) * 100;
-        return `
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Pi Context Map</title>
-    <style>
-        :root {
-            --bg: #0f172a;
-            --card-bg: #1e293b;
-            --text: #f1f5f9;
-            --text-dim: #94a3b8;
-            --primary: #38bdf8;
-            --active: #22c55e;
-            --stale: #eab308;
-            --legacy: #ef4444;
-            --border: #334155;
-        }
-        body {
-            background: var(--bg);
-            color: var(--text);
-            font-family: 'Inter', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
-            margin: 0;
-            padding: 2rem;
-            line-height: 1.5;
-        }
-        .container { max-width: 1200px; margin: 0 auto; }
-        header { margin-bottom: 3rem; border-bottom: 1px solid var(--border); padding-bottom: 2rem; }
-        h1 { font-size: 2rem; margin: 0; color: var(--primary); }
-        .stats-grid {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
-            gap: 1.5rem;
-            margin-top: 2rem;
-        }
-        .stat-card {
-            background: var(--card-bg);
-            padding: 1.5rem;
-            border-radius: 12px;
-            border: 1px solid var(--border);
-            text-align: center;
-        }
-        .stat-value { font-size: 1.5rem; font-weight: bold; display: block; }
-        .stat-label { color: var(--text-dim); font-size: 0.875rem; text-transform: uppercase; }
-        
-        .budget-container {
-            margin: 2rem 0;
-            background: var(--card-bg);
-            padding: 1rem;
-            border-radius: 12px;
-            border: 1px solid var(--border);
-        }
-        .budget-bar {
-            height: 24px;
-            background: #020617;
-            border-radius: 12px;
-            display: flex;
-            overflow: hidden;
-            margin-bottom: 0.5rem;
-        }
-        .budget-segment { height: 100%; transition: width 0.3s ease; }
-        .seg-system { background: #6366f1; }
-        .seg-history { background: #a855f7; }
-        .seg-files { background: var(--primary); }
-        .seg-tools { background: #ec4899; }
-        
-        .budget-legend {
-            display: flex;
-            gap: 1rem;
-            justify-content: center;
-            font-size: 0.75rem;
-            color: var(--text-dim);
-        }
-        .legend-item { display: flex; align-items: center; gap: 0.5rem; }
-        .dot { width: 8px; height: 8px; border-radius: 50%; }
-
-        .file-grid {
-            display: grid;
-            grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
-            gap: 1rem;
-        }
-        .file-card {
-            background: var(--card-bg);
-            border: 1px solid var(--border);
-            border-radius: 12px;
-            padding: 1rem;
-            transition: transform 0.2s ease, border-color 0.2s ease;
-            display: flex;
-            flex-direction: column;
-            justify-content: space-between;
-        }
-        .file-card:hover { transform: translateY(-4px); border-color: var(--primary); }
-        .file-header {
-            display: flex;
-            justify-content: space-between;
-            align-items: flex-start;
-            margin-bottom: 1rem;
-        }
-        .file-path {
-            font-family: 'Fira Code', monospace;
-            font-size: 0.875rem;
-            word-break: break-all;
-            margin-right: 1rem;
-            color: var(--text);
-        }
-        .file-weight { font-size: 0.75rem; color: var(--text-dim); white-space: nowrap; }
-        .file-footer {
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-            margin-top: 1rem;
-            font-size: 0.75rem;
-        }
-        .op-badge {
-            background: #0f172a;
-            padding: 2px 6px;
-            border-radius: 4px;
-            color: var(--text-dim);
-        }
-        .turn-badge { color: var(--text-dim); }
-        .status-text { font-weight: bold; text-transform: uppercase; }
-
-        /* Status Colors */
-        .active { border-left: 4px solid var(--active); }
-        .active .status-text { color: var(--active); }
-        .stale { border-left: 4px solid var(--stale); }
-        .stale .status-text { color: var(--stale); }
-        .legacy { border-left: 4px solid var(--legacy); }
-        .legacy .status-text { color: var(--legacy); }
-
-        .weight-bar {
-            height: 4px;
-            background: #020617;
-            border-radius: 2px;
-            margin-top: 1rem;
-            overflow: hidden;
-        }
-        .weight-fill {
-            height: 100%;
-            background: var(--primary);
-            transition: width 0.3s ease;
-        }
-    </style>
-</head>
-<body>
-    <div class="container">
-        <header>
-            <h1>Pi Context Map</h1>
-            <p style="color: var(--text-dim)">Session context window visualization and token distribution.</p>
-            
-            <div class="stats-grid">
-                <div class="stat-card">
-                    <span class="stat-value">${map.totalTokens.toLocaleString()}</span>
-                    <span class="stat-label">Total Tokens</span>
-                </div>
-                <div class="stat-card">
-                    <span class="stat-value">${map.files.length}</span>
-                    <span class="stat-label">Files in Context</span>
-                </div>
-                <div class="stat-card">
-                    <span class="stat-value">${map.fileTokens.toLocaleString()}</span>
-                    <span class="stat-label">File Tokens</span>
-                </div>
-                <div class="stat-card">
-                    <span class="stat-value">${Math.round(budgetPercent)}%</span>
-                    <span class="stat-label">File Load</span>
-                </div>
-            </div>
-
-            <div class="budget-container">
-                <div class="budget-bar">
-                    <div class="budget-segment seg-system" style="width: ${(map.systemTokens / map.totalTokens) * 100 || 0}%"></div>
-                    <div class="budget-segment seg-history" style="width: ${(map.historyTokens / map.totalTokens) * 100 || 0}%"></div>
-                    <div class="budget-segment seg-files" style="width: ${(map.fileTokens / map.totalTokens) * 100 || 0}%"></div>
-                    <div class="budget-segment seg-tools" style="width: ${(map.toolTokens / map.totalTokens) * 100 || 0}%"></div>
-                </div>
-                <div class="budget-legend">
-                    <div class="legend-item"><span class="dot seg-system"></span> System</div>
-                    <div class="legend-item"><span class="dot seg-history"></span> History</div>
-                    <div class="legend-item"><span class="dot seg-files"></span> Files</div>
-                    <div class="legend-item"><span class="dot seg-tools"></span> Tools</div>
-                </div>
-            </div>
-        </header>
-
-        <div class="file-grid">
-            ${fileCards}
-        </div>
-    </div>
-</body>
-</html>
-`;
-    }
-    static writeReport(html) {
-        const reportDir = (0, node_path_1.join)((0, node_os_1.homedir)(), ".pi", "context-map");
-        (0, node_fs_1.mkdirSync)(reportDir, { recursive: true });
-        const reportPath = (0, node_path_1.join)(reportDir, "report.html");
-        (0, node_fs_1.writeFileSync)(reportPath, html, "utf8");
-        return reportPath;
-    }
-    static getOpIcon(type) {
-        switch (type) {
-            case "read":
-                return "👁️";
-            case "write":
-                return "📝";
-            case "edit":
-                return "✍️";
-            case "delete":
-                return "🗑️";
-            default:
-                return "📄";
-        }
-    }
-    static escapeHtml(text) {
-        const map = {
-            "&": "&amp;",
-            "<": "&lt;",
-            ">": "&gt;",
-            '"': "&quot;",
-            "'": "&#039;",
-        };
-        return text.replace(/[&<>"']/g, (m) => map[m]);
-    }
-}
-exports.ReportGenerator = ReportGenerator;

package/dist/index.d.ts

@@ -1,6 +0,0 @@
-/**
- * pi-context-map
- * Pi extension to visualize session context window and token distribution.
- */
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-export default function (pi: ExtensionAPI): void;