contextforllm 0.1.0__py3-none-any.whl

contextforllm/__main__.py

@@ -0,0 +1,34 @@
+import os
+import sys
+import subprocess
+import webbrowser
+import time
+
+def main():
+    app_dir = os.path.dirname(os.path.abspath(__file__))
+    app_path = os.path.join(app_dir, "app.py")
+
+    print("")
+    print("Starting ContextForLLM...")
+    print("")
+
+    process = subprocess.Popen(
+        [sys.executable, app_path],
+        cwd=app_dir
+    )
+
+    time.sleep(2)
+    webbrowser.open("http://127.0.0.1:5000")
+
+    try:
+        process.wait()
+    except KeyboardInterrupt:
+        process.terminate()
+        print("")
+        print("ContextForLLM stopped.")
+        print("")
+
+if __name__ == "__main__":
+    main()
+
+    

contextforllm/app.py

@@ -0,0 +1,225 @@
+import os
+import json
+from flask import Flask, request, jsonify, send_from_directory
+from context_builder import (
+    build_folder_tree,
+    collect_files,
+    build_file_block,
+    build_header,
+    split_into_prompts,
+    save_prompts,
+    count_tokens,
+    load_contextignore,
+)
+from project_summary import (
+    generate_project_summary,
+    save_summary,
+    load_summary,
+    delete_summary,
+)
+
+app = Flask(__name__, static_folder=os.path.join(os.path.dirname(__file__), "ui"))
+
+TOOL_DIR = os.path.dirname(os.path.abspath(__file__))
+RECENT_FILE = os.path.join(TOOL_DIR, "recent_projects.json")
+MAX_RECENT = 8
+
+# Session-only Groq key (not saved to disk)
+session_groq_key = ""
+
+# ── Recent projects ────────────────────────────────────────────
+def load_recent():
+    if os.path.isfile(RECENT_FILE):
+        try:
+            with open(RECENT_FILE, "r") as f:
+                return json.load(f)
+        except Exception:
+            return []
+    return []
+
+def save_recent(project_path, project_name):
+    recent = load_recent()
+    recent = [r for r in recent if r["path"] != project_path]
+    recent.insert(0, {"path": project_path, "name": project_name})
+    recent = recent[:MAX_RECENT]
+    with open(RECENT_FILE, "w") as f:
+        json.dump(recent, f, indent=2)
+
+# ── Routes ─────────────────────────────────────────────────────
+@app.route("/")
+def index():
+    return send_from_directory("ui", "index.html")
+
+@app.route("/api/recent", methods=["GET"])
+def get_recent():
+    recent = load_recent()
+    recent = [r for r in recent if os.path.isdir(r["path"])]
+    return jsonify({"recent": recent})
+
+@app.route("/api/recent/remove", methods=["POST"])
+def remove_recent():
+    data = request.json
+    path = data.get("path", "")
+    recent = load_recent()
+    recent = [r for r in recent if r["path"] != path]
+    with open(RECENT_FILE, "w") as f:
+        json.dump(recent, f, indent=2)
+    delete_summary(TOOL_DIR, path)
+    return jsonify({"ok": True})
+
+@app.route("/api/groq-key", methods=["POST"])
+def set_groq_key():
+    global session_groq_key
+    data = request.json
+    key = data.get("key", "").strip()
+    if not key:
+        return jsonify({"error": "Key is empty"}), 400
+    if not key.startswith("gsk_"):
+        return jsonify({"error": "Invalid key — Groq keys start with gsk_"}), 400
+    session_groq_key = key
+    return jsonify({"ok": True})
+
+@app.route("/api/groq-key/status", methods=["GET"])
+def groq_key_status():
+    return jsonify({"has_key": bool(session_groq_key)})
+
+@app.route("/api/scan", methods=["POST"])
+def scan():
+    data = request.json
+    project_path = data.get("project_path", "").strip()
+    project_path = os.path.expanduser(project_path)
+    project_path = os.path.abspath(project_path)
+
+    if not os.path.isdir(project_path):
+        return jsonify({"error": f"Folder not found: {project_path}"}), 400
+
+    project_name = os.path.basename(project_path)
+    tree_lines = build_folder_tree(project_path)
+    files = collect_files(project_path)
+    patterns = load_contextignore(project_path)
+    save_recent(project_path, project_name)
+
+    file_blocks = [build_file_block(path, content) for path, content in files]
+    header = build_header(project_name, project_path, "\n".join(tree_lines), 1)
+    full_text = header + "\n".join(file_blocks)
+    total_tokens = count_tokens(full_text)
+
+    existing_summary = load_summary(TOOL_DIR, project_path)
+
+    return jsonify({
+        "project_name": project_name,
+        "project_path": project_path,
+        "tree": tree_lines,
+        "files": [{"path": p, "tokens": count_tokens(c)} for p, c in files],
+        "total_tokens": total_tokens,
+        "file_count": len(files),
+        "contextignore_rules": patterns,
+        "existing_summary": existing_summary,
+        "groq_key_set": bool(session_groq_key)
+    })
+
+@app.route("/api/summary/generate", methods=["POST"])
+def generate_summary():
+    global session_groq_key
+    data = request.json
+    project_path = os.path.abspath(
+        os.path.expanduser(data.get("project_path", "")))
+
+    if not session_groq_key:
+        return jsonify({"error": "No Groq API key set for this session"}), 400
+
+    if not os.path.isdir(project_path):
+        return jsonify({"error": "Folder not found"}), 400
+
+    project_name = os.path.basename(project_path)
+    files = collect_files(project_path)
+
+    try:
+        summary = generate_project_summary(project_name, files, session_groq_key)
+        save_summary(TOOL_DIR, project_path, summary)
+        return jsonify({"summary": summary})
+    except Exception as e:
+        return jsonify({"error": str(e)}), 500
+
+@app.route("/api/summary/save", methods=["POST"])
+def save_summary_route():
+    data = request.json
+    project_path = os.path.abspath(
+        os.path.expanduser(data.get("project_path", "")))
+    summary = data.get("summary", "").strip()
+    if not summary:
+        return jsonify({"error": "Summary is empty"}), 400
+    save_summary(TOOL_DIR, project_path, summary)
+    return jsonify({"ok": True})
+
+@app.route("/api/summary/delete", methods=["POST"])
+def delete_summary_route():
+    data = request.json
+    project_path = os.path.abspath(
+        os.path.expanduser(data.get("project_path", "")))
+    delete_summary(TOOL_DIR, project_path)
+    return jsonify({"ok": True})
+
+@app.route("/api/generate", methods=["POST"])
+def generate():
+    data = request.json
+    project_path = os.path.abspath(
+        os.path.expanduser(data.get("project_path", "")))
+    task = data.get("task", "").strip()
+    excluded = set(data.get("excluded", []))
+    annotations = data.get("annotations", {})
+    include_summary = data.get("include_summary", True)
+
+    if not os.path.isdir(project_path):
+        return jsonify({"error": "Folder not found"}), 400
+
+    project_name = os.path.basename(project_path)
+    tree_lines = build_folder_tree(project_path)
+    all_files = collect_files(project_path)
+    files = [(p, c) for p, c in all_files if p not in excluded]
+
+    summary_text = ""
+    if include_summary:
+        summary_text = load_summary(TOOL_DIR, project_path) or ""
+
+    output_dir = os.path.join(TOOL_DIR, "output")
+    os.makedirs(output_dir, exist_ok=True)
+
+    file_blocks = [
+        build_file_block(path, content, annotations.get(path, ""))
+        for path, content in files
+    ]
+    header = build_header(project_name, project_path, "\n".join(tree_lines), 1)
+
+    if summary_text:
+        header = f"## PROJECT SUMMARY\n\n{summary_text}\n\n{header}"
+
+    prompts = split_into_prompts(header, file_blocks, task, project_name)
+    saved_files = save_prompts(prompts, output_dir)
+
+    parts = []
+    for i, filepath in enumerate(saved_files):
+        with open(filepath, "r") as f:
+            content = f.read()
+        parts.append({
+            "part": i + 1,
+            "filename": os.path.basename(filepath),
+            "tokens": count_tokens(content),
+            "content": content
+        })
+
+    return jsonify({
+        "total_parts": len(parts),
+        "output_dir": output_dir,
+        "parts": parts
+    })
+
+if __name__ == "__main__":
+    print("\nContextForLLM is running.")
+    print("Open this in your browser: http://localhost:5000\n")
+    app.run(debug=False, port=5000)
+
+
+
+
+    

contextforllm/context_builder.py

@@ -0,0 +1,178 @@
+import os
+import fnmatch
+import tiktoken
+
+INCLUDE_EXTENSIONS = [
+    ".py", ".js", ".ts", ".jsx", ".tsx",
+    ".html", ".css", ".json", ".md",
+    ".txt", ".env.example", ".yaml", ".yml",
+    ".sh", ".sql"
+]
+
+SKIP_FOLDERS = [
+    "venv", ".venv", "env",
+    "node_modules", ".git",
+    "__pycache__", ".next",
+    "dist", "build", ".idea",
+    ".vscode", "coverage",
+    "output", "summaries"
+]
+
+MAX_TOKENS_PER_PART = 80000
+
+# ── Token counting ─────────────────────────────────────────────
+def count_tokens(text):
+    try:
+        enc = tiktoken.get_encoding("cl100k_base")
+        return len(enc.encode(text))
+    except Exception:
+        return len(text) // 4
+
+# ── .contextignore ─────────────────────────────────────────────
+def load_contextignore(project_path):
+    ignore_file = os.path.join(project_path, ".contextignore")
+    if not os.path.isfile(ignore_file):
+        return []
+    patterns = []
+    with open(ignore_file, "r") as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith("#"):
+                patterns.append(line)
+    return patterns
+
+def is_ignored(rel_path, patterns):
+    for pattern in patterns:
+        if fnmatch.fnmatch(rel_path, pattern):
+            return True
+        if fnmatch.fnmatch(os.path.basename(rel_path), pattern):
+            return True
+    return False
+
+# ── File collection ────────────────────────────────────────────
+def collect_files(project_path):
+    patterns = load_contextignore(project_path)
+    result = []
+    for root, dirs, files in os.walk(project_path):
+        dirs[:] = [
+            d for d in dirs
+            if d not in SKIP_FOLDERS and not d.startswith(".")
+        ]
+        for fname in sorted(files):
+            _, ext = os.path.splitext(fname)
+            if ext.lower() not in INCLUDE_EXTENSIONS:
+                continue
+            full_path = os.path.join(root, fname)
+            rel_path = os.path.relpath(full_path, project_path)
+            if is_ignored(rel_path, patterns):
+                continue
+            try:
+                with open(full_path, "r", encoding="utf-8", errors="ignore") as f:
+                    content = f.read()
+                result.append((rel_path, content))
+            except Exception:
+                continue
+    return result
+
+# ── Folder tree ────────────────────────────────────────────────
+def build_folder_tree(project_path):
+    lines = []
+    for root, dirs, files in os.walk(project_path):
+        dirs[:] = sorted([
+            d for d in dirs
+            if d not in SKIP_FOLDERS and not d.startswith(".")
+        ])
+        level = os.path.relpath(root, project_path).count(os.sep)
+        if os.path.relpath(root, project_path) == ".":
+            level = 0
+        indent = "    " * level
+        if level > 0:
+            lines.append(f"{indent}— {os.path.basename(root)}/")
+        for fname in sorted(files):
+            _, ext = os.path.splitext(fname)
+            if ext.lower() in INCLUDE_EXTENSIONS:
+                lines.append(f"{'    ' * (level+1 if level > 0 else 1)}— {fname}")
+    return lines
+
+# ── File block ─────────────────────────────────────────────────
+def build_file_block(rel_path, content, annotation=""):
+    lines = []
+    lines.append(f"\n{'='*60}")
+    lines.append(f"FILE: {rel_path}")
+    if annotation:
+        lines.append(f"# NOTE: {annotation}")
+    lines.append("="*60)
+    lines.append(content)
+    return "\n".join(lines)
+
+# ── Header ─────────────────────────────────────────────────────
+def build_header(project_name, project_path, tree_text, part_num):
+    return f"""{'='*60}
+CONTEXT DOCUMENT — {project_name}
+Part {part_num}
+{'='*60}
+
+This document contains the full source code of the project '{project_name}'.
+Read all files carefully before responding.
+
+PROJECT PATH: {project_path}
+
+FOLDER STRUCTURE:
+{tree_text}
+
+{'='*60}
+SOURCE FILES
+{'='*60}
+"""
+
+# ── Prompt splitter ────────────────────────────────────────────
+def split_into_prompts(header, file_blocks, task, project_name):
+    parts = []
+    current_blocks = []
+    current_tokens = count_tokens(header)
+
+    for block in file_blocks:
+        block_tokens = count_tokens(block)
+        if current_tokens + block_tokens > MAX_TOKENS_PER_PART and current_blocks:
+            parts.append(current_blocks)
+            current_blocks = [block]
+            current_tokens = count_tokens(header) + block_tokens
+        else:
+            current_blocks.append(block)
+            current_tokens += block_tokens
+
+    if current_blocks:
+        parts.append(current_blocks)
+
+    total = len(parts)
+    prompts = []
+    for i, blocks in enumerate(parts):
+        part_num = i + 1
+        h = build_header(project_name, "", "", part_num)
+        body = h + "\n".join(blocks)
+        if part_num < total:
+            body += f"\n\n{'='*60}\nEND OF PART {part_num} OF {total}\n"
+            body += "This is not all the code. Wait for the next part before responding.\n"
+            body += f"Reply only with: 'Part {part_num} received. Send part {part_num+1}.'\n{'='*60}"
+        else:
+            if task:
+                body += f"\n\n{'='*60}\nYOUR TASK\n{'='*60}\n{task}"
+            body += f"\n\n{'='*60}\nEND OF CONTEXT"
+            if total > 1:
+                body += f" (Final part {part_num} of {total})"
+            body += f"\n{'='*60}"
+            if total > 1:
+                body += "\nAll parts have been sent. Please complete the task above."
+        prompts.append(body)
+
+    return prompts
+
+def save_prompts(prompts, output_dir):
+    os.makedirs(output_dir, exist_ok=True)
+    saved = []
+    for i, prompt in enumerate(prompts):
+        path = os.path.join(output_dir, f"prompt_part_{i+1}.txt")
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(prompt)
+        saved.append(path)
+    return saved

contextforllm/project_summary.py

@@ -0,0 +1,87 @@
+import os
+from groq import Groq
+
+SUMMARY_MODEL = "llama-3.3-70b-versatile"
+
+def get_client(api_key):
+    if not api_key:
+        raise ValueError("No Groq API key provided")
+    return Groq(api_key=api_key)
+
+def build_condensed_context(files, max_chars_per_file=800):
+    lines = []
+    for path, content in files:
+        lines.append(f"\n--- FILE: {path} ---")
+        preview = content[:max_chars_per_file]
+        if len(content) > max_chars_per_file:
+            preview += "\n... (truncated)"
+        lines.append(preview)
+    return "\n".join(lines)
+
+def generate_project_summary(project_name, files, api_key):
+    client = get_client(api_key)
+    condensed = build_condensed_context(files)
+    prompt = f"""You are analyzing a software project called '{project_name}'.
+
+Here is a condensed view of the project files:
+
+{condensed}
+
+Write a concise project summary with exactly these sections:
+
+**What this project does:**
+One or two sentences describing what the project is and what problem it solves.
+
+**Tech stack:**
+List the main languages, frameworks, and libraries used.
+
+**Main files:**
+A short description of what each key file does. One line per file.
+
+**Notes for the LLM:**
+Any important things an LLM should know before working on this codebase — conventions, patterns, or things to be careful about.
+
+Keep the entire summary under 300 words. Be specific and factual. Do not make things up."""
+
+    response = client.chat.completions.create(
+        model=SUMMARY_MODEL,
+        messages=[
+            {
+                "role": "system",
+                "content": "You are a technical assistant that writes clear, accurate project summaries for developers."
+            },
+            {
+                "role": "user",
+                "content": prompt
+            }
+        ],
+        temperature=0.3,
+        max_tokens=600
+    )
+    return response.choices[0].message.content.strip()
+
+# ── Summary storage ────────────────────────────────────────────
+def get_summary_path(tool_dir, project_path):
+    import hashlib
+    summaries_dir = os.path.join(tool_dir, "summaries")
+    os.makedirs(summaries_dir, exist_ok=True)
+    path_hash = hashlib.md5(project_path.encode()).hexdigest()
+    return os.path.join(summaries_dir, f"{path_hash}.txt")
+
+def save_summary(tool_dir, project_path, summary):
+    summary_path = get_summary_path(tool_dir, project_path)
+    with open(summary_path, "w", encoding="utf-8") as f:
+        f.write(summary)
+
+def load_summary(tool_dir, project_path):
+    summary_path = get_summary_path(tool_dir, project_path)
+    if os.path.isfile(summary_path):
+        with open(summary_path, "r", encoding="utf-8") as f:
+            return f.read()
+    return None
+
+def delete_summary(tool_dir, project_path):
+    summary_path = get_summary_path(tool_dir, project_path)
+    if os.path.isfile(summary_path):
+        os.remove(summary_path)
+