docpilot-cli 1.0.0__py3-none-any.whl

docpilot/chat.py

@@ -0,0 +1,66 @@
+from langchain_ollama.llms import OllamaLLM as Ollama
+from langchain_core.prompts import ChatPromptTemplate
+from .embed import vectorstore
+import os
+from . import store
+
+config = store.load_config()
+retrieval_k = int(config.get("retrieval_k", 6))
+max_context_chars = int(config.get("max_context_chars", 3500))
+max_doc_chars = int(config.get("max_doc_chars", 700))
+
+model = Ollama(
+    model=config.get("default_model", "deepseek-coder-v2"),
+    num_predict=int(config.get("num_predict", 192)),
+    num_ctx=int(config.get("num_ctx", 2048)),
+    num_thread=int(config.get("num_thread", max(1, (os.cpu_count() or 4) - 1))),
+    temperature=float(config.get("temperature", 0.1)),
+)
+
+retriever = vectorstore.as_retriever(search_kwargs={"k": retrieval_k})
+template = """
+You are an assistant for answering questions based on the following ingested documents. 
+Use the information in the documents to answer the question as best as you can. 
+If you don't know the answer, say you don't know.
+Always use the information in the documents and never make up an answer.
+Here are some relevant docs: {reviews}
+Here is the question to answer: {question}
+"""
+prompt = ChatPromptTemplate.from_template(template)
+
+chain = prompt | model
+
+
+def _build_bounded_context(docs, max_total_chars: int = 6000, max_doc_chars: int = 1200) -> str:
+    lines = []
+    total = 0
+    for doc in docs:
+        doc_text = doc.page_content.strip()
+        if len(doc_text) > max_doc_chars:
+            doc_text = doc_text[:max_doc_chars].rsplit(" ", 1)[0].strip() or doc_text[:max_doc_chars]
+        line = f"[ID: {doc.id}] {doc_text}"
+        if total + len(line) > max_total_chars:
+            break
+        lines.append(line)
+        total += len(line)
+    return "\n".join(lines)
+
+
+def askai(question):
+    rag = retriever.invoke(question)
+    rag_text = _build_bounded_context(
+        rag,
+        max_total_chars=max_context_chars,
+        max_doc_chars=max_doc_chars,
+    )
+    result = chain.invoke({"reviews": rag_text, "question": question})
+    return result
+
+
+if __name__ == "__main__":
+    while True:
+        question = input("Ask a question (or type 'exit'): ")
+        if question.lower() == "exit":
+            break
+        result = askai(question)
+        print(result, "\n\n")

docpilot/cli.py

@@ -0,0 +1,313 @@
+import ollama
+import typer
+import importlib
+from pathlib import Path
+from .chat import askai
+from .scrape import scrape_sitemap, scrape_site
+from .embed import embed_texts
+from . import store
+from rich.console import Console
+from rich.markdown import Markdown
+from rich.panel import Panel
+from rich.text import Text
+from rich.align import Align
+
+# Initialize config at startup
+store.init_config()
+
+# Rich console for beautiful output
+console = Console()
+
+app = typer.Typer()
+
+# Project metadata
+PROJECT_VERSION = "0.0.1"
+PROJECT_NAME = "docpilot"
+PROJECT_DESCRIPTION = "A local-first RAG pipeline CLI tool"
+
+
+def loadpdf(file_path: Path) -> list[str]:
+    try:
+        PdfReader = importlib.import_module("pypdf").PdfReader
+    except Exception:
+        print("PDF support is unavailable. Install dependencies again to include pypdf.")
+        return []
+    reader = PdfReader(str(file_path))
+    texts: list[str] = []
+    for page in reader.pages:
+        page_text = (page.extract_text() or "").strip()
+        if page_text:
+            texts.append(page_text)
+    return texts
+
+
+def loadcsv(file_path: Path) -> list[str]:
+    import csv
+
+    try:
+        with open(file_path, newline="", encoding="utf-8") as f:
+            sample = f.read(2048)
+            f.seek(0)
+            try:
+                dialect = csv.Sniffer().sniff(sample)
+            except csv.Error:
+                dialect = csv.excel
+
+            reader = csv.DictReader(f, dialect=dialect)
+            if not reader.fieldnames:
+                return []
+
+            texts: list[str] = []
+            for row in reader:
+                row_text = " | ".join(f"{col}: {val if val else ''}" for col, val in row.items() if col)
+                if row_text.strip():
+                    texts.append(row_text)
+            return texts
+    except Exception as e:
+        print(f"Error reading CSV: {e}")
+        return []
+
+
+def generate_ascii_art() -> str:
+    """Generate beautiful ASCII art for the project."""
+    try:
+        import pyfiglet
+
+        ascii_text = pyfiglet.figlet_format(PROJECT_NAME, font="banner")
+    except ImportError:
+        # Fallback to simple ASCII if pyfiglet not available
+        ascii_text = """
+    ____            ____  _ _       _   
+   / __ \\___  _____/ __ \\(_) |     | |  
+  / / / / _ \\/ ___/ /_/ / /| |     | |  
+ / /_/ /  __/ /__/ ____/ / | |___  |_|  
+/_____/\\___/\\___/_/   /_/  |_____/  (_) 
+"""
+    return ascii_text
+
+
+@app.command()
+def show():
+    """Display version and project information in beautiful ASCII art."""
+    ascii_art = generate_ascii_art()
+    print(ascii_art)
+    print("=" * 60)
+    print(f"Version:     {PROJECT_VERSION}")
+    print(f"Description: {PROJECT_DESCRIPTION}")
+    print("=" * 60)
+
+
+def parse_markdown(content: str) -> None:
+    """Parse and render markdown content beautifully in the terminal."""
+    markdown = Markdown(content)
+    console.print(markdown)
+
+
+@app.command()
+def render(
+    source: str = typer.Argument(..., help="Path to markdown file or inline markdown text"),
+    is_file: bool = typer.Option(True, "--file/--text", "-f/-t", help="Is source a file or inline text?"),
+):
+    """Parse and beautifully render markdown content in the terminal."""
+    try:
+        if is_file:
+            source_path = Path(source).expanduser().resolve()
+            if not source_path.exists():
+                console.print(f"[red]Error: File not found: {source}[/red]")
+                return
+            if not source_path.is_file():
+                console.print(f"[red]Error: Not a file: {source}[/red]")
+                return
+            content = source_path.read_text()
+        else:
+            content = source
+
+        parse_markdown(content)
+    except Exception as e:
+        console.print(f"[red]Error parsing markdown: {e}[/red]")
+
+
+@app.command()
+def ingest(
+    source: str,
+    max_pages: int = typer.Option(20, "--max-pages", "-p", help="Max pages to crawl for websites."),
+    workers: int = typer.Option(16, "--workers", "-w", help="Concurrent workers for scraping."),
+    batch_size: int = typer.Option(32, "--batch-size", "-b", min=1, help="Embedding batch size for faster ingest."),
+    embed_workers: int = typer.Option(
+        0, "--embed-workers", "-e", min=0, help="Threads for chunking before embedding (0 = auto)."
+    ),
+):
+    """Ingest a website, sitemap, PDF, or CSV by chunking and embedding content."""
+    store.check_ollama_connection()
+    if source.endswith("sitemap.xml"):
+        texts = scrape_sitemap(source, max_workers=workers)
+    elif source.startswith("http"):
+        texts = scrape_site(source, max_pages=max_pages, max_workers=workers)
+    else:
+        source_path = Path(source).expanduser().resolve()
+        if not source_path.exists() or not source_path.is_file():
+            print("Unsupported source")
+            return
+
+        suffix = source_path.suffix.lower()
+        if suffix == ".pdf":
+            texts = loadpdf(source_path)
+        elif suffix == ".csv":
+            texts = loadcsv(source_path)
+        else:
+            print("Unsupported local file type. Use .pdf or .csv")
+            return
+
+        if not texts:
+            print("No extractable text found in file.")
+            return
+        source = str(source_path)
+
+    embed_texts(texts, source=source, batch_size=batch_size, embed_workers=embed_workers)
+    print("Ingestion complete.")
+
+
+@app.command()
+def ask(question: str):
+    """Ask a question against the ingested docs and display beautifully rendered response."""
+    store.check_ollama_connection()
+    # Question panel
+    question_text = Text(question, style="bold white")
+    question_panel = Panel(
+        question_text,
+        title="[bold cyan]❓ QUESTION[/bold cyan]",
+        border_style="cyan",
+        padding=(1, 2),
+    )
+    console.print(question_panel)
+
+    try:
+        console.print("[dim]⏳ Thinking...[/dim]\n")
+        response = askai(question)
+
+        # Response panel with markdown
+        response_markdown = Markdown(response)
+        response_panel = Panel(
+            response_markdown,
+            title="[bold green]✨ RESPONSE[/bold green]",
+            border_style="green",
+            padding=(1, 2),
+        )
+        console.print(response_panel)
+
+        # Success footer
+        footer = Align.center(Text("✓ Done", style="dim yellow"))
+        console.print(footer)
+        console.print()
+
+    except Exception as e:
+        error_panel = Panel(
+            Text(str(e), style="bold red"),
+            title="[bold red]❌ ERROR[/bold red]",
+            border_style="red",
+            padding=(1, 2),
+        )
+        console.print(error_panel)
+
+
+@app.command()
+def clear():
+    """Clear the local vector database."""
+    config = store.load_config()
+    db_path = Path(config.get("db_path", ""))
+
+    if not db_path.exists():
+        console.print("[yellow]Database is already empty or does not exist.[/yellow]")
+        return
+
+    confirm = typer.confirm(f"Are you sure you want to delete the database at {db_path}?")
+    if not confirm:
+        console.print("Operation cancelled.")
+        return
+
+    import shutil
+
+    try:
+        shutil.rmtree(db_path)
+        console.print("[bold green]✓ Database cleared successfully![/bold green]")
+    except Exception as e:
+        console.print(f"[bold red]❌ Error clearing database: {e}[/bold red]")
+
+
+@app.command()
+def setup():
+    """Run the interactive model setup wizard."""
+    store.interactive_setup(first_time=False)
+
+
+@app.command()
+def model(action: str, model_name=typer.Argument(None, help="Name of the model to set")):
+    """List available Ollama models or set a default model for Q&A or embeddings."""
+    models = ollama.list()
+    config = store.load_config()
+    """List available models or set a default model."""
+    if action == "list":
+        for i in models.models:
+            print(i.model)
+    elif action == "set":
+        if model_name in [m.model for m in models.models]:
+            config["default_model"] = model_name
+            store.save_config(config)
+            print(f"Model set to: {model_name}")
+        else:
+            print(f"Model '{model_name}' not found. Use 'list' to see available models.")
+    elif action == "setembed":
+        if model_name in [m.model for m in models.models]:
+            config["default_embed_model"] = model_name
+            store.save_config(config)
+            print(f"Embed model set to: {model_name}")
+        else:
+            print(f"Model '{model_name}' not found. Use 'list' to see available models.")
+    else:
+        print("Unsupported action. Use 'list' to see available models.")
+
+
+@app.command()
+def speed(profile: str = typer.Argument("balanced", help="fast, balanced, or quality")):
+    """Set reply speed profile by tuning retrieval and Ollama generation settings."""
+    config = store.load_config()
+
+    profiles = {
+        "fast": {
+            "retrieval_k": 4,
+            "max_context_chars": 2200,
+            "max_doc_chars": 500,
+            "num_predict": 128,
+            "num_ctx": 1536,
+            "temperature": 0.1,
+        },
+        "balanced": {
+            "retrieval_k": 6,
+            "max_context_chars": 3500,
+            "max_doc_chars": 700,
+            "num_predict": 192,
+            "num_ctx": 2048,
+            "temperature": 0.1,
+        },
+        "quality": {
+            "retrieval_k": 10,
+            "max_context_chars": 5500,
+            "max_doc_chars": 1000,
+            "num_predict": 320,
+            "num_ctx": 4096,
+            "temperature": 0.2,
+        },
+    }
+
+    selected = profiles.get(profile.lower())
+    if selected is None:
+        print("Unsupported profile. Use: fast, balanced, or quality")
+        return
+
+    config.update(selected)
+    store.save_config(config)
+    print(f"Speed profile set to: {profile.lower()}")
+
+
+if __name__ == "__main__":
+    app()

docpilot/embed.py

@@ -0,0 +1,172 @@
+from langchain_chroma import Chroma
+from langchain_core.documents import Document
+from langchain_ollama import OllamaEmbeddings
+import hashlib
+import os
+from collections import deque
+from concurrent.futures import ThreadPoolExecutor
+from . import store
+
+
+def _is_context_length_error(exc: Exception) -> bool:
+    msg = str(exc).lower()
+    return ("context" in msg and "length" in msg) or "input length" in msg or "too many tokens" in msg
+
+
+def _split_text_by_chars(text: str, max_chars: int, overlap: int) -> list[str]:
+    text = text.strip()
+    if not text:
+        return []
+    if len(text) <= max_chars:
+        return [text]
+
+    parts: list[str] = []
+    step = max(1, max_chars - overlap)
+    start = 0
+    while start < len(text):
+        part = text[start : start + max_chars].strip()
+        if part:
+            parts.append(part)
+        start += step
+    return parts
+
+
+def _safe_add_documents(batch_docs: list[Document], batch_ids: list[str]) -> tuple[int, int]:
+    """Try batch insert first; on context overflow, split individual docs and retry."""
+    try:
+        vectorstore.add_documents(documents=batch_docs, ids=batch_ids)
+        return len(batch_docs), 0
+    except Exception as e:
+        if not _is_context_length_error(e):
+            raise
+
+    inserted = 0
+    skipped = 0
+    for doc, doc_id in zip(batch_docs, batch_ids):
+        queue: deque[tuple[str, str]] = deque([(doc.page_content, doc_id)])
+        while queue:
+            text, current_id = queue.popleft()
+            try:
+                vectorstore.add_documents(
+                    documents=[Document(page_content=text, metadata=doc.metadata, id=current_id)],
+                    ids=[current_id],
+                )
+                inserted += 1
+                continue
+            except Exception as inner_exc:
+                if not _is_context_length_error(inner_exc):
+                    raise
+
+                # Keep splitting until chunks are small enough; skip pathological inputs.
+                if len(text) <= 220:
+                    skipped += 1
+                    print(f"Skipped chunk due to context limit: {current_id}")
+                    continue
+
+                next_size = max(220, len(text) // 2)
+                overlap = min(80, max(20, next_size // 5))
+                sub_parts = _split_text_by_chars(text, max_chars=next_size, overlap=overlap)
+                if len(sub_parts) <= 1:
+                    skipped += 1
+                    print(f"Skipped chunk due to repeated overflow: {current_id}")
+                    continue
+
+                for idx, sub in enumerate(sub_parts):
+                    queue.append((sub, f"{current_id}-s{idx}"))
+
+    return inserted, skipped
+
+
+def _print_progress(label: str, current: int, total: int, width: int = 28) -> None:
+    if total <= 0:
+        return
+    ratio = min(max(current / total, 0.0), 1.0)
+    filled = int(width * ratio)
+    bar = "#" * filled + "-" * (width - filled)
+    end = "\n" if current >= total else "\r"
+    print(f"{label}: [{bar}] {current}/{total}", end=end, flush=True)
+
+
+embeddings = OllamaEmbeddings(model=store.load_config().get("default_embed_model", "mxbai-embed-large:335m"))
+db_location = store.path / "chroma_langchain_db"
+
+vectorstore = Chroma(collection_name="documents", persist_directory=db_location, embedding_function=embeddings)
+
+
+def chunk_text(
+    text: str,
+    chunk_size: int = 120,
+    overlap: int = 20,
+    max_chars: int = 1200,
+) -> list[str]:
+    words = text.split()
+    chunks = []
+    i = 0
+    step = max(1, chunk_size - overlap)
+    while i < len(words):
+        chunk = " ".join(words[i : i + chunk_size]).strip()
+        if len(chunk) > max_chars:
+            chunk = chunk[:max_chars].rsplit(" ", 1)[0].strip() or chunk[:max_chars]
+        if chunk:
+            chunks.append(chunk)
+        i += step
+    return chunks
+
+
+def _resolve_embed_workers(embed_workers: int, text_count: int) -> int:
+    if text_count <= 1:
+        return 1
+    if embed_workers > 0:
+        return min(embed_workers, text_count)
+
+    cpu_count = os.cpu_count() or 4
+    auto_workers = max(2, cpu_count // 2)
+    auto_workers = min(8, auto_workers)
+    return min(auto_workers, text_count)
+
+
+def embed_texts(texts: list[str], source: str = "web", batch_size: int = 32, embed_workers: int = 0):
+    documents = []
+    ids = []
+    chunk_counter = 0
+    seen_chunks = set()  # Track chunk content to avoid duplicates
+
+    workers = _resolve_embed_workers(embed_workers, len(texts))
+    if workers > 1:
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            chunked_texts = list(executor.map(chunk_text, texts))
+    else:
+        chunked_texts = [chunk_text(text) for text in texts]
+
+    for chunks in chunked_texts:
+        for chunk in chunks:
+            # Skip duplicate chunks
+            if chunk in seen_chunks:
+                continue
+            seen_chunks.add(chunk)
+
+            doc_id = hashlib.md5(f"{source}:{chunk_counter}:{chunk}".encode()).hexdigest()
+            doc = Document(page_content=chunk + "source" + source, metadata={"source": source}, id=doc_id)
+            documents.append(doc)
+            ids.append(doc_id)
+            chunk_counter += 1
+
+    if not documents:
+        print(f"No chunks to embed from {source}")
+        return
+
+    total_batches = (len(documents) + batch_size - 1) // batch_size
+    inserted_total = 0
+    skipped_total = 0
+    for batch_num, i in enumerate(range(0, len(documents), batch_size), start=1):
+        batch_docs = documents[i : i + batch_size]
+        batch_ids = ids[i : i + batch_size]
+        inserted, skipped = _safe_add_documents(batch_docs, batch_ids)
+        inserted_total += inserted
+        skipped_total += skipped
+        _print_progress("Embedding batches", batch_num, total_batches)
+
+    print(f"Embedded {inserted_total} chunks from {source} (skipped {skipped_total})")
+
+
+retriever = vectorstore.as_retriever(search_kwargs={"k": 20})

docpilot/scrape.py

@@ -0,0 +1,234 @@
+import httpx
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections import deque
+import time
+
+
+REQUEST_HEADERS = {"User-Agent": "docpilot/0.0.1 (+https://github.com/foss-hack/docpilot)"}
+MAX_RETRIES = 4
+BACKOFF_SECONDS = 0.8
+
+
+def _print_progress(label: str, current: int, total: int, width: int = 28) -> None:
+    if total <= 0:
+        return
+    ratio = min(max(current / total, 0.0), 1.0)
+    filled = int(width * ratio)
+    bar = "#" * filled + "-" * (width - filled)
+    end = "\n" if current >= total else "\r"
+    print(f"{label}: [{bar}] {current}/{total}", end=end, flush=True)
+
+
+def _extract_text(html: str) -> str:
+    soup = BeautifulSoup(html, "html.parser")
+    return _extract_text_from_soup(soup)
+
+
+def _extract_text_from_soup(soup: BeautifulSoup) -> str:
+    for tag in soup(["nav", "footer", "script", "style"]):
+        tag.decompose()
+    return soup.get_text(separator="\n", strip=True)
+
+
+def _normalize_url(url: str) -> str:
+    parsed = urlparse(url)
+    cleaned = parsed._replace(fragment="")
+    normalized = cleaned.geturl()
+    if normalized.endswith("/") and len(normalized) > len(f"{cleaned.scheme}://{cleaned.netloc}/"):
+        normalized = normalized.rstrip("/")
+    return normalized
+
+
+def _should_skip_url(url: str) -> bool:
+    path = urlparse(url).path
+    # Skip utility/search pages and language index variants that trigger many low-value requests.
+    if "/title/Special:" in path:
+        return True
+    if "/title/Main_page_(" in path:
+        return True
+    return False
+
+
+def _retry_delay_seconds(response: httpx.Response | None, attempt: int) -> float:
+    if response is not None:
+        retry_after = response.headers.get("Retry-After")
+        if retry_after:
+            try:
+                return max(0.2, float(retry_after))
+            except ValueError:
+                pass
+    return BACKOFF_SECONDS * (2**attempt)
+
+
+def _get_with_retries(client: httpx.Client, url: str, timeout: float) -> httpx.Response:
+    last_exc: Exception | None = None
+    for attempt in range(MAX_RETRIES):
+        try:
+            res = client.get(url, follow_redirects=True, timeout=timeout)
+            if res.status_code == 429:
+                if attempt < MAX_RETRIES - 1:
+                    time.sleep(_retry_delay_seconds(res, attempt))
+                    continue
+            if 500 <= res.status_code < 600 and attempt < MAX_RETRIES - 1:
+                time.sleep(_retry_delay_seconds(res, attempt))
+                continue
+            res.raise_for_status()
+            return res
+        except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout, httpx.RemoteProtocolError) as e:
+            last_exc = e
+            if attempt < MAX_RETRIES - 1:
+                time.sleep(_retry_delay_seconds(None, attempt))
+                continue
+            raise
+        except httpx.HTTPStatusError as e:
+            last_exc = e
+            if e.response.status_code in (429, 500, 502, 503, 504) and attempt < MAX_RETRIES - 1:
+                time.sleep(_retry_delay_seconds(e.response, attempt))
+                continue
+            raise
+
+    if last_exc is not None:
+        raise last_exc
+    raise RuntimeError(f"Request failed for {url}")
+
+
+def scrape_url(url: str, client: httpx.Client | None = None) -> str:
+    if client is None:
+        with httpx.Client(headers=REQUEST_HEADERS) as local_client:
+            res = _get_with_retries(local_client, url, timeout=30.0)
+    else:
+        res = _get_with_retries(client, url, timeout=30.0)
+    return _extract_text(res.text)
+
+
+def _fetch_html(url: str, client: httpx.Client) -> tuple[str, str]:
+    res = _get_with_retries(client, url, timeout=20.0)
+    return url, res.text
+
+
+def _collect_sitemap_urls(
+    sitemap_url: str,
+    seen: set[str] | None = None,
+    client: httpx.Client | None = None,
+) -> list[str]:
+    if seen is None:
+        seen = set()
+    if sitemap_url in seen:
+        return []
+    seen.add(sitemap_url)
+
+    try:
+        if client is None:
+            with httpx.Client(headers=REQUEST_HEADERS) as local_client:
+                res = _get_with_retries(local_client, sitemap_url, timeout=30.0)
+        else:
+            res = _get_with_retries(client, sitemap_url, timeout=30.0)
+    except httpx.HTTPStatusError as e:
+        if e.response.status_code == 404:
+            print(f"Sitemap not found (404): {sitemap_url}")
+        else:
+            print(f"Failed to fetch sitemap (HTTP {e.response.status_code}): {sitemap_url}")
+        return []
+    except Exception as e:
+        print(f"Error fetching sitemap: {sitemap_url} - {e}")
+        return []
+
+    soup = BeautifulSoup(res.text, "xml")
+    if not soup.find():
+        print(f"Invalid sitemap XML from: {sitemap_url}")
+        return []
+
+    urls: list[str] = []
+    for loc in soup.find_all("loc"):
+        if not loc.text:
+            continue
+        target = loc.text.strip()
+        if not target:
+            continue
+        if target.endswith(".xml") or target.endswith(".xml.gz"):
+            urls.extend(_collect_sitemap_urls(target, seen, client=client))
+        else:
+            urls.append(target)
+    return urls
+
+
+def scrape_sitemap(sitemap_url: str, max_workers: int = 16) -> list[str]:
+    limits = httpx.Limits(max_keepalive_connections=20, max_connections=100)
+    with httpx.Client(limits=limits, headers=REQUEST_HEADERS) as client:
+        urls = _collect_sitemap_urls(sitemap_url, client=client)
+    texts: list[str] = []
+    total = len(urls)
+    if total == 0:
+        return texts
+
+    workers = max(1, min(max_workers, total))
+    with httpx.Client(limits=limits, headers=REQUEST_HEADERS) as client:
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = [executor.submit(scrape_url, u, client) for u in urls]
+            completed = 0
+            for future in as_completed(futures):
+                completed += 1
+                try:
+                    texts.append(future.result())
+                except Exception:
+                    pass
+                _print_progress("Scraping sitemap pages", completed, total)
+    return texts
+
+
+def scrape_site(base_url: str, max_pages: int = 100, max_workers: int = 16) -> list[str]:
+    base_url = _normalize_url(base_url)
+    base_netloc = urlparse(base_url).netloc
+
+    visited: set[str] = set()
+    queued: set[str] = {base_url}
+    to_visit: deque[str] = deque([base_url])
+    texts = []
+    limits = httpx.Limits(max_keepalive_connections=20, max_connections=100)
+    effective_workers = max(1, max_workers)
+
+    with httpx.Client(limits=limits, headers=REQUEST_HEADERS) as client:
+        with ThreadPoolExecutor(max_workers=effective_workers) as executor:
+            while to_visit and len(visited) < max_pages:
+                batch: list[str] = []
+                while to_visit and len(batch) < effective_workers and (len(visited) + len(batch)) < max_pages:
+                    candidate = to_visit.popleft()
+                    if candidate in visited:
+                        continue
+                    if _should_skip_url(candidate):
+                        continue
+                    batch.append(candidate)
+
+                futures = {executor.submit(_fetch_html, url, client): url for url in batch}
+                for future in as_completed(futures):
+                    url = futures[future]
+                    visited.add(url)
+                    _print_progress("Crawling pages", len(visited), max_pages)
+                    try:
+                        _, html = future.result()
+                    except httpx.HTTPStatusError as e:
+                        if e.response.status_code == 404:
+                            print(f"  Page not found (404): {url}")
+                        else:
+                            print(f"  HTTP error {e.response.status_code}: {url}")
+                        continue
+                    except Exception as e:
+                        print(f"  Error scraping {url}: {type(e).__name__}")
+                        continue
+
+                    soup = BeautifulSoup(html, "html.parser")
+                    texts.append(_extract_text_from_soup(soup))
+                    for a in soup.find_all("a", href=True):
+                        full = _normalize_url(urljoin(url, a["href"]))
+                        parsed = urlparse(full)
+                        if parsed.netloc != base_netloc:
+                            continue
+                        if _should_skip_url(full):
+                            continue
+                        if full in visited or full in queued:
+                            continue
+                        queued.add(full)
+                        to_visit.append(full)
+    return texts

docpilot/store.py

@@ -0,0 +1,113 @@
+import tomllib
+import tomli_w
+from pathlib import Path
+import ollama
+import os
+
+path = Path.home() / ".docpilot"
+CONFIG_PATH = path / "config.toml"
+
+
+def _get_available_models():
+    try:
+        models = ollama.list().models
+        return [m.model for m in models]
+    except Exception:
+        return []
+
+
+_AVAILABLE_MODELS = _get_available_models()
+
+
+DEFAULT_CONFIG = {
+    "default_embed_model": _AVAILABLE_MODELS[0] if _AVAILABLE_MODELS else "mxbai-embed-large:335m",
+    "default_model": _AVAILABLE_MODELS[1] if len(_AVAILABLE_MODELS) > 1 else "llama2",
+    "db_path": str(path / "chroma_langchain_db"),
+    "log_level": "info",
+    "retrieval_k": 6,
+    "max_context_chars": 3500,
+    "max_doc_chars": 700,
+    "num_predict": 192,
+    "num_ctx": 2048,
+    "num_thread": max(1, (os.cpu_count() or 4) - 1),
+    "temperature": 0.1,
+}
+
+
+def check_ollama_connection():
+    try:
+        ollama.list()
+    except Exception:
+        from rich.console import Console
+
+        console = Console()
+        console.print("\n[bold red]❌ Error: Could not connect to Ollama.[/bold red]")
+        console.print("[yellow]Please ensure the Ollama application is running and try again.[/yellow]\n")
+        import sys
+
+        sys.exit(1)
+
+
+def init_config():
+    """Initialize config file with defaults if it doesn't exist."""
+    if not CONFIG_PATH.exists():
+        CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+        interactive_setup(first_time=True)
+    return load_config()
+
+
+def interactive_setup(first_time=False):
+    from rich.console import Console
+    from rich.prompt import Prompt
+
+    console = Console()
+
+    if first_time:
+        console.print("\n[bold cyan]🚀 Welcome to Docpilot! Let's do a quick setup.[/bold cyan]")
+    else:
+        console.print("\n[bold cyan]⚙️  Docpilot Model Setup[/bold cyan]")
+
+    models = _get_available_models()
+    if not models:
+        console.print("[yellow]No local Ollama models found! Using default fallbacks.[/yellow]")
+        console.print("Please remember to run `ollama pull mxbai-embed-large` and `ollama pull qwen2.5:latest` later.")
+        embed_model = Prompt.ask("Enter embedding model name", default="mxbai-embed-large:335m")
+        chat_model = Prompt.ask("Enter chat model name", default="qwen2.5:latest")
+    else:
+        # Smart defaults based on names
+        embed_guess = next((m for m in models if "embed" in m.lower()), models[0])
+        chat_guess = next((m for m in models if "embed" not in m.lower()), models[1] if len(models) > 1 else models[0])
+
+        embed_model = Prompt.ask(
+            "Select your [bold green]embedding model[/bold green]", choices=models, default=embed_guess
+        )
+        chat_model = Prompt.ask("Select your [bold blue]chat model[/bold blue]", choices=models, default=chat_guess)
+
+    config = load_config() if not first_time else DEFAULT_CONFIG.copy()
+    config["default_embed_model"] = embed_model
+    config["default_model"] = chat_model
+    save_config(config)
+    console.print(f"[bold green]✓ Configuration saved to {CONFIG_PATH}[/bold green]\n")
+    return load_config()
+
+
+def load_config():
+    if not CONFIG_PATH.exists():
+        return DEFAULT_CONFIG.copy()
+    try:
+        with open(CONFIG_PATH, "rb") as f:
+            loaded = tomllib.load(f)
+    except tomllib.TOMLDecodeError:
+        print(f"Invalid config at {CONFIG_PATH}. Recreating with defaults.")
+        save_config(DEFAULT_CONFIG.copy())
+        return DEFAULT_CONFIG.copy()
+
+    merged = DEFAULT_CONFIG.copy()
+    merged.update(loaded)
+    return merged
+
+
+def save_config(config: dict):
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with open(CONFIG_PATH, "wb") as f:
+        tomli_w.dump(config, f)

docpilot_cli-1.0.0.dist-info/METADATA

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: docpilot-cli
+Version: 1.0.0
+Summary: A local-first RAG pipeline CLI tool
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: bs4>=0.0.2
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: langchain>=0.2.0
+Requires-Dist: langchain-chroma>=0.1.0
+Requires-Dist: langchain-ollama>=0.1.0
+Requires-Dist: ollama>=0.2.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: typer>=0.12.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: tomli-w>=1.0.0
+Requires-Dist: pypdf>=5.0.0
+Requires-Dist: pyfiglet>=0.8.0
+Requires-Dist: rich>=13.0.0
+Dynamic: license-file
+
+# 🚀 DocPilot CLI
+
+![PyPI - Version](https://img.shields.io/pypi/v/docpilot-cli?color=blue&style=flat-square)
+![Python Version](https://img.shields.io/badge/python-3.12%2B-blue?style=flat-square)
+![License](https://img.shields.io/badge/license-MIT-green?style=flat-square)
+
+**DocPilot** is a lightning-fast, local-first CLI for document ingestion and interactive question-answering. Powered by [Ollama](https://ollama.com/) and Chroma, it allows you to ingest websites, PDFs, and CSVs directly from your terminal and chat with your documents—keeping 100% of your data safely on your own machine.
+
+It’s built for practical developer workflows: crawl sites concurrently, prepare chunks with multi-threading, and iterate rapidly without ever paying for a cloud API.
+
+---
+
+## ✨ Features
+
+- **🔒 100% Local**: No data ever leaves your machine. Powered by Ollama.
+- **⚡ Interactive Setup Wizard**: Get up and running instantly with smart model auto-detection.
+- **🌐 Universal Ingestion**: Seamlessly ingest Website URLs, XML Sitemaps, PDFs, and CSVs.
+- **🚀 Concurrent Processing**: Lightning-fast crawling and multi-threaded document chunking.
+- **🎛️ Performance Profiles**: Switch between `fast`, `balanced`, and `quality` inference speeds on the fly.
+- **🎨 Beautiful Terminal UI**: Rich markdown rendering, ASCII art, and intuitive progress bars.
+
+---
+
+## 📦 Installation
+
+DocPilot is available on PyPI! You can install it globally using `pip`, `uv`, or `pipx`.
+
+```bash
+# Recommended: Install using uv or pipx
+uv tool install docpilot-cli
+
+# Or via standard pip
+pip install docpilot-cli
+
+# Optional: Add PDF parsing support
+pip install "docpilot-cli[pdf]"
+```
+
+### Prerequisites
+1. **Python 3.12+**
+2. **[Ollama](https://ollama.com/)**: Installed and running in the background.
+3. Pull your preferred models:
+```bash
+ollama pull qwen2.5:latest
+ollama pull mxbai-embed-large:335m
+```
+
+---
+
+## 🛠️ Quick Start
+
+The very first time you run a DocPilot command, it will launch the **Interactive Setup Wizard** to help you configure your chat and embedding models. 
+
+### 1. Ingest Knowledge
+Point DocPilot to any documentation site, sitemap, PDF, or CSV:
+
+```bash
+# Crawl a website
+docpilot ingest "https://docs.python.org/3/" --max-pages 100 --workers 16
+
+# Ingest a local PDF
+docpilot ingest "./docs/engineering_handbook.pdf"
+
+# Ingest a CSV
+docpilot ingest "./data/faq.csv"
+```
+
+### 2. Ask Questions
+Query your newly created local knowledge base:
+
+```bash
+docpilot ask "How do I create a virtual environment?"
+```
+
+---
+
+## 🧰 CLI Command Reference
+
+Manage your configuration, models, and local database with ease.
+
+### `docpilot setup`
+Re-run the interactive setup wizard at any time to change your default models.
+
+### `docpilot clear`
+Wipe your local Chroma vector database to start fresh. Prompts for safety confirmation.
+
+### `docpilot speed [profile]`
+Adjust the retrieval and generation settings for your desired use case.
+- `fast`: Lower latency, shorter context limits.
+- `balanced`: Default trade-off.
+- `quality`: Larger context, more comprehensive answers, slower inference.
+
+### `docpilot model`
+Manually manage your Ollama models without the interactive setup wizard.
+```bash
+docpilot model list
+docpilot model set <chat-model>
+docpilot model setembed <embedding-model>
+```
+
+### `docpilot render`
+Parse and beautifully render any markdown file or text string directly in your terminal.
+
+### `docpilot show`
+Display your current project version and configuration in beautiful ASCII art.
+
+---
+
+## 🏗️ Architecture Under the Hood
+
+DocPilot employs an optimized RAG (Retrieval-Augmented Generation) pipeline:
+
+1. **Ingestion**: Native Python extractors (BeautifulSoup4, `csv`, `pypdf`) parse the raw data.
+2. **Chunking**: Multi-threaded chunkers slice the documents into semantically coherent pieces.
+3. **Embedding**: `langchain-ollama` creates local vector embeddings via Ollama.
+4. **Storage**: `chromadb` persistently stores vectors on disk at `~/.docpilot/chroma_langchain_db`.
+5. **Retrieval**: User queries are embedded, matched via similarity search, and fed into a system prompt.
+6. **Generation**: The designated Ollama chat model generates the final response streamed to the terminal using `rich`.
+
+---
+
+## 🤝 Contributing
+
+Contributions are welcome! If you are using DocPilot for your daily workflows or in a hackathon, feel free to open issues and pull requests.
+
+To set up a local development environment:
+```bash
+git clone https://github.com/yourusername/docpilot.git
+cd docpilot
+uv pip install -e ".[dev]"
+uv run pytest
+```
+
+---
+
+## 📄 License
+
+This project is licensed under the MIT License.

docpilot_cli-1.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+docpilot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docpilot/chat.py,sha256=DxNUPcPpbkjmsVNUeZDzs5PZxH2zmsr577DNYKW9UVM,2263
+docpilot/cli.py,sha256=DB7zlVjtFlPMqvWQYTv7oQuAVAePkqcjcK5q4-DIZpo,9862
+docpilot/embed.py,sha256=3Y-BmVi0W8Jgt2G1eEKrKdjwPBZUPZyqAls5_VGd-Mw,5948
+docpilot/scrape.py,sha256=jNUJi1G9Jx1-kT25sTHU7v-ZxmCdC0YfNTnaz6PPpNU,8776
+docpilot/store.py,sha256=wrYezb1lJaeRYwzs1WIl4e-rRd51wAvAbTJZeZTKzn0,3720
+docpilot_cli-1.0.0.dist-info/licenses/LICENSE,sha256=X4mftAPvc1sAnTsGkUXLxr13OO6ieIzNTCaxpLOLgPc,1068
+docpilot_cli-1.0.0.dist-info/METADATA,sha256=NXfGnUH0_pkZ-eYHQ9Hoq8H0TYaZYNY_SyOxJaNbmRU,5272
+docpilot_cli-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+docpilot_cli-1.0.0.dist-info/entry_points.txt,sha256=4xdYvYBz1-YIujMQ466UqTOrNYDPgy7vLFZhiuhLtlw,46
+docpilot_cli-1.0.0.dist-info/top_level.txt,sha256=kTb5lJ_m1AkLt-9BLs6UKxf-FmvQLYT9ticqTXKO3d8,9
+docpilot_cli-1.0.0.dist-info/RECORD,,

docpilot_cli-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

docpilot_cli-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+docpilot = docpilot.cli:app

docpilot_cli-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ASWIN ASHOK
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docpilot_cli-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+docpilot