docugraph 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. docugraph/__init__.py +24 -0
  2. docugraph/_version.py +28 -0
  3. docugraph/agents/__init__.py +27 -0
  4. docugraph/agents/coding_agent.py +532 -0
  5. docugraph/agents/tools/__init__.py +62 -0
  6. docugraph/agents/tools/graph_query.py +203 -0
  7. docugraph/agents/tools/memory_ops.py +305 -0
  8. docugraph/agents/tools/search_code.py +305 -0
  9. docugraph/agents/tools/search_docs.py +148 -0
  10. docugraph/core/__init__.py +34 -0
  11. docugraph/core/config.py +183 -0
  12. docugraph/core/embeddings.py +394 -0
  13. docugraph/core/llm.py +892 -0
  14. docugraph/core/models.py +145 -0
  15. docugraph/ingestion/__init__.py +64 -0
  16. docugraph/ingestion/api_docs.py +646 -0
  17. docugraph/ingestion/chunker.py +382 -0
  18. docugraph/ingestion/crawler.py +305 -0
  19. docugraph/ingestion/git_indexer.py +556 -0
  20. docugraph/ingestion/local_files.py +516 -0
  21. docugraph/interfaces/__init__.py +1 -0
  22. docugraph/interfaces/cli.py +983 -0
  23. docugraph/interfaces/mcp_server.py +918 -0
  24. docugraph/retrieval/__init__.py +54 -0
  25. docugraph/retrieval/context_builder.py +522 -0
  26. docugraph/retrieval/hybrid_search.py +534 -0
  27. docugraph/retrieval/reranker.py +532 -0
  28. docugraph/storage/__init__.py +14 -0
  29. docugraph/storage/graph_store.py +602 -0
  30. docugraph/storage/memory_store.py +529 -0
  31. docugraph/storage/vector_store.py +367 -0
  32. docugraph-0.1.0.dist-info/METADATA +354 -0
  33. docugraph-0.1.0.dist-info/RECORD +37 -0
  34. docugraph-0.1.0.dist-info/WHEEL +4 -0
  35. docugraph-0.1.0.dist-info/entry_points.txt +3 -0
  36. docugraph-0.1.0.dist-info/licenses/LICENSE +201 -0
  37. docugraph-0.1.0.dist-info/licenses/NOTICE +10 -0
@@ -0,0 +1,983 @@
1
+ """Command-line interface for DocuGraph AI."""
2
+
3
+ import asyncio
4
+ from pathlib import Path
5
+
6
+ import click
7
+ from rich.console import Console
8
+ from rich.progress import Progress, SpinnerColumn, TextColumn
9
+ from rich.table import Table
10
+
11
+ from docugraph._version import __version__
12
+
13
+ console = Console()
14
+
15
+ # Exit code for "the command needed an LLM and could not reach a usable one".
16
+ # Distinct from 1 so a script can tell "not set up yet" from "failed", and
17
+ # from 0 so a wrapped call cannot mistake it for success.
18
+ EXIT_LLM_UNAVAILABLE = 3
19
+
20
+
21
+ @click.group()
22
+ @click.version_option(version=__version__, prog_name="docugraph")
23
+ def main() -> None:
24
+ """DocuGraph AI - Intelligent documentation RAG for coding agents."""
25
+ pass
26
+
27
+
28
+ @main.command()
29
+ @click.argument("url")
30
+ @click.option(
31
+ "--max-pages",
32
+ "-n",
33
+ default=1,
34
+ help="Maximum number of pages to crawl (default: 1 for single page)",
35
+ )
36
+ @click.option(
37
+ "--pattern",
38
+ "-p",
39
+ default=None,
40
+ help="URL pattern to filter links (regex)",
41
+ )
42
+ @click.option(
43
+ "--no-cache",
44
+ is_flag=True,
45
+ help="Disable caching",
46
+ )
47
+ def crawl(url: str, max_pages: int, pattern: str | None, no_cache: bool) -> None:
48
+ """Crawl a URL and index its content.
49
+
50
+ Examples:
51
+ docugraph crawl https://fastapi.tiangolo.com/tutorial/
52
+ docugraph crawl https://docs.python.org --max-pages 50
53
+
54
+ This project uses Crawl4AI (https://github.com/unclecode/crawl4ai) for web
55
+ data extraction.
56
+ """
57
+ from docugraph.ingestion.chunker import Chunker
58
+ from docugraph.ingestion.crawler import DocCrawler
59
+ from docugraph.storage.vector_store import VectorStore
60
+
61
+ async def _crawl() -> None:
62
+ crawler = DocCrawler()
63
+ chunker = Chunker()
64
+ vector_store = VectorStore()
65
+
66
+ with Progress(
67
+ SpinnerColumn(),
68
+ TextColumn("[progress.description]{task.description}"),
69
+ console=console,
70
+ ) as progress:
71
+ # Crawl
72
+ if max_pages == 1:
73
+ task = progress.add_task(f"Crawling {url}...", total=None)
74
+ documents = [await crawler.crawl_single(url, use_cache=not no_cache)]
75
+ else:
76
+ task = progress.add_task(
77
+ f"Crawling up to {max_pages} pages from {url}...", total=None
78
+ )
79
+ documents = await crawler.crawl_site(
80
+ url,
81
+ max_pages=max_pages,
82
+ url_pattern=pattern,
83
+ use_cache=not no_cache,
84
+ )
85
+
86
+ progress.update(task, description=f"Crawled {len(documents)} pages")
87
+
88
+ # Chunk
89
+ progress.update(task, description="Chunking documents...")
90
+ all_chunks = []
91
+ for doc in documents:
92
+ chunks = chunker.chunk_document(doc)
93
+ all_chunks.extend(chunks)
94
+
95
+ progress.update(
96
+ task, description=f"Created {len(all_chunks)} chunks from {len(documents)} pages"
97
+ )
98
+
99
+ # Index
100
+ progress.update(task, description="Indexing chunks (generating embeddings)...")
101
+ count = vector_store.add_chunks(all_chunks)
102
+
103
+ progress.update(task, description="Done!")
104
+
105
+ console.print(
106
+ f"\n[green]Successfully indexed {count} chunks from {len(documents)} pages[/green]"
107
+ )
108
+ console.print(f"[dim]Total chunks in store: {vector_store.count()}[/dim]")
109
+
110
+ asyncio.run(_crawl())
111
+
112
+
113
+ @main.command()
114
+ @click.argument("query")
115
+ @click.option(
116
+ "--top-k",
117
+ "-k",
118
+ default=5,
119
+ help="Number of results to return (default: 5)",
120
+ )
121
+ @click.option(
122
+ "--show-content",
123
+ "-c",
124
+ is_flag=True,
125
+ help="Show full chunk content",
126
+ )
127
+ @click.option(
128
+ "--rerank",
129
+ is_flag=True,
130
+ help=(
131
+ "Rerank results with the local cross-encoder "
132
+ "(BAAI/bge-reranker-v2-m3, multilingual; downloads ~1.8 GB on first use)"
133
+ ),
134
+ )
135
+ def search(query: str, top_k: int, show_content: bool, rerank: bool) -> None:
136
+ """Search indexed documentation.
137
+
138
+ Examples:
139
+ docugraph search "how to create a FastAPI endpoint"
140
+ docugraph search "async error handling" --top-k 10
141
+ docugraph search "path operations" --rerank
142
+ """
143
+ from docugraph.storage.vector_store import VectorStore
144
+
145
+ vector_store = VectorStore()
146
+
147
+ with Progress(
148
+ SpinnerColumn(),
149
+ TextColumn("[progress.description]{task.description}"),
150
+ console=console,
151
+ ) as progress:
152
+ task = progress.add_task("Searching...", total=None)
153
+ results = vector_store.search(query, top_k=top_k)
154
+ if rerank and results:
155
+ progress.update(task, description="Reranking...")
156
+ from docugraph.core.models import SearchResult
157
+ from docugraph.retrieval.reranker import FastembedReranker
158
+
159
+ reranker = FastembedReranker()
160
+ reranked = reranker.rerank(query=query, results=results, top_k=top_k)
161
+ # Surface the blended rerank score (not the stale retrieval score),
162
+ # so displayed scores match the reranked order.
163
+ results = [SearchResult(chunk=rr.chunk, score=rr.final_score) for rr in reranked]
164
+ progress.update(task, description="Done!")
165
+
166
+ if not results:
167
+ console.print("[yellow]No results found.[/yellow]")
168
+ return
169
+
170
+ console.print(f"\n[bold]Found {len(results)} results for:[/bold] {query}\n")
171
+
172
+ for i, result in enumerate(results, 1):
173
+ chunk = result.chunk
174
+ score = result.score
175
+
176
+ # Get metadata
177
+ source = chunk.metadata.get("source_url") or chunk.metadata.get("source_path", "Unknown")
178
+ title = chunk.metadata.get("title", "Untitled")
179
+ section = chunk.metadata.get("section_title", "")
180
+
181
+ # Create display
182
+ console.print(f"[bold cyan]{i}. {title}[/bold cyan]")
183
+ if section:
184
+ console.print(f" [dim]Section: {section}[/dim]")
185
+ console.print(f" [dim]Source: {source}[/dim]")
186
+ console.print(f" [dim]Score: {score:.4f}[/dim]")
187
+
188
+ if show_content:
189
+ # Truncate content for display
190
+ content = chunk.content
191
+ if len(content) > 500:
192
+ content = content[:500] + "..."
193
+ console.print(f"\n [italic]{content}[/italic]")
194
+
195
+ console.print()
196
+
197
+
198
+ @main.command()
199
+ def stats() -> None:
200
+ """Show statistics about indexed content."""
201
+ from docugraph.core.config import get_config
202
+ from docugraph.storage.vector_store import VectorStore
203
+
204
+ config = get_config()
205
+ vector_store = VectorStore()
206
+
207
+ table = Table(title="DocuGraph Statistics")
208
+ table.add_column("Metric", style="cyan")
209
+ table.add_column("Value", style="green")
210
+
211
+ table.add_row("Total Chunks", str(vector_store.count()))
212
+ table.add_row("Data Directory", str(config.storage.data_dir))
213
+ table.add_row("Embedding Model", vector_store.embedding_model)
214
+ table.add_row("Embedding Provider", config.embeddings.provider.value)
215
+
216
+ console.print(table)
217
+
218
+
219
+ @main.command()
220
+ @click.confirmation_option(prompt="Are you sure you want to clear all indexed data?")
221
+ def clear() -> None:
222
+ """Clear all indexed content."""
223
+ from docugraph.storage.vector_store import VectorStore
224
+
225
+ vector_store = VectorStore()
226
+ vector_store.clear()
227
+
228
+ console.print("[green]All indexed content has been cleared.[/green]")
229
+
230
+
231
+ @main.group()
232
+ def config() -> None:
233
+ """Manage configuration."""
234
+ pass
235
+
236
+
237
+ @config.command("show")
238
+ def config_show() -> None:
239
+ """Show current configuration."""
240
+ from docugraph.core.config import get_config
241
+
242
+ cfg = get_config()
243
+
244
+ console.print("[bold]Current Configuration[/bold]\n")
245
+ console.print(cfg.model_dump_json(indent=2))
246
+
247
+
248
+ @config.command("init")
249
+ @click.option(
250
+ "--path",
251
+ "-p",
252
+ default=None,
253
+ help="Path for config file (default: ~/.docugraph/config.yaml)",
254
+ )
255
+ def config_init(path: str | None) -> None:
256
+ """Initialize a configuration file."""
257
+ from docugraph.core.config import Config
258
+
259
+ config = Config()
260
+ save_path = Path(path) if path else None
261
+ config.save(save_path)
262
+
263
+ actual_path = save_path or Path.home() / ".docugraph" / "config.yaml"
264
+ console.print(f"[green]Configuration saved to {actual_path}[/green]")
265
+
266
+
267
+ @main.command("mcp-server")
268
+ def mcp_server() -> None:
269
+ """Start the MCP server for Claude Code integration."""
270
+ from docugraph.interfaces.mcp_server import run_server
271
+
272
+ console.print("[green]Starting DocuGraph MCP server...[/green]")
273
+ run_server()
274
+
275
+
276
+ # =============================================================================
277
+ # Memory Commands
278
+ # =============================================================================
279
+
280
+
281
+ @main.group()
282
+ def memory() -> None:
283
+ """Manage agent memory (key-value store with repo/branch awareness)."""
284
+ pass
285
+
286
+
287
+ @memory.command("set")
288
+ @click.argument("key")
289
+ @click.argument("value")
290
+ @click.option("--repo", "-r", default="default", help="Repository scope")
291
+ @click.option("--branch", "-b", default="main", help="Branch scope")
292
+ def memory_set(key: str, value: str, repo: str, branch: str) -> None:
293
+ """Store a value in memory.
294
+
295
+ Examples:
296
+ docugraph memory set auth_method "JWT tokens"
297
+ docugraph memory set db_choice "PostgreSQL" --repo myapp --branch feature/db
298
+ """
299
+ import json
300
+
301
+ from docugraph.storage.memory_store import MemoryStore
302
+
303
+ store = MemoryStore()
304
+
305
+ # Try to parse as JSON, otherwise store as string
306
+ try:
307
+ parsed_value = json.loads(value)
308
+ except json.JSONDecodeError:
309
+ parsed_value = value
310
+
311
+ store.set(key, parsed_value, repository=repo, branch=branch)
312
+ console.print(f"[green]Stored:[/green] {key} = {value}")
313
+ console.print(f"[dim]Scope: {repo}/{branch}[/dim]")
314
+
315
+
316
+ @memory.command("get")
317
+ @click.argument("key")
318
+ @click.option("--repo", "-r", default="default", help="Repository scope")
319
+ @click.option("--branch", "-b", default="main", help="Branch scope")
320
+ def memory_get(key: str, repo: str, branch: str) -> None:
321
+ """Retrieve a value from memory.
322
+
323
+ Examples:
324
+ docugraph memory get auth_method
325
+ docugraph memory get db_choice --repo myapp --branch feature/db
326
+ """
327
+ import json
328
+
329
+ from docugraph.storage.memory_store import MemoryStore
330
+
331
+ store = MemoryStore()
332
+ value = store.get(key, repository=repo, branch=branch)
333
+
334
+ if value is None:
335
+ console.print(f"[yellow]No value found for key: {key}[/yellow]")
336
+ console.print(f"[dim]Scope: {repo}/{branch}[/dim]")
337
+ return
338
+
339
+ # Format output
340
+ if isinstance(value, (dict, list)):
341
+ console.print(f"[bold]{key}[/bold] =")
342
+ console.print(json.dumps(value, indent=2))
343
+ else:
344
+ console.print(f"[bold]{key}[/bold] = {value}")
345
+
346
+ console.print(f"[dim]Scope: {repo}/{branch}[/dim]")
347
+
348
+
349
+ @memory.command("list")
350
+ @click.option("--repo", "-r", default="default", help="Repository scope")
351
+ @click.option("--branch", "-b", default="main", help="Branch scope")
352
+ @click.option("--pattern", "-p", default="%", help="Key pattern (SQL LIKE, use % for wildcard)")
353
+ def memory_list(repo: str, branch: str, pattern: str) -> None:
354
+ """List stored memory keys.
355
+
356
+ Examples:
357
+ docugraph memory list
358
+ docugraph memory list --repo myapp
359
+ docugraph memory list --pattern "auth%"
360
+ """
361
+ from docugraph.storage.memory_store import MemoryStore
362
+
363
+ store = MemoryStore()
364
+
365
+ if pattern == "%":
366
+ keys = store.list_keys(repository=repo, branch=branch)
367
+ else:
368
+ entries = store.search(pattern, repository=repo, branch=branch)
369
+ keys = [e.key for e in entries]
370
+
371
+ if not keys:
372
+ console.print("[yellow]No memories found.[/yellow]")
373
+ console.print(f"[dim]Scope: {repo}/{branch}[/dim]")
374
+ return
375
+
376
+ table = Table(title=f"Memories ({repo}/{branch})")
377
+ table.add_column("Key", style="cyan")
378
+
379
+ for key in keys:
380
+ table.add_row(key)
381
+
382
+ console.print(table)
383
+ console.print(f"[dim]Total: {len(keys)} entries[/dim]")
384
+
385
+
386
+ @memory.command("delete")
387
+ @click.argument("key")
388
+ @click.option("--repo", "-r", default="default", help="Repository scope")
389
+ @click.option("--branch", "-b", default="main", help="Branch scope")
390
+ def memory_delete(key: str, repo: str, branch: str) -> None:
391
+ """Delete a memory entry.
392
+
393
+ Examples:
394
+ docugraph memory delete auth_method
395
+ docugraph memory delete db_choice --repo myapp
396
+ """
397
+ from docugraph.storage.memory_store import MemoryStore
398
+
399
+ store = MemoryStore()
400
+ deleted = store.delete(key, repository=repo, branch=branch)
401
+
402
+ if deleted:
403
+ console.print(f"[green]Deleted: {key}[/green]")
404
+ else:
405
+ console.print(f"[yellow]Key not found: {key}[/yellow]")
406
+
407
+ console.print(f"[dim]Scope: {repo}/{branch}[/dim]")
408
+
409
+
410
+ @memory.command("clear")
411
+ @click.option("--repo", "-r", default=None, help="Repository scope (omit for all)")
412
+ @click.option("--branch", "-b", default=None, help="Branch scope")
413
+ @click.confirmation_option(prompt="Are you sure you want to clear memories?")
414
+ def memory_clear(repo: str | None, branch: str | None) -> None:
415
+ """Clear memory entries.
416
+
417
+ Examples:
418
+ docugraph memory clear # Clear all
419
+ docugraph memory clear --repo myapp # Clear repo
420
+ docugraph memory clear --repo myapp --branch main # Clear branch
421
+ """
422
+ from docugraph.storage.memory_store import MemoryStore
423
+
424
+ store = MemoryStore()
425
+ count = store.clear(repository=repo, branch=branch)
426
+
427
+ scope = "all repositories"
428
+ if repo and branch:
429
+ scope = f"{repo}/{branch}"
430
+ elif repo:
431
+ scope = f"repository {repo}"
432
+
433
+ console.print(f"[green]Cleared {count} entries from {scope}[/green]")
434
+
435
+
436
+ @memory.command("show")
437
+ @click.option("--repo", "-r", default="default", help="Repository scope")
438
+ @click.option("--branch", "-b", default="main", help="Branch scope")
439
+ def memory_show(repo: str, branch: str) -> None:
440
+ """Show all memories as key-value pairs.
441
+
442
+ Examples:
443
+ docugraph memory show
444
+ docugraph memory show --repo myapp --branch feature/auth
445
+ """
446
+ import json
447
+
448
+ from docugraph.storage.memory_store import MemoryStore
449
+
450
+ store = MemoryStore()
451
+ memories = store.get_all(repository=repo, branch=branch)
452
+
453
+ if not memories:
454
+ console.print("[yellow]No memories found.[/yellow]")
455
+ console.print(f"[dim]Scope: {repo}/{branch}[/dim]")
456
+ return
457
+
458
+ console.print(f"[bold]Memories ({repo}/{branch})[/bold]\n")
459
+ console.print(json.dumps(memories, indent=2))
460
+
461
+
462
+ @memory.command("repos")
463
+ def memory_repos() -> None:
464
+ """List all repositories with stored memories."""
465
+ from docugraph.storage.memory_store import MemoryStore
466
+
467
+ store = MemoryStore()
468
+ repos = store.list_repositories()
469
+
470
+ if not repos:
471
+ console.print("[yellow]No repositories found.[/yellow]")
472
+ return
473
+
474
+ table = Table(title="Repositories with Memories")
475
+ table.add_column("Repository", style="cyan")
476
+ table.add_column("Branches", style="green")
477
+
478
+ for repo in repos:
479
+ branches = store.list_branches(repo)
480
+ table.add_row(repo, ", ".join(branches))
481
+
482
+ console.print(table)
483
+
484
+
485
+ # =============================================================================
486
+ # Graph Commands
487
+ # =============================================================================
488
+
489
+
490
+ @main.group()
491
+ def graph() -> None:
492
+ """Manage knowledge graph (entities and relationships)."""
493
+ pass
494
+
495
+
496
+ @graph.command("add")
497
+ @click.argument("content")
498
+ @click.option("--name", "-n", default=None, help="Episode name")
499
+ @click.option("--group", "-g", default="default", help="Group ID for scoping searches")
500
+ @click.option("--type", "-t", "source_type", default="text", help="Content type (text/json)")
501
+ def graph_add(content: str, name: str | None, group: str, source_type: str) -> None:
502
+ """Add content to extract entities and relationships.
503
+
504
+ Graphiti automatically extracts entities and relationships from text, which
505
+ needs an LLM: Ollama locally, or an OPENAI_API_KEY / ANTHROPIC_API_KEY.
506
+ Exits 3 if none can be reached.
507
+
508
+ Examples:
509
+ docugraph graph add "FastAPI is a Python web framework. It uses Starlette for routing."
510
+ docugraph graph add '{"name": "Alice", "role": "Developer"}' --type json
511
+ """
512
+ from docugraph.storage.graph_store import GraphStore
513
+
514
+ async def _add() -> bool:
515
+ store = GraphStore()
516
+ try:
517
+ episode_name = name or f"episode_{hash(content) % 10000}"
518
+ result = await store.add_episode(
519
+ name=episode_name,
520
+ content=content,
521
+ source_type=source_type,
522
+ group_id=group,
523
+ )
524
+
525
+ console.print(f"[green]Added episode: {result['episode_uuid']}[/green]")
526
+ console.print(f"[dim]Extracted {result['entities_count']} entities[/dim]")
527
+ console.print(f"[dim]Extracted {result['relationships_count']} relationships[/dim]")
528
+
529
+ if result["entities"]:
530
+ console.print("\n[bold]Entities:[/bold]")
531
+ for e in result["entities"][:5]:
532
+ console.print(f" - {e['name']}")
533
+ if len(result["entities"]) > 5:
534
+ console.print(f" ... and {len(result['entities']) - 5} more")
535
+ return True
536
+
537
+ except RuntimeError as e:
538
+ # LLMUnavailableError carries its own full explanation; anything
539
+ # else is reported as-is. Either way: no traceback, and a non-zero
540
+ # exit so the caller does not read a failure as success.
541
+ console.print(f"[red]{e}[/red]")
542
+ return False
543
+ finally:
544
+ await store.close()
545
+
546
+ if not asyncio.run(_add()):
547
+ raise SystemExit(EXIT_LLM_UNAVAILABLE)
548
+
549
+
550
+ @graph.command("search")
551
+ @click.argument("query")
552
+ @click.option("--limit", "-l", default=10, help="Maximum results")
553
+ @click.option("--group", "-g", default=None, help="Group ID to filter by")
554
+ def graph_search(query: str, limit: int, group: str | None) -> None:
555
+ """Search the knowledge graph.
556
+
557
+ Needs an LLM: Ollama locally, or an OPENAI_API_KEY / ANTHROPIC_API_KEY.
558
+ Exits 3 if none can be reached.
559
+
560
+ Examples:
561
+ docugraph graph search "Python web frameworks"
562
+ docugraph graph search "authentication" --group security
563
+ """
564
+ from docugraph.storage.graph_store import GraphStore
565
+
566
+ async def _search() -> bool:
567
+ store = GraphStore()
568
+ try:
569
+ group_ids = [group] if group else None
570
+ results = await store.search(query, group_ids=group_ids, num_results=limit)
571
+
572
+ if not results:
573
+ console.print("[yellow]No results found.[/yellow]")
574
+ return True
575
+
576
+ console.print(f"\n[bold]Found {len(results)} facts for:[/bold] {query}\n")
577
+
578
+ for i, r in enumerate(results, 1):
579
+ console.print(f"[cyan]{i}.[/cyan] {r['fact']}")
580
+ if r.get("valid_at"):
581
+ console.print(f" [dim]Valid: {r['valid_at']}[/dim]")
582
+ console.print()
583
+ return True
584
+
585
+ except RuntimeError as e:
586
+ console.print(f"[red]{e}[/red]")
587
+ return False
588
+ finally:
589
+ await store.close()
590
+
591
+ if not asyncio.run(_search()):
592
+ raise SystemExit(EXIT_LLM_UNAVAILABLE)
593
+
594
+
595
+ @graph.command("clear")
596
+ @click.confirmation_option(prompt="Are you sure you want to clear the knowledge graph?")
597
+ def graph_clear() -> None:
598
+ """Clear all data from the knowledge graph."""
599
+ from docugraph.storage.graph_store import GraphStore
600
+
601
+ async def _clear() -> None:
602
+ store = GraphStore()
603
+ await store.clear()
604
+ console.print("[green]Knowledge graph has been cleared.[/green]")
605
+
606
+ asyncio.run(_clear())
607
+
608
+
609
+ # =============================================================================
610
+ # Index Commands (Local Files and Git Repositories)
611
+ # =============================================================================
612
+
613
+
614
+ @main.group()
615
+ def index() -> None:
616
+ """Index local files and git repositories."""
617
+ pass
618
+
619
+
620
+ @index.command("local")
621
+ @click.argument("path", type=click.Path(exists=True, file_okay=False, dir_okay=True))
622
+ @click.option(
623
+ "--recursive/--no-recursive",
624
+ "-r/-R",
625
+ default=True,
626
+ help="Recursively index subdirectories (default: yes)",
627
+ )
628
+ @click.option(
629
+ "--pattern",
630
+ "-p",
631
+ multiple=True,
632
+ help="Include patterns (e.g., '*.md', '*.rst'). Can be specified multiple times.",
633
+ )
634
+ @click.option(
635
+ "--exclude",
636
+ "-e",
637
+ multiple=True,
638
+ help="Exclude patterns. Can be specified multiple times.",
639
+ )
640
+ @click.option(
641
+ "--include-hidden",
642
+ is_flag=True,
643
+ help="Include hidden files (starting with .)",
644
+ )
645
+ @click.option(
646
+ "--stats-only",
647
+ is_flag=True,
648
+ help="Only show statistics, don't index",
649
+ )
650
+ def index_local(
651
+ path: str,
652
+ recursive: bool,
653
+ pattern: tuple[str, ...],
654
+ exclude: tuple[str, ...],
655
+ include_hidden: bool,
656
+ stats_only: bool,
657
+ ) -> None:
658
+ """Index local documentation files (Markdown, RST, text).
659
+
660
+ Examples:
661
+ docugraph index local ./docs
662
+ docugraph index local ./my-project --pattern "*.md" --pattern "README*"
663
+ docugraph index local ./docs --exclude "*.draft.md"
664
+ docugraph index local ./project --stats-only
665
+ """
666
+ from docugraph.ingestion.chunker import Chunker
667
+ from docugraph.ingestion.local_files import LocalFileConfig, LocalFileIndexer
668
+ from docugraph.storage.vector_store import VectorStore
669
+
670
+ # Build config
671
+ config = LocalFileConfig(include_hidden=include_hidden)
672
+
673
+ if pattern:
674
+ config.include_patterns = list(pattern)
675
+
676
+ if exclude:
677
+ config.exclude_patterns = list(config.exclude_patterns) + list(exclude)
678
+
679
+ indexer = LocalFileIndexer(config=config)
680
+
681
+ # Stats only mode
682
+ if stats_only:
683
+ stats = indexer.get_stats(path, recursive=recursive)
684
+
685
+ table = Table(title=f"Index Statistics: {path}")
686
+ table.add_column("Metric", style="cyan")
687
+ table.add_column("Value", style="green")
688
+
689
+ table.add_row("Total Indexable Files", str(stats["total_files"]))
690
+ table.add_row("Total Size", f"{stats['total_size'] / 1024:.1f} KB")
691
+ table.add_row("Skipped Files", str(stats["skipped"]))
692
+
693
+ for file_type, count in stats.get("by_type", {}).items():
694
+ table.add_row(f" {file_type}", str(count))
695
+
696
+ console.print(table)
697
+ return
698
+
699
+ # Index and store
700
+ chunker = Chunker()
701
+ vector_store = VectorStore()
702
+
703
+ with Progress(
704
+ SpinnerColumn(),
705
+ TextColumn("[progress.description]{task.description}"),
706
+ console=console,
707
+ ) as progress:
708
+ task = progress.add_task(f"Indexing {path}...", total=None)
709
+
710
+ # Index files
711
+ documents = indexer.index_to_documents(path, recursive=recursive)
712
+ progress.update(task, description=f"Found {len(documents)} files")
713
+
714
+ # Chunk documents
715
+ progress.update(task, description="Chunking documents...")
716
+ all_chunks = []
717
+ for doc in documents:
718
+ chunks = chunker.chunk_document(doc)
719
+ all_chunks.extend(chunks)
720
+
721
+ progress.update(task, description=f"Created {len(all_chunks)} chunks")
722
+
723
+ # Index chunks
724
+ progress.update(task, description="Indexing chunks (generating embeddings)...")
725
+ count = vector_store.add_chunks(all_chunks)
726
+
727
+ progress.update(task, description="Done!")
728
+
729
+ console.print(
730
+ f"\n[green]Successfully indexed {count} chunks from {len(documents)} files[/green]"
731
+ )
732
+ console.print(f"[dim]Total chunks in store: {vector_store.count()}[/dim]")
733
+
734
+
735
+ @index.command("git")
736
+ @click.argument("url")
737
+ @click.option(
738
+ "--name",
739
+ "-n",
740
+ default=None,
741
+ help="Custom name for the repository",
742
+ )
743
+ @click.option(
744
+ "--ref",
745
+ "-r",
746
+ default=None,
747
+ help="Branch, tag, or commit to checkout",
748
+ )
749
+ @click.option(
750
+ "--shallow",
751
+ is_flag=True,
752
+ help="Shallow clone (depth=1) for faster cloning",
753
+ )
754
+ @click.option(
755
+ "--include-code",
756
+ is_flag=True,
757
+ help="Include code files (not just documentation)",
758
+ )
759
+ @click.option(
760
+ "--cleanup",
761
+ is_flag=True,
762
+ help="Remove cloned repository after indexing",
763
+ )
764
+ @click.option(
765
+ "--stats-only",
766
+ is_flag=True,
767
+ help="Only show statistics, don't index",
768
+ )
769
+ def index_git(
770
+ url: str,
771
+ name: str | None,
772
+ ref: str | None,
773
+ shallow: bool,
774
+ include_code: bool,
775
+ cleanup: bool,
776
+ stats_only: bool,
777
+ ) -> None:
778
+ """Clone and index a git repository.
779
+
780
+ Examples:
781
+ docugraph index git https://github.com/fastapi/fastapi
782
+ docugraph index git https://github.com/user/repo --ref main --shallow
783
+ docugraph index git git@github.com:user/repo.git --include-code
784
+ docugraph index git https://github.com/user/repo --stats-only
785
+ """
786
+ from docugraph.ingestion.chunker import Chunker
787
+ from docugraph.ingestion.git_indexer import GitIndexer, GitIndexerConfig
788
+ from docugraph.storage.vector_store import VectorStore
789
+
790
+ config = GitIndexerConfig(
791
+ clone_depth=1 if shallow else None,
792
+ ref=ref,
793
+ include_code=include_code,
794
+ cleanup_after_index=cleanup,
795
+ )
796
+
797
+ indexer = GitIndexer(config=config)
798
+
799
+ with Progress(
800
+ SpinnerColumn(),
801
+ TextColumn("[progress.description]{task.description}"),
802
+ console=console,
803
+ ) as progress:
804
+ task = progress.add_task(f"Cloning {url}...", total=None)
805
+
806
+ try:
807
+ # Clone repository
808
+ repo_info = indexer.clone(url, name=name, ref=ref)
809
+ progress.update(task, description=f"Cloned to {repo_info.local_path.name}")
810
+
811
+ # Stats only mode
812
+ if stats_only:
813
+ progress.update(task, description="Analyzing repository...")
814
+ stats = indexer.get_repo_stats(repo_info.local_path)
815
+ progress.update(task, description="Done!")
816
+
817
+ table = Table(title=f"Repository: {repo_info.name}")
818
+ table.add_column("Metric", style="cyan")
819
+ table.add_column("Value", style="green")
820
+
821
+ table.add_row("URL", repo_info.url)
822
+ table.add_row("Branch", repo_info.branch or "N/A")
823
+ table.add_row("Commit", (repo_info.commit_hash or "N/A")[:12])
824
+ table.add_row("Indexable Files", str(stats["total_files"]))
825
+ table.add_row("Total Size", f"{stats['total_size'] / 1024:.1f} KB")
826
+
827
+ for file_type, count in stats.get("by_type", {}).items():
828
+ table.add_row(f" {file_type}", str(count))
829
+
830
+ console.print(table)
831
+
832
+ if cleanup:
833
+ indexer.cleanup(repo_info.local_path)
834
+ console.print("[dim]Repository cleaned up.[/dim]")
835
+ return
836
+
837
+ # Index repository
838
+ progress.update(task, description="Indexing files...")
839
+ documents = []
840
+ for indexed_file in indexer.index_repo(repo_info.local_path, repo_info):
841
+ documents.append(indexed_file.to_document())
842
+
843
+ progress.update(task, description=f"Found {len(documents)} files")
844
+
845
+ # Chunk documents
846
+ progress.update(task, description="Chunking documents...")
847
+ chunker = Chunker()
848
+ all_chunks = []
849
+ for doc in documents:
850
+ chunks = chunker.chunk_document(doc)
851
+ all_chunks.extend(chunks)
852
+
853
+ progress.update(task, description=f"Created {len(all_chunks)} chunks")
854
+
855
+ # Index chunks
856
+ progress.update(task, description="Indexing chunks (generating embeddings)...")
857
+ vector_store = VectorStore()
858
+ count = vector_store.add_chunks(all_chunks)
859
+
860
+ progress.update(task, description="Done!")
861
+
862
+ # Cleanup if requested
863
+ if cleanup:
864
+ indexer.cleanup(repo_info.local_path)
865
+
866
+ except RuntimeError as e:
867
+ console.print(f"[red]Error: {e}[/red]")
868
+ return
869
+
870
+ console.print(
871
+ f"\n[green]Successfully indexed {count} chunks from {len(documents)} files[/green]"
872
+ )
873
+ console.print(f"[dim]Repository: {repo_info.name} @ {repo_info.branch or 'HEAD'}[/dim]")
874
+ console.print(f"[dim]Commit: {(repo_info.commit_hash or 'N/A')[:12]}[/dim]")
875
+ console.print(f"[dim]Total chunks in store: {vector_store.count()}[/dim]")
876
+
877
+
878
+ @index.command("list")
879
+ def index_list() -> None:
880
+ """List all cloned repositories."""
881
+ from docugraph.ingestion.git_indexer import GitIndexer
882
+
883
+ indexer = GitIndexer()
884
+ repos = indexer.list_repos()
885
+
886
+ if not repos:
887
+ console.print("[yellow]No cloned repositories found.[/yellow]")
888
+ return
889
+
890
+ table = Table(title="Cloned Repositories")
891
+ table.add_column("Name", style="cyan")
892
+ table.add_column("Branch", style="green")
893
+ table.add_column("Commit", style="dim")
894
+ table.add_column("URL")
895
+
896
+ for repo in repos:
897
+ table.add_row(
898
+ repo.name,
899
+ repo.branch or "N/A",
900
+ (repo.commit_hash or "N/A")[:12],
901
+ repo.url[:50] + "..." if len(repo.url) > 50 else repo.url,
902
+ )
903
+
904
+ console.print(table)
905
+
906
+
907
+ @index.command("update")
908
+ @click.argument("name")
909
+ def index_update(name: str) -> None:
910
+ """Update a cloned repository (git pull).
911
+
912
+ Examples:
913
+ docugraph index update fastapi
914
+ """
915
+ from docugraph.ingestion.git_indexer import GitIndexer
916
+
917
+ indexer = GitIndexer()
918
+ repos = indexer.list_repos()
919
+
920
+ # Find repo by name
921
+ repo_path = None
922
+ for repo in repos:
923
+ if repo.name == name:
924
+ repo_path = repo.local_path
925
+ break
926
+
927
+ if repo_path is None:
928
+ console.print(f"[red]Repository not found: {name}[/red]")
929
+ console.print("[dim]Use 'docugraph index list' to see cloned repositories.[/dim]")
930
+ return
931
+
932
+ with Progress(
933
+ SpinnerColumn(),
934
+ TextColumn("[progress.description]{task.description}"),
935
+ console=console,
936
+ ) as progress:
937
+ task = progress.add_task(f"Updating {name}...", total=None)
938
+
939
+ try:
940
+ repo_info = indexer.update(repo_path)
941
+ progress.update(task, description="Done!")
942
+
943
+ console.print(f"[green]Updated repository: {name}[/green]")
944
+ console.print(f"[dim]Branch: {repo_info.branch}[/dim]")
945
+ console.print(f"[dim]Commit: {(repo_info.commit_hash or 'N/A')[:12]}[/dim]")
946
+
947
+ except RuntimeError as e:
948
+ console.print(f"[red]Error: {e}[/red]")
949
+
950
+
951
+ @index.command("remove")
952
+ @click.argument("name")
953
+ @click.confirmation_option(prompt="Are you sure you want to remove this repository?")
954
+ def index_remove(name: str) -> None:
955
+ """Remove a cloned repository.
956
+
957
+ Examples:
958
+ docugraph index remove fastapi
959
+ """
960
+ from docugraph.ingestion.git_indexer import GitIndexer
961
+
962
+ indexer = GitIndexer()
963
+ repos = indexer.list_repos()
964
+
965
+ # Find repo by name
966
+ repo_path = None
967
+ for repo in repos:
968
+ if repo.name == name:
969
+ repo_path = repo.local_path
970
+ break
971
+
972
+ if repo_path is None:
973
+ console.print(f"[red]Repository not found: {name}[/red]")
974
+ return
975
+
976
+ if indexer.cleanup(repo_path):
977
+ console.print(f"[green]Removed repository: {name}[/green]")
978
+ else:
979
+ console.print(f"[red]Failed to remove repository: {name}[/red]")
980
+
981
+
982
+ if __name__ == "__main__":
983
+ main()