haiku.rag-slim 0.16.0__py3-none-any.whl → 0.24.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of haiku.rag-slim might be problematic. Click here for more details.

Files changed (94) hide show
  1. haiku/rag/app.py +430 -72
  2. haiku/rag/chunkers/__init__.py +31 -0
  3. haiku/rag/chunkers/base.py +31 -0
  4. haiku/rag/chunkers/docling_local.py +164 -0
  5. haiku/rag/chunkers/docling_serve.py +179 -0
  6. haiku/rag/cli.py +207 -24
  7. haiku/rag/cli_chat.py +489 -0
  8. haiku/rag/client.py +1251 -266
  9. haiku/rag/config/__init__.py +16 -10
  10. haiku/rag/config/loader.py +5 -44
  11. haiku/rag/config/models.py +126 -17
  12. haiku/rag/converters/__init__.py +31 -0
  13. haiku/rag/converters/base.py +63 -0
  14. haiku/rag/converters/docling_local.py +193 -0
  15. haiku/rag/converters/docling_serve.py +229 -0
  16. haiku/rag/converters/text_utils.py +237 -0
  17. haiku/rag/embeddings/__init__.py +123 -24
  18. haiku/rag/embeddings/voyageai.py +175 -20
  19. haiku/rag/graph/__init__.py +0 -11
  20. haiku/rag/graph/agui/__init__.py +8 -2
  21. haiku/rag/graph/agui/cli_renderer.py +1 -1
  22. haiku/rag/graph/agui/emitter.py +219 -31
  23. haiku/rag/graph/agui/server.py +20 -62
  24. haiku/rag/graph/agui/stream.py +1 -2
  25. haiku/rag/graph/research/__init__.py +5 -2
  26. haiku/rag/graph/research/dependencies.py +12 -126
  27. haiku/rag/graph/research/graph.py +390 -135
  28. haiku/rag/graph/research/models.py +91 -112
  29. haiku/rag/graph/research/prompts.py +99 -91
  30. haiku/rag/graph/research/state.py +35 -27
  31. haiku/rag/inspector/__init__.py +8 -0
  32. haiku/rag/inspector/app.py +259 -0
  33. haiku/rag/inspector/widgets/__init__.py +6 -0
  34. haiku/rag/inspector/widgets/chunk_list.py +100 -0
  35. haiku/rag/inspector/widgets/context_modal.py +89 -0
  36. haiku/rag/inspector/widgets/detail_view.py +130 -0
  37. haiku/rag/inspector/widgets/document_list.py +75 -0
  38. haiku/rag/inspector/widgets/info_modal.py +209 -0
  39. haiku/rag/inspector/widgets/search_modal.py +183 -0
  40. haiku/rag/inspector/widgets/visual_modal.py +126 -0
  41. haiku/rag/mcp.py +106 -102
  42. haiku/rag/monitor.py +33 -9
  43. haiku/rag/providers/__init__.py +5 -0
  44. haiku/rag/providers/docling_serve.py +108 -0
  45. haiku/rag/qa/__init__.py +12 -10
  46. haiku/rag/qa/agent.py +43 -61
  47. haiku/rag/qa/prompts.py +35 -57
  48. haiku/rag/reranking/__init__.py +9 -6
  49. haiku/rag/reranking/base.py +1 -1
  50. haiku/rag/reranking/cohere.py +5 -4
  51. haiku/rag/reranking/mxbai.py +5 -2
  52. haiku/rag/reranking/vllm.py +3 -4
  53. haiku/rag/reranking/zeroentropy.py +6 -5
  54. haiku/rag/store/__init__.py +2 -1
  55. haiku/rag/store/engine.py +242 -42
  56. haiku/rag/store/exceptions.py +4 -0
  57. haiku/rag/store/models/__init__.py +8 -2
  58. haiku/rag/store/models/chunk.py +190 -0
  59. haiku/rag/store/models/document.py +46 -0
  60. haiku/rag/store/repositories/chunk.py +141 -121
  61. haiku/rag/store/repositories/document.py +25 -84
  62. haiku/rag/store/repositories/settings.py +11 -14
  63. haiku/rag/store/upgrades/__init__.py +19 -3
  64. haiku/rag/store/upgrades/v0_10_1.py +1 -1
  65. haiku/rag/store/upgrades/v0_19_6.py +65 -0
  66. haiku/rag/store/upgrades/v0_20_0.py +68 -0
  67. haiku/rag/store/upgrades/v0_23_1.py +100 -0
  68. haiku/rag/store/upgrades/v0_9_3.py +3 -3
  69. haiku/rag/utils.py +371 -146
  70. {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/METADATA +15 -12
  71. haiku_rag_slim-0.24.0.dist-info/RECORD +78 -0
  72. {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/WHEEL +1 -1
  73. haiku/rag/chunker.py +0 -65
  74. haiku/rag/embeddings/base.py +0 -25
  75. haiku/rag/embeddings/ollama.py +0 -28
  76. haiku/rag/embeddings/openai.py +0 -26
  77. haiku/rag/embeddings/vllm.py +0 -29
  78. haiku/rag/graph/agui/events.py +0 -254
  79. haiku/rag/graph/common/__init__.py +0 -5
  80. haiku/rag/graph/common/models.py +0 -42
  81. haiku/rag/graph/common/nodes.py +0 -265
  82. haiku/rag/graph/common/prompts.py +0 -46
  83. haiku/rag/graph/common/utils.py +0 -44
  84. haiku/rag/graph/deep_qa/__init__.py +0 -1
  85. haiku/rag/graph/deep_qa/dependencies.py +0 -27
  86. haiku/rag/graph/deep_qa/graph.py +0 -243
  87. haiku/rag/graph/deep_qa/models.py +0 -20
  88. haiku/rag/graph/deep_qa/prompts.py +0 -59
  89. haiku/rag/graph/deep_qa/state.py +0 -56
  90. haiku/rag/graph/research/common.py +0 -87
  91. haiku/rag/reader.py +0 -135
  92. haiku_rag_slim-0.16.0.dist-info/RECORD +0 -71
  93. {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/entry_points.txt +0 -0
  94. {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/licenses/LICENSE +0 -0
haiku/rag/cli.py CHANGED
@@ -1,6 +1,7 @@
1
1
  import asyncio
2
2
  import json
3
3
  import warnings
4
+ from datetime import datetime
4
5
  from importlib.metadata import version
5
6
  from pathlib import Path
6
7
  from typing import Any
@@ -26,6 +27,10 @@ cli = typer.Typer(
26
27
  context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
27
28
  )
28
29
 
30
+ # Module-level flags set by callback
31
+ _read_only: bool = False
32
+ _before: datetime | None = None
33
+
29
34
 
30
35
  def create_app(db: Path | None = None) -> HaikuRAGApp:
31
36
  """Create HaikuRAGApp with loaded config and resolved database path.
@@ -38,7 +43,9 @@ def create_app(db: Path | None = None) -> HaikuRAGApp:
38
43
  """
39
44
  config = get_config()
40
45
  db_path = db if db else config.storage.data_dir / "haiku.rag.lancedb"
41
- return HaikuRAGApp(db_path=db_path, config=config)
46
+ return HaikuRAGApp(
47
+ db_path=db_path, config=config, read_only=_read_only, before=_before
48
+ )
42
49
 
43
50
 
44
51
  async def check_version():
@@ -72,8 +79,33 @@ def main(
72
79
  "--config",
73
80
  help="Path to YAML configuration file",
74
81
  ),
82
+ read_only: bool = typer.Option(
83
+ False,
84
+ "--read-only",
85
+ help="Open database in read-only mode",
86
+ ),
87
+ before: str | None = typer.Option(
88
+ None,
89
+ "--before",
90
+ help="Query database as it existed before this datetime (implies --read-only). "
91
+ "Accepts ISO 8601 format (e.g., 2025-01-15T14:30:00) or date (e.g., 2025-01-15)",
92
+ ),
75
93
  ):
76
94
  """haiku.rag CLI - Vector database RAG system"""
95
+ global _read_only, _before
96
+ _read_only = read_only
97
+
98
+ # Parse and store before datetime
99
+ if before is not None:
100
+ from haiku.rag.utils import parse_datetime, to_utc
101
+
102
+ try:
103
+ _before = to_utc(parse_datetime(before))
104
+ except ValueError as e:
105
+ typer.echo(f"Error: {e}")
106
+ raise typer.Exit(1)
107
+ else:
108
+ _before = None
77
109
  # Load config from --config, local folder, or default directory
78
110
  config_path = find_config_file(cli_path=config)
79
111
  if config_path:
@@ -81,18 +113,24 @@ def main(
81
113
  loaded_config = AppConfig.model_validate(yaml_data)
82
114
  set_config(loaded_config)
83
115
 
84
- # Configure logging minimally for CLI context
85
- if get_config().environment == "development":
86
- # Lazy import logfire only in development
87
- try:
88
- import logfire # type: ignore
116
+ # Configure logging for CLI context
117
+ configure_cli_logging()
89
118
 
90
- logfire.configure(send_to_logfire="if-token-present")
91
- logfire.instrument_pydantic_ai()
92
- except Exception:
93
- pass
94
- else:
95
- configure_cli_logging()
119
+ # Configure logfire (only sends data if token is present)
120
+ try:
121
+ import logfire
122
+
123
+ is_production = get_config().environment != "development"
124
+ logfire.configure(
125
+ send_to_logfire="if-token-present",
126
+ console=False if is_production else None,
127
+ )
128
+ logfire.instrument_pydantic_ai()
129
+ except Exception:
130
+ pass
131
+
132
+ if get_config().environment != "development":
133
+ # Suppress warnings in production
96
134
  warnings.filterwarnings("ignore")
97
135
 
98
136
  # Run version check before any command
@@ -237,11 +275,11 @@ def search(
237
275
  query: str = typer.Argument(
238
276
  help="The search query to use",
239
277
  ),
240
- limit: int = typer.Option(
241
- 5,
278
+ limit: int | None = typer.Option(
279
+ None,
242
280
  "--limit",
243
281
  "-l",
244
- help="Maximum number of results to return",
282
+ help="Maximum number of results to return (default: config search.default_limit)",
245
283
  ),
246
284
  filter: str | None = typer.Option(
247
285
  None,
@@ -259,6 +297,21 @@ def search(
259
297
  asyncio.run(app.search(query=query, limit=limit, filter=filter))
260
298
 
261
299
 
300
+ @cli.command("visualize", help="Show visual grounding for a chunk")
301
+ def visualize(
302
+ chunk_id: str = typer.Argument(
303
+ help="The ID of the chunk to visualize",
304
+ ),
305
+ db: Path | None = typer.Option(
306
+ None,
307
+ "--db",
308
+ help="Path to the LanceDB database file",
309
+ ),
310
+ ):
311
+ app = create_app(db)
312
+ asyncio.run(app.visualize_chunk(chunk_id=chunk_id))
313
+
314
+
262
315
  @cli.command("ask", help="Ask a question using the QA agent")
263
316
  def ask(
264
317
  question: str = typer.Argument(
@@ -284,15 +337,24 @@ def ask(
284
337
  "--verbose",
285
338
  help="Show verbose progress output (only with --deep)",
286
339
  ),
340
+ filter: str | None = typer.Option(
341
+ None,
342
+ "--filter",
343
+ "-f",
344
+ help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")",
345
+ ),
287
346
  ):
288
347
  app = create_app(db)
289
- asyncio.run(app.ask(question=question, cite=cite, deep=deep, verbose=verbose))
348
+ asyncio.run(
349
+ app.ask(question=question, cite=cite, deep=deep, verbose=verbose, filter=filter)
350
+ )
290
351
 
291
352
 
292
353
  @cli.command("research", help="Run multi-agent research and output a concise report")
293
354
  def research(
294
355
  question: str = typer.Argument(
295
- help="The research question to investigate",
356
+ None,
357
+ help="The research question to investigate (required unless --interactive)",
296
358
  ),
297
359
  db: Path | None = typer.Option(
298
360
  None,
@@ -304,9 +366,42 @@ def research(
304
366
  "--verbose",
305
367
  help="Show planning, searching previews, evaluation summary, and stop reason",
306
368
  ),
369
+ filter: str | None = typer.Option(
370
+ None,
371
+ "--filter",
372
+ "-f",
373
+ help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")",
374
+ ),
375
+ interactive: bool = typer.Option(
376
+ False,
377
+ "--interactive",
378
+ "-i",
379
+ help="Start interactive research mode with human-in-the-loop",
380
+ ),
307
381
  ):
308
382
  app = create_app(db)
309
- asyncio.run(app.research(question=question, verbose=verbose))
383
+
384
+ if interactive:
385
+ from haiku.rag.cli_chat import interactive_research
386
+ from haiku.rag.client import HaikuRAG
387
+
388
+ client = HaikuRAG(
389
+ db_path=app.db_path, config=app.config, read_only=_read_only, before=_before
390
+ )
391
+ try:
392
+ interactive_research(
393
+ client=client,
394
+ config=app.config,
395
+ search_filter=filter,
396
+ question=question,
397
+ )
398
+ finally:
399
+ client.close()
400
+ else:
401
+ if question is None:
402
+ typer.echo("Error: Question is required unless using --interactive mode")
403
+ raise typer.Exit(1)
404
+ asyncio.run(app.research(question=question, verbose=verbose, filter=filter))
310
405
 
311
406
 
312
407
  @cli.command("settings", help="Display current configuration settings")
@@ -358,9 +453,32 @@ def rebuild(
358
453
  "--db",
359
454
  help="Path to the LanceDB database file",
360
455
  ),
456
+ embed_only: bool = typer.Option(
457
+ False,
458
+ "--embed-only",
459
+ help="Only regenerate embeddings, keep existing chunks",
460
+ ),
461
+ rechunk: bool = typer.Option(
462
+ False,
463
+ "--rechunk",
464
+ help="Re-chunk from existing content without accessing source files",
465
+ ),
361
466
  ):
467
+ from haiku.rag.client import RebuildMode
468
+
469
+ if embed_only and rechunk:
470
+ typer.echo("Error: --embed-only and --rechunk are mutually exclusive")
471
+ raise typer.Exit(1)
472
+
473
+ if embed_only:
474
+ mode = RebuildMode.EMBED_ONLY
475
+ elif rechunk:
476
+ mode = RebuildMode.RECHUNK
477
+ else:
478
+ mode = RebuildMode.FULL
479
+
362
480
  app = create_app(db)
363
- asyncio.run(app.rebuild())
481
+ asyncio.run(app.rebuild(mode=mode))
364
482
 
365
483
 
366
484
  @cli.command("vacuum", help="Optimize and clean up all tables to reduce disk usage")
@@ -375,7 +493,31 @@ def vacuum(
375
493
  asyncio.run(app.vacuum())
376
494
 
377
495
 
378
- @cli.command("info", help="Show read-only database info (no upgrades or writes)")
496
+ @cli.command("create-index", help="Create vector index for efficient similarity search")
497
+ def create_index(
498
+ db: Path | None = typer.Option(
499
+ None,
500
+ "--db",
501
+ help="Path to the LanceDB database file",
502
+ ),
503
+ ):
504
+ app = create_app(db)
505
+ asyncio.run(app.create_index())
506
+
507
+
508
+ @cli.command("init", help="Initialize a new database")
509
+ def init_db(
510
+ db: Path | None = typer.Option(
511
+ None,
512
+ "--db",
513
+ help="Path to the LanceDB database file",
514
+ ),
515
+ ):
516
+ app = create_app(db)
517
+ asyncio.run(app.init())
518
+
519
+
520
+ @cli.command("info", help="Show database info")
379
521
  def info(
380
522
  db: Path | None = typer.Option(
381
523
  None,
@@ -387,18 +529,59 @@ def info(
387
529
  asyncio.run(app.info())
388
530
 
389
531
 
532
+ @cli.command("history", help="Show version history for database tables")
533
+ def history(
534
+ db: Path | None = typer.Option(
535
+ None,
536
+ "--db",
537
+ help="Path to the LanceDB database file",
538
+ ),
539
+ table: str | None = typer.Option(
540
+ None,
541
+ "--table",
542
+ "-t",
543
+ help="Specific table to show history for (documents, chunks, settings)",
544
+ ),
545
+ limit: int | None = typer.Option(
546
+ None,
547
+ "--limit",
548
+ "-l",
549
+ help="Maximum number of versions to show per table",
550
+ ),
551
+ ):
552
+ app = create_app(db)
553
+ asyncio.run(app.history(table=table, limit=limit))
554
+
555
+
390
556
  @cli.command("download-models", help="Download Docling and Ollama models per config")
391
557
  def download_models_cmd():
392
- from haiku.rag.utils import prefetch_models
393
-
558
+ app = HaikuRAGApp(db_path=Path(), config=get_config())
394
559
  try:
395
- prefetch_models()
396
- typer.echo("Models downloaded successfully.")
560
+ asyncio.run(app.download_models())
397
561
  except Exception as e:
398
562
  typer.echo(f"Error downloading models: {e}")
399
563
  raise typer.Exit(1)
400
564
 
401
565
 
566
+ @cli.command("inspect", help="Launch interactive TUI to inspect database contents")
567
+ def inspect(
568
+ db: Path | None = typer.Option(
569
+ None,
570
+ "--db",
571
+ help="Path to the LanceDB database file",
572
+ ),
573
+ ):
574
+ """Launch the inspector TUI for browsing documents and chunks."""
575
+ try:
576
+ from haiku.rag.inspector import run_inspector
577
+ except ImportError as e:
578
+ typer.echo(f"Error: {e}", err=True)
579
+ raise typer.Exit(1) from e
580
+
581
+ db_path = db if db else get_config().storage.data_dir / "haiku.rag.lancedb"
582
+ run_inspector(db_path, read_only=_read_only, before=_before)
583
+
584
+
402
585
  @cli.command(
403
586
  "serve",
404
587
  help="Start haiku.rag server. Use --monitor, --mcp, and/or --agui to enable services.",