haiku.rag-slim 0.16.0__py3-none-any.whl → 0.24.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of haiku.rag-slim might be problematic. Click here for more details.
- haiku/rag/app.py +430 -72
- haiku/rag/chunkers/__init__.py +31 -0
- haiku/rag/chunkers/base.py +31 -0
- haiku/rag/chunkers/docling_local.py +164 -0
- haiku/rag/chunkers/docling_serve.py +179 -0
- haiku/rag/cli.py +207 -24
- haiku/rag/cli_chat.py +489 -0
- haiku/rag/client.py +1251 -266
- haiku/rag/config/__init__.py +16 -10
- haiku/rag/config/loader.py +5 -44
- haiku/rag/config/models.py +126 -17
- haiku/rag/converters/__init__.py +31 -0
- haiku/rag/converters/base.py +63 -0
- haiku/rag/converters/docling_local.py +193 -0
- haiku/rag/converters/docling_serve.py +229 -0
- haiku/rag/converters/text_utils.py +237 -0
- haiku/rag/embeddings/__init__.py +123 -24
- haiku/rag/embeddings/voyageai.py +175 -20
- haiku/rag/graph/__init__.py +0 -11
- haiku/rag/graph/agui/__init__.py +8 -2
- haiku/rag/graph/agui/cli_renderer.py +1 -1
- haiku/rag/graph/agui/emitter.py +219 -31
- haiku/rag/graph/agui/server.py +20 -62
- haiku/rag/graph/agui/stream.py +1 -2
- haiku/rag/graph/research/__init__.py +5 -2
- haiku/rag/graph/research/dependencies.py +12 -126
- haiku/rag/graph/research/graph.py +390 -135
- haiku/rag/graph/research/models.py +91 -112
- haiku/rag/graph/research/prompts.py +99 -91
- haiku/rag/graph/research/state.py +35 -27
- haiku/rag/inspector/__init__.py +8 -0
- haiku/rag/inspector/app.py +259 -0
- haiku/rag/inspector/widgets/__init__.py +6 -0
- haiku/rag/inspector/widgets/chunk_list.py +100 -0
- haiku/rag/inspector/widgets/context_modal.py +89 -0
- haiku/rag/inspector/widgets/detail_view.py +130 -0
- haiku/rag/inspector/widgets/document_list.py +75 -0
- haiku/rag/inspector/widgets/info_modal.py +209 -0
- haiku/rag/inspector/widgets/search_modal.py +183 -0
- haiku/rag/inspector/widgets/visual_modal.py +126 -0
- haiku/rag/mcp.py +106 -102
- haiku/rag/monitor.py +33 -9
- haiku/rag/providers/__init__.py +5 -0
- haiku/rag/providers/docling_serve.py +108 -0
- haiku/rag/qa/__init__.py +12 -10
- haiku/rag/qa/agent.py +43 -61
- haiku/rag/qa/prompts.py +35 -57
- haiku/rag/reranking/__init__.py +9 -6
- haiku/rag/reranking/base.py +1 -1
- haiku/rag/reranking/cohere.py +5 -4
- haiku/rag/reranking/mxbai.py +5 -2
- haiku/rag/reranking/vllm.py +3 -4
- haiku/rag/reranking/zeroentropy.py +6 -5
- haiku/rag/store/__init__.py +2 -1
- haiku/rag/store/engine.py +242 -42
- haiku/rag/store/exceptions.py +4 -0
- haiku/rag/store/models/__init__.py +8 -2
- haiku/rag/store/models/chunk.py +190 -0
- haiku/rag/store/models/document.py +46 -0
- haiku/rag/store/repositories/chunk.py +141 -121
- haiku/rag/store/repositories/document.py +25 -84
- haiku/rag/store/repositories/settings.py +11 -14
- haiku/rag/store/upgrades/__init__.py +19 -3
- haiku/rag/store/upgrades/v0_10_1.py +1 -1
- haiku/rag/store/upgrades/v0_19_6.py +65 -0
- haiku/rag/store/upgrades/v0_20_0.py +68 -0
- haiku/rag/store/upgrades/v0_23_1.py +100 -0
- haiku/rag/store/upgrades/v0_9_3.py +3 -3
- haiku/rag/utils.py +371 -146
- {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/METADATA +15 -12
- haiku_rag_slim-0.24.0.dist-info/RECORD +78 -0
- {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/WHEEL +1 -1
- haiku/rag/chunker.py +0 -65
- haiku/rag/embeddings/base.py +0 -25
- haiku/rag/embeddings/ollama.py +0 -28
- haiku/rag/embeddings/openai.py +0 -26
- haiku/rag/embeddings/vllm.py +0 -29
- haiku/rag/graph/agui/events.py +0 -254
- haiku/rag/graph/common/__init__.py +0 -5
- haiku/rag/graph/common/models.py +0 -42
- haiku/rag/graph/common/nodes.py +0 -265
- haiku/rag/graph/common/prompts.py +0 -46
- haiku/rag/graph/common/utils.py +0 -44
- haiku/rag/graph/deep_qa/__init__.py +0 -1
- haiku/rag/graph/deep_qa/dependencies.py +0 -27
- haiku/rag/graph/deep_qa/graph.py +0 -243
- haiku/rag/graph/deep_qa/models.py +0 -20
- haiku/rag/graph/deep_qa/prompts.py +0 -59
- haiku/rag/graph/deep_qa/state.py +0 -56
- haiku/rag/graph/research/common.py +0 -87
- haiku/rag/reader.py +0 -135
- haiku_rag_slim-0.16.0.dist-info/RECORD +0 -71
- {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/entry_points.txt +0 -0
- {haiku_rag_slim-0.16.0.dist-info → haiku_rag_slim-0.24.0.dist-info}/licenses/LICENSE +0 -0
haiku/rag/cli.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import json
|
|
3
3
|
import warnings
|
|
4
|
+
from datetime import datetime
|
|
4
5
|
from importlib.metadata import version
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
from typing import Any
|
|
@@ -26,6 +27,10 @@ cli = typer.Typer(
|
|
|
26
27
|
context_settings={"help_option_names": ["-h", "--help"]}, no_args_is_help=True
|
|
27
28
|
)
|
|
28
29
|
|
|
30
|
+
# Module-level flags set by callback
|
|
31
|
+
_read_only: bool = False
|
|
32
|
+
_before: datetime | None = None
|
|
33
|
+
|
|
29
34
|
|
|
30
35
|
def create_app(db: Path | None = None) -> HaikuRAGApp:
|
|
31
36
|
"""Create HaikuRAGApp with loaded config and resolved database path.
|
|
@@ -38,7 +43,9 @@ def create_app(db: Path | None = None) -> HaikuRAGApp:
|
|
|
38
43
|
"""
|
|
39
44
|
config = get_config()
|
|
40
45
|
db_path = db if db else config.storage.data_dir / "haiku.rag.lancedb"
|
|
41
|
-
return HaikuRAGApp(
|
|
46
|
+
return HaikuRAGApp(
|
|
47
|
+
db_path=db_path, config=config, read_only=_read_only, before=_before
|
|
48
|
+
)
|
|
42
49
|
|
|
43
50
|
|
|
44
51
|
async def check_version():
|
|
@@ -72,8 +79,33 @@ def main(
|
|
|
72
79
|
"--config",
|
|
73
80
|
help="Path to YAML configuration file",
|
|
74
81
|
),
|
|
82
|
+
read_only: bool = typer.Option(
|
|
83
|
+
False,
|
|
84
|
+
"--read-only",
|
|
85
|
+
help="Open database in read-only mode",
|
|
86
|
+
),
|
|
87
|
+
before: str | None = typer.Option(
|
|
88
|
+
None,
|
|
89
|
+
"--before",
|
|
90
|
+
help="Query database as it existed before this datetime (implies --read-only). "
|
|
91
|
+
"Accepts ISO 8601 format (e.g., 2025-01-15T14:30:00) or date (e.g., 2025-01-15)",
|
|
92
|
+
),
|
|
75
93
|
):
|
|
76
94
|
"""haiku.rag CLI - Vector database RAG system"""
|
|
95
|
+
global _read_only, _before
|
|
96
|
+
_read_only = read_only
|
|
97
|
+
|
|
98
|
+
# Parse and store before datetime
|
|
99
|
+
if before is not None:
|
|
100
|
+
from haiku.rag.utils import parse_datetime, to_utc
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
_before = to_utc(parse_datetime(before))
|
|
104
|
+
except ValueError as e:
|
|
105
|
+
typer.echo(f"Error: {e}")
|
|
106
|
+
raise typer.Exit(1)
|
|
107
|
+
else:
|
|
108
|
+
_before = None
|
|
77
109
|
# Load config from --config, local folder, or default directory
|
|
78
110
|
config_path = find_config_file(cli_path=config)
|
|
79
111
|
if config_path:
|
|
@@ -81,18 +113,24 @@ def main(
|
|
|
81
113
|
loaded_config = AppConfig.model_validate(yaml_data)
|
|
82
114
|
set_config(loaded_config)
|
|
83
115
|
|
|
84
|
-
# Configure logging
|
|
85
|
-
|
|
86
|
-
# Lazy import logfire only in development
|
|
87
|
-
try:
|
|
88
|
-
import logfire # type: ignore
|
|
116
|
+
# Configure logging for CLI context
|
|
117
|
+
configure_cli_logging()
|
|
89
118
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
119
|
+
# Configure logfire (only sends data if token is present)
|
|
120
|
+
try:
|
|
121
|
+
import logfire
|
|
122
|
+
|
|
123
|
+
is_production = get_config().environment != "development"
|
|
124
|
+
logfire.configure(
|
|
125
|
+
send_to_logfire="if-token-present",
|
|
126
|
+
console=False if is_production else None,
|
|
127
|
+
)
|
|
128
|
+
logfire.instrument_pydantic_ai()
|
|
129
|
+
except Exception:
|
|
130
|
+
pass
|
|
131
|
+
|
|
132
|
+
if get_config().environment != "development":
|
|
133
|
+
# Suppress warnings in production
|
|
96
134
|
warnings.filterwarnings("ignore")
|
|
97
135
|
|
|
98
136
|
# Run version check before any command
|
|
@@ -237,11 +275,11 @@ def search(
|
|
|
237
275
|
query: str = typer.Argument(
|
|
238
276
|
help="The search query to use",
|
|
239
277
|
),
|
|
240
|
-
limit: int = typer.Option(
|
|
241
|
-
|
|
278
|
+
limit: int | None = typer.Option(
|
|
279
|
+
None,
|
|
242
280
|
"--limit",
|
|
243
281
|
"-l",
|
|
244
|
-
help="Maximum number of results to return",
|
|
282
|
+
help="Maximum number of results to return (default: config search.default_limit)",
|
|
245
283
|
),
|
|
246
284
|
filter: str | None = typer.Option(
|
|
247
285
|
None,
|
|
@@ -259,6 +297,21 @@ def search(
|
|
|
259
297
|
asyncio.run(app.search(query=query, limit=limit, filter=filter))
|
|
260
298
|
|
|
261
299
|
|
|
300
|
+
@cli.command("visualize", help="Show visual grounding for a chunk")
|
|
301
|
+
def visualize(
|
|
302
|
+
chunk_id: str = typer.Argument(
|
|
303
|
+
help="The ID of the chunk to visualize",
|
|
304
|
+
),
|
|
305
|
+
db: Path | None = typer.Option(
|
|
306
|
+
None,
|
|
307
|
+
"--db",
|
|
308
|
+
help="Path to the LanceDB database file",
|
|
309
|
+
),
|
|
310
|
+
):
|
|
311
|
+
app = create_app(db)
|
|
312
|
+
asyncio.run(app.visualize_chunk(chunk_id=chunk_id))
|
|
313
|
+
|
|
314
|
+
|
|
262
315
|
@cli.command("ask", help="Ask a question using the QA agent")
|
|
263
316
|
def ask(
|
|
264
317
|
question: str = typer.Argument(
|
|
@@ -284,15 +337,24 @@ def ask(
|
|
|
284
337
|
"--verbose",
|
|
285
338
|
help="Show verbose progress output (only with --deep)",
|
|
286
339
|
),
|
|
340
|
+
filter: str | None = typer.Option(
|
|
341
|
+
None,
|
|
342
|
+
"--filter",
|
|
343
|
+
"-f",
|
|
344
|
+
help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")",
|
|
345
|
+
),
|
|
287
346
|
):
|
|
288
347
|
app = create_app(db)
|
|
289
|
-
asyncio.run(
|
|
348
|
+
asyncio.run(
|
|
349
|
+
app.ask(question=question, cite=cite, deep=deep, verbose=verbose, filter=filter)
|
|
350
|
+
)
|
|
290
351
|
|
|
291
352
|
|
|
292
353
|
@cli.command("research", help="Run multi-agent research and output a concise report")
|
|
293
354
|
def research(
|
|
294
355
|
question: str = typer.Argument(
|
|
295
|
-
|
|
356
|
+
None,
|
|
357
|
+
help="The research question to investigate (required unless --interactive)",
|
|
296
358
|
),
|
|
297
359
|
db: Path | None = typer.Option(
|
|
298
360
|
None,
|
|
@@ -304,9 +366,42 @@ def research(
|
|
|
304
366
|
"--verbose",
|
|
305
367
|
help="Show planning, searching previews, evaluation summary, and stop reason",
|
|
306
368
|
),
|
|
369
|
+
filter: str | None = typer.Option(
|
|
370
|
+
None,
|
|
371
|
+
"--filter",
|
|
372
|
+
"-f",
|
|
373
|
+
help="SQL WHERE clause to filter documents (e.g., \"uri LIKE '%arxiv%'\")",
|
|
374
|
+
),
|
|
375
|
+
interactive: bool = typer.Option(
|
|
376
|
+
False,
|
|
377
|
+
"--interactive",
|
|
378
|
+
"-i",
|
|
379
|
+
help="Start interactive research mode with human-in-the-loop",
|
|
380
|
+
),
|
|
307
381
|
):
|
|
308
382
|
app = create_app(db)
|
|
309
|
-
|
|
383
|
+
|
|
384
|
+
if interactive:
|
|
385
|
+
from haiku.rag.cli_chat import interactive_research
|
|
386
|
+
from haiku.rag.client import HaikuRAG
|
|
387
|
+
|
|
388
|
+
client = HaikuRAG(
|
|
389
|
+
db_path=app.db_path, config=app.config, read_only=_read_only, before=_before
|
|
390
|
+
)
|
|
391
|
+
try:
|
|
392
|
+
interactive_research(
|
|
393
|
+
client=client,
|
|
394
|
+
config=app.config,
|
|
395
|
+
search_filter=filter,
|
|
396
|
+
question=question,
|
|
397
|
+
)
|
|
398
|
+
finally:
|
|
399
|
+
client.close()
|
|
400
|
+
else:
|
|
401
|
+
if question is None:
|
|
402
|
+
typer.echo("Error: Question is required unless using --interactive mode")
|
|
403
|
+
raise typer.Exit(1)
|
|
404
|
+
asyncio.run(app.research(question=question, verbose=verbose, filter=filter))
|
|
310
405
|
|
|
311
406
|
|
|
312
407
|
@cli.command("settings", help="Display current configuration settings")
|
|
@@ -358,9 +453,32 @@ def rebuild(
|
|
|
358
453
|
"--db",
|
|
359
454
|
help="Path to the LanceDB database file",
|
|
360
455
|
),
|
|
456
|
+
embed_only: bool = typer.Option(
|
|
457
|
+
False,
|
|
458
|
+
"--embed-only",
|
|
459
|
+
help="Only regenerate embeddings, keep existing chunks",
|
|
460
|
+
),
|
|
461
|
+
rechunk: bool = typer.Option(
|
|
462
|
+
False,
|
|
463
|
+
"--rechunk",
|
|
464
|
+
help="Re-chunk from existing content without accessing source files",
|
|
465
|
+
),
|
|
361
466
|
):
|
|
467
|
+
from haiku.rag.client import RebuildMode
|
|
468
|
+
|
|
469
|
+
if embed_only and rechunk:
|
|
470
|
+
typer.echo("Error: --embed-only and --rechunk are mutually exclusive")
|
|
471
|
+
raise typer.Exit(1)
|
|
472
|
+
|
|
473
|
+
if embed_only:
|
|
474
|
+
mode = RebuildMode.EMBED_ONLY
|
|
475
|
+
elif rechunk:
|
|
476
|
+
mode = RebuildMode.RECHUNK
|
|
477
|
+
else:
|
|
478
|
+
mode = RebuildMode.FULL
|
|
479
|
+
|
|
362
480
|
app = create_app(db)
|
|
363
|
-
asyncio.run(app.rebuild())
|
|
481
|
+
asyncio.run(app.rebuild(mode=mode))
|
|
364
482
|
|
|
365
483
|
|
|
366
484
|
@cli.command("vacuum", help="Optimize and clean up all tables to reduce disk usage")
|
|
@@ -375,7 +493,31 @@ def vacuum(
|
|
|
375
493
|
asyncio.run(app.vacuum())
|
|
376
494
|
|
|
377
495
|
|
|
378
|
-
@cli.command("
|
|
496
|
+
@cli.command("create-index", help="Create vector index for efficient similarity search")
|
|
497
|
+
def create_index(
|
|
498
|
+
db: Path | None = typer.Option(
|
|
499
|
+
None,
|
|
500
|
+
"--db",
|
|
501
|
+
help="Path to the LanceDB database file",
|
|
502
|
+
),
|
|
503
|
+
):
|
|
504
|
+
app = create_app(db)
|
|
505
|
+
asyncio.run(app.create_index())
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
@cli.command("init", help="Initialize a new database")
|
|
509
|
+
def init_db(
|
|
510
|
+
db: Path | None = typer.Option(
|
|
511
|
+
None,
|
|
512
|
+
"--db",
|
|
513
|
+
help="Path to the LanceDB database file",
|
|
514
|
+
),
|
|
515
|
+
):
|
|
516
|
+
app = create_app(db)
|
|
517
|
+
asyncio.run(app.init())
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
@cli.command("info", help="Show database info")
|
|
379
521
|
def info(
|
|
380
522
|
db: Path | None = typer.Option(
|
|
381
523
|
None,
|
|
@@ -387,18 +529,59 @@ def info(
|
|
|
387
529
|
asyncio.run(app.info())
|
|
388
530
|
|
|
389
531
|
|
|
532
|
+
@cli.command("history", help="Show version history for database tables")
|
|
533
|
+
def history(
|
|
534
|
+
db: Path | None = typer.Option(
|
|
535
|
+
None,
|
|
536
|
+
"--db",
|
|
537
|
+
help="Path to the LanceDB database file",
|
|
538
|
+
),
|
|
539
|
+
table: str | None = typer.Option(
|
|
540
|
+
None,
|
|
541
|
+
"--table",
|
|
542
|
+
"-t",
|
|
543
|
+
help="Specific table to show history for (documents, chunks, settings)",
|
|
544
|
+
),
|
|
545
|
+
limit: int | None = typer.Option(
|
|
546
|
+
None,
|
|
547
|
+
"--limit",
|
|
548
|
+
"-l",
|
|
549
|
+
help="Maximum number of versions to show per table",
|
|
550
|
+
),
|
|
551
|
+
):
|
|
552
|
+
app = create_app(db)
|
|
553
|
+
asyncio.run(app.history(table=table, limit=limit))
|
|
554
|
+
|
|
555
|
+
|
|
390
556
|
@cli.command("download-models", help="Download Docling and Ollama models per config")
|
|
391
557
|
def download_models_cmd():
|
|
392
|
-
|
|
393
|
-
|
|
558
|
+
app = HaikuRAGApp(db_path=Path(), config=get_config())
|
|
394
559
|
try:
|
|
395
|
-
|
|
396
|
-
typer.echo("Models downloaded successfully.")
|
|
560
|
+
asyncio.run(app.download_models())
|
|
397
561
|
except Exception as e:
|
|
398
562
|
typer.echo(f"Error downloading models: {e}")
|
|
399
563
|
raise typer.Exit(1)
|
|
400
564
|
|
|
401
565
|
|
|
566
|
+
@cli.command("inspect", help="Launch interactive TUI to inspect database contents")
|
|
567
|
+
def inspect(
|
|
568
|
+
db: Path | None = typer.Option(
|
|
569
|
+
None,
|
|
570
|
+
"--db",
|
|
571
|
+
help="Path to the LanceDB database file",
|
|
572
|
+
),
|
|
573
|
+
):
|
|
574
|
+
"""Launch the inspector TUI for browsing documents and chunks."""
|
|
575
|
+
try:
|
|
576
|
+
from haiku.rag.inspector import run_inspector
|
|
577
|
+
except ImportError as e:
|
|
578
|
+
typer.echo(f"Error: {e}", err=True)
|
|
579
|
+
raise typer.Exit(1) from e
|
|
580
|
+
|
|
581
|
+
db_path = db if db else get_config().storage.data_dir / "haiku.rag.lancedb"
|
|
582
|
+
run_inspector(db_path, read_only=_read_only, before=_before)
|
|
583
|
+
|
|
584
|
+
|
|
402
585
|
@cli.command(
|
|
403
586
|
"serve",
|
|
404
587
|
help="Start haiku.rag server. Use --monitor, --mcp, and/or --agui to enable services.",
|