doclex-cli 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- doclex/__init__.py +0 -0
- doclex/cache/__init__.py +0 -0
- doclex/cache/store.py +196 -0
- doclex/cli.py +430 -0
- doclex/config.py +43 -0
- doclex/extraction/__init__.py +0 -0
- doclex/extraction/extractor.py +72 -0
- doclex/index/__init__.py +0 -0
- doclex/index/bm25_index.py +120 -0
- doclex/ingestion/__init__.py +0 -0
- doclex/ingestion/scanner.py +87 -0
- doclex/ingestion/watcher.py +94 -0
- doclex/metadata/__init__.py +0 -0
- doclex/metadata/db.py +250 -0
- doclex/metadata/models.py +55 -0
- doclex/pipeline.py +257 -0
- doclex/registry.py +103 -0
- doclex/segmentation/__init__.py +0 -0
- doclex/segmentation/segmenter.py +398 -0
- doclex_cli-0.1.1.dist-info/METADATA +221 -0
- doclex_cli-0.1.1.dist-info/RECORD +24 -0
- doclex_cli-0.1.1.dist-info/WHEEL +4 -0
- doclex_cli-0.1.1.dist-info/entry_points.txt +2 -0
- doclex_cli-0.1.1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from kreuzberg import ExtractionConfig, extract_file_sync
|
|
8
|
+
from kreuzberg.exceptions import KreuzbergError
|
|
9
|
+
|
|
10
|
+
from doclex.metadata.models import ExtractedDoc, Segment
|
|
11
|
+
|
|
12
|
+
# Minimum chars for a segment to be worth indexing
|
|
13
|
+
_MIN_CHARS = 30
|
|
14
|
+
|
|
15
|
+
# Maximum chars per segment — recursive splitter enforces this
|
|
16
|
+
_MAX_CHARS = 1024
|
|
17
|
+
|
|
18
|
+
# Headings: Markdown / plain text
|
|
19
|
+
_HEADING_RE = re.compile(
|
|
20
|
+
r"(?:^|\n)(#{1,3}\s+.+|(?=\S).+\n[=\-]{3,})", re.MULTILINE
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
# Separators tried in order by the recursive splitter
|
|
24
|
+
_SEPARATORS = ["\n\n", "\n", " "]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def segment(doc: ExtractedDoc) -> list[Segment]:
|
|
28
|
+
"""Segment an ExtractedDoc into indexable units based on file type.
|
|
29
|
+
|
|
30
|
+
Strategy:
|
|
31
|
+
1. Structure-based segmentation (headings, pages, sheets, tables)
|
|
32
|
+
2. Any segment > _MAX_CHARS is recursively split preserving semantic boundaries
|
|
33
|
+
"""
|
|
34
|
+
suffix = Path(doc.path).suffix.lower()
|
|
35
|
+
|
|
36
|
+
if suffix == ".pdf":
|
|
37
|
+
raw = _segment_pdf(doc)
|
|
38
|
+
elif suffix in (".xlsx",):
|
|
39
|
+
raw = _segment_spreadsheet(doc)
|
|
40
|
+
elif suffix == ".csv":
|
|
41
|
+
raw = _segment_csv(doc)
|
|
42
|
+
elif suffix in (".docx", ".md", ".txt"):
|
|
43
|
+
raw = _segment_text(doc)
|
|
44
|
+
else:
|
|
45
|
+
raw = _segment_fallback(doc)
|
|
46
|
+
|
|
47
|
+
return _apply_size_limit(raw, _MAX_CHARS)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ------------------------------------------------------------------
|
|
51
|
+
# PDF — page-by-page via form-feed split, fallback to full text
|
|
52
|
+
# ------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
def _segment_pdf(doc: ExtractedDoc) -> list[Segment]:
|
|
55
|
+
"""Segment PDF: markdown-based when precise mode produced headings, otherwise page-based."""
|
|
56
|
+
segments: list[Segment] = []
|
|
57
|
+
|
|
58
|
+
if _has_markdown_headings(doc.text):
|
|
59
|
+
# precise mode: Kreuzberg returned markdown with heading structure
|
|
60
|
+
segments = _segment_text(doc)
|
|
61
|
+
else:
|
|
62
|
+
pages = doc.text.split("\f")
|
|
63
|
+
segments = _make_segments(doc, pages, "page", "Page") if len(pages) > 1 else _segment_fallback(doc)
|
|
64
|
+
|
|
65
|
+
# Append table segments from precise mode (result.tables)
|
|
66
|
+
if doc.tables:
|
|
67
|
+
offset = len(segments)
|
|
68
|
+
for i, table in enumerate(doc.tables):
|
|
69
|
+
content = _table_to_text(table)
|
|
70
|
+
if len(content) >= _MIN_CHARS:
|
|
71
|
+
label = getattr(table, "name", None) or f"Table {i + 1}"
|
|
72
|
+
segments.append(_make_segment(doc, content, "table", label, offset + i))
|
|
73
|
+
|
|
74
|
+
return segments or _segment_fallback(doc)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _has_markdown_headings(text: str) -> bool:
|
|
78
|
+
return bool(re.search(r"(?:^|\n)#{1,3} ", text))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ------------------------------------------------------------------
|
|
82
|
+
# XLSX — sheet-by-sheet via Kreuzberg tables
|
|
83
|
+
# ------------------------------------------------------------------
|
|
84
|
+
|
|
85
|
+
def _segment_spreadsheet(doc: ExtractedDoc) -> list[Segment]:
|
|
86
|
+
try:
|
|
87
|
+
config = ExtractionConfig()
|
|
88
|
+
result = extract_file_sync(doc.path, None, config)
|
|
89
|
+
if result.tables:
|
|
90
|
+
segments: list[Segment] = []
|
|
91
|
+
for i, table in enumerate(result.tables):
|
|
92
|
+
label = getattr(table, "name", None) or f"Sheet {i + 1}"
|
|
93
|
+
content = _table_to_text(table)
|
|
94
|
+
if len(content) >= _MIN_CHARS:
|
|
95
|
+
segments.append(_make_segment(doc, content, "sheet", label, i))
|
|
96
|
+
if segments:
|
|
97
|
+
return segments
|
|
98
|
+
except KreuzbergError:
|
|
99
|
+
pass
|
|
100
|
+
|
|
101
|
+
return _segment_fallback(doc)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# ------------------------------------------------------------------
|
|
105
|
+
# CSV — table or chunked rows
|
|
106
|
+
# ------------------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
def _segment_csv(doc: ExtractedDoc) -> list[Segment]:
|
|
109
|
+
try:
|
|
110
|
+
config = ExtractionConfig()
|
|
111
|
+
result = extract_file_sync(doc.path, None, config)
|
|
112
|
+
if result.tables:
|
|
113
|
+
content = _table_to_text(result.tables[0])
|
|
114
|
+
return [_make_segment(doc, content, "table", "Table", 0)]
|
|
115
|
+
except KreuzbergError:
|
|
116
|
+
pass
|
|
117
|
+
|
|
118
|
+
lines = [l for l in doc.text.splitlines() if l.strip()]
|
|
119
|
+
if not lines:
|
|
120
|
+
return []
|
|
121
|
+
|
|
122
|
+
chunk_size = 50
|
|
123
|
+
segments: list[Segment] = []
|
|
124
|
+
for i in range(0, len(lines), chunk_size):
|
|
125
|
+
chunk = "\n".join(lines[i : i + chunk_size])
|
|
126
|
+
label = f"Rows {i + 1}–{min(i + chunk_size, len(lines))}"
|
|
127
|
+
segments.append(_make_segment(doc, chunk, "table", label, i // chunk_size))
|
|
128
|
+
return segments
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# ------------------------------------------------------------------
|
|
132
|
+
# DOCX / MD / TXT — heading-based segmentation
|
|
133
|
+
# ------------------------------------------------------------------
|
|
134
|
+
|
|
135
|
+
def _segment_text(doc: ExtractedDoc) -> list[Segment]:
|
|
136
|
+
text = doc.text.strip()
|
|
137
|
+
if not text:
|
|
138
|
+
return []
|
|
139
|
+
|
|
140
|
+
# Split on Markdown headings (# / ## / ###)
|
|
141
|
+
parts = re.split(r"\n(?=#{1,3} )", text)
|
|
142
|
+
|
|
143
|
+
if len(parts) <= 1:
|
|
144
|
+
# No headings — split on blank lines, no tree structure
|
|
145
|
+
parts = re.split(r"\n{2,}", text)
|
|
146
|
+
segments: list[Segment] = []
|
|
147
|
+
for i, part in enumerate(parts):
|
|
148
|
+
part = part.strip()
|
|
149
|
+
if len(part) < _MIN_CHARS:
|
|
150
|
+
continue
|
|
151
|
+
first_line = part.splitlines()[0].lstrip("#").strip()
|
|
152
|
+
label = first_line[:80] if first_line else f"Block {i + 1}"
|
|
153
|
+
segments.append(_make_segment(doc, part, "block", label, i))
|
|
154
|
+
return segments or _segment_fallback(doc)
|
|
155
|
+
|
|
156
|
+
# Parse heading parts
|
|
157
|
+
parsed: list[tuple[int, str, str, str, int]] = [] # (level, title, node_id, content, index)
|
|
158
|
+
for i, part in enumerate(parts):
|
|
159
|
+
part = part.strip()
|
|
160
|
+
if not part:
|
|
161
|
+
continue
|
|
162
|
+
first_line = part.splitlines()[0]
|
|
163
|
+
m = re.match(r"^(#{1,3}) ", first_line)
|
|
164
|
+
level = len(m.group(1)) if m else 0
|
|
165
|
+
title = first_line.lstrip("#").strip()[:80] or f"Block {i + 1}"
|
|
166
|
+
node_id = _segment_id(doc.path, title, i)
|
|
167
|
+
parsed.append((level, title, node_id, part, i))
|
|
168
|
+
|
|
169
|
+
# Create segments with level and section_id
|
|
170
|
+
segments = []
|
|
171
|
+
for level, title, node_id, content, i in parsed:
|
|
172
|
+
if len(content) < _MIN_CHARS:
|
|
173
|
+
continue
|
|
174
|
+
location_type = "section" if level <= 2 else "block"
|
|
175
|
+
segments.append(Segment(
|
|
176
|
+
id=node_id,
|
|
177
|
+
file_path=doc.path,
|
|
178
|
+
content=content,
|
|
179
|
+
location_type=location_type,
|
|
180
|
+
location_label=title,
|
|
181
|
+
location_index=i,
|
|
182
|
+
char_count=len(content),
|
|
183
|
+
section_id=node_id,
|
|
184
|
+
level=level,
|
|
185
|
+
))
|
|
186
|
+
|
|
187
|
+
return segments or _segment_fallback(doc)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ------------------------------------------------------------------
|
|
191
|
+
# Fallback — whole text as single block
|
|
192
|
+
# ------------------------------------------------------------------
|
|
193
|
+
|
|
194
|
+
def _segment_fallback(doc: ExtractedDoc) -> list[Segment]:
|
|
195
|
+
text = doc.text.strip()
|
|
196
|
+
if len(text) < _MIN_CHARS:
|
|
197
|
+
return []
|
|
198
|
+
return [_make_segment(doc, text, "block", "Document", 0)]
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# ------------------------------------------------------------------
|
|
202
|
+
# Size enforcement — recursive character splitter
|
|
203
|
+
# ------------------------------------------------------------------
|
|
204
|
+
|
|
205
|
+
def _apply_size_limit(segments: list[Segment], max_chars: int) -> list[Segment]:
|
|
206
|
+
"""Post-process segments: split on internal headings first, then by size."""
|
|
207
|
+
result: list[Segment] = []
|
|
208
|
+
for seg in segments:
|
|
209
|
+
for s in _split_on_internal_headings(seg):
|
|
210
|
+
result.extend(_split_if_too_long(s, max_chars))
|
|
211
|
+
return result
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _split_on_internal_headings(seg: Segment) -> list[Segment]:
|
|
215
|
+
"""Split a segment on any internal markdown headings it contains.
|
|
216
|
+
|
|
217
|
+
Prevents nesting content from different sections under a single heading node.
|
|
218
|
+
Only applies to heading segments (level > 0).
|
|
219
|
+
"""
|
|
220
|
+
if seg.level == 0:
|
|
221
|
+
return [seg]
|
|
222
|
+
|
|
223
|
+
parts = re.split(r"\n(?=#{1,3} )", seg.content)
|
|
224
|
+
if len(parts) <= 1:
|
|
225
|
+
return [seg]
|
|
226
|
+
|
|
227
|
+
result: list[Segment] = []
|
|
228
|
+
for i, part in enumerate(parts):
|
|
229
|
+
part = part.strip()
|
|
230
|
+
if not part:
|
|
231
|
+
continue
|
|
232
|
+
first_line = part.splitlines()[0]
|
|
233
|
+
m = re.match(r"^(#{1,3}) ", first_line)
|
|
234
|
+
|
|
235
|
+
if i == 0:
|
|
236
|
+
# Original segment — keep id and level, update content
|
|
237
|
+
result.append(Segment(
|
|
238
|
+
id=seg.id,
|
|
239
|
+
file_path=seg.file_path,
|
|
240
|
+
content=part,
|
|
241
|
+
location_type=seg.location_type,
|
|
242
|
+
location_label=seg.location_label,
|
|
243
|
+
location_index=seg.location_index,
|
|
244
|
+
char_count=len(part),
|
|
245
|
+
section_id=seg.section_id,
|
|
246
|
+
level=seg.level,
|
|
247
|
+
))
|
|
248
|
+
elif m:
|
|
249
|
+
# Internal heading — new sibling segment with its own level
|
|
250
|
+
level = len(m.group(1))
|
|
251
|
+
title = first_line.lstrip("#").strip()[:80]
|
|
252
|
+
node_id = _segment_id(seg.file_path, title, seg.location_index * 1000 + i)
|
|
253
|
+
result.append(Segment(
|
|
254
|
+
id=node_id,
|
|
255
|
+
file_path=seg.file_path,
|
|
256
|
+
content=part,
|
|
257
|
+
location_type="section" if level <= 2 else "block",
|
|
258
|
+
location_label=title,
|
|
259
|
+
location_index=seg.location_index * 1000 + i,
|
|
260
|
+
char_count=len(part),
|
|
261
|
+
section_id=node_id,
|
|
262
|
+
level=level,
|
|
263
|
+
))
|
|
264
|
+
|
|
265
|
+
return result if result else [seg]
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _split_if_too_long(seg: Segment, max_chars: int) -> list[Segment]:
|
|
269
|
+
"""Return the segment as-is if within limit, or split it recursively."""
|
|
270
|
+
if len(seg.content) <= max_chars:
|
|
271
|
+
return [seg]
|
|
272
|
+
|
|
273
|
+
chunks = _recursive_split(seg.content, max_chars, _SEPARATORS)
|
|
274
|
+
|
|
275
|
+
result: list[Segment] = []
|
|
276
|
+
for i, chunk in enumerate(chunks):
|
|
277
|
+
chunk = chunk.strip()
|
|
278
|
+
if len(chunk) < _MIN_CHARS:
|
|
279
|
+
continue
|
|
280
|
+
sub_label = f"{seg.location_label} › {i + 1}"
|
|
281
|
+
sub_index = seg.location_index * 1000 + i
|
|
282
|
+
result.append(Segment(
|
|
283
|
+
id=_segment_id(seg.file_path, sub_label, sub_index),
|
|
284
|
+
file_path=seg.file_path,
|
|
285
|
+
content=chunk,
|
|
286
|
+
location_type=seg.location_type,
|
|
287
|
+
location_label=sub_label,
|
|
288
|
+
location_index=sub_index,
|
|
289
|
+
char_count=len(chunk),
|
|
290
|
+
section_id=seg.section_id,
|
|
291
|
+
level=0, # sub-segments are continuations, not new heading nodes
|
|
292
|
+
))
|
|
293
|
+
|
|
294
|
+
# Guard: never drop a valid segment entirely
|
|
295
|
+
return result if result else [seg]
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _recursive_split(text: str, max_chars: int, separators: list[str]) -> list[str]:
|
|
299
|
+
"""Split text into chunks of at most max_chars using the coarsest separator first.
|
|
300
|
+
|
|
301
|
+
Strategy:
|
|
302
|
+
1. Try to split on the coarsest separator.
|
|
303
|
+
2. Greedily merge consecutive pieces back together up to max_chars.
|
|
304
|
+
3. Pieces that are still too large are recursively split with the next separator.
|
|
305
|
+
"""
|
|
306
|
+
if len(text) <= max_chars:
|
|
307
|
+
return [text]
|
|
308
|
+
|
|
309
|
+
if not separators:
|
|
310
|
+
# Hard character slice as last resort
|
|
311
|
+
return [text[i : i + max_chars] for i in range(0, len(text), max_chars)]
|
|
312
|
+
|
|
313
|
+
sep, *remaining = separators
|
|
314
|
+
raw_parts = text.split(sep)
|
|
315
|
+
|
|
316
|
+
if len(raw_parts) == 1:
|
|
317
|
+
# Separator not present in this text — try the next one
|
|
318
|
+
return _recursive_split(text, max_chars, remaining)
|
|
319
|
+
|
|
320
|
+
# First pass: recursively handle pieces that are individually too large
|
|
321
|
+
good_parts: list[str] = []
|
|
322
|
+
for part in raw_parts:
|
|
323
|
+
part = part.strip()
|
|
324
|
+
if not part:
|
|
325
|
+
continue
|
|
326
|
+
if len(part) <= max_chars:
|
|
327
|
+
good_parts.append(part)
|
|
328
|
+
else:
|
|
329
|
+
good_parts.extend(_recursive_split(part, max_chars, remaining))
|
|
330
|
+
|
|
331
|
+
# Second pass: greedy merge — combine consecutive parts up to max_chars
|
|
332
|
+
chunks: list[str] = []
|
|
333
|
+
current = ""
|
|
334
|
+
for part in good_parts:
|
|
335
|
+
candidate = (current + sep + part).strip() if current else part
|
|
336
|
+
if len(candidate) <= max_chars:
|
|
337
|
+
current = candidate
|
|
338
|
+
else:
|
|
339
|
+
if current:
|
|
340
|
+
chunks.append(current)
|
|
341
|
+
current = part
|
|
342
|
+
if current:
|
|
343
|
+
chunks.append(current)
|
|
344
|
+
|
|
345
|
+
return chunks
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
# ------------------------------------------------------------------
|
|
349
|
+
# Helpers
|
|
350
|
+
# ------------------------------------------------------------------
|
|
351
|
+
|
|
352
|
+
def _make_segments(
|
|
353
|
+
doc: ExtractedDoc,
|
|
354
|
+
parts: list[str],
|
|
355
|
+
location_type: str,
|
|
356
|
+
label_prefix: str,
|
|
357
|
+
) -> list[Segment]:
|
|
358
|
+
segments: list[Segment] = []
|
|
359
|
+
for i, part in enumerate(parts):
|
|
360
|
+
part = part.strip()
|
|
361
|
+
if len(part) < _MIN_CHARS:
|
|
362
|
+
continue
|
|
363
|
+
label = f"{label_prefix} {i + 1}"
|
|
364
|
+
segments.append(_make_segment(doc, part, location_type, label, i))
|
|
365
|
+
return segments
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _make_segment(
|
|
369
|
+
doc: ExtractedDoc,
|
|
370
|
+
content: str,
|
|
371
|
+
location_type: str,
|
|
372
|
+
location_label: str,
|
|
373
|
+
location_index: int,
|
|
374
|
+
) -> Segment:
|
|
375
|
+
seg_id = _segment_id(doc.path, location_label, location_index)
|
|
376
|
+
return Segment(
|
|
377
|
+
id=seg_id,
|
|
378
|
+
file_path=doc.path,
|
|
379
|
+
content=content,
|
|
380
|
+
location_type=location_type,
|
|
381
|
+
location_label=location_label,
|
|
382
|
+
location_index=location_index,
|
|
383
|
+
char_count=len(content),
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _table_to_text(table: object) -> str:
|
|
388
|
+
"""Convert a Kreuzberg ExtractedTable to plain text."""
|
|
389
|
+
if hasattr(table, "content") and table.content:
|
|
390
|
+
return str(table.content)
|
|
391
|
+
if hasattr(table, "rows") and table.rows:
|
|
392
|
+
return "\n".join("\t".join(str(c) for c in row) for row in table.rows)
|
|
393
|
+
return str(table)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _segment_id(path: str, label: str, index: int) -> str:
|
|
397
|
+
raw = f"{path}|{label}|{index}"
|
|
398
|
+
return hashlib.sha256(raw.encode()).hexdigest()[:32]
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: doclex-cli
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Local document search — BM25 lexical search over PDFs, DOCX, XLSX, CSV, Markdown and TXT files. No embeddings, no vector DB, no cloud.
|
|
5
|
+
Project-URL: Repository, https://github.com/EnPaiva93/doclex-cli
|
|
6
|
+
Author-email: EnPaiva_93 <enpaiva93@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Python: >=3.12
|
|
10
|
+
Requires-Dist: aiosqlite
|
|
11
|
+
Requires-Dist: bm25x
|
|
12
|
+
Requires-Dist: kreuzberg==4.8.2
|
|
13
|
+
Requires-Dist: typer
|
|
14
|
+
Requires-Dist: watchdog
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
17
|
+
Requires-Dist: pytest-asyncio; extra == 'dev'
|
|
18
|
+
Requires-Dist: ranx; extra == 'dev'
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
<h1 align="center">Doclex</h1>
|
|
22
|
+
|
|
23
|
+
<p align="center"><em>English · <a href="README.es.md">Español</a></em></p>
|
|
24
|
+
|
|
25
|
+
<p align="center">
|
|
26
|
+
<img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="License: MIT">
|
|
27
|
+
<img src="https://img.shields.io/badge/python-3.12%2B-blue.svg" alt="Python 3.12+">
|
|
28
|
+
</p>
|
|
29
|
+
|
|
30
|
+
<p align="center">A CLI that turns a local folder of documents into a searchable, citable corpus. Point it at a folder; extraction, segmentation, indexing, and change tracking happen automatically.</p>
|
|
31
|
+
|
|
32
|
+
## Features
|
|
33
|
+
|
|
34
|
+
Doclex is the orchestration layer that ties them together so you never have to think about the pipeline yourself:
|
|
35
|
+
|
|
36
|
+
- **Plug and play** — point it at a folder. Extraction, structural segmentation, indexing, and change tracking all happen automatically; there's no pipeline to wire up.
|
|
37
|
+
- **SQLite is the source of truth** — every file's hash, mtime, and indexing status live in one place, so Doclex always knows exactly what changed and reprocesses only that.
|
|
38
|
+
- **Local-first** — everything runs on your machine, nothing leaves it. No embeddings, no vector database, no cloud.
|
|
39
|
+
- **Deterministic** — BM25 lexical scoring, no black-box similarity.
|
|
40
|
+
- **Traceable** — every result cites its file, page/sheet/section, and score.
|
|
41
|
+
- **Incremental** — only reprocesses files that actually changed.
|
|
42
|
+
|
|
43
|
+
## Install
|
|
44
|
+
|
|
45
|
+
Requires Python ≥3.12 and [uv](https://docs.astral.sh/uv/).
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
git clone https://github.com/EnPaiva93/doclex-cli.git
|
|
49
|
+
cd doclex-cli
|
|
50
|
+
uv sync
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Quickstart
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
# register a folder as a project
|
|
57
|
+
uv run doclex add mydocs ./path/to/documents
|
|
58
|
+
|
|
59
|
+
# build the index
|
|
60
|
+
uv run doclex index --project mydocs
|
|
61
|
+
|
|
62
|
+
# search it
|
|
63
|
+
uv run doclex search "quarterly budget" --project mydocs
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
[1] budget_2024.pdf — page 3 (score: 4.812)
|
|
68
|
+
"...the quarterly budget was approved with a 12% increase in..."
|
|
69
|
+
|
|
70
|
+
[2] notes.md — Planning (score: 2.107)
|
|
71
|
+
"...need to revisit the budget assumptions before..."
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Walkthrough
|
|
75
|
+
|
|
76
|
+
A typical day-to-day flow, from an empty project to keeping it in sync:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
# 1. register the folder you want to search
|
|
80
|
+
doclex add contracts ~/Documents/legal/contracts
|
|
81
|
+
|
|
82
|
+
# 2. build the index — extracts, segments, and indexes every supported file
|
|
83
|
+
doclex index --project contracts
|
|
84
|
+
```
|
|
85
|
+
```
|
|
86
|
+
Processing: 42 new, 0 modified, 0 deleted
|
|
87
|
+
✓ Done.
|
|
88
|
+
```
|
|
89
|
+
```bash
|
|
90
|
+
# 3. search it
|
|
91
|
+
doclex search "termination clause" --project contracts
|
|
92
|
+
```
|
|
93
|
+
```
|
|
94
|
+
[1] MSA_2023_AcmeCorp.pdf — Section 9.2 (score: 6.104)
|
|
95
|
+
"...either party may terminate this Agreement upon 30 days written notice..."
|
|
96
|
+
|
|
97
|
+
[2] vendor_agreement_v3.docx — Termination (score: 5.887)
|
|
98
|
+
"...termination for convenience requires 60 days advance notice to..."
|
|
99
|
+
```
|
|
100
|
+
```bash
|
|
101
|
+
# 4. check what's indexed
|
|
102
|
+
doclex status --project contracts
|
|
103
|
+
```
|
|
104
|
+
```
|
|
105
|
+
Folder ~/Documents/legal/contracts
|
|
106
|
+
Indexed 42 files
|
|
107
|
+
Segments 318
|
|
108
|
+
BM25 index 318 docs
|
|
109
|
+
Disk state up to date
|
|
110
|
+
```
|
|
111
|
+
```bash
|
|
112
|
+
# 5. a new contract shows up on disk — reindexing picks it up automatically
|
|
113
|
+
doclex index --project contracts
|
|
114
|
+
```
|
|
115
|
+
```
|
|
116
|
+
Processing: 1 new, 0 modified, 0 deleted
|
|
117
|
+
✓ Done.
|
|
118
|
+
```
|
|
119
|
+
```bash
|
|
120
|
+
# 6. see every registered project
|
|
121
|
+
doclex list
|
|
122
|
+
```
|
|
123
|
+
```
|
|
124
|
+
Name Folder
|
|
125
|
+
contracts /home/you/Documents/legal/contracts
|
|
126
|
+
```
|
|
127
|
+
```bash
|
|
128
|
+
# 7. done with it — deregister and purge its cache/index
|
|
129
|
+
doclex remove contracts
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Each project is independent — register as many non-overlapping folders as you need, each with its own cache and index:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
doclex add hr-policies ~/Documents/hr
|
|
136
|
+
doclex index --project hr-policies
|
|
137
|
+
doclex search "vacation days" --project hr-policies --top 5
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Projects
|
|
141
|
+
|
|
142
|
+
Every command operates on a **registered project** — a name bound to a folder — instead of a raw `--folder` flag. This keeps folder resolution unambiguous and lets Doclex reject overlapping folders (a subfolder registered as its own project would silently duplicate indexing and caching):
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
doclex add mydocs ./path/to/documents
|
|
146
|
+
doclex add mydocs ./other # error: name already exists
|
|
147
|
+
doclex add sub ./path/to/documents/subdir # error: overlaps with 'mydocs'
|
|
148
|
+
doclex remove mydocs # deregisters and purges its cache/index
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Commands
|
|
152
|
+
|
|
153
|
+
| Command | Description |
|
|
154
|
+
|---|---|
|
|
155
|
+
| `doclex add <name> <folder>` | Register a folder under a project name |
|
|
156
|
+
| `doclex list` | List registered projects |
|
|
157
|
+
| `doclex remove <name>` | Remove a project and purge its cache/index |
|
|
158
|
+
| `doclex index --project <name>` | Scan and index (incremental by default) |
|
|
159
|
+
| `doclex search <query> --project <name>` | Search and print results with citations |
|
|
160
|
+
| `doclex status --project <name>` | Show corpus/index status |
|
|
161
|
+
|
|
162
|
+
Run `doclex <command> --help` for all flags and examples.
|
|
163
|
+
|
|
164
|
+
### Flags in practice
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
# force a full re-extraction
|
|
168
|
+
doclex index --project mydocs --force
|
|
169
|
+
|
|
170
|
+
# reindex specific files after editing them
|
|
171
|
+
doclex index --project mydocs --file report.pdf --file notes.docx
|
|
172
|
+
|
|
173
|
+
# more results, full segment text instead of a snippet
|
|
174
|
+
doclex search "risk assessment" --project mydocs --top 20 --full
|
|
175
|
+
|
|
176
|
+
# list every indexed file and its status
|
|
177
|
+
doclex status --project mydocs --all --filter error
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## Architecture
|
|
181
|
+
|
|
182
|
+
```
|
|
183
|
+
folder → scan/watch → extract (Kreuzberg) → segment → BM25X index
|
|
184
|
+
↕
|
|
185
|
+
SQLite (files, segments)
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
| Layer | Responsibility |
|
|
189
|
+
|---|---|
|
|
190
|
+
| **Ingestion** | Scans a project's folder, hashes files to detect new/modified/deleted ones, skips its own cache directory |
|
|
191
|
+
| **Extraction** | Delegates to [Kreuzberg](https://github.com/Goldziher/kreuzberg) to turn each file into plain text, normalized to an internal format |
|
|
192
|
+
| **Segmentation** | Splits text along the file's natural structure — never blind fixed-size chunking (PDF → page, XLSX → sheet, DOCX → section, TXT/MD → block) |
|
|
193
|
+
| **Index** | Indexes segments (not whole files) with BM25X; supports incremental add/update/delete |
|
|
194
|
+
| **Metadata** | SQLite tracks each file's hash, mtime, size, MIME type, and indexing status, plus which segments belong to it |
|
|
195
|
+
|
|
196
|
+
A local cache stores extracted text and generated segments per file, so unchanged files are never reprocessed.
|
|
197
|
+
|
|
198
|
+
## Supported file types
|
|
199
|
+
|
|
200
|
+
PDF · TXT · Markdown · DOCX · XLSX · CSV
|
|
201
|
+
|
|
202
|
+
## Design principles
|
|
203
|
+
|
|
204
|
+
Doclex is built to be local-first, incremental, explainable, deterministic, and cheap to run. It deliberately does **not**:
|
|
205
|
+
|
|
206
|
+
- use embeddings or a vector database
|
|
207
|
+
- upload files anywhere
|
|
208
|
+
- depend on external services to search
|
|
209
|
+
- run OCR by default
|
|
210
|
+
- generate answers — it retrieves evidence, it doesn't interpret it
|
|
211
|
+
|
|
212
|
+
## Stack
|
|
213
|
+
|
|
214
|
+
Python · [Typer](https://typer.tiangolo.com/) · [Kreuzberg](https://github.com/Goldziher/kreuzberg) · BM25X · Watchdog · SQLite
|
|
215
|
+
|
|
216
|
+
## Development
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
uv sync --extra dev
|
|
220
|
+
uv run pytest
|
|
221
|
+
```
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
doclex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
doclex/cli.py,sha256=rmbwL4XvGcmzTYlOaxnEqZQ8IOTuekvTeJDITWMfwLg,15989
|
|
3
|
+
doclex/config.py,sha256=ZXgzFt7ONfvPHstdr53dtb4YB_3zEeO-XV3QoOrh_Zw,1338
|
|
4
|
+
doclex/pipeline.py,sha256=fXxVdfY79Wv8Ud5sou4tbGDs_HIJKNl6JW6bXStDI7k,9644
|
|
5
|
+
doclex/registry.py,sha256=G768iMsk4TR7FQQEHWTC7cu4uFQA-eZ4STRQCR8m_gA,3102
|
|
6
|
+
doclex/cache/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
doclex/cache/store.py,sha256=g05wNUtxRf_0PsWYhEfoZJwh_jupjd33swSkUpVKz_E,7319
|
|
8
|
+
doclex/extraction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
doclex/extraction/extractor.py,sha256=zOaWODRqr5I8bDnocQCGWu73j6Xcw5Aa2bjnFIzs-Ms,2602
|
|
10
|
+
doclex/index/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
doclex/index/bm25_index.py,sha256=11qzuQUu8l_FOc125A7NzOtaMuslUS_R-Y35OubdxhU,4474
|
|
12
|
+
doclex/ingestion/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
doclex/ingestion/scanner.py,sha256=J2fb5m2ulIS53Jr3aBfnBIGvQvy1md4ckl4h0td7rZ0,2620
|
|
14
|
+
doclex/ingestion/watcher.py,sha256=dsyURJtrp4OfBkUPcrYOIfMMKklMlYFYqdqnytIzMHA,2993
|
|
15
|
+
doclex/metadata/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
16
|
+
doclex/metadata/db.py,sha256=41871wIiKa1-iLu0GV6omoyW0vRYFOAxv0ms6hiuEuc,8736
|
|
17
|
+
doclex/metadata/models.py,sha256=e2fc6mkhwKxYeLXeys7ZwEhTKJOzewzHPyFxzpjmUnA,1427
|
|
18
|
+
doclex/segmentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
|
+
doclex/segmentation/segmenter.py,sha256=m0MChmyEGV9HSZLjVHXz7DPOPOi03lxg88Pn4PezgyA,13572
|
|
20
|
+
doclex_cli-0.1.1.dist-info/METADATA,sha256=0Vrfl-T89v3qJaOdbmRB4zzQApnb8wkYUgPeCOKE_0s,7448
|
|
21
|
+
doclex_cli-0.1.1.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
|
|
22
|
+
doclex_cli-0.1.1.dist-info/entry_points.txt,sha256=D2nc0RMXcbn6oElT0xqj-pzpEC2YBQP5aXkK6ydKCKU,42
|
|
23
|
+
doclex_cli-0.1.1.dist-info/licenses/LICENSE,sha256=VtEVThH2Pajb8nQ0ibXjL50aaHdJwwWJPDZBa8Aoxq8,1067
|
|
24
|
+
doclex_cli-0.1.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 EnPaiva_93
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|