everalgo-knowledge 0.2.0__tar.gz → 0.2.1__tar.gz

{everalgo_knowledge-0.2.0 → everalgo_knowledge-0.2.1}/CHANGELOG.md

@@ -6,6 +6,20 @@ follows [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+## [0.2.1] - 2026-09-16
+
+### Changed
+
+- The minimum `everalgo-core` version is now `0.5.1`, because knowledge splitting relies on that release's special-token-safe `count_tokens` behavior.
+
+### Fixed
+
+- `split_content_to_blocks` and `split_and_batch_content` no longer raise `ValueError` on a document that quotes an `o200k_base` special-token literal such as `<|endoftext|>` — tokenizer documentation and bug reports could not be ingested at all. `_block_split` had grown a second, independent tokenizer; it now counts through `everalgo._tokenize.count_tokens`, which is the shared entry point fixed in `everalgo-core` (see that changelog for the rationale). Block boundaries and batch budgets for documents without such a literal are unchanged.
+
+### Removed
+
+- The direct `tiktoken` dependency. The package no longer imports it; token counting goes through `everalgo-core`, which declares it. No action is needed from callers — `tiktoken` still arrives transitively.
+
 ## [0.2.0] - 2026-09-15
 
 ### Fixed
@@ -31,4 +45,5 @@ Previously a reserved namespace with `Private :: Do Not Upload`.
 [0.2.0]: https://github.com/EverMind-AI/EverAlgo/releases/tag/everalgo-knowledge/v0.2.0
 [0.2.0rc1]: https://github.com/EverMind-AI/EverAlgo/releases/tag/everalgo-knowledge/v0.2.0rc1
 [0.1.1]: https://github.com/EverMind-AI/EverAlgo/releases/tag/everalgo-knowledge/v0.1.1
-[Unreleased]: https://github.com/EverMind-AI/EverAlgo/compare/everalgo-knowledge/v0.2.0...HEAD
+[Unreleased]: https://github.com/EverMind-AI/EverAlgo/compare/everalgo-knowledge/v0.2.1...HEAD
+[0.2.1]: https://github.com/EverMind-AI/EverAlgo/compare/everalgo-knowledge/v0.2.0...everalgo-knowledge/v0.2.1

{everalgo_knowledge-0.2.0 → everalgo_knowledge-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: everalgo-knowledge
-Version: 0.2.0
+Version: 0.2.1
 Summary: EverAlgo knowledge: file-based knowledge extractors producing KnowledgeMemory.
 Project-URL: Homepage, https://github.com/EverMind-AI/EverAlgo
 Project-URL: Repository, https://github.com/EverMind-AI/EverAlgo
@@ -19,9 +19,8 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.12
 Requires-Dist: asgiref>=3.0
-Requires-Dist: everalgo-core<2.0.0,>=0.3.0
+Requires-Dist: everalgo-core<2.0.0,>=0.5.1
 Requires-Dist: everalgo-parser<2.0.0,>=0.2.0
-Requires-Dist: tiktoken>=0.7
 Description-Content-Type: text/markdown
 
 # everalgo-knowledge

{everalgo_knowledge-0.2.0 → everalgo_knowledge-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "everalgo-knowledge"
-version = "0.2.0"
+version = "0.2.1"
 description = "EverAlgo knowledge: file-based knowledge extractors producing KnowledgeMemory."
 requires-python = ">=3.12"
 license = "Apache-2.0"
@@ -21,9 +21,8 @@ classifiers = [
 ]
 dependencies = [
   "asgiref>=3.0",
-  "everalgo-core>=0.3.0,<2.0.0",
+  "everalgo-core>=0.5.1,<2.0.0",
   "everalgo-parser>=0.2.0,<2.0.0",
-  "tiktoken>=0.7",
 ]
 
 [project.urls]

{everalgo_knowledge-0.2.0 → everalgo_knowledge-0.2.1}/src/everalgo/knowledge/_block_split.py

@@ -17,9 +17,8 @@ the knowledge extractor pipeline.
 from __future__ import annotations
 
 import re
-from functools import lru_cache
 
-import tiktoken
+from everalgo._tokenize import count_tokens
 
 __all__ = [
     "DEFAULT_MAX_TOKENS_PER_ATOM",
@@ -37,8 +36,6 @@ __all__ = [
 TABLE_START_MARKER = "TABLE_START"
 TABLE_END_MARKER = "TABLE_END"
 
-_O200K_ENCODING_NAME = "o200k_base"
-
 # Default token budget per LLM-window batch. 80K leaves headroom on a 128K-context
 # model after the topic-extraction prompt + JSON response. Tunable per call.
 DEFAULT_MAX_TOKENS_PER_BATCH = 80_000
@@ -52,12 +49,6 @@ DEFAULT_MAX_TOKENS_PER_BATCH = 80_000
 DEFAULT_MAX_TOKENS_PER_ATOM = DEFAULT_MAX_TOKENS_PER_BATCH // 4
 
 
-@lru_cache(maxsize=1)
-def _get_tokenizer() -> tiktoken.Encoding:
-    """Return the shared ``o200k_base`` encoding, initialising on first call."""
-    return tiktoken.get_encoding(_O200K_ENCODING_NAME)
-
-
 def preprocess_content(content: str) -> str:
     """Normalize parsed text before block splitting.
 
@@ -144,8 +135,7 @@ def _split_long_line(text: str, max_tokens: int) -> list[str]:
     66,000-character CJK line, which came back 66,006 characters with mojibake at
     every seam. Character slices concatenate back to the original exactly.
     """
-    enc = _get_tokenizer()
-    tokens_per_char = len(enc.encode(text)) / len(text)
+    tokens_per_char = count_tokens(text) / len(text)
     # 0.9 leaves margin for a denser stretch than the whole-line average.
     window = max(1, int(max_tokens / tokens_per_char * 0.9))
 
@@ -153,7 +143,7 @@ def _split_long_line(text: str, max_tokens: int) -> list[str]:
     start = 0
     while start < len(text):
         end = min(start + window, len(text))
-        while end - start > 1 and len(enc.encode(text[start:end])) > max_tokens:
+        while end - start > 1 and count_tokens(text[start:end]) > max_tokens:
             end = start + (end - start) * 3 // 4
         pieces.append(text[start:end])
         start = end
@@ -168,11 +158,10 @@ def _split_oversized_atom(text: str, max_tokens: int) -> list[str]:
     The pieces stay adjacent in the atom list, so a topic that claims all of them
     reassembles the original text through ``_topic_build._rebuild_content``.
     """
-    enc = _get_tokenizer()
     lines = text.split("\n")
     header = _table_header(lines)
     header_text = "\n".join(header)
-    header_tokens = len(enc.encode(header_text)) if header else 0
+    header_tokens = count_tokens(header_text) if header else 0
 
     pieces: list[str] = []
     current: list[str] = []
@@ -182,12 +171,12 @@ def _split_oversized_atom(text: str, max_tokens: int) -> list[str]:
         if current:
             pieces.append("\n".join(header + current) if header else "\n".join(current))
 
-    newline_tokens = len(enc.encode("\n"))
+    newline_tokens = count_tokens("\n")
     for line in lines[len(header) :]:
         # Count the "\n" that ``join`` will insert. Omitting it undercounts a piece by
         # roughly one token per line, which pushes long runs over the budget and drops
         # them into the character-boundary fallback — splitting list items mid-item.
-        line_tokens = len(enc.encode(line)) + newline_tokens
+        line_tokens = count_tokens(line) + newline_tokens
         if current and current_tokens + line_tokens > max_tokens:
             flush()
             current, current_tokens = [line], header_tokens + line_tokens
@@ -204,7 +193,7 @@ def _split_oversized_atom(text: str, max_tokens: int) -> list[str]:
     # PDF often arrives as one very long unwrapped line.
     bounded: list[str] = []
     for piece in pieces:
-        if len(enc.encode(piece)) <= max_tokens:
+        if count_tokens(piece) <= max_tokens:
             bounded.append(piece)
         else:
             bounded.extend(_split_long_line(piece, max_tokens))
@@ -297,13 +286,12 @@ def split_and_batch_content(
     if not atoms:
         return []
 
-    enc = _get_tokenizer()
     batches: list[list[tuple[int, str]]] = []
     current_batch: list[tuple[int, str]] = []
     current_tokens = 0
 
     for atom in atoms:
-        atom_tokens = len(enc.encode(atom[1]))
+        atom_tokens = count_tokens(atom[1])
         if current_batch and current_tokens + atom_tokens > max_tokens:
             batches.append(current_batch)
             current_batch = [atom]

{everalgo_knowledge-0.2.0 → everalgo_knowledge-0.2.1}/tests/knowledge/test_block_split.py

@@ -4,12 +4,12 @@ from __future__ import annotations
 
 import pytest
 
+from everalgo._tokenize import count_tokens
 from everalgo.knowledge._block_split import (
     DEFAULT_MAX_TOKENS_PER_ATOM,
     DEFAULT_MAX_TOKENS_PER_BATCH,
     TABLE_END_MARKER,
     TABLE_START_MARKER,
-    _get_tokenizer,
     format_numbered_paragraphs,
     preprocess_content,
     split_and_batch_content,
@@ -143,13 +143,12 @@ def _table(rows: int, *, header: bool = True) -> str:
 
 def test_unbounded_table_merge_no_longer_yields_an_over_budget_atom() -> None:
     """A page that is mostly one table used to collapse into a single mega-atom."""
-    enc = _get_tokenizer()
     doc = f"# Title\n\n## Variants\n\n{_table(6000)}\n\n## References\n\nSee the index.\n"
 
     atoms = split_content_to_blocks(doc)
 
     assert len(atoms) > 6, "the table must no longer be one atom"
-    assert all(len(enc.encode(t)) <= DEFAULT_MAX_TOKENS_PER_ATOM for _, t in atoms)
+    assert all(count_tokens(t) <= DEFAULT_MAX_TOKENS_PER_ATOM for _, t in atoms)
     assert [i for i, _ in atoms] == list(range(len(atoms))), "ids stay dense and sequential"
 
 
@@ -173,13 +172,12 @@ def test_oversized_atom_pieces_preserve_every_row() -> None:
 
 
 def test_long_list_is_split_without_a_header() -> None:
-    enc = _get_tokenizer()
     doc = "\n".join(f"- Item {i}: the polished lens recipe was introduced here." for i in range(8000))
 
     atoms = split_content_to_blocks(doc)
 
     assert len(atoms) > 1
-    assert all(len(enc.encode(t)) <= DEFAULT_MAX_TOKENS_PER_ATOM for _, t in atoms)
+    assert all(count_tokens(t) <= DEFAULT_MAX_TOKENS_PER_ATOM for _, t in atoms)
     assert [ln for _, t in atoms for ln in t.splitlines()] == doc.splitlines()
 
 
@@ -191,13 +189,12 @@ def test_ordinary_document_is_untouched_by_the_cap() -> None:
 
 
 def test_cap_is_configurable_and_zero_disables_it() -> None:
-    enc = _get_tokenizer()
     doc = _table(2000)
 
     assert len(split_content_to_blocks(doc, max_atom_tokens=0)) == 1
     tight = split_content_to_blocks(doc, max_atom_tokens=5_000)
     assert len(tight) > 1
-    assert all(len(enc.encode(t)) <= 5_000 for _, t in tight)
+    assert all(count_tokens(t) <= 5_000 for _, t in tight)
 
 
 # ── adversarial: ways an atom could still slip past the cap ──────────
@@ -216,64 +213,59 @@ def test_cap_is_configurable_and_zero_disables_it() -> None:
 )
 def test_cap_holds_for_every_script(label: str, unit: str) -> None:
     """A character-count pre-filter is unsound: CJK and emoji exceed 1 token/char."""
-    enc = _get_tokenizer()
     cap = 4_000
     doc = "\n".join(f"- {unit}" for _ in range(2_000))
 
     atoms = split_content_to_blocks(doc, max_atom_tokens=cap)
 
-    oversized = [(i, len(enc.encode(t))) for i, t in atoms if len(enc.encode(t)) > cap]
+    oversized = [(i, count_tokens(t)) for i, t in atoms if count_tokens(t) > cap]
     assert not oversized, f"{label}: atoms over cap {cap}: {oversized}"
 
 
 def test_single_line_longer_than_the_cap_is_split_on_token_boundaries() -> None:
     """CJK from a PDF often arrives as one very long unwrapped line."""
-    enc = _get_tokenizer()
     cap = 1_000
     doc = "洗涤剂的总活性物含量决定去污能力和溶解性能。" * 3_000  # one line, no breaks
 
     atoms = split_content_to_blocks(doc, max_atom_tokens=cap)
 
     assert len(atoms) > 1
-    assert all(len(enc.encode(t)) <= cap for _, t in atoms)
+    assert all(count_tokens(t) <= cap for _, t in atoms)
     assert "".join(t for _, t in atoms) == doc.strip(), "no content may be lost"
 
 
 def test_table_marker_path_is_also_capped() -> None:
     """Rule 1 (TABLE_START/END) is unbounded too, not just the native-row merge."""
-    enc = _get_tokenizer()
     cap = 4_000
     rows = "\n".join(f"<tr><td>Variant {i}</td><td>Wandering merchant</td></tr>" for i in range(3_000))
     doc = f"# Title\n\n{TABLE_START_MARKER}\n{rows}\n{TABLE_END_MARKER}\n\nAfter.\n"
 
     atoms = split_content_to_blocks(doc, max_atom_tokens=cap)
 
-    assert all(len(enc.encode(t)) <= cap for _, t in atoms)
+    assert all(count_tokens(t) <= cap for _, t in atoms)
     assert atoms[-1][1] == "After."
 
 
 def test_table_without_a_separator_row_still_splits() -> None:
     """Header detection must not be a precondition for the cap to apply."""
-    enc = _get_tokenizer()
     cap = 4_000
     doc = "\n".join(f"| Variant {i} | Wandering merchant, biome {i % 40} |" for i in range(3_000))
 
     atoms = split_content_to_blocks(doc, max_atom_tokens=cap)
 
     assert len(atoms) > 1
-    assert all(len(enc.encode(t)) <= cap for _, t in atoms)
+    assert all(count_tokens(t) <= cap for _, t in atoms)
 
 
 def test_header_larger_than_the_cap_does_not_loop_or_drop_content() -> None:
     """Degenerate: the repeated header alone already exceeds the budget."""
-    enc = _get_tokenizer()
     header = "| " + " | ".join(f"column_{i}" for i in range(600)) + " |"
     doc = header + "\n|" + "---|" * 600 + "\n" + "\n".join(f"| row {i} |" for i in range(500))
 
     atoms = split_content_to_blocks(doc, max_atom_tokens=500)
 
     assert atoms, "must not return empty"
-    assert all(len(enc.encode(t)) <= 500 for _, t in atoms)
+    assert all(count_tokens(t) <= 500 for _, t in atoms)
 
 
 def test_ids_stay_dense_after_splitting_several_atoms() -> None:
@@ -290,7 +282,6 @@ def test_ids_stay_dense_after_splitting_several_atoms() -> None:
 
 def test_split_atoms_batch_within_budget_end_to_end() -> None:
     """The whole point: no batch may exceed the window after splitting."""
-    enc = _get_tokenizer()
     rows = "\n".join(
         f"| Shiny variant #{i:04d} | Trading with the wandering merchant in biome {i % 40} "
         f"| {'Rare' if i % 3 else 'Common'} | Introduced in the winter update. |"
@@ -300,5 +291,16 @@ def test_split_atoms_batch_within_budget_end_to_end() -> None:
 
     batches = split_and_batch_content(split_content_to_blocks(preprocess_content(doc)))
 
-    over = [len(enc.encode(format_numbered_paragraphs(b))) for b in batches]
+    over = [count_tokens(format_numbered_paragraphs(b)) for b in batches]
     assert all(n <= DEFAULT_MAX_TOKENS_PER_BATCH for n in over), f"over-budget batches: {over}"
+
+
+def test_split_handles_special_token_literals() -> None:
+    """Block splitting counts tokens, so it shared the tokenizer defect.
+
+    A knowledge document about tokenizers quotes these literals; before the shared encode
+    wrapper that document could not be ingested at all.
+    """
+    content = "The tokenizer reserves the literal `<|endoftext|>` as a control token.\n\n" * 20
+    blocks = split_content_to_blocks(content)
+    assert blocks  # split, not raised