graphrag-chunking 3.0.1__tar.gz → 3.0.3__tar.gz

{graphrag_chunking-3.0.1 → graphrag_chunking-3.0.3}/.gitignore

@@ -63,3 +63,7 @@ docsite/
 
 # Root build assets
 packages/*/LICENSE
+
+# Notebooks outputs
+packages/graphrag-*/example_notebooks/**/output
+packages/graphrag-*/example_notebooks/**/cache

{graphrag_chunking-3.0.1 → graphrag_chunking-3.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: graphrag-chunking
-Version: 3.0.1
+Version: 3.0.3
 Summary: Chunking utilities for GraphRAG
 Project-URL: Source, https://github.com/microsoft/graphrag
 Author: Mónica Carvajal
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: <3.14,>=3.11
-Requires-Dist: graphrag-common==3.0.1
+Requires-Dist: graphrag-common==3.0.2
 Requires-Dist: pydantic~=2.10
 Description-Content-Type: text/markdown
 
@@ -26,34 +26,17 @@ This package contains a collection of text chunkers, a core config model, and a
 
 The SentenceChunker class splits text into individual sentences by identifying sentence boundaries. It takes input text and returns a list where each element is a separate sentence, making it easy to process text at the sentence level.
 
-```python
-chunker = SentenceChunker()
-chunks = chunker.chunk("This is a test. Another sentence.")
-print(chunks) # ["This is a test.", "Another sentence."]
-```
+[Open the notebook to explore the basic sentence example code](example_notebooks/basic_sentence_example.ipynb)
 
 ### Token chunking
 
 The TokenChunker splits text into fixed-size chunks based on token count rather than sentence boundaries. It uses a tokenizer to encode text into tokens, then creates chunks of a specified size with configurable overlap between chunks.
 
-```python
-tokenizer = tiktoken.get_encoding("o200k_base")
-chunker = TokenChunker(size=3, overlap=0, encode=tokenizer.encode, decode=tokenizer.decode)
-chunks = chunker.chunk("This is a random test fragment of some text")
-print(chunks) # ["This is a", " random test fragment", " of some text"]
-```
+[Open the notebook to explore the token chunking example code](example_notebooks/token_chunking_example.ipynb)
+
 
 ### Using the factory via helper util
 
 The create_chunker factory function provides a configuration-driven approach to instantiate chunkers by accepting a ChunkingConfig object that specifies the chunking strategy and parameters. This allows for more flexible and maintainable code by separating chunker configuration from direct instantiation.
 
-```python
-tokenizer = tiktoken.get_encoding("o200k_base")
-config = ChunkingConfig(
-    strategy="tokens",
-    size=3,
-    overlap=0
-)
-chunker = create_chunker(config, tokenizer.encode, tokenizer.decode)
-...
-```
+[Open the notebook to explore the factory helper util example code](example_notebooks/factory_helper_util_example.ipynb)

{graphrag_chunking-3.0.1 → graphrag_chunking-3.0.3}/README.md

@@ -8,34 +8,17 @@ This package contains a collection of text chunkers, a core config model, and a
 
 The SentenceChunker class splits text into individual sentences by identifying sentence boundaries. It takes input text and returns a list where each element is a separate sentence, making it easy to process text at the sentence level.
 
-```python
-chunker = SentenceChunker()
-chunks = chunker.chunk("This is a test. Another sentence.")
-print(chunks) # ["This is a test.", "Another sentence."]
-```
+[Open the notebook to explore the basic sentence example code](example_notebooks/basic_sentence_example.ipynb)
 
 ### Token chunking
 
 The TokenChunker splits text into fixed-size chunks based on token count rather than sentence boundaries. It uses a tokenizer to encode text into tokens, then creates chunks of a specified size with configurable overlap between chunks.
 
-```python
-tokenizer = tiktoken.get_encoding("o200k_base")
-chunker = TokenChunker(size=3, overlap=0, encode=tokenizer.encode, decode=tokenizer.decode)
-chunks = chunker.chunk("This is a random test fragment of some text")
-print(chunks) # ["This is a", " random test fragment", " of some text"]
-```
+[Open the notebook to explore the token chunking example code](example_notebooks/token_chunking_example.ipynb)
+
 
 ### Using the factory via helper util
 
 The create_chunker factory function provides a configuration-driven approach to instantiate chunkers by accepting a ChunkingConfig object that specifies the chunking strategy and parameters. This allows for more flexible and maintainable code by separating chunker configuration from direct instantiation.
 
-```python
-tokenizer = tiktoken.get_encoding("o200k_base")
-config = ChunkingConfig(
-    strategy="tokens",
-    size=3,
-    overlap=0
-)
-chunker = create_chunker(config, tokenizer.encode, tokenizer.decode)
-...
-```
+[Open the notebook to explore the factory helper util example code](example_notebooks/factory_helper_util_example.ipynb)

graphrag_chunking-3.0.3/example_notebooks/basic_sentence_example.ipynb

@@ -0,0 +1,60 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a91f3280",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Copyright (c) 2026 Microsoft Corporation.\n",
+    "# Licensed under the MIT License."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3f76dc60",
+   "metadata": {},
+   "source": [
+    "## Basic sentence chunking with nltk\n",
+    "\n",
+    "The SentenceChunker class splits text into individual sentences by identifying sentence boundaries. It takes input text and returns a list where each element is a separate sentence, making it easy to process text at the sentence level."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4c96d3cc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from graphrag_chunking.sentence_chunker import SentenceChunker\n",
+    "\n",
+    "chunker = SentenceChunker()\n",
+    "chunks = chunker.chunk(\"This is a test. Another sentence.\")\n",
+    "print(chunks)  # [\"This is a test.\", \"Another sentence.\"]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

graphrag_chunking-3.0.3/example_notebooks/factory_helper_util_example.ipynb

@@ -0,0 +1,77 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "cf52b7c7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Copyright (c) 2026 Microsoft Corporation.\n",
+    "# Licensed under the MIT License."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "37a05202",
+   "metadata": {},
+   "source": [
+    "## Using the factory via helper util example\n",
+    "\n",
+    "\n",
+    "The create_chunker factory function provides a configuration-driven approach to instantiate chunkers by accepting a ChunkingConfig object that specifies the chunking strategy and parameters. This allows for more flexible and maintainable code by separating chunker configuration from direct instantiation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "027e9c45",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[TextChunk(original='This is a', text='This is a', index=0, start_char=0, end_char=8, token_count=3), TextChunk(original=' test. Another', text=' test. Another', index=1, start_char=9, end_char=22, token_count=3), TextChunk(original=' sentence. And', text=' sentence. And', index=2, start_char=23, end_char=36, token_count=3), TextChunk(original=' another one.', text=' another one.', index=3, start_char=37, end_char=49, token_count=3)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import tiktoken\n",
+    "from graphrag_chunking.chunker_factory import create_chunker\n",
+    "from graphrag_chunking.chunking_config import ChunkingConfig\n",
+    "\n",
+    "tokenizer = tiktoken.get_encoding(\"o200k_base\")\n",
+    "config = ChunkingConfig(\n",
+    "    strategy=\"tokens\",  # type: ignore\n",
+    "    size=3,\n",
+    "    overlap=0,\n",
+    ")\n",
+    "chunker = create_chunker(config, tokenizer.encode, tokenizer.decode)\n",
+    "\n",
+    "print(chunker.chunk(\"This is a test. Another sentence. And another one.\"))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

graphrag_chunking-3.0.3/example_notebooks/token_chunking_example.ipynb

@@ -0,0 +1,72 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "41bf35b3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Copyright (c) 2026 Microsoft Corporation.\n",
+    "# Licensed under the MIT License."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c59c030",
+   "metadata": {},
+   "source": [
+    "## Token chunking example\n",
+    "\n",
+    "The TokenChunker splits text into fixed-size chunks based on token count rather than sentence boundaries. It uses a tokenizer to encode text into tokens, then creates chunks of a specified size with configurable overlap between chunks."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "cd653d39",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[TextChunk(original='This is a', text='This is a', index=0, start_char=0, end_char=8, token_count=3), TextChunk(original=' random test fragment', text=' random test fragment', index=1, start_char=9, end_char=29, token_count=3), TextChunk(original=' of some text', text=' of some text', index=2, start_char=30, end_char=42, token_count=3)]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import tiktoken\n",
+    "from graphrag_chunking.token_chunker import TokenChunker\n",
+    "\n",
+    "tokenizer = tiktoken.get_encoding(\"o200k_base\")\n",
+    "chunker = TokenChunker(\n",
+    "    size=3, overlap=0, encode=tokenizer.encode, decode=tokenizer.decode\n",
+    ")\n",
+    "chunks = chunker.chunk(\"This is a random test fragment of some text\")\n",
+    "print(chunks)  # [\"This is a\", \" random test fragment\", \" of some text\"]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{graphrag_chunking-3.0.1 → graphrag_chunking-3.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "graphrag-chunking"
-version = "3.0.1"
+version = "3.0.3"
 description = "Chunking utilities for GraphRAG"
 authors = [
     {name = "Alonso Guevara Fernández", email = "alonsog@microsoft.com"},
@@ -30,7 +30,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "graphrag-common==3.0.1",
+    "graphrag-common==3.0.2",
     "pydantic~=2.10",
 ]