xgen-doc2chunk 0.1.0__tar.gz → 0.1.1__tar.gz

{xgen_doc2chunk-0.1.0 → xgen_doc2chunk-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xgen-doc2chunk
-Version: 0.1.0
+Version: 0.1.1
 Summary: Convert raw documents into AI-understandable context with intelligent text extraction, table detection, and semantic chunking
 Project-URL: Homepage, https://github.com/master0419/doc2chunk
 Project-URL: Documentation, https://github.com/master0419/doc2chunk#readme

{xgen_doc2chunk-0.1.0 → xgen_doc2chunk-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "xgen-doc2chunk"
-version = "0.1.0"
+version = "0.1.1"
 description = "Convert raw documents into AI-understandable context with intelligent text extraction, table detection, and semantic chunking"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -86,11 +86,11 @@ Issues = "https://github.com/master0419/doc2chunk/issues"
 Changelog = "https://github.com/master0419/doc2chunk/releases"
 
 [tool.hatch.build.targets.wheel]
-packages = ["xgen-doc2chunk"]
+packages = ["xgen_doc2chunk"]
 
 [tool.hatch.build.targets.sdist]
 include = [
-    "xgen-doc2chunk/",
+    "xgen_doc2chunk/",
     "README.md",
     "LICENSE",
     "pyproject.toml",

xgen_doc2chunk-0.1.1/xgen_doc2chunk/__init__.py

@@ -0,0 +1,42 @@
+# xgen_doc2chunk/__init__.py
+"""
+xgen_doc2chunk Library
+
+A document processing and chunking library for AI applications.
+
+Package Structure:
+- core: Document processing core module
+    - DocumentProcessor: Main document processing class
+    - processor: Individual document type handlers (PDF, DOCX, PPT, Excel, HWP, etc.)
+    - functions: Utility functions
+
+- chunking: Text chunking module
+    - Text splitting and chunking logic
+    - Table-preserving chunking
+    - Page-based chunking
+
+Usage:
+    from xgen_doc2chunk import DocumentProcessor
+    
+    processor = DocumentProcessor()
+    text = processor.extract_text("document.pdf")
+    result = processor.extract_chunks("document.pdf", chunk_size=1000)
+"""
+
+__version__ = "0.1.0"
+
+# Expose core classes at top level
+from xgen_doc2chunk.core import DocumentProcessor
+
+# Explicit subpackages
+from xgen_doc2chunk import core
+from xgen_doc2chunk import chunking
+
+__all__ = [
+    "__version__",
+    # Core classes
+    "DocumentProcessor",
+    # Subpackages
+    "core",
+    "chunking",
+]

xgen_doc2chunk-0.1.1/xgen_doc2chunk/chunking/__init__.py

@@ -0,0 +1,168 @@
+# xgen_doc2chunk/chunking/__init__.py
+"""
+Chunking - Text Chunking Module
+
+This package provides functionality to split document text into appropriately sized chunks.
+
+Module Structure:
+- chunking: Main chunking functions (split_text_preserving_html_blocks, etc.)
+- constants: Constants, patterns, and data classes
+- table_parser: HTML table parsing
+- table_chunker: Table chunking core logic
+- protected_regions: Protected region handling
+- page_chunker: Page-based chunking
+- text_chunker: Text chunking
+- sheet_processor: Sheet and metadata processing
+
+Usage:
+    from xgen_doc2chunk.chunking import split_text_preserving_html_blocks, chunk_plain_text
+    from xgen_doc2chunk.chunking import TableRow, ParsedTable
+"""
+
+# === Main Chunking Functions (chunking.py) ===
+from xgen_doc2chunk.chunking.chunking import (
+    create_chunks,
+)
+
+# constants
+from xgen_doc2chunk.chunking.constants import (
+    # Constants
+    LANGCHAIN_CODE_LANGUAGE_MAP,
+    HTML_TABLE_PATTERN,
+    CHART_BLOCK_PATTERN,
+    TEXTBOX_BLOCK_PATTERN,
+    IMAGE_TAG_PATTERN,
+    MARKDOWN_TABLE_PATTERN,
+    TABLE_WRAPPER_OVERHEAD,
+    CHUNK_INDEX_OVERHEAD,
+    TABLE_SIZE_THRESHOLD_MULTIPLIER,
+    TABLE_BASED_FILE_TYPES,
+    # Data classes
+    TableRow,
+    ParsedTable,
+)
+
+# table_parser
+from xgen_doc2chunk.chunking.table_parser import (
+    parse_html_table,
+    extract_cell_spans,
+    extract_cell_spans_with_positions,
+    has_complex_spans,
+)
+
+# table_chunker
+from xgen_doc2chunk.chunking.table_chunker import (
+    calculate_available_space,
+    adjust_rowspan_in_chunk,
+    build_table_chunk,
+    update_chunk_metadata,
+    split_table_into_chunks,
+    split_table_preserving_rowspan,
+    chunk_large_table,
+)
+
+# protected_regions
+from xgen_doc2chunk.chunking.protected_regions import (
+    find_protected_regions,
+    get_protected_region_positions,
+    ensure_protected_region_integrity,
+    split_with_protected_regions,
+    split_large_chunk_with_protected_regions,
+    # Backward compatibility aliases
+    ensure_table_integrity,
+    split_large_chunk_with_table_protection,
+)
+
+# page_chunker
+from xgen_doc2chunk.chunking.page_chunker import (
+    split_into_pages,
+    merge_pages,
+    get_overlap_content,
+    chunk_by_pages,
+)
+
+# text_chunker
+from xgen_doc2chunk.chunking.text_chunker import (
+    chunk_plain_text,
+    chunk_text_without_tables,
+    chunk_with_row_protection,
+    chunk_with_row_protection_simple,
+    clean_chunks,
+    chunk_code_text,
+    reconstruct_text_from_chunks,
+    find_overlap_length,
+    estimate_chunks_count,
+)
+
+# sheet_processor
+from xgen_doc2chunk.chunking.sheet_processor import (
+    extract_document_metadata,
+    prepend_metadata_to_chunks,
+    extract_sheet_sections,
+    extract_content_segments,
+    chunk_multi_sheet_content,
+    chunk_single_table_content,
+)
+
+
+__all__ = [
+    # === Primary API ===
+    "create_chunks",
+    # constants
+    "LANGCHAIN_CODE_LANGUAGE_MAP",
+    "HTML_TABLE_PATTERN",
+    "CHART_BLOCK_PATTERN",
+    "TEXTBOX_BLOCK_PATTERN",
+    "IMAGE_TAG_PATTERN",
+    "MARKDOWN_TABLE_PATTERN",
+    "TABLE_WRAPPER_OVERHEAD",
+    "CHUNK_INDEX_OVERHEAD",
+    "TABLE_SIZE_THRESHOLD_MULTIPLIER",
+    "TABLE_BASED_FILE_TYPES",
+    "TableRow",
+    "ParsedTable",
+    # table_parser
+    "parse_html_table",
+    "extract_cell_spans",
+    "extract_cell_spans_with_positions",
+    "has_complex_spans",
+    # table_chunker
+    "calculate_available_space",
+    "adjust_rowspan_in_chunk",
+    "build_table_chunk",
+    "update_chunk_metadata",
+    "split_table_into_chunks",
+    "split_table_preserving_rowspan",
+    "chunk_large_table",
+    # protected_regions
+    "find_protected_regions",
+    "get_protected_region_positions",
+    "ensure_protected_region_integrity",
+    "split_with_protected_regions",
+    "split_large_chunk_with_protected_regions",
+    "ensure_table_integrity",
+    "split_large_chunk_with_table_protection",
+    # page_chunker
+    "split_into_pages",
+    "merge_pages",
+    "get_overlap_content",
+    "chunk_by_pages",
+    # text_chunker
+    "chunk_plain_text",
+    "chunk_text_without_tables",
+    "chunk_with_row_protection",
+    "chunk_with_row_protection_simple",
+    "clean_chunks",
+    "chunk_code_text",
+    "reconstruct_text_from_chunks",
+    "find_overlap_length",
+    "estimate_chunks_count",
+    # sheet_processor
+    "extract_document_metadata",
+    "prepend_metadata_to_chunks",
+    "extract_sheet_sections",
+    "extract_content_segments",
+    "chunk_multi_sheet_content",
+    "chunk_single_table_content",
+]
+