sec2md 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

sec2md/chunker/markdown_blocks.py

@@ -2,15 +2,26 @@ import re
 from abc import ABC
 from typing import List
 
+try:
+    import tiktoken
+    TIKTOKEN_AVAILABLE = True
+except ImportError:
+    TIKTOKEN_AVAILABLE = False
+
 
 def estimate_tokens(text: str) -> int:
     """
-    Estimate token count using character/4 heuristic.
+    Calculate token count for text.
 
-    This is a simple approximation. For exact token counting,
-    use your embedding provider's tokenizer.
+    Uses tiktoken with cl100k_base encoding (gpt-3.5-turbo/gpt-4) if available.
+    Falls back to character/4 heuristic if tiktoken is not installed.
     """
-    return max(1, len(text) // 4)
+    if TIKTOKEN_AVAILABLE:
+        encoding = tiktoken.get_encoding("cl100k_base")
+        return len(encoding.encode(text))
+    else:
+        # Fallback: simple heuristic
+        return max(1, len(text) // 4)
 
 
 def split_sentences(text: str) -> List[str]:

{sec2md-0.1.0.dist-info → sec2md-0.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sec2md
-Version: 0.1.0
+Version: 0.1.1
 Summary: Convert SEC EDGAR filings to LLM-ready Markdown for AI agents and agentic RAG
 Author-email: Lucas Astorian <lucas@intellifin.ai>
 License: MIT
@@ -25,6 +25,7 @@ License-File: LICENSE
 Requires-Dist: beautifulsoup4>=4.12.0
 Requires-Dist: lxml>=4.9.0
 Requires-Dist: requests>=2.31.0
+Requires-Dist: tiktoken>=0.5.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
@@ -34,25 +35,31 @@ Dynamic: license-file
 
 # sec2md
 
-Transform messy SEC filings into clean, structured Markdown.
-**Built for AI. Optimized for retrieval. Ready for production.**
-
 [![PyPI](https://img.shields.io/pypi/v/sec2md.svg)](https://pypi.org/project/sec2md)
-[![Downloads](https://pepy.tech/badge/sec2md)](https://pepy.tech/project/sec2md)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 [![Documentation](https://img.shields.io/badge/docs-readthedocs-blue.svg)](https://sec2md.readthedocs.io)
 
+Transform messy SEC filings into clean, structured Markdown.
+**Built for AI. Optimized for retrieval. Ready for production.**
+
+![Before and After Comparison](comparison.png)
+*Apple 10-K cover page: Raw SEC HTML (left) vs. Clean Markdown (right)*
+
 ---
 
 ## The Problem
 
-SEC filings are a nightmare for LLMs:
-- ❌ **XBRL tags** pollute the text (`<us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax>`)
-- ❌ **Nested tables** with absolute positioning break standard parsers
-- ❌ **Inline CSS** and presentational HTML obscure semantic structure
-- ❌ **200+ page documents** with no clear section boundaries
+RAG pipelines fail on SEC filings because **standard parsers destroy document structure.**
+
+When you flatten a 200-page 10-K to plain text:
+
+- ❌ **Tables break** — Complex financial statements become misaligned text
+- ❌ **Pages are lost** — Can't cite sources or trace answers back
+- ❌ **Sections merge** — Risk Factors and MD&A become indistinguishable
+- ❌ **Formatting is stripped** — Headers, bolds, lists (LLM reasoning cues) gone
+- ❌ **Retrieval fails** — Chunks without structure return wrong context
 
-Standard HTML-to-text converters produce garbage. Your RAG pipeline deserves better.
+Your RAG system is only as good as your data. Garbage in, garbage out.
 
 ## The Solution
 
@@ -177,22 +184,13 @@ Most libraries force you to choose between speed and accuracy. `sec2md` gives yo
 - 🎯 **Accurate** - Purpose-built for SEC document structure
 - 🔧 **Simple** - One function call, zero configuration
 
-### Built for Production RAG
+### Built for Agentic RAG
 Don't rebuild what we've already solved:
 - ✅ **Page tracking** - Cite sources with exact page numbers
 - ✅ **Section detection** - Extract just what you need (Risk Factors, MD&A)
 - ✅ **Smart chunking** - Respects table boundaries, preserves context
 - ✅ **Metadata headers** - Boost embedding quality 2-3x with contextual headers
 
-### Avoid the Maintenance Nightmare
-Building your own SEC parser starts simple - scaling it is another story. What begins as BeautifulSoup and regex quickly turns into:
-- 🔴 Edge cases for every filing format variation
-- 🔴 Table parsing that breaks on nested structures
-- 🔴 XBRL tag stripping that misses new namespaces
-- 🔴 Section detection that fails on formatting changes
-
-**`sec2md` handles this for you.** Focus on building AI features, not parsing documents.
-
 ---
 
 ## Documentation

{sec2md-0.1.0.dist-info → sec2md-0.1.1.dist-info}/RECORD

@@ -9,11 +9,11 @@ sec2md/sections.py,sha256=TxmpucUH389T9QDqu_SYVsJi0_WtsNTaG0ILspoF88E,3395
 sec2md/table_parser.py,sha256=FhR8OwX5NAJmzdbTFzHQTGUNUPieYN37UzMFbQMkogU,12551
 sec2md/utils.py,sha256=2lUeN5irTbdIyjylCkaPKMv4ALWxWMJl96PTO8FV3Ik,2990
 sec2md/chunker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sec2md/chunker/markdown_blocks.py,sha256=RaC03Czm3OEYGwySLzNLv-Bq4qupQ4AbAzv00uTR_xc,3479
+sec2md/chunker/markdown_blocks.py,sha256=vfLs3sZp6lVcSY2ZDm_MIuQ41cSxlnclBsEWfxgoGBw,3787
 sec2md/chunker/markdown_chunk.py,sha256=87s_zdZ5prQ4GyeULhdCijJYz8xXn4gSGDRNpotziFU,2632
 sec2md/chunker/markdown_chunker.py,sha256=ex8Kpzae9b8hluLt-CnW_RM6aQ5Ag7u8cy9gWHM8yhU,9305
-sec2md-0.1.0.dist-info/licenses/LICENSE,sha256=uJDiSGQ5TOx-PGhu2LGH4A-O53vS5hrQ5sc3j2Ps_Rk,1071
-sec2md-0.1.0.dist-info/METADATA,sha256=qJLRPZiCME3MyDIH3pin32_EJGfwk9KTQTfgfDoOYBs,7840
-sec2md-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sec2md-0.1.0.dist-info/top_level.txt,sha256=Jpmw3laEWwS9fljtAEg4sExjFw3zP8dGarjIknyh1v8,7
-sec2md-0.1.0.dist-info/RECORD,,
+sec2md-0.1.1.dist-info/licenses/LICENSE,sha256=uJDiSGQ5TOx-PGhu2LGH4A-O53vS5hrQ5sc3j2Ps_Rk,1071
+sec2md-0.1.1.dist-info/METADATA,sha256=QcLiN6MRfjo_VQyyruCwn15RVnrsJqUxW_AgViPjNJ4,7594
+sec2md-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sec2md-0.1.1.dist-info/top_level.txt,sha256=Jpmw3laEWwS9fljtAEg4sExjFw3zP8dGarjIknyh1v8,7
+sec2md-0.1.1.dist-info/RECORD,,