doc2struct 0.1.0__tar.gz → 0.1.3__tar.gz

doc2struct-0.1.3/.gitattributes

@@ -0,0 +1,2 @@
+# Keep vendored patches reproducible; blank context lines carry a space marker.
+c/vendor/pdfium/text_extraction.patch text eol=lf -whitespace

{doc2struct-0.1.0 → doc2struct-0.1.3}/.github/workflows/release.yml

@@ -48,7 +48,7 @@ jobs:
           - runner: windows-2022
             runtime: win_amd64
             args: ""
-          - runner: macos-14
+          - runner: macos-15
             runtime: macos_arm64
             args: ""
           - runner: macos-15-intel
@@ -67,7 +67,7 @@ jobs:
         id: runtime-cache
         with:
           path: c/vendor/pdfium/runtime/${{ matrix.runtime }}
-          key: pdfium-${{ matrix.runner }}-${{ matrix.runtime }}-${{ hashFiles('build_scripts/build_pdfium.py', 'c/vendor/pdfium/bridge.cpp', 'c/vendor/pdfium/bridge.h') }}
+          key: pdfium-${{ matrix.runner }}-${{ matrix.runtime }}-${{ hashFiles('build_scripts/build_pdfium.py', 'build_scripts/validate_pdfium.py', 'c/vendor/pdfium/bridge.cpp', 'c/vendor/pdfium/bridge.h', 'c/vendor/pdfium/text_mode.h', 'c/vendor/pdfium/text_extraction.patch') }}
       - name: Build pinned private PDFium once per platform
         if: steps.runtime-cache.outputs.cache-hit != 'true'
         run: python build_scripts/build_pdfium.py --fetch --jobs 4 ${{ matrix.args }}
@@ -99,7 +99,7 @@ jobs:
           - runner: windows-2022
             runtime: win_amd64
             arch: AMD64
-          - runner: macos-14
+          - runner: macos-15
             runtime: macos_arm64
             arch: arm64
           - runner: macos-15-intel
@@ -139,7 +139,7 @@ jobs:
           - runner: windows-2022
             runtime: win_amd64
             platform: win_amd64
-          - runner: macos-14
+          - runner: macos-15
             runtime: macos_arm64
             platform: macos_arm64
           - runner: macos-15-intel

{doc2struct-0.1.0 → doc2struct-0.1.3}/.gitignore

@@ -28,7 +28,7 @@ venv/
 *.exp
 cmake-build-*/
 
-# run.py output
+# Local experiments and generated outputs
 testing/
 
 # local filing data
@@ -39,3 +39,4 @@ data/
 .idea/
 .DS_Store
 Thumbs.db
+

{doc2struct-0.1.0 → doc2struct-0.1.3}/CMakeLists.txt

@@ -1,6 +1,17 @@
 cmake_minimum_required(VERSION 3.18)
 project(doc2struct LANGUAGES C)
 option(DOC2STRUCT_BUILD_PYTHON "Build the Python extension" OFF)
+option(DOC2STRUCT_ENABLE_IPO "Enable supported Release link-time optimization" ${DOC2STRUCT_BUILD_PYTHON})
+if(DOC2STRUCT_ENABLE_IPO AND NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION
+   AND NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE)
+    include(CheckIPOSupported)
+    check_ipo_supported(RESULT DOC2STRUCT_IPO_SUPPORTED OUTPUT DOC2STRUCT_IPO_ERROR LANGUAGES C)
+    if(DOC2STRUCT_IPO_SUPPORTED)
+        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE ON)
+    else()
+        message(STATUS "Link-time optimization unavailable: ${DOC2STRUCT_IPO_ERROR}")
+    endif()
+endif()
 add_subdirectory(c)
 
 if(DOC2STRUCT_BUILD_PYTHON)

{doc2struct-0.1.0 → doc2struct-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: doc2struct
-Version: 0.1.0
+Version: 0.1.3
 Summary: Parse HTML, PDF, text, and partial SGML into structured documents in C and Python.
 Author: John Friedman
 License-Expression: MIT
@@ -24,6 +24,7 @@ doc2struct is an algorithm that can parse documents fast on one thread. It has b
 ![Parsing a document](image.png)
 
 **installation**
+
 python 
 ```
 pip install doc2struct
@@ -38,52 +39,19 @@ cmake --build build --config Release
 cmake --install build --config Release --prefix ./install
 ```
 
-**functions**
+**quickstart**
 
 ```python
-# Parse directly to standardized text
-standardized_text = parse_html(html, rules=None)
-standardized_text = parse_pdf(pdf, rules=None)
-standardized_text = parse_text(text, rules=None)
-standardized_text = parse_partialsgml(text, rules=None)
-
-# Convert to instructions
-instructions = convert_html_to_instructions(html, rules=None)
-instructions = convert_pdf_to_instructions(pdf, rules=None)
-instructions = convert_text_to_instructions(text, rules=None)
-instructions = convert_partialsgml_to_instructions(text, rules=None)
-
-# Convert instructions to standardized text
-standardized_text = convert_instructions_to_standardized_text(
-    instructions, rules=None
-)
-
-# Visualize
-visualize_instructions(
-    instructions,
-    filename="instructions_visualization.html",
-    open_browser=True,
-)
-visualize_standardized_text(
-    standardized_text,
-    filename="standardized_text_visualization.html",
-    open_browser=True,
-)
-
-# Saving output
-import polars as pl
-from doc2struct import parse_html
+from doc2struct import parse_html, visualize_standardized_text
+# e.g. https://www.sec.gov/Archives/edgar/data/1318605/000095017022000796/tsla-20211231.htm
+with open("10-K.html","rb") as f:
+    html = f.read()
 
 standardized_text = parse_html(html, rules="SEC:10-K")
-pl.from_dicts(
-    standardized_text, infer_schema_length=None
-).write_parquet("document.parquet")
+visualize_standardized_text(standardized_text)
 ```
 
 
-
-
-
 It emits a compact data structure, preserving nesting, suitable for storage as parquet.
 
 | Column | Meaning | Example |
@@ -154,6 +122,49 @@ The ideal use case for doc2struct is as part of a [document router](https://data
 
 The first usecase of doc2struct will be processing every SEC html, pdf, and text file (10TB+) for about $0.04 (AWS spot estimate using c6gd.medium).
 
+## Functions
+
+```python
+# Parse directly to standardized text
+standardized_text = parse_html(html, rules=None)
+standardized_text = parse_pdf(pdf, rules=None)
+standardized_text = parse_text(text, rules=None)
+standardized_text = parse_partialsgml(text, rules=None)
+
+# Convert to instructions
+instructions = convert_html_to_instructions(html, rules=None)
+instructions = convert_pdf_to_instructions(pdf, rules=None)
+instructions = convert_text_to_instructions(text, rules=None)
+instructions = convert_partialsgml_to_instructions(text, rules=None)
+
+# Convert instructions to standardized text
+standardized_text = convert_instructions_to_standardized_text(
+    instructions, rules=None
+)
+
+# Visualize
+visualize_instructions(
+    instructions,
+    filename="instructions_visualization.html",
+    open_browser=True,
+)
+visualize_standardized_text(
+    standardized_text,
+    filename="standardized_text_visualization.html",
+    open_browser=True,
+)
+
+# Saving output
+import polars as pl
+from doc2struct import parse_html
+
+standardized_text = parse_html(html, rules="SEC:10-K")
+pl.from_dicts(
+    standardized_text, infer_schema_length=None
+).write_parquet("document.parquet")
+```
+
+
 ## Modalities supported
 
 - HTML
@@ -457,7 +468,7 @@ It should be fairly easy to add new adapters for different modalities. OCR -> in
     - HTML parsing speed benchmarked at 500 MB/s 
     - PDF parsing speed benchmarked at 25 MB/s 
     - TXT at 60 MB/s
-    - Rulesets defined for [n][ of sec corpus]
+    - Rulesets defined for 900+ SEC document types.
 
 ## Process
 
@@ -471,6 +482,8 @@ It should be fairly easy to add new adapters for different modalities. OCR -> in
 
 ## Benchmarks
 
+> Note: since this benchmark was first run, performance has been increased by ~50% on x86.
+
 | data | type | size | time | throughput |
 |---|---|---|---|---|
 | Every 2025 10-K and 10-K/A | html | 19,085.105 MB | 39,962 ms | 477.6 MB/s |

doc2struct-0.1.3/agents.md

@@ -0,0 +1 @@
+We don't need tests stop adding them.

{doc2struct-0.1.0 → doc2struct-0.1.3}/build_scripts/build_pdfium.py

@@ -15,6 +15,8 @@ import shutil
 import subprocess
 import sys
 
+from validate_pdfium import BRIDGE_ABI
+
 ROOT=Path(__file__).resolve().parents[1]
 HERE=ROOT/'c/vendor/pdfium'
 SOURCE=ROOT/'build/pdfium'
@@ -64,6 +66,26 @@ def replace(path, old, new):
     path.write_text(text.replace(old,new,1))
 
 
+def patch_text_extraction():
+    patch=HERE/'text_extraction.patch'
+    command=['git','apply','--ignore-space-change','--whitespace=nowarn']
+    check=subprocess.run([*command,'--check',str(patch)],cwd=SOURCE,capture_output=True,text=True)
+    if check.returncode==0:
+        subprocess.run([*command,str(patch)],cwd=SOURCE,check=True)
+    else:
+        applied=subprocess.run([*command,'--reverse','--check',str(patch)],
+                               cwd=SOURCE,capture_output=True,text=True)
+        if applied.returncode:
+            raise RuntimeError('Cannot apply the text extraction patch; use a clean checkout of '
+                               'the pinned PDFium revision.\n'+check.stderr)
+    target=SOURCE/'core/fpdfapi/page/doc2struct_text_mode.h'
+    contents=(HERE/'text_mode.h').read_text(encoding='utf-8')
+    if not target.exists() or target.read_text(encoding='utf-8')!=contents:
+        target.write_text(contents,encoding='utf-8')
+    replace(SOURCE/'core/fpdfapi/page/BUILD.gn','source_set("page") {\n  sources = [',
+        'source_set("page") {\n  sources = [\n    "doc2struct_text_mode.h",')
+
+
 def runtime_key():
     arch=platform.machine().lower()
     arch={'amd64':'x86_64','aarch64':'arm64'}.get(arch,arch)
@@ -76,7 +98,7 @@ def bundle(binary, args, key, *, cross=False):
     if not cross:
         dll=ctypes.CDLL(str(binary.resolve()))
         dll.DS_PDF_BridgeVersion.restype=ctypes.c_uint
-        if dll.DS_PDF_BridgeVersion()!=1:
+        if dll.DS_PDF_BridgeVersion()!=BRIDGE_ABI:
             raise RuntimeError('Unexpected PDFium bridge ABI')
     target=HERE/'runtime'/key
     target.mkdir(parents=True,exist_ok=True)
@@ -97,7 +119,8 @@ def bundle(binary, args, key, *, cross=False):
                 parts.append(str(path.relative_to(SOURCE))+'\n\n'+path.read_text(encoding='utf-8',errors='replace'))
     (notices/'third_party.txt').write_text('\n\n'.join(parts),encoding='utf-8')
     metadata=dict(origin='doc2struct',major=149,minor=0,build=7825,patch=0,
-        revision=PDFIUM_REVISION,bridge_abi=1,platform=key,
+        revision=PDFIUM_REVISION,bridge_abi=BRIDGE_ABI,platform=key,
+        text_patch_sha256=hashlib.sha256((HERE/'text_extraction.patch').read_bytes()).hexdigest(),
         sha256=hashlib.sha256(binary.read_bytes()).hexdigest(),gn_args=args)
     (target/'version.json').write_text(json.dumps(metadata,indent=2)+'\n')
     print('Bundled '+str(target),flush=True)
@@ -119,6 +142,7 @@ def main():
     if options.fetch: fetch()
     revision=subprocess.check_output(['git','rev-parse','HEAD'],cwd=SOURCE,text=True).strip()
     if revision!=PDFIUM_REVISION: raise RuntimeError('Unexpected PDFium source revision: '+revision)
+    patch_text_extraction()
     replace(SOURCE/'.gn','script_executable = "python3"','script_executable = "'+Path(sys.executable).as_posix()+'"')
     for name in ['bridge.cpp','bridge.h']:
         contents=(HERE/name).read_text()

{doc2struct-0.1.0 → doc2struct-0.1.3}/build_scripts/validate_pdfium.py

@@ -5,12 +5,14 @@ import hashlib
 import json
 from pathlib import Path
 
+BRIDGE_ABI = 2
+
 
 def validate(binary, *, load=True, licenses=None):
     binary = Path(binary).resolve()
     metadata = json.loads(binary.with_name('version.json').read_text(encoding='utf-8'))
-    if metadata.get('bridge_abi') != 1:
-        raise ValueError('PDFium bridge ABI must be 1')
+    if metadata.get('bridge_abi') != BRIDGE_ABI:
+        raise ValueError(f'PDFium bridge ABI must be {BRIDGE_ABI}')
     if metadata.get('sha256') != hashlib.sha256(binary.read_bytes()).hexdigest():
         raise ValueError('PDFium binary does not match version.json')
     licenses = Path(licenses) if licenses is not None else binary.parent / 'licenses'
@@ -20,8 +22,9 @@ def validate(binary, *, load=True, licenses=None):
     if load:
         runtime = ctypes.CDLL(str(binary))
         runtime.DS_PDF_BridgeVersion.restype = ctypes.c_uint
-        if runtime.DS_PDF_BridgeVersion() != 1:
+        if runtime.DS_PDF_BridgeVersion() != BRIDGE_ABI:
             raise ValueError('Unsupported PDFium bridge ABI')
+        getattr(runtime, 'DS_PDF_LoadTextPage')
     return metadata
 
 

{doc2struct-0.1.0 → doc2struct-0.1.3}/c/CMakeLists.txt

@@ -50,6 +50,7 @@ endif()
 add_library(doc2struct STATIC
     src/preset_data.h
     src/api.c
+    src/html_pipeline.c
     src/pdf.c
     src/pdf_layout.c
     src/config.c
@@ -57,13 +58,9 @@ add_library(doc2struct STATIC
     src/memory.c
     src/text.c
     src/text_layout.c
-    src/tokenizer.c
-    src/normalizer.c
     src/rules.c
-    src/parser.c
     src/standardized_text.c
     src/lists.c
-    src/flow.c
     src/json.c
     src/tables.c
     vendor/pcre2/pcre2_auto_possess.c

{doc2struct-0.1.0 → doc2struct-0.1.3}/c/include/doc2struct/doc2struct.h

@@ -6,7 +6,7 @@
 extern "C" {
 #endif
 
-#define DOC2STRUCT_VERSION "0.1.0"
+#define DOC2STRUCT_VERSION "0.1.3"
 #define DOC2STRUCT_DEFAULT_CACHE_SIZE ((size_t)8 * 1024 * 1024)
 typedef struct doc2struct_rules doc2struct_rules;
 typedef struct doc2struct_parser doc2struct_parser;

{doc2struct-0.1.0 → doc2struct-0.1.3}/c/src/doc2struct.h

@@ -3,6 +3,7 @@
 #include <doc2struct/doc2struct.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <string.h>
 #define PCRE2_CODE_UNIT_WIDTH 8
 #include "pcre2.h"
 
@@ -115,9 +116,15 @@ typedef struct doc2struct_rules {
 typedef struct doc2struct_item {
     Run run;
     int fake_table;
+    unsigned casing; /* Lazy facts for assembled, immutable instruction text. */
     Vec *hrefs;
 } Item;
 typedef doc2struct_matrix Matrix;
+typedef struct {
+    Item *items;
+    size_t nitems;
+    Str display, rule;
+} TextView;
 typedef struct {
     Item *items;
     size_t nitems;
@@ -132,6 +139,7 @@ typedef struct {
     size_t ntable_cells;
     int layout_text;
     int redundant;
+    TextView *text_view;
 } Line;
 
 typedef struct ListLines {
@@ -170,21 +178,27 @@ typedef struct doc2struct_document {
 } Document;
 typedef int (*TokenSink)(void *, int, Str, const Attrs *);
 
-Str ds_str(const char *s);
-int ds_eq(Str a, Str b);
-int ds_is(Str a, const char *s);
+/* Keep these private helpers visible so literal lengths and short comparisons
+ * can be folded at each call site without link-time optimization. */
+static inline Str ds_str(const char *s) { return (Str){s, strlen(s)}; }
+static inline int ds_eq(Str a, Str b) {
+    return a.n == b.n && (!a.n || !memcmp(a.p, b.p, a.n));
+}
+static inline int ds_is(Str a, const char *s) { return a.p && ds_eq(a, ds_str(s)); }
 void *ds_alloc(Arena *, size_t);
 void *ds_alloc_uninit(Arena *, size_t);
 void ds_arena_free(Arena *);
 void ds_pool_clear(ArenaPool *);
 void *ds_push(Arena *, Vec *, size_t);
 void *ds_map_get(Map *, Str);
+void **ds_map_slot(Arena *, Map *, Str);
 int ds_map_put(Arena *, Map *, Str, void *);
 Str ds_copy(Arena *, Str);
 Str ds_cat(Arena *, Str, Str);
 Str ds_join(Arena *, Str *, size_t, Str);
 Str ds_strip(Str, int);
 Str ds_lower(Arena *, Str);
+Str ds_rule_text(Arena *, Str);
 int ds_space(uint32_t);
 int ds_upper(Str);
 int ds_proper(Str);

{doc2struct-0.1.0 → doc2struct-0.1.3}/c/src/flow.c

@@ -24,6 +24,8 @@ typedef struct {
 typedef struct {
     Document *document;
     Vec items, cells, parts;
+    FlowCell **grid;
+    size_t grid_cap;
     size_t block;
     size_t max_row, max_col, last_row, last_col;
     int active, table, has_coordinate, unordered;
@@ -280,9 +282,17 @@ static int table_finish(Flow *flow, Line *line) {
     }
     if (nc && nr > SIZE_MAX / nc / sizeof(Str))
         goto large;
-    grid = ds_alloc(&d->arena, nr * nc * sizeof(FlowCell *));
-    if (!grid)
-        return 0;
+    /* This grid is scratch storage. Output items and matrices own their arrays. */
+    size_t grid_size = nr * nc;
+    if (!flow->grid || grid_size > flow->grid_cap) {
+        grid = ds_alloc_uninit(&d->arena, grid_size * sizeof(*grid));
+        if (!grid)
+            return 0;
+        flow->grid = grid;
+        flow->grid_cap = grid_size;
+    }
+    grid = flow->grid;
+    memset(grid, 0, grid_size * sizeof(*grid));
     for (i = 0; i < flow->cells.n; i++) {
         FlowCell *cell = cells + i;
         for (y = 0; y < cell->rs; y++)

doc2struct-0.1.3/c/src/html_pipeline.c

@@ -0,0 +1,14 @@
+/* Keep the HTML stages together so the tokenizer can call the small text,
+ * start and end handlers directly, without changing normalization rules. */
+#if defined(_MSC_VER)
+#define DS_HTML_INLINE __forceinline
+#elif defined(__GNUC__) || defined(__clang__)
+#define DS_HTML_INLINE inline __attribute__((always_inline))
+#else
+#define DS_HTML_INLINE inline
+#endif
+#include "flow.c"
+#include "normalizer.c"
+#include "tokenizer.c"
+#include "parser.c"
+#undef DS_HTML_INLINE

{doc2struct-0.1.0 → doc2struct-0.1.3}/c/src/memory.c

@@ -15,12 +15,6 @@ struct Block {
     Alignment alignment;
     unsigned char data[];
 };
-Str ds_str(const char *s) {
-    Str r = {s, strlen(s)};
-    return r;
-}
-int ds_eq(Str a, Str b) { return a.n == b.n && (!a.n || !memcmp(a.p, b.p, a.n)); }
-int ds_is(Str a, const char *s) { return a.p && ds_eq(a, ds_str(s)); }
 static Block *pool_take(ArenaPool *pool, size_t capacity) {
     Block **link, **best = NULL, *block;
     if (!pool || !pool->blocks)
@@ -119,7 +113,46 @@ void *ds_push(Arena *a, Vec *v, size_t width) {
     }
     return (char *)v->p + v->n++ * width;
 }
+#if defined(_M_X64)
+#include <intrin.h>
+#include <nmmintrin.h>
+static int crc_available(void) {
+    static volatile long cached = -1;
+    long result = cached;
+    if (result < 0) {
+        int registers[4];
+        __cpuid(registers, 1);
+        result = (registers[2] >> 20) & 1;
+        _InterlockedCompareExchange(&cached, result, -1);
+    }
+    return result != 0;
+}
+static uint64_t crc_string(Str s) {
+    uint64_t a = s.n, b = 0x9e3779b1U, c = 0x85ebca77U, d = 0xc2b2ae3dU;
+    size_t i = 0;
+    for (; s.n - i >= 32; i += 32) {
+        uint64_t words[4];
+        memcpy(words, s.p + i, sizeof(words));
+        a = _mm_crc32_u64(a, words[0]); b = _mm_crc32_u64(b, words[1]);
+        c = _mm_crc32_u64(c, words[2]); d = _mm_crc32_u64(d, words[3]);
+    }
+    for (; s.n - i >= 8; i += 8) {
+        uint64_t word;
+        memcpy(&word, s.p + i, sizeof(word));
+        a = _mm_crc32_u64(a, word);
+    }
+    for (; i < s.n; i++) a = _mm_crc32_u8((unsigned)a, (unsigned char)s.p[i]);
+    uint64_t hash = a ^ (b << 32) ^ ((c << 17) | (c >> 47)) ^ ((d << 43) | (d >> 21));
+    hash ^= hash >> 33; hash *= 0xff51afd7ed558ccdULL;
+    hash ^= hash >> 33; hash *= 0xc4ceb9fe1a85ec53ULL;
+    hash ^= hash >> 33;
+    return hash ? hash : 1; /* Zero marks an unused Map entry. */
+}
+#endif
 static uint64_t hash(Str s) {
+#if defined(_M_X64)
+    if (s.n >= 32 && crc_available()) return crc_string(s);
+#endif
     uint64_t h = 0x9e3779b97f4a7c15ULL ^ (uint64_t)s.n, word;
     size_t i = 0;
     /* Compare complete keys after hashing; process unaligned words without
@@ -183,6 +216,47 @@ int ds_map_put(Arena *a, Map *m, Str k, void *value) {
     m->entries[i] = (Entry){k, value, h};
     return 1;
 }
+
+void **ds_map_slot(Arena *a, Map *m, Str k) {
+    size_t i;
+    uint64_t h = hash(k);
+    /* Resolve existing keys before considering growth; one hash per key. */
+    if (m->cap) {
+        i = (size_t)h & (m->cap - 1);
+        while (m->entries[i].hash) {
+            if (m->entries[i].hash == h && ds_eq(m->entries[i].key, k))
+                return &m->entries[i].value;
+            i = (i + 1) & (m->cap - 1);
+        }
+    }
+    if (!m->cap || (m->n + 1) * 10 > m->cap * 7) {
+        size_t cap = m->cap ? m->cap * 2 : 32, j;
+        Entry *entries;
+        if (cap < m->cap || cap > SIZE_MAX / sizeof(Entry)) {
+            snprintf(a->error, sizeof(a->error), "map too large");
+            return 0;
+        }
+        entries = ds_alloc(a, cap * sizeof(Entry));
+        if (!entries)
+            return 0;
+        for (j = 0; j < m->cap; j++)
+            if (m->entries[j].hash) {
+                i = (size_t)m->entries[j].hash & (cap - 1);
+                while (entries[i].hash)
+                    i = (i + 1) & (cap - 1);
+                entries[i] = m->entries[j];
+            }
+        m->entries = entries;
+        m->cap = cap;
+    }
+    i = (size_t)h & (m->cap - 1);
+    while (m->entries[i].hash && !ds_eq(m->entries[i].key, k))
+        i = (i + 1) & (m->cap - 1);
+    if (!m->entries[i].hash)
+        m->n++;
+    m->entries[i] = (Entry){k, NULL, h};
+    return &m->entries[i].value;
+}
 Str ds_copy(Arena *a, Str s) {
     char *p = ds_alloc_uninit(a, s.n + 1);
     if (!p)