headson 0.4.0__tar.gz → 0.5.0__tar.gz

{headson-0.4.0 → headson-0.5.0}/Cargo.lock

@@ -266,7 +266,7 @@ dependencies = [
 
 [[package]]
 name = "headson"
-version = "0.4.0"
+version = "0.5.0"
 dependencies = [
  "anyhow",
  "assert_cmd",

{headson-0.4.0 → headson-0.5.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "headson"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2024"
 description = "Budget‑constrained JSON preview renderer"
 readme = "README.md"

{headson-0.4.0 → headson-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: headson
-Version: 0.4.0
+Version: 0.5.0
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
@@ -66,7 +66,8 @@ Common flags:
 - `--no-space`: no space after `:` in objects
 - `--indent <STR>`: indentation unit (default: two spaces)
 - `--string-cap <N>`: max graphemes to consider per string (default: 500)
-- `--tail`: prefer the end of arrays when truncating. Strings are unaffected. In `pseudo`/`js` templates the omission marker appears at the start; `json` remains strict JSON with no annotations.
+- `--head`: prefer the beginning of arrays when truncating (keep first N). Strings are unaffected. In `pseudo`/`js` templates the omission marker appears near the end; `json` remains strict. Mutually exclusive with `--tail`.
+- `--tail`: prefer the end of arrays when truncating (keep last N). Strings are unaffected. In `pseudo`/`js` templates the omission marker appears at the start; `json` remains strict. Mutually exclusive with `--head`.
 
 Notes:
 
@@ -77,6 +78,7 @@ Notes:
   - Using `--global-budget` may truncate or omit entire files to respect the total budget.
   - The tool finds the largest preview that fits the budget; if even the tiniest preview exceeds it, you still get a minimal, valid preview.
   - When passing file paths, directories and binary files are ignored; a notice is printed to stderr for each (e.g., `Ignored binary file: ./path/to/file`). Stdin mode reads the stream as-is.
+  - Head vs Tail sampling: these options bias which part of arrays are kept before rendering. They guarantee the kept segment is contiguous at the chosen side (prefix for `--head`, suffix for `--tail`). Display templates may still insert additional internal gap markers inside that kept segment to honor very small budgets; `json` remains strict and unannotated.
 
 Quick one‑liners:
 
@@ -137,10 +139,10 @@ A thin Python extension module is available on PyPI as `headson`.
 
  - Install: `pip install headson` (ABI3 wheels for Python 3.10+ on Linux/macOS/Windows).
 - API:
-  - `headson.summarize(text: str, *, template: str = "pseudo", character_budget: int | None = None, tail: bool = False) -> str`
+  - `headson.summarize(text: str, *, template: str = "pseudo", character_budget: int | None = None, skew: str = "balanced") -> str`
     - `template`: one of `"json" | "pseudo" | "js"`
     - `character_budget`: maximum output size in characters (default: 500)
-  - `tail`: prefer the end of arrays when truncating; strings unaffected. Affects only display templates (`pseudo`/`js`); `json` remains strict.
+    - `skew`: one of `"balanced" | "head" | "tail"` (focus arrays on start vs end; only affects display templates; `json` remains strict).
 
 Example:
 
@@ -158,11 +160,56 @@ print(
         json.dumps(list(range(100))),
         template="pseudo",
         character_budget=80,
-        tail=True,
+        skew="tail",
     )
 )
 ```
 
+# Algorithm
+
+```mermaid
+%%{init: {"themeCSS": ".cluster > rect { fill: transparent; stroke: transparent; } .clusterLabel > text { font-size: 16px; font-weight: 600; } .clusterLabel span { padding: 6px 10px; font-size: 16px; font-weight: 600; }"}}%%
+flowchart TD
+    subgraph Deserialization
+        direction TB
+        A["Input file(s)"]
+        A -- Single --> C["Parse into optimized tree (with array pre‑sampling) ¹"]
+        A -- Multiple --> D["Parse each file and wrap into a fileset object"]
+        D --> C
+    end
+    subgraph Prioritization
+        direction TB
+        E["Build priority order ²"]
+        F["Choose top N nodes ³"]
+    end
+    subgraph Serialization
+        direction TB
+        G["Render attempt ⁴"]
+        H["Output preview string"]
+    end
+    C --> E
+    E --> F
+    F --> G
+    G --> F
+    F --> H
+    %% Color classes for categories
+    classDef des fill:#eaf2ff,stroke:#3b82f6,stroke-width:1px,color:#0f172a;
+    classDef prio fill:#ecfdf5,stroke:#10b981,stroke-width:1px,color:#064e3b;
+    classDef ser fill:#fff1f2,stroke:#f43f5e,stroke-width:1px,color:#7f1d1d;
+    class A,C,D des;
+    class E,F prio;
+    class G,H ser;
+    style Deserialization fill:transparent,stroke:transparent
+    style Prioritization fill:transparent,stroke:transparent
+    style Serialization fill:transparent,stroke:transparent
+```
+
+## Footnotes
+ - <sup><b>[1]</b></sup> <b>Optimized tree representation</b>: An arena‑style tree stored in flat, contiguous buffers. Each node records its kind and value plus index ranges into shared child and key arrays. Arrays are ingested in a single pass and may be deterministically pre‑sampled: the first element is always kept; additional elements are selected via a fixed per‑index inclusion test; for kept elements, original indices are stored and full lengths are counted. This enables accurate omission info and internal gap markers later, while minimizing pointer chasing.
+ - <sup><b>[2]</b></sup> <b>Priority order</b>: Nodes are scored so previews surface representative structure and values first. Arrays can favor head/mid/tail coverage (default) or strictly the head; tail preference flips head/tail when configured. Object properties are ordered by key, and strings expand by grapheme with early characters prioritized over very deep expansions.
+ - <sup><b>[3]</b></sup> <b>Choose top N nodes (binary search)</b>: Iteratively picks N so that the rendered preview fits within the character budget, looping between “choose N” and a render attempt to converge quickly.
+ - <sup><b>[4]</b></sup> <b>Render attempt</b>: Serializes the currently included nodes using the selected template. Omission summaries and per-file section headers appear in display templates (pseudo/js); json remains strict. For arrays, display templates may insert internal gap markers between non‑contiguous kept items using original indices.
+
 ## License
 
 MIT

{headson-0.4.0 → headson-0.5.0}/README.md

@@ -51,7 +51,8 @@ Common flags:
 - `--no-space`: no space after `:` in objects
 - `--indent <STR>`: indentation unit (default: two spaces)
 - `--string-cap <N>`: max graphemes to consider per string (default: 500)
-- `--tail`: prefer the end of arrays when truncating. Strings are unaffected. In `pseudo`/`js` templates the omission marker appears at the start; `json` remains strict JSON with no annotations.
+- `--head`: prefer the beginning of arrays when truncating (keep first N). Strings are unaffected. In `pseudo`/`js` templates the omission marker appears near the end; `json` remains strict. Mutually exclusive with `--tail`.
+- `--tail`: prefer the end of arrays when truncating (keep last N). Strings are unaffected. In `pseudo`/`js` templates the omission marker appears at the start; `json` remains strict. Mutually exclusive with `--head`.
 
 Notes:
 
@@ -62,6 +63,7 @@ Notes:
   - Using `--global-budget` may truncate or omit entire files to respect the total budget.
   - The tool finds the largest preview that fits the budget; if even the tiniest preview exceeds it, you still get a minimal, valid preview.
   - When passing file paths, directories and binary files are ignored; a notice is printed to stderr for each (e.g., `Ignored binary file: ./path/to/file`). Stdin mode reads the stream as-is.
+  - Head vs Tail sampling: these options bias which part of arrays are kept before rendering. They guarantee the kept segment is contiguous at the chosen side (prefix for `--head`, suffix for `--tail`). Display templates may still insert additional internal gap markers inside that kept segment to honor very small budgets; `json` remains strict and unannotated.
 
 Quick one‑liners:
 
@@ -122,10 +124,10 @@ A thin Python extension module is available on PyPI as `headson`.
 
  - Install: `pip install headson` (ABI3 wheels for Python 3.10+ on Linux/macOS/Windows).
 - API:
-  - `headson.summarize(text: str, *, template: str = "pseudo", character_budget: int | None = None, tail: bool = False) -> str`
+  - `headson.summarize(text: str, *, template: str = "pseudo", character_budget: int | None = None, skew: str = "balanced") -> str`
     - `template`: one of `"json" | "pseudo" | "js"`
     - `character_budget`: maximum output size in characters (default: 500)
-  - `tail`: prefer the end of arrays when truncating; strings unaffected. Affects only display templates (`pseudo`/`js`); `json` remains strict.
+    - `skew`: one of `"balanced" | "head" | "tail"` (focus arrays on start vs end; only affects display templates; `json` remains strict).
 
 Example:
 
@@ -143,11 +145,56 @@ print(
         json.dumps(list(range(100))),
         template="pseudo",
         character_budget=80,
-        tail=True,
+        skew="tail",
     )
 )
 ```
 
+# Algorithm
+
+```mermaid
+%%{init: {"themeCSS": ".cluster > rect { fill: transparent; stroke: transparent; } .clusterLabel > text { font-size: 16px; font-weight: 600; } .clusterLabel span { padding: 6px 10px; font-size: 16px; font-weight: 600; }"}}%%
+flowchart TD
+    subgraph Deserialization
+        direction TB
+        A["Input file(s)"]
+        A -- Single --> C["Parse into optimized tree (with array pre‑sampling) ¹"]
+        A -- Multiple --> D["Parse each file and wrap into a fileset object"]
+        D --> C
+    end
+    subgraph Prioritization
+        direction TB
+        E["Build priority order ²"]
+        F["Choose top N nodes ³"]
+    end
+    subgraph Serialization
+        direction TB
+        G["Render attempt ⁴"]
+        H["Output preview string"]
+    end
+    C --> E
+    E --> F
+    F --> G
+    G --> F
+    F --> H
+    %% Color classes for categories
+    classDef des fill:#eaf2ff,stroke:#3b82f6,stroke-width:1px,color:#0f172a;
+    classDef prio fill:#ecfdf5,stroke:#10b981,stroke-width:1px,color:#064e3b;
+    classDef ser fill:#fff1f2,stroke:#f43f5e,stroke-width:1px,color:#7f1d1d;
+    class A,C,D des;
+    class E,F prio;
+    class G,H ser;
+    style Deserialization fill:transparent,stroke:transparent
+    style Prioritization fill:transparent,stroke:transparent
+    style Serialization fill:transparent,stroke:transparent
+```
+
+## Footnotes
+ - <sup><b>[1]</b></sup> <b>Optimized tree representation</b>: An arena‑style tree stored in flat, contiguous buffers. Each node records its kind and value plus index ranges into shared child and key arrays. Arrays are ingested in a single pass and may be deterministically pre‑sampled: the first element is always kept; additional elements are selected via a fixed per‑index inclusion test; for kept elements, original indices are stored and full lengths are counted. This enables accurate omission info and internal gap markers later, while minimizing pointer chasing.
+ - <sup><b>[2]</b></sup> <b>Priority order</b>: Nodes are scored so previews surface representative structure and values first. Arrays can favor head/mid/tail coverage (default) or strictly the head; tail preference flips head/tail when configured. Object properties are ordered by key, and strings expand by grapheme with early characters prioritized over very deep expansions.
+ - <sup><b>[3]</b></sup> <b>Choose top N nodes (binary search)</b>: Iteratively picks N so that the rendered preview fits within the character budget, looping between “choose N” and a render attempt to converge quickly.
+ - <sup><b>[4]</b></sup> <b>Render attempt</b>: Serializes the currently included nodes using the selected template. Omission summaries and per-file section headers appear in display templates (pseudo/js); json remains strict. For arrays, display templates may insert internal gap markers between non‑contiguous kept items using original indices.
+
 ## License
 
 MIT

{headson-0.4.0 → headson-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "headson"
-version = "0.4.0"
+version = "0.5.0"
 description = "Budget‑constrained JSON preview renderer (Python bindings)"
 readme = "README.md"
 requires-python = ">=3.10"

{headson-0.4.0 → headson-0.5.0}/python/Cargo.lock

@@ -169,7 +169,7 @@ dependencies = [
 
 [[package]]
 name = "headson"
-version = "0.4.0"
+version = "0.5.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -182,7 +182,7 @@ dependencies = [
 
 [[package]]
 name = "headson-python"
-version = "0.4.0"
+version = "0.5.0"
 dependencies = [
  "anyhow",
  "headson",

{headson-0.4.0 → headson-0.5.0}/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "headson-python"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2021"
 publish = false
 readme = "README.md"

{headson-0.4.0 → headson-0.5.0}/python/README.md

@@ -4,10 +4,13 @@ Minimal Python API for the `headson` JSON preview renderer.
 
 Currently exported function:
 
-- `headson.summarize(text: str, *, template: str = "pseudo", character_budget: int | None = None, tail: bool = False) -> str`
+- `headson.summarize(text: str, *, template: str = "pseudo", character_budget: int | None = None, skew: str = "balanced") -> str`
   - `template`: one of `"json" | "pseudo" | "js"`.
   - `character_budget`: maximum output size in characters (defaults to 500 if not set).
-  - `tail`: prefer the end of arrays when truncating; strings are unaffected. Only affects display templates (`pseudo`/`js`); `json` remains strict JSON with no annotations.
+  - `skew`: one of `"balanced" | "head" | "tail"`.
+    - `balanced`: default behavior.
+    - `head`: focus arrays on the beginning (keep first N).
+    - `tail`: focus arrays on the end (keep last N); pseudo/js place omission markers at the start.
 
 Examples:
 
@@ -20,9 +23,9 @@ print(headson.summarize('{"a": 1, "b": [1,2,3]}', template="pseudo", character_b
 # Strict JSON template preserves valid JSON output
 print(headson.summarize('{"a": 1, "b": {"c": 2}}', template="json", character_budget=10_000))
 
-# JS template with tail preference: prefer the end of arrays when truncating
+# JS template with tail skew: prefer the end of arrays when truncating
 arr = ','.join(str(i) for i in range(100))
-print(headson.summarize('{"arr": [' + arr + ']}', template="js", character_budget=60, tail=True))
+print(headson.summarize('{"arr": [' + arr + ']}', template="js", character_budget=60, skew="tail"))
 
 # Note: tail mode affects only pseudo/js display templates; the json template stays strict.
 ```

headson-0.5.0/python/headson/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+# Directly re-export the compiled extension function with the final signature.
+from .headson import summarize  # type: ignore
+
+__all__ = ["summarize"]

{headson-0.4.0 → headson-0.5.0}/python/src/lib.rs

@@ -2,7 +2,7 @@ use anyhow::{bail, Result};
 use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::*;
 use pyo3::types::PyModule;
-use headson_core::{OutputTemplate, RenderConfig, PriorityConfig};
+use headson_core::{ArraySamplerStrategy, OutputTemplate, PriorityConfig, RenderConfig};
 
 fn to_template(s: &str) -> Result<OutputTemplate> {
     match s.to_ascii_lowercase().as_str() {
@@ -13,25 +13,41 @@ fn to_template(s: &str) -> Result<OutputTemplate> {
     }
 }
 
-fn render_config(template: &str, prefer_tail_arrays: bool) -> Result<RenderConfig> {
+fn render_config_with_sampler(
+    template: &str,
+    sampler: ArraySamplerStrategy,
+) -> Result<RenderConfig> {
     let t = to_template(template)?;
     let space = " ".to_string();
     let newline = "\n".to_string();
     let indent_unit = "  ".to_string();
-    Ok(RenderConfig {
-        template: t,
-        indent_unit,
-        space,
-        newline,
-        prefer_tail_arrays,
-    })
+    let prefer_tail_arrays = matches!(sampler, ArraySamplerStrategy::Tail);
+    Ok(RenderConfig { template: t, indent_unit, space, newline, prefer_tail_arrays })
+}
+
+fn parse_skew(skew: &str) -> Result<ArraySamplerStrategy> {
+    match skew.to_ascii_lowercase().as_str() {
+        "balanced" => Ok(ArraySamplerStrategy::Default),
+        "head" => Ok(ArraySamplerStrategy::Head),
+        "tail" => Ok(ArraySamplerStrategy::Tail),
+        other => bail!(
+            "unknown skew: {} (expected 'balanced' | 'head' | 'tail')",
+            other
+        ),
+    }
 }
 
-fn priority_config(per_file_budget: usize, prefer_tail_arrays: bool) -> PriorityConfig {
+fn priority_config(
+    per_file_budget: usize,
+    sampler: ArraySamplerStrategy,
+) -> PriorityConfig {
+    let prefer_tail_arrays = matches!(sampler, ArraySamplerStrategy::Tail);
     PriorityConfig {
         max_string_graphemes: 500,
         array_max_items: (per_file_budget / 2).max(1),
         prefer_tail_arrays,
+        array_bias: headson_core::ArrayBias::HeadMidTail,
+        array_sampler: sampler,
     }
 }
 
@@ -40,18 +56,19 @@ fn to_pyerr(e: anyhow::Error) -> PyErr {
 }
 
 #[pyfunction]
-#[pyo3(signature = (text, *, template="pseudo", character_budget=None, tail=false))]
+#[pyo3(signature = (text, *, template="pseudo", character_budget=None, skew="balanced"))]
 fn summarize(
     py: Python<'_>,
     text: &str,
     template: &str,
     character_budget: Option<usize>,
-    tail: bool,
+    skew: &str,
 ) -> PyResult<String> {
-    let cfg = render_config(template, tail).map_err(to_pyerr)?;
+    let sampler = parse_skew(skew).map_err(to_pyerr)?;
+    let cfg = render_config_with_sampler(template, sampler).map_err(to_pyerr)?;
     let budget = character_budget.unwrap_or(500);
     let per_file_for_priority = budget.max(1);
-    let prio = priority_config(per_file_for_priority, tail);
+    let prio = priority_config(per_file_for_priority, sampler);
     let input = text.as_bytes().to_vec();
     py.detach(|| headson_core::headson(input, &cfg, &prio, budget).map_err(to_pyerr))
 }

{headson-0.4.0 → headson-0.5.0}/src/json_ingest/builder.rs

@@ -1,21 +1,28 @@
 use serde::Deserializer;
-use serde::de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor};
+use serde::de::{DeserializeSeed, MapAccess, SeqAccess, Visitor};
 use std::cell::RefCell;
 
 use crate::order::NodeKind;
 use crate::utils::tree_arena::{JsonTreeArena, JsonTreeNode};
 
+use super::samplers::ArraySamplerKind;
+
 #[derive(Default)]
 pub(crate) struct JsonTreeBuilder {
     arena: RefCell<JsonTreeArena>,
     pub(crate) array_cap: usize,
+    sampler: ArraySamplerKind,
 }
 
 impl JsonTreeBuilder {
-    pub(crate) fn new(array_cap: usize) -> Self {
+    pub(crate) fn new(
+        array_cap: usize,
+        sampler: super::samplers::ArraySamplerKind,
+    ) -> Self {
         Self {
             arena: RefCell::new(JsonTreeArena::default()),
             array_cap,
+            sampler,
         }
     }
 
@@ -88,15 +95,34 @@ impl JsonTreeBuilder {
         kept: usize,
         total: usize,
         local_children: Vec<usize>,
+        local_indices: Vec<usize>,
     ) {
         let mut a = self.arena.borrow_mut();
         let children_start = a.children.len();
         a.children.extend(local_children);
+
+        // Detect contiguous indices 0..kept-1 to skip storing arr_indices data
+        let contiguous = local_indices.len() == kept
+            && local_indices.iter().enumerate().all(|(i, &idx)| idx == i);
+
+        let (arr_indices_start, pushed_len) =
+            if kept == 0 || contiguous || local_indices.is_empty() {
+                (0usize, 0usize)
+            } else {
+                let start = a.arr_indices.len();
+                a.arr_indices.extend(local_indices);
+                let pushed = a.arr_indices.len().saturating_sub(start);
+                (start, pushed)
+            };
+
         let n = &mut a.nodes[id];
         n.kind = NodeKind::Array;
         n.children_start = children_start;
         n.children_len = kept;
         n.array_len = Some(total);
+        n.arr_indices_start = arr_indices_start;
+        // When no indices were pushed, mark len=0 to indicate contiguous 0..kept
+        n.arr_indices_len = pushed_len.min(kept);
     }
 
     fn finish_object(
@@ -205,29 +231,19 @@ impl<'de> Visitor<'de> for NodeVisitor<'_> {
         A: SeqAccess<'de>,
     {
         let id = self.b.push_default();
-        let mut local_children: Vec<usize> = Vec::new();
-        let low = seq.size_hint().unwrap_or(0);
-        local_children.reserve(low.min(self.b.array_cap));
-        let mut kept = 0usize;
-        let mut total = 0usize;
-        while kept < self.b.array_cap {
-            let next = {
-                let seed = self.b.seed();
-                seq.next_element_seed(seed)?
-            };
-            match next {
-                Some(cid) => {
-                    local_children.push(cid);
-                    kept += 1;
-                    total += 1;
-                }
-                None => break,
-            }
-        }
-        while (seq.next_element::<IgnoredAny>()?).is_some() {
-            total += 1;
-        }
-        self.b.finish_array(id, kept, total, local_children);
+        let sampled = self.b.sampler.sample_stream(
+            &mut seq,
+            self.b,
+            self.b.array_cap,
+        )?;
+        let kept = sampled.children.len();
+        self.b.finish_array(
+            id,
+            kept,
+            sampled.total_len,
+            sampled.children,
+            sampled.indices,
+        );
         Ok(id)
     }
 
@@ -255,3 +271,5 @@ impl<'de> Visitor<'de> for NodeVisitor<'_> {
         Ok(id)
     }
 }
+
+impl JsonTreeBuilder {}

{headson-0.4.0 → headson-0.5.0}/src/json_ingest/mod.rs

@@ -1,7 +1,8 @@
 mod builder;
+mod samplers;
 use serde::de::DeserializeSeed;
 
-use crate::order::PriorityConfig;
+use crate::PriorityConfig;
 use crate::utils::tree_arena::JsonTreeArena;
 use anyhow::Result;
 use builder::JsonTreeBuilder;
@@ -19,7 +20,10 @@ pub fn build_json_tree_arena_from_bytes(
     config: &PriorityConfig,
 ) -> Result<JsonTreeArena> {
     let mut de = simd_json::Deserializer::from_slice(&mut bytes)?;
-    let builder = JsonTreeBuilder::new(config.array_max_items);
+    let builder = JsonTreeBuilder::new(
+        config.array_max_items,
+        config.array_sampler.into(),
+    );
     let root_id: usize = {
         let seed = builder.seed();
         seed.deserialize(&mut de)?
@@ -33,7 +37,10 @@ pub fn build_json_tree_arena_from_many(
     mut inputs: Vec<(String, Vec<u8>)>,
     config: &PriorityConfig,
 ) -> Result<JsonTreeArena> {
-    let builder = JsonTreeBuilder::new(config.array_max_items);
+    let builder = JsonTreeBuilder::new(
+        config.array_max_items,
+        config.array_sampler.into(),
+    );
     let mut child_ids: Vec<usize> = Vec::with_capacity(inputs.len());
     let mut keys: Vec<String> = Vec::with_capacity(inputs.len());
     for (key, mut bytes) in inputs.drain(..) {