softschema 0.4.0__tar.gz → 0.5.0__tar.gz

{softschema-0.4.0 → softschema-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: softschema
-Version: 0.4.0
+Version: 0.5.0
 Summary: Soft schema conventions and validation tools for Markdown/YAML artifacts
 Project-URL: Homepage, https://github.com/jlevy/softschema
 Project-URL: Repository, https://github.com/jlevy/softschema

{softschema-0.4.0 → softschema-0.5.0}/docs/softschema-python-design.md

@@ -169,6 +169,16 @@ free-form mappings are unaffected.
 The overlay is validation-time only; compiled schemas never change.
 `validate_structural` exposes the same behavior via its `strict_extras` keyword.
 
+Compiling a schema is a pure function of the schema text and `strict_extras`, so
+`validate_structural` memoizes it rather than reparsing and recompiling on every call.
+The cache is keyed on the schema text itself, not on the file’s path or stat, so a
+regenerated schema can never be served from a stale entry and identical bytes at two
+paths share one entry.
+Validation that passes `resources` builds a fresh validator, because that mapping is
+neither hashable nor cheap to fingerprint, and compile failures are not cached.
+`clear_validator_cache` drops every entry, for a long-lived process that regenerates
+compiled schemas in place; ordinary callers never need it.
+
 There are two public entry points: `validate_artifact` (above) for Markdown/YAML
 documents, and `validate_values` for an already-extracted mapping (a body-form runtime,
 a structured-output adapter, a fixture).
@@ -287,7 +297,6 @@ The current first-release kinds:
 | --- | --- |
 | `artifact_unreadable` | The artifact cannot be opened. |
 | `artifact_invalid_utf8` | The artifact is not valid UTF-8. |
-| `artifact_too_large` | The artifact exceeds the bounded input size. |
 | `yaml_parse_error` | YAML or frontmatter could not be parsed. |
 | `no_frontmatter` | Frontmatter block is missing in a Markdown artifact. |
 | `frontmatter_not_mapping` | Frontmatter parsed but is not a mapping at the top level. |

{softschema-0.4.0 → softschema-0.5.0}/docs/softschema-spec.md

@@ -110,10 +110,16 @@ date or timestamp; a semantic model or explicit structural assertion may impose
 constraint. An explicit tag such as `!!timestamp` remains unsupported under the
 explicit-tag rule.
 
-One input is at most 1 MiB, one scalar is at most 256 KiB, and one YAML document has at
-most 100,000 scalar or collection nodes and 64 simultaneously open collections,
-including the root. Exceeding a structure limit is `yaml_limit`, including a host YAML
-parser stack overflow caused by hostile nesting.
+One YAML document has at most 64 simultaneously open collections, including the root.
+Exceeding that depth is `yaml_limit`, as is a host YAML parser stack overflow.
+The depth rule is a portability rule rather than a resource ceiling: left to the host,
+the two runtimes disagree by an order of magnitude about how deep a document may be, so
+a fixed bound is what makes depth mean the same thing in both.
+
+Input size, scalar size, and node count are not bounded.
+Those ceilings only answered whether a hostile document could exhaust the parser, and
+softschema reads artifacts its own callers just wrote; a cap that can only fire after
+the artifact is written destroys completed work rather than rejecting bad input.
 
 For `frontmatter-md`, the opening and closing delimiters begin at column one and contain
 `---` plus optional trailing whitespace.

{softschema-0.4.0 → softschema-0.5.0}/packages/python/src/softschema/__init__.py

@@ -27,6 +27,7 @@ from softschema.validate import (
     SemanticResult,
     StructuralResult,
     ValidationResult,
+    clear_validator_cache,
     infer_envelope_key,
     validate_artifact,
     validate_semantic,
@@ -56,6 +57,7 @@ __all__ = [
     "StructuralResult",
     "ValidationResult",
     "WarningCode",
+    "clear_validator_cache",
     "compile_model",
     "infer_envelope_key",
     "parse_schema_metadata",

{softschema-0.4.0 → softschema-0.5.0}/packages/python/src/softschema/_portable.py

@@ -19,11 +19,30 @@ from ruamel.yaml.events import (
 )
 from ruamel.yaml.nodes import ScalarNode
 
-MAX_INPUT_BYTES = 1_048_576
-MAX_DEPTH = 64
-MAX_NODES = 100_000
-MAX_SCALAR_BYTES = 262_144
 MAX_SAFE_INTEGER = 9_007_199_254_740_991
+"""Largest integer that survives a round trip through a JS number."""
+
+MAX_DEPTH = 64
+"""Simultaneously open collections, including the root.
+
+A portability rule, not a resource guard, which is why it survives while the input,
+node, and scalar ceilings did not. Those three only answered whether a hostile document
+could exhaust the parser, and softschema reads artifacts its own callers just wrote.
+Depth answers a different question: how deep a document still means the same thing in
+both runtimes.
+
+Without an explicit rule the host stack decides, and the two hosts disagree by an order
+of magnitude. CPython's default recursion limit of 1,000 lets this constructor reach
+depth 491 from an empty caller stack — less from a real one, since the caller's own
+frames come out of the same budget — while V8 parses past 10,000. A document between
+those two bounds would be valid in TypeScript and a crash in Python, which is exactly
+what the shared vectors exist to prevent.
+
+64 is far above any real artifact and leaves roughly 7x headroom under the measured
+ceiling, so the check below always fires before the recursion does. Raising it is safe
+only while that headroom holds; past a few hundred the Python guard stops being reachable
+and `RecursionError` takes over.
+"""
 
 
 def _construct_timestamp_as_string(_constructor: SafeConstructor, node: ScalarNode) -> str:
@@ -48,12 +67,7 @@ class PortableInputError(ValueError):
 
 
 def read_utf8(path: Path) -> str:
-    """Read one modest UTF-8 file after checking its byte size."""
-    size = path.stat().st_size
-    if size > MAX_INPUT_BYTES:
-        raise PortableInputError(
-            "input_too_large", f"input is {size} bytes; limit is {MAX_INPUT_BYTES}"
-        )
+    """Read one UTF-8 artifact."""
     try:
         return path.read_bytes().decode("utf-8", errors="strict")
     except UnicodeDecodeError as exc:
@@ -65,7 +79,6 @@ def parse_yaml(text: str) -> Any:
     yaml = YAML(typ="safe")
     yaml.Constructor = _PortableConstructor
     stack: list[tuple[str, bool]] = []
-    nodes = 0
     has_alias = False
 
     def consume_parent_slot() -> None:
@@ -80,14 +93,10 @@ def parse_yaml(text: str) -> Any:
             if getattr(event, "tag", None) is not None:
                 raise PortableInputError("yaml_custom_tag", "explicit YAML tags are not supported")
             if isinstance(event, ScalarEvent):
-                nodes += 1
-                if len(event.value.encode("utf-8", errors="surrogatepass")) > MAX_SCALAR_BYTES:
-                    raise PortableInputError("yaml_limit", "YAML scalar exceeds the size limit")
                 if stack and stack[-1] == ("map", True) and event.value == "<<":
                     raise PortableInputError("yaml_merge_key", "YAML merge keys are not supported")
                 consume_parent_slot()
             elif isinstance(event, (MappingStartEvent, SequenceStartEvent)):
-                nodes += 1
                 consume_parent_slot()
                 stack.append(
                     ("map", True) if isinstance(event, MappingStartEvent) else ("seq", False)
@@ -96,13 +105,17 @@ def parse_yaml(text: str) -> Any:
                     raise PortableInputError("yaml_limit", "YAML nesting exceeds the depth limit")
             elif isinstance(event, (MappingEndEvent, SequenceEndEvent)):
                 stack.pop()
-            if nodes > MAX_NODES:
-                raise PortableInputError("yaml_limit", "YAML exceeds the node limit")
         if has_alias:
             raise PortableInputError("yaml_alias", "YAML aliases and anchors are not supported")
         value = yaml.load(text)
     except PortableInputError:
         raise
+    except RecursionError as exc:
+        # The preflight above rejects anything past MAX_DEPTH, so construction should
+        # never recurse this far. Mirror the TypeScript RangeError handler anyway: a
+        # caller already deep in its own stack can exhaust the budget at a legal depth,
+        # and that has to stay a structured result rather than escape validate_artifact.
+        raise PortableInputError("yaml_limit", "YAML nesting exhausted the parser stack") from exc
     except YAMLError as exc:
         code = "yaml_duplicate_key" if "duplicate key" in str(exc).lower() else "yaml_parse_error"
         raise PortableInputError(code, str(exc)) from exc
@@ -113,13 +126,9 @@ def parse_yaml(text: str) -> Any:
 
 
 def _check_value(root: Any) -> None:
-    stack: list[tuple[Any, int]] = [(root, 0)]
-    nodes = 0
+    stack: list[Any] = [root]
     while stack:
-        value, depth = stack.pop()
-        nodes += 1
-        if nodes > MAX_NODES or depth > MAX_DEPTH:
-            raise PortableInputError("yaml_limit", "YAML value exceeds the structure limit")
+        value = stack.pop()
         if value is None or isinstance(value, bool):
             continue
         if isinstance(value, str):
@@ -144,13 +153,13 @@ def _check_value(root: Any) -> None:
                 "host-native date and datetime values are not portable; use an ISO string",
             )
         if isinstance(value, list):
-            stack.extend((item, depth + 1) for item in value)
+            stack.extend(value)
             continue
         if isinstance(value, dict):
             for key, item in value.items():
                 if not isinstance(key, str):
                     raise PortableInputError("yaml_non_string_key", "mapping keys must be strings")
-                stack.append((item, depth + 1))
+                stack.append(item)
             continue
         raise PortableInputError(
             "yaml_unsupported_scalar", f"unsupported YAML value: {type(value).__name__}"

{softschema-0.4.0 → softschema-0.5.0}/packages/python/src/softschema/validate.py

@@ -17,6 +17,7 @@ from __future__ import annotations
 import re
 from collections.abc import Iterator, Mapping
 from dataclasses import dataclass, field
+from functools import lru_cache
 from pathlib import Path
 from typing import Any, Literal
 
@@ -83,7 +84,7 @@ class ArtifactValidationResult:
     outcome: Literal["valid", "invalid", "input_error"] = field(init=False)
 
     def __post_init__(self) -> None:
-        input_codes = {"artifact_unreadable", "artifact_invalid_utf8", "artifact_too_large"}
+        input_codes = {"artifact_unreadable", "artifact_invalid_utf8"}
         first_kind = self.structural.errors[0].get("kind") if self.structural.errors else None
         outcome = "valid" if self.ok else "input_error" if first_kind in input_codes else "invalid"
         object.__setattr__(self, "outcome", outcome)
@@ -97,6 +98,96 @@ class ArtifactValidationResult:
         return [warning.code for warning in self.warnings]
 
 
+class _SchemaRootNotAMapping(Exception):
+    """A compiled schema file did not parse to a mapping."""
+
+
+_VALIDATOR_CACHE_SIZE = 256
+"""Compiled validators to retain.
+
+Sized well above the number of distinct compiled schemas a large run touches — a
+downstream suite validated 521 documents against far fewer schemas — so a whole
+run reuses entries rather than evicting them.
+"""
+
+
+def _build_validator(
+    schema_text: str,
+    strict_extras: bool,
+    resources: Mapping[str, dict[str, Any]] | None,
+) -> Draft202012Validator:
+    """Build the validator a compiled schema describes.
+
+    Every step here is a pure function of ``schema_text`` plus ``strict_extras``
+    and ``resources`` — nothing depends on the document being validated.
+    """
+    schema = parse_yaml(schema_text)
+    if not isinstance(schema, dict):
+        raise _SchemaRootNotAMapping
+    _check_patterns(schema)
+    Draft202012Validator.check_schema(schema)
+    _check_schema_identities(schema, resources or {})
+    if strict_extras:
+        schema = apply_enforced_extras(schema)
+    registry: Registry[Any] = Registry()
+    for key, resource_schema in (resources or {}).items():
+        resource_id = str(resource_schema.get("$id", key))
+        registry = registry.with_resource(
+            resource_id,
+            Resource.from_contents(resource_schema, default_specification=DRAFT202012),
+        )
+    return Draft202012Validator(schema, registry=registry.crawl())
+
+
+@lru_cache(maxsize=_VALIDATOR_CACHE_SIZE)
+def _cached_validator(schema_text: str, strict_extras: bool) -> Draft202012Validator:
+    """Memoize :func:`_build_validator` for the no-``resources`` case.
+
+    A compiled schema is a build output, so parsing it, checking it against the
+    dialect, applying the ``enforced`` overlay, and crawling the registry produce
+    the same validator every time. Doing that work per validation call dominated
+    large suites: one schema file was recompiled 137 times in a single run, and
+    schema compilation accounted for roughly a quarter of a consumer's total test
+    wall clock.
+
+    Keyed on the schema text itself rather than on the file's path and stat, so a
+    rewritten schema can never be served from a stale entry: a file whose mtime
+    and size both survive an edit — a same-length rewrite, or any restore that
+    preserves timestamps, such as ``cp -p``, ``rsync -t``, ``tar -x``, or a CI
+    cache — still hashes differently and misses. Two paths holding identical
+    bytes correctly share one entry.
+
+    Reading and hashing the text is negligible against what a hit avoids: the
+    YAML parse, the dialect check, the overlay, and the registry crawl.
+    """
+    return _build_validator(schema_text, strict_extras, None)
+
+
+def clear_validator_cache() -> None:
+    """Drop every memoized validator.
+
+    Only needed by a long-lived process that regenerates compiled schemas in
+    place, such as a watch mode or a language server; ordinary callers never have
+    to call this, because a rewritten schema misses the cache on its own.
+    """
+    _cached_validator.cache_clear()
+
+
+def _validator_for(
+    schema_yaml_path: Path,
+    strict_extras: bool,
+    resources: Mapping[str, dict[str, Any]] | None,
+) -> Draft202012Validator:
+    """Return a validator, reusing a cached one when it is safe to do so."""
+    schema_text = read_utf8(schema_yaml_path)
+    if resources:
+        # `resources` is a plain mapping of dicts, so it is neither hashable
+        # nor cheap to fingerprint. Registry-backed validation is the rare
+        # path; build it fresh rather than risk a wrong cache key.
+        return _build_validator(schema_text, strict_extras, resources)
+    return _cached_validator(schema_text, strict_extras)
+
+
 def validate_structural(
     values: Any,
     schema_yaml_path: Path,
@@ -112,23 +203,7 @@ def validate_structural(
     :func:`softschema.canonicalize.apply_enforced_extras`.
     """
     try:
-        schema = _read_yaml(schema_yaml_path)
-        if not isinstance(schema, dict):
-            return _schema_invalid("syntax", "compiled schema root must be a mapping")
-        _check_patterns(schema)
-        Draft202012Validator.check_schema(schema)
-        _check_schema_identities(schema, resources or {})
-        if strict_extras:
-            schema = apply_enforced_extras(schema)
-        registry: Registry[Any] = Registry()
-        for key, resource_schema in (resources or {}).items():
-            resource_id = str(resource_schema.get("$id", key))
-            registry = registry.with_resource(
-                resource_id,
-                Resource.from_contents(resource_schema, default_specification=DRAFT202012),
-            )
-        registry = registry.crawl()
-        validator = Draft202012Validator(schema, registry=registry)
+        validator = _validator_for(schema_yaml_path, strict_extras, resources)
         errors = [
             structural_error_record(
                 path=list(error.absolute_path),
@@ -138,6 +213,8 @@ def validate_structural(
             )
             for error in validator.iter_errors(values)
         ]
+    except _SchemaRootNotAMapping:
+        return _schema_invalid("syntax", "compiled schema root must be a mapping")
     except PortableInputError as exc:
         return _schema_invalid("syntax", str(exc))
     except SchemaError as exc:
@@ -747,8 +824,6 @@ def _portable_error_kind(error: Exception) -> str:
         return "yaml_parse_error"
     if error.code == "invalid_utf8":
         return "artifact_invalid_utf8"
-    if error.code == "input_too_large":
-        return "artifact_too_large"
     return error.code
 
 

{softschema-0.4.0 → softschema-0.5.0}/packages/python/tests/test_core.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+import os
 from datetime import date, datetime
 from pathlib import Path
+from typing import Any
 
 import pytest
 from pydantic import BaseModel, ConfigDict, ValidationError, model_validator
@@ -13,6 +15,7 @@ from softschema import (
     Contracts,
     SchemaProfile,
     SchemaStatus,
+    clear_validator_cache,
     compile_model,
     parse_schema_metadata,
     validate_artifact,
@@ -20,6 +23,7 @@ from softschema import (
     validate_structural,
     validate_values,
 )
+from softschema import validate as validate_module
 from softschema._portable import parse_yaml
 
 
@@ -73,6 +77,7 @@ def test_package_root_exports_only_the_supported_surface() -> None:
         "StructuralResult",
         "ValidationResult",
         "WarningCode",
+        "clear_validator_cache",
         "compile_model",
         "infer_envelope_key",
         "parse_schema_metadata",
@@ -514,8 +519,6 @@ def test_shared_artifact_input_vectors(tmp_path: Path) -> None:
         path = tmp_path / f"{case['id']}.yaml"
         if case.get("source") == "invalid_utf8":
             path.write_bytes(b"value: \xff")
-        elif case.get("source") == "too_large":
-            path.write_bytes(b"x" * 1_048_577)
         elif "text" in case:
             path.write_text(case["text"])
         result = validate_artifact(path, contract=contract)
@@ -639,3 +642,152 @@ def test_contract_grammar_accepts_valid_ids(contract_id: str) -> None:
 def test_contract_grammar_rejects_malformed_ids(contract_id: str) -> None:
     with pytest.raises(ValidationError):
         parse_schema_metadata(contract_id)
+
+
+def _write_schema(path: Path, *, required_field: str) -> None:
+    path.write_text(
+        "$schema: https://json-schema.org/draft/2020-12/schema\n"
+        "type: object\n"
+        "properties:\n"
+        f"  {required_field}:\n"
+        "    type: string\n"
+        "required:\n"
+        f"  - {required_field}\n",
+        encoding="utf-8",
+    )
+
+
+def test_repeated_validation_reuses_one_compiled_validator(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Compiling a schema is pure, so it should happen once per schema file.
+
+    Recompiling per call — parse, dialect check, enforced overlay, registry
+    crawl — dominated large suites well ahead of the actual validation work.
+    """
+    clear_validator_cache()
+    schema = tmp_path / "sample.schema.yaml"
+    _write_schema(schema, required_field="name")
+
+    parses: list[str] = []
+    original = parse_yaml
+
+    def counting_parse(text: str) -> Any:
+        parses.append(text)
+        return original(text)
+
+    monkeypatch.setattr(validate_module, "parse_yaml", counting_parse)
+    for _ in range(5):
+        assert validate_structural({"name": "ok"}, schema).ok
+
+    assert parses.count(schema.read_text()) == 1, "compiled schema should be parsed once"
+
+
+def test_rewriting_a_schema_is_not_served_from_the_cache(tmp_path: Path) -> None:
+    """A regenerated schema must take effect, never a stale compiled copy."""
+    schema = tmp_path / "sample.schema.yaml"
+    _write_schema(schema, required_field="name")
+    assert validate_structural({"name": "ok"}, schema).ok
+    assert not validate_structural({"other": "ok"}, schema).ok
+
+    _write_schema(schema, required_field="other_name")
+
+    assert validate_structural({"other_name": "ok"}, schema).ok
+    assert not validate_structural({"name": "ok"}, schema).ok
+
+
+def test_same_size_rewrite_with_preserved_mtime_is_not_served_from_the_cache(
+    tmp_path: Path,
+) -> None:
+    """Content decides the entry, so no stat pair can alias two schemas.
+
+    A key of (path, mtime, size) served a stale validator here: the rewrite is
+    the same length, and restoring the original mtime is what ``cp -p``,
+    ``rsync -t``, ``tar -x``, and a restored CI cache all do.
+    """
+    clear_validator_cache()
+    schema = tmp_path / "sample.schema.yaml"
+    _write_schema(schema, required_field="aaaa")
+    before = schema.stat()
+    assert validate_structural({"aaaa": "ok"}, schema).ok
+
+    _write_schema(schema, required_field="cccc")  # identical length
+    os.utime(schema, ns=(before.st_mtime_ns, before.st_mtime_ns))
+    after = schema.stat()
+    assert (after.st_size, after.st_mtime_ns) == (before.st_size, before.st_mtime_ns)
+
+    assert validate_structural({"cccc": "ok"}, schema).ok
+    assert not validate_structural({"aaaa": "ok"}, schema).ok
+
+
+def test_identical_schemas_at_two_paths_share_one_entry(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Keying on content means a copied schema costs nothing to compile twice."""
+    clear_validator_cache()
+    first = tmp_path / "first.schema.yaml"
+    second = tmp_path / "second.schema.yaml"
+    _write_schema(first, required_field="name")
+    _write_schema(second, required_field="name")
+
+    parses: list[str] = []
+    original = parse_yaml
+
+    def counting_parse(text: str) -> Any:
+        parses.append(text)
+        return original(text)
+
+    monkeypatch.setattr(validate_module, "parse_yaml", counting_parse)
+    assert validate_structural({"name": "ok"}, first).ok
+    assert validate_structural({"name": "ok"}, second).ok
+
+    assert parses.count(first.read_text()) == 1
+
+
+def test_clear_validator_cache_forces_a_rebuild(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The escape hatch exists for processes that regenerate schemas in place."""
+    clear_validator_cache()
+    schema = tmp_path / "sample.schema.yaml"
+    _write_schema(schema, required_field="name")
+    assert validate_structural({"name": "ok"}, schema).ok
+
+    parses: list[str] = []
+    original = parse_yaml
+
+    def counting_parse(text: str) -> Any:
+        parses.append(text)
+        return original(text)
+
+    monkeypatch.setattr(validate_module, "parse_yaml", counting_parse)
+    assert validate_structural({"name": "ok"}, schema).ok
+    assert parses == [], "a warm entry should not reparse"
+
+    clear_validator_cache()
+    assert validate_structural({"name": "ok"}, schema).ok
+    assert parses.count(schema.read_text()) == 1
+
+
+def test_strict_extras_is_part_of_the_cache_key(tmp_path: Path) -> None:
+    """The enforced overlay changes the validator, so it cannot share an entry."""
+    schema = tmp_path / "sample.schema.yaml"
+    _write_schema(schema, required_field="name")
+    payload = {"name": "ok", "unexpected": 1}
+
+    assert validate_structural(payload, schema, strict_extras=False).ok
+    assert not validate_structural(payload, schema, strict_extras=True).ok
+    # Re-run both to prove neither direction poisoned the other's entry.
+    assert validate_structural(payload, schema, strict_extras=False).ok
+    assert not validate_structural(payload, schema, strict_extras=True).ok
+
+
+def test_a_schema_whose_root_is_not_a_mapping_still_reports_syntax(tmp_path: Path) -> None:
+    """The cached path must preserve the original structured failure."""
+    schema = tmp_path / "bad.schema.yaml"
+    schema.write_text("- not\n- a mapping\n", encoding="utf-8")
+
+    result = validate_structural({"name": "ok"}, schema)
+
+    assert not result.ok
+    assert result.errors[0]["kind"] == "schema_invalid"