gramson 0.1.0__py3-none-any.whl

gramson/__init__.py

@@ -0,0 +1 @@
+"""Gramson: declaratively defined JSON expression languages."""

gramson/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+raise SystemExit(main())

gramson/cli.py

@@ -0,0 +1,195 @@
+import argparse
+import json
+import math
+import sys
+from functools import lru_cache
+from importlib.resources import files
+from pathlib import Path
+from urllib.parse import quote, urlsplit
+from urllib.request import url2pathname
+
+import yaml
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import SchemaError, best_match
+from referencing import Registry, Resource
+from referencing.exceptions import NoSuchResource, Unresolvable
+from referencing.jsonschema import DRAFT202012
+
+from .generation import (
+    generate_schemas,
+    load_core,
+    load_language,
+    read_yaml,
+    write_schemas,
+)
+
+SCHEMA_BASE = "https://gramson.invalid/schema/"
+
+
+def unique_object(pairs):
+    result = {}
+    for key, value in pairs:
+        if not isinstance(key, str):
+            raise ValueError("Object keys must be strings.")
+        if key in result:
+            raise ValueError(f"Duplicate object key: {key!r}.")
+        result[key] = value
+    return result
+
+
+class ExpressionLoader(yaml.SafeLoader):
+    def construct_mapping(self, node, deep=False):
+        self.flatten_mapping(node)
+        return unique_object(self.construct_pairs(node, deep=deep))
+
+
+def check_json_value(value, ancestors=None):
+    """Reject YAML values that cannot be represented as finite JSON data."""
+    if value is None or isinstance(value, (str, bool, int)):
+        return
+    if isinstance(value, float):
+        if not math.isfinite(value):
+            raise ValueError("Numbers must be finite JSON numbers.")
+        return
+    if not isinstance(value, (list, dict)):
+        raise ValueError(f"Unsupported JSON value type: {type(value).__name__}.")
+    ancestors = set() if ancestors is None else ancestors
+    if id(value) in ancestors:
+        raise ValueError("Cyclic YAML aliases cannot represent JSON values.")
+    ancestors.add(id(value))
+    if isinstance(value, dict):
+        if any(not isinstance(key, str) for key in value):
+            raise ValueError("Object keys must be strings.")
+        children = value.values()
+    else:
+        children = value
+    for child in children:
+        check_json_value(child, ancestors)
+    ancestors.remove(id(value))
+
+
+def expression_validator(schema_path=None):
+    """Load bundled or generated local schemas with no remote retrieval."""
+    schemas = {}
+
+    def load_schema(source):
+        if source.name.endswith(".json"):
+            schema = json.loads(source.read_text(encoding="utf-8"), object_pairs_hook=unique_object)
+        else:
+            schema = read_yaml(source)
+        Draft202012Validator.check_schema(schema)
+        return schema
+
+    if schema_path is not None:
+        schema_path = Path(schema_path).resolve()
+        root = schema_path.parent
+
+        @lru_cache(maxsize=None)
+        def retrieve_local(uri):
+            parsed = urlsplit(uri)
+            if parsed.scheme != "file" or parsed.netloc:
+                raise NoSuchResource(ref=uri)
+            source = Path(url2pathname(parsed.path)).resolve()
+            if not source.is_relative_to(root):
+                raise NoSuchResource(ref=uri)
+            return Resource.from_contents(load_schema(source), default_specification=DRAFT202012)
+
+        # Resolve only referenced files, without scanning unrelated directories
+        # or falling back to package resources or network retrieval.
+        registry = Registry(retrieve=retrieve_local).with_resource(
+            schema_path.as_uri(),
+            Resource.from_contents(load_schema(schema_path), default_specification=DRAFT202012),
+        )
+        return Draft202012Validator({"$ref": schema_path.as_uri()}, registry=registry)
+
+    def read_directory(directory, prefix=""):
+        for item in directory.iterdir():
+            relative = prefix + item.name
+            if item.is_dir():
+                read_directory(item, relative + "/")
+            elif item.name.endswith((".schema.yaml", ".schema.yml", ".schema.json")):
+                schemas[relative] = load_schema(item)
+
+    read_directory(files("gramson.schemas"))
+    registry = Registry().with_resources(
+        (SCHEMA_BASE + quote(name, safe="/"), Resource.from_contents(schema, default_specification=DRAFT202012))
+        for name, schema in schemas.items()
+    )
+    return Draft202012Validator({
+        "$ref": SCHEMA_BASE + "expression.schema.yaml",
+    }, registry=registry)
+
+
+def error_detail(error):
+    """Select the matching operator's failure from recursive oneOf branches."""
+    while error.context:
+        candidates = list(error.context)
+        if isinstance(error.instance, dict):
+            if len(error.instance) != 1:
+                return error.json_path, "Expected exactly one registered operator key."
+            operator = next(iter(error.instance))
+            matching = [child for child in candidates if child.path and child.path[0] == operator]
+            if not matching:
+                return error.json_path, f"Unknown operator {operator!r}."
+            candidates = matching
+        elif isinstance(error.instance, list):
+            relevant = [child for child in candidates if child.validator not in ("type", "not")]
+            if relevant:
+                candidates = relevant
+        error = best_match(candidates)
+    return error.json_path, error.message
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="gramson", description="Generate grammars and validate Gramson expression syntax offline.")
+    commands = parser.add_subparsers(dest="command", required=True)
+    generate = commands.add_parser("generate", help="Generate a self-contained expression grammar.")
+    generate.add_argument("--manifest", type=Path, help="Language manifest; defaults to the bundled core language.")
+    generate.add_argument("--spec-path", type=Path, action="append", default=[], help="Search root for external namespace:group includes (repeatable).")
+    generate.add_argument("--output-directory", type=Path, default=Path("schema"), help="Output directory (default: schema).")
+    validate = commands.add_parser("validate", help="Validate one JSON or YAML expression.")
+    validate.add_argument("expression", help="Expression file, or - to read stdin.")
+    validate.add_argument("--format", choices=("json", "yaml"), help="Override file format; stdin defaults to JSON.")
+    validate.add_argument("--schema", type=Path, help="Generated schema entry point; defaults to the bundled grammar.")
+    options = parser.parse_args(argv)
+    if options.command == "generate":
+        try:
+            if options.spec_path and options.manifest is None:
+                raise ValueError("--spec-path requires --manifest.")
+            groups = load_core() if options.manifest is None else load_language(options.manifest, spec_paths=options.spec_path)
+            schemas = generate_schemas(groups)
+            write_schemas(schemas, options.output_directory)
+        except (OSError, UnicodeError, ValueError, yaml.YAMLError, RecursionError) as error:
+            print(f"gramson generate: {error}", file=sys.stderr)
+            return 2
+        print(f"Generated {len(schemas)} schemas in {options.output_directory}")
+        return 0
+    label = "stdin" if options.expression == "-" else options.expression
+    input_format = options.format or (
+        "yaml" if Path(options.expression).suffix.lower() in (".yaml", ".yml") else "json"
+    )
+    try:
+        source = sys.stdin.read() if options.expression == "-" else Path(options.expression).read_text(encoding="utf-8")
+        if not source.strip():
+            raise ValueError("Expected one expression; input is empty.")
+        expression = (
+            yaml.load(source, Loader=ExpressionLoader)
+            if input_format == "yaml"
+            else json.loads(source, object_pairs_hook=unique_object)
+        )
+        check_json_value(expression)
+    except (OSError, UnicodeError, ValueError, yaml.YAMLError, RecursionError) as error:
+        print(f"{label}: input error: {error}", file=sys.stderr)
+        return 2
+
+    try:
+        error = next(expression_validator(options.schema).iter_errors(expression), None)
+    except (OSError, UnicodeError, ValueError, yaml.YAMLError, SchemaError, Unresolvable, RecursionError) as error:
+        print(f"{label}: schema error: {error}", file=sys.stderr)
+        return 2
+    if error is not None:
+        location, message = error_detail(error)
+        print(f"{label}: invalid expression at {location}: {message}", file=sys.stderr)
+        return 1
+    print(f"{label}: valid expression")
+    return 0

gramson/generation.py

@@ -0,0 +1,366 @@
+import posixpath
+import re
+from importlib.resources import files
+from pathlib import Path
+from urllib.parse import quote
+
+import yaml
+
+BUNDLED_INCLUDES = {
+    "gramson:core": Path("core"),
+    "gramson:bitwise": Path("extensions/bitwise.yaml"),
+}
+INCLUDE_NAME = re.compile(r"[a-z][a-z0-9_-]*:[a-z][a-z0-9_-]*")
+
+
+def generate_schema(operators):
+    if not isinstance(operators, dict) or not operators:
+        raise ValueError("The registry must be a nonempty object.")
+
+    alternatives = [
+        {"$ref": "#/$defs/scalarLiteral"},
+        {"$ref": "#/$defs/arrayExpression"},
+    ]
+    definitions = {
+        "expression": {"oneOf": alternatives},
+        "scalarLiteral": {"type": ["null", "boolean", "number", "string"]},
+        "arrayExpression": {
+            "type": "array", "items": {"$ref": "#/$defs/expression"}
+        },
+    }
+    for name, definition in operators.items():
+        if not isinstance(name, str) or not name:
+            raise ValueError("Operator names must be nonempty strings.")
+        if not isinstance(definition, dict):
+            raise ValueError(f"{name}: the definition must be an object.")
+        allowed = {
+            "argumentKind", "minArgs", "maxArgs", "shorthand", "schemaName",
+            "title", "summary", "description",
+        }
+        if definition.keys() - allowed:
+            raise ValueError(f"{name}: unsupported definition fields.")
+        for field in ("title", "summary", "description"):
+            if field in definition and (
+                not isinstance(definition[field], str) or not definition[field].strip()
+            ):
+                raise ValueError(f"{name}: {field} must be a nonempty string.")
+
+        schema_name = definition.get("schemaName", f"operator_{name}")
+        if not isinstance(schema_name, str) or not schema_name:
+            raise ValueError(f"{name}: schemaName must be a nonempty string.")
+        if schema_name in definitions:
+            raise ValueError(f"{name}: duplicate or reserved schemaName {schema_name!r}.")
+
+        kind = definition.get("argumentKind")
+        minimum = definition.get("minArgs")
+        maximum = definition.get("maxArgs")
+        shorthand = definition.get("shorthand", False)
+        if kind not in ("string", "expression"):
+            raise ValueError(f"{name}: argumentKind must be string or expression.")
+        if type(minimum) is not int or minimum < 0:
+            raise ValueError(f"{name}: minArgs must be a nonnegative integer.")
+        if "maxArgs" in definition and (
+            type(maximum) is not int or maximum < minimum
+        ):
+            raise ValueError(f"{name}: maxArgs must be an integer >= minArgs.")
+        if type(shorthand) is not bool:
+            raise ValueError(f"{name}: shorthand must be a boolean.")
+        if shorthand and (minimum > 1 or maximum == 0):
+            raise ValueError(f"{name}: shorthand requires an allowed arity of one.")
+
+        argument = (
+            {"type": "string"}
+            if kind == "string"
+            else {"$ref": "#/$defs/expression"}
+        )
+        arguments = {"type": "array", "items": argument, "minItems": minimum}
+        if maximum is not None:
+            arguments["maxItems"] = maximum
+        unwrapped = {**argument, "not": {"type": "array"}}
+        value = {"oneOf": [unwrapped, arguments]} if shorthand else arguments
+        definitions[schema_name] = {
+            "title": definition.get("title", f"{name} expression"),
+            "type": "object",
+            "required": [name],
+            "additionalProperties": False,
+            "properties": {name: value},
+        }
+        documentation = [
+            definition[field] for field in ("summary", "description")
+            if field in definition
+        ]
+        if documentation:
+            definitions[schema_name]["description"] = "\n\n".join(documentation)
+        pointer_token = schema_name.replace("~", "~0").replace("/", "~1")
+        alternatives.append({"$ref": f"#/$defs/{quote(pointer_token, safe='')}"})
+
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "title": "Gramson expression grammar",
+        "description": "Generated structural grammar; no evaluation or semantic typing.",
+        "$ref": "#/$defs/expression",
+        "$defs": definitions,
+    }
+
+class UniqueKeyLoader(yaml.SafeLoader):
+    def construct_mapping(self, node, deep=False):
+        self.flatten_mapping(node)
+        mapping = {}
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in mapping:
+                raise ValueError(
+                    f"{self.name}:{key_node.start_mark.line + 1}: duplicate YAML key {key!r}"
+                )
+            mapping[key] = self.construct_object(value_node, deep=deep)
+        return mapping
+
+
+def read_yaml(path):
+    with path.open(encoding="utf-8") as stream:
+        return yaml.load(stream, Loader=UniqueKeyLoader)
+
+
+def _load_bundled(name):
+    """Load installed declarations with portable output paths."""
+    groups = {}
+
+    def read_directory(directory, relative):
+        for source in sorted(directory.iterdir(), key=lambda item: item.name):
+            name = relative / source.name
+            if source.is_dir():
+                read_directory(source, name)
+            elif source.name.endswith((".yaml", ".yml")):
+                groups[name] = read_yaml(source)
+
+    relative = BUNDLED_INCLUDES[name]
+    resource = files("gramson.specs").joinpath(*relative.parts)
+    if resource.is_dir():
+        read_directory(resource, relative)
+    else:
+        groups[relative] = read_yaml(resource)
+    groups = dict(sorted(groups.items()))
+    merge_operators(groups)
+    return groups
+
+
+def load_core():
+    """Load the default bundled core language."""
+    return _load_bundled("gramson:core")
+
+
+def _load_external(name, spec_paths):
+    namespace, group_name = name.split(":")
+    matches = []
+    for root in spec_paths:
+        candidate = root / namespace / group_name
+        if candidate.is_dir():
+            resolved = candidate.resolve()
+            if not resolved.is_relative_to(root):
+                raise ValueError(f"Include {name!r} escapes its spec search path: {candidate}.")
+            matches.append(resolved)
+    if not matches:
+        searched = ", ".join(str(root / namespace / group_name) for root in spec_paths)
+        hint = f"searched {searched}" if searched else "provide --spec-path to search for external specifications"
+        raise ValueError(f"Unknown include {name!r}; {hint}.")
+    if len(matches) != 1:
+        raise ValueError(f"Ambiguous include {name!r}: {', '.join(str(path) for path in matches)}.")
+    directory = matches[0]
+    sources = sorted(
+        source for source in directory.rglob("*")
+        if source.is_file() and source.suffix in (".yaml", ".yml")
+    )
+    if not sources:
+        raise ValueError(f"Include {name!r}: no YAML operator files in {directory}.")
+    groups = {}
+    for source in sources:
+        if not source.resolve().is_relative_to(directory):
+            raise ValueError(f"{source}: operator files must stay inside include {name!r}.")
+        relative = Path(namespace) / group_name / source.relative_to(directory)
+        groups[relative] = read_yaml(source)
+    return groups
+
+
+def merge_operators(groups):
+    operators = {}
+    operator_sources = {}
+    definition_sources = {}
+    for source, definitions in groups.items():
+        try:
+            generate_schema(definitions)
+        except ValueError as error:
+            raise ValueError(f"{source}: {error}") from error
+        for name, definition in definitions.items():
+            if name in operator_sources:
+                raise ValueError(
+                    f"Duplicate operator {name!r} in {operator_sources[name]} and {source}"
+                )
+            schema_name = definition.get("schemaName", f"operator_{name}")
+            if schema_name in definition_sources:
+                raise ValueError(
+                    f"Duplicate schemaName {schema_name!r} in "
+                    f"{definition_sources[schema_name]} and {source}"
+                )
+            operators[name] = definition
+            operator_sources[name] = source
+            definition_sources[schema_name] = source
+    if not operators:
+        raise ValueError("The language must contain at least one operator group.")
+    return operators
+
+
+def load_language(manifest_path, spec_paths=()):
+    manifest_path = Path(manifest_path).resolve()
+    manifest = read_yaml(manifest_path)
+    if not isinstance(manifest, dict) or set(manifest) - {"operatorDirectories", "include"}:
+        raise ValueError(f"{manifest_path}: expected include and/or operatorDirectories.")
+    includes = manifest.get("include", [])
+    if not isinstance(includes, list):
+        raise ValueError(f"{manifest_path}: include must be a list of namespace:group names.")
+    roots = list(dict.fromkeys(Path(path).resolve() for path in spec_paths))
+    for root in roots:
+        if not root.is_dir():
+            raise ValueError(f"{manifest_path}: spec search path must be an existing directory: {root}.")
+    groups = {}
+    targets = {}
+
+    def add_groups(additions, origin):
+        for relative, definitions in additions.items():
+            target = relative.with_suffix(".schema.yaml").as_posix().casefold()
+            if target in targets:
+                raise ValueError(f"Output collision between {targets[target]} and {origin} {relative}.")
+            targets[target] = f"{origin} {relative}"
+            groups[relative] = definitions
+
+    selected = set()
+    for name in includes:
+        if not isinstance(name, str) or not INCLUDE_NAME.fullmatch(name):
+            raise ValueError(f"{manifest_path}: invalid include {name!r}; expected lowercase namespace:group identifiers.")
+        if name in selected:
+            raise ValueError(f"{manifest_path}: repeated include {name!r}.")
+        selected.add(name)
+        if name.startswith("gramson:"):
+            if name not in BUNDLED_INCLUDES:
+                raise ValueError(f"{manifest_path}: unknown bundled include {name!r}; gramson: is reserved for this package.")
+            add_groups(_load_bundled(name), "bundled")
+        else:
+            try:
+                additions = _load_external(name, roots)
+            except ValueError as error:
+                raise ValueError(f"{manifest_path}: {error}") from error
+            add_groups(additions, f"include {name!r}")
+    directories = manifest.get("operatorDirectories", [])
+    if not isinstance(directories, list) or (not directories and not groups):
+        raise ValueError(f"{manifest_path}: operatorDirectories must be a nonempty list unless include selects definitions.")
+    source_root = manifest_path.parent
+    sources = set()
+    seen_directories = set()
+    for directory in directories:
+        if not isinstance(directory, str) or not directory.strip():
+            raise ValueError(f"{manifest_path}: directory names must be nonempty strings.")
+        relative = Path(directory)
+        if relative.is_absolute() or ".." in relative.parts or relative == Path("."):
+            raise ValueError(f"{manifest_path}: directories must be relative subdirectories.")
+        path = (source_root / relative).resolve()
+        if not path.is_relative_to(source_root) or not path.is_dir():
+            raise ValueError(f"{manifest_path}: missing or out-of-tree directory {directory!r}.")
+        if path in seen_directories:
+            raise ValueError(f"{manifest_path}: repeated directory {directory!r}.")
+        seen_directories.add(path)
+        operator_files = sorted(
+            source for source in path.rglob("*")
+            if source.is_file() and source.suffix in (".yaml", ".yml")
+        )
+        if not operator_files:
+            raise ValueError(f"{manifest_path}: no YAML operator files in {directory!r}.")
+        for source in operator_files:
+            if not source.resolve().is_relative_to(source_root):
+                raise ValueError(f"{source}: operator files must stay inside the specification.")
+            sources.add(source)
+    for source in sorted(sources):
+        relative = source.relative_to(source_root)
+        add_groups({relative: read_yaml(source)}, "local")
+    groups = dict(sorted(groups.items()))
+    merge_operators(groups)
+    return groups
+
+def definition_reference(name, document=""):
+    pointer_token = name.replace("~", "~0").replace("/", "~1")
+    return f"{document}#/$defs/{quote(pointer_token, safe='')}"
+
+
+def generate_schemas(groups):
+    compiled = generate_schema(merge_operators(groups))
+    dialect = compiled["$schema"]
+    alternatives = [
+        {"$ref": "shared.schema.yaml#/$defs/scalarLiteral"},
+        {"$ref": "#/$defs/arrayExpression"},
+    ]
+    schemas = {
+        Path("expression.schema.yaml"): {
+            "$schema": dialect,
+            "title": "Gramson expression grammar",
+            "description": compiled["description"],
+            "$ref": "#/$defs/expression",
+            "$defs": {
+                "expression": {"oneOf": alternatives},
+                "arrayExpression": compiled["$defs"]["arrayExpression"],
+            },
+        },
+        Path("shared.schema.yaml"): {
+            "$schema": dialect,
+            "title": "Shared expression definitions",
+            "$defs": {"scalarLiteral": compiled["$defs"]["scalarLiteral"]},
+        },
+    }
+
+    def relocate(value, expression_reference):
+        if isinstance(value, list):
+            return [relocate(item, expression_reference) for item in value]
+        if isinstance(value, dict):
+            return {
+                key: (
+                    expression_reference
+                    if key == "$ref" and item == "#/$defs/expression"
+                    else relocate(item, expression_reference)
+                )
+                for key, item in value.items()
+            }
+        return value
+
+    for source, operators in groups.items():
+        source = Path(source)
+        if source.is_absolute() or ".." in source.parts:
+            raise ValueError(f"{source}: expected a relative operator source path.")
+        target = source.with_suffix(".schema.yaml")
+        if target in schemas:
+            raise ValueError(f"{source}: generated schema path collision at {target}.")
+        expression_reference = quote(
+            posixpath.relpath("expression.schema.yaml", target.parent.as_posix()), safe="/"
+        ) + "#/$defs/expression"
+        definitions = {}
+        for name, operator in operators.items():
+            schema_name = operator.get("schemaName", f"operator_{name}")
+            definitions[schema_name] = relocate(
+                compiled["$defs"][schema_name], expression_reference
+            )
+            alternatives.append({
+                "$ref": definition_reference(schema_name, quote(target.as_posix(), safe="/"))
+            })
+        schemas[target] = {
+            "$schema": dialect,
+            "title": f"{source.stem} operators",
+            "$defs": definitions,
+        }
+    return schemas
+
+
+def write_schemas(schemas, output_directory):
+    """Write an already compiled grammar without modifying unrelated files."""
+    output_directory = Path(output_directory)
+    for relative, schema in schemas.items():
+        target = output_directory / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(
+            yaml.safe_dump(schema, sort_keys=False, allow_unicode=True), encoding="utf-8"
+        )

gramson/schemas/__init__.py

@@ -0,0 +1 @@
+"""Generated grammars packaged as gramson.schemas for offline CLI validation."""