yaqpy 0.6.0__py3-none-any.whl

yaqpy/__init__.py

@@ -0,0 +1,40 @@
+"""yaqpy - a pure-Python (standard library only) implementation of mikefarah/yq.
+
+>>> import yaqpy
+>>> yaqpy.evaluate(".a.b", "a:\\n  b: 3\\n")
+'3\\n'
+>>> yaqpy.query(".items[] | select(. > 1)", {"items": [1, 2, 3]})
+[2, 3]
+"""
+
+from yaqpy.api import (
+    Yq, apply_recipe, compile, detect_format, dump, evaluate, evaluate_all, list_recipes, load,
+    query, update,
+)
+from yaqpy.app.recipe_service import RecipeRun
+from yaqpy.core.lang.parser import Expression
+from yaqpy.core.model.node import Kind, Node, Style
+from yaqpy.errors import (
+    EvaluationError, EvaluationLimitError, ExpressionSyntaxError, FormatError, RecipeError,
+    SecurityError, UnknownFormatError, YamlSyntaxError, YqError,
+)
+from yaqpy.options import (
+    CsvOptions, JsonOptions, Limits, Options, PropertiesOptions, SchemaOptions, SecurityPolicy,
+    TomlOptions, ToonOptions, XmlOptions, YamlOptions,
+)
+from yaqpy.recipes import Recipe, build_recipe
+from yaqpy.recipes.analysis import RecipeReport
+
+__version__ = "0.6.0"
+
+__all__ = [
+    "Yq", "compile", "dump", "evaluate", "evaluate_all", "load", "query", "update",
+    "apply_recipe", "list_recipes", "build_recipe", "Recipe", "RecipeReport", "RecipeRun",
+    "detect_format",
+    "Expression", "Kind", "Node", "Style",
+    "EvaluationError", "EvaluationLimitError", "ExpressionSyntaxError", "FormatError", "RecipeError",
+    "SecurityError", "UnknownFormatError", "YamlSyntaxError", "YqError",
+    "CsvOptions", "JsonOptions", "Limits", "Options", "PropertiesOptions", "SchemaOptions",
+    "SecurityPolicy", "TomlOptions", "ToonOptions", "XmlOptions", "YamlOptions",
+    "__version__",
+]

yaqpy/__main__.py

@@ -0,0 +1,8 @@
+"""``python -m yaqpy`` entry point."""
+
+import sys
+
+from yaqpy.cli.main import main
+
+if __name__ == "__main__":
+    sys.exit(main())

yaqpy/api.py

@@ -0,0 +1,189 @@
+"""Public library API (design doc section 10)."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from datetime import datetime
+from typing import Any
+
+from yaqpy.app.dto import EvalMode, EvaluateRequest, InputSource
+from yaqpy.app.ports import SandboxFileSystem, StaticEnvironment
+from yaqpy.app.printer import MemorySink
+from yaqpy.app.recipe_service import RecipeRun, RecipeService
+from yaqpy.app.service import YqService
+from yaqpy.core.lang.parser import Expression
+from yaqpy.core.model.convert import from_python, to_python
+from yaqpy.core.model.node import Node
+from yaqpy.core.operators import OperatorRegistry, builtin_registry
+from yaqpy.formats.registry import FormatRegistry, builtin_formats
+from yaqpy.options import Options
+from yaqpy.recipes import Recipe, builtin_recipes
+
+
+class Yq:
+    """Stateless facade. Safe to share between threads."""
+
+    def __init__(
+        self,
+        options: Options | None = None,
+        *,
+        operators: OperatorRegistry | None = None,
+        formats: FormatRegistry | None = None,
+        environ: Mapping[str, str] | None = None,
+        clock: Callable[[], datetime] | None = None,
+    ) -> None:
+        self.options = options or Options()
+        if environ is None:
+            import os
+
+            environ = os.environ
+        self._service = YqService(
+            SandboxFileSystem(), StaticEnvironment(environ),
+            operators=operators or builtin_registry(), formats=formats or builtin_formats(),
+            clock=clock,
+        )
+
+    # ------------------------------------------------------------------ compile
+
+    def compile(self, expression: str) -> Expression:
+        return self._service.compile(expression)
+
+    # ------------------------------------------------------------------ text in / text out
+
+    def _request(self, expression: str | Expression, texts: Iterable[str], mode: EvalMode,
+                 options: Options | None) -> EvaluateRequest:
+        source = expression.source if isinstance(expression, Expression) else expression
+        inputs = tuple(InputSource("<text>", text) for text in texts)
+        return EvaluateRequest(expression=source, inputs=inputs, mode=mode,
+                               options=options or self.options)
+
+    def evaluate(self, expression: str | Expression, text: str = "", *,
+                 options: Options | None = None) -> str:
+        request = self._request(expression, [text] if text != "" else [], EvalMode.STREAM, options)
+        result = self._service.evaluate(request, MemorySink())
+        return result.output or ""
+
+    def evaluate_all(self, expression: str | Expression, texts: Iterable[str], *,
+                     options: Options | None = None) -> str:
+        request = self._request(expression, texts, EvalMode.ALL, options)
+        result = self._service.evaluate(request, MemorySink())
+        return result.output or ""
+
+    # ------------------------------------------------------------------ nodes / python objects
+
+    def evaluate_nodes(self, expression: str | Expression, documents: Sequence[Node], *,
+                       options: Options | None = None) -> list[Node]:
+        return self._service.evaluate_nodes(expression, documents, options or self.options)
+
+    def iter_results(self, expression: str | Expression, text: str, *,
+                     options: Options | None = None) -> Iterator[Node]:
+        options = options or self.options
+        decoder = self._service.formats.decoder_for(options.input_format, options)
+        for doc in decoder.decode_documents(text):
+            yield from self.evaluate_nodes(expression, [doc], options=options)
+
+    def query(self, expression: str | Expression, data: Any, *,
+              options: Options | None = None) -> list[Any]:
+        root = from_python(data)
+        root.evaluate_together = True
+        results = self.evaluate_nodes(expression, [root], options=options)
+        return [to_python(n) for n in results]
+
+    def update(self, expression: str | Expression, data: Any, *,
+               options: Options | None = None) -> Any:
+        root = from_python(data)
+        root.evaluate_together = True
+        self.evaluate_nodes(expression, [root], options=options)
+        return to_python(root)
+
+    def load(self, text: str, *, format: str = "yaml", options: Options | None = None) -> list[Node]:
+        options = options or self.options
+        decoder = self._service.formats.decoder_for(format, options)
+        return list(decoder.decode_documents(text))
+
+    def dump(self, documents: Iterable[Node], *, format: str = "yaml",
+             options: Options | None = None) -> str:
+        from yaqpy.app.printer import ResultPrinter
+
+        options = options or self.options
+        spec = self._service.formats.get(format)
+        unwrap = options.unwrap_scalar if options.unwrap_scalar is not None else spec.unwrap_scalar_default
+        encoder = self._service.formats.encoder_for(format, options, unwrap)
+        sink = MemorySink()
+        ResultPrinter(encoder, sink).print_results(list(documents))
+        return sink.finish() or ""
+
+
+    # ------------------------------------------------------------------ recipes (a yaqpy extension)
+
+    def apply_recipe(self, recipe: str | Recipe, text: str, *, input_format: str = "json",
+                     output_format: str = "json", prune_null: bool = False, prune_empty: bool = False,
+                     options: Options | None = None) -> RecipeRun:
+        """Convert ``text`` with a recipe: a bundled one by name, or a ``Recipe`` you built.
+
+        Never reads files or environment variables, whatever the options say. The result holds the
+        converted text (``output``) and a ``report`` of what was dropped, added or does not fit the
+        target schema.
+        """
+        service = RecipeService(self._service)
+        if isinstance(recipe, str):
+            recipe = service.load(recipe)
+        return service.run(recipe, InputSource("<text>", text), options or self.options,
+                           input_format=service.input_format_for(recipe, "", input_format),
+                           output_format=service.output_format_for(recipe, output_format),
+                           prune_null=prune_null, prune_empty=prune_empty)
+
+
+_DEFAULT = Yq()
+
+
+def compile(expression: str) -> Expression:  # noqa: A001 - mirrors the design doc
+    return _DEFAULT.compile(expression)
+
+
+def evaluate(expression: str, text: str = "", *, options: Options | None = None) -> str:
+    return _DEFAULT.evaluate(expression, text, options=options)
+
+
+def evaluate_all(expression: str, texts: Iterable[str], *, options: Options | None = None) -> str:
+    return _DEFAULT.evaluate_all(expression, texts, options=options)
+
+
+def query(expression: str, data: Any, *, options: Options | None = None) -> list[Any]:
+    return _DEFAULT.query(expression, data, options=options)
+
+
+def update(expression: str, data: Any, *, options: Options | None = None) -> Any:
+    return _DEFAULT.update(expression, data, options=options)
+
+
+def load(text: str, *, format: str = "yaml", options: Options | None = None) -> list[Node]:
+    return _DEFAULT.load(text, format=format, options=options)
+
+
+def dump(documents: Iterable[Node], *, format: str = "yaml", options: Options | None = None) -> str:
+    return _DEFAULT.dump(documents, format=format, options=options)
+
+
+def detect_format(text: str) -> str:
+    """Guess the format of ``text`` from its content alone (a yaqpy extension; Go yq has no such
+    thing - it only ever looks at a file's extension).
+
+    Meant for text with no filename to go by, or none whose extension names a format: the same
+    guess ``Options(input_format="auto")`` falls back to once a filename's extension gives no
+    answer. Returns a format name ``Options(input_format=...)`` accepts; "yaml" is the fallback
+    when nothing in the content looks confident enough (never an error).
+    """
+    return builtin_formats().guess("", text).name
+
+
+def list_recipes() -> dict[str, Recipe]:
+    """The bundled recipes by name."""
+    return dict(builtin_recipes())
+
+
+def apply_recipe(recipe: str | Recipe, text: str, *, input_format: str = "json",
+                 output_format: str = "json", prune_null: bool = False, prune_empty: bool = False,
+                 options: Options | None = None) -> RecipeRun:
+    return _DEFAULT.apply_recipe(recipe, text, input_format=input_format, output_format=output_format,
+                                 prune_null=prune_null, prune_empty=prune_empty, options=options)

yaqpy/app/__init__.py

@@ -0,0 +1,11 @@
+"""Application layer: use cases shared by CLI, GUI and API adapters."""
+
+from yaqpy.app.dto import EvalMode, EvaluateRequest, EvaluateResult, InputSource
+from yaqpy.app.printer import InPlaceSink, MemorySink, ResultPrinter, StreamSink
+from yaqpy.app.service import PRETTY_PRINT_EXP, YqService
+
+__all__ = [
+    "EvalMode", "EvaluateRequest", "EvaluateResult", "InputSource",
+    "InPlaceSink", "MemorySink", "ResultPrinter", "StreamSink",
+    "PRETTY_PRINT_EXP", "YqService",
+]

yaqpy/app/dto.py

@@ -0,0 +1,58 @@
+"""Request / result objects shared by every adapter (design doc 11-1)."""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass, field
+
+from yaqpy.options import Options
+
+
+class EvalMode(enum.Enum):
+    STREAM = "stream"
+    ALL = "all"
+
+
+@dataclass(frozen=True, slots=True)
+class InputSource:
+    name: str                       # file name, "-" for stdin, "<text>" for in-memory input
+    text: str | None = None         # None -> read through the FileSystemPort
+
+
+@dataclass(frozen=True, slots=True, kw_only=True)
+class EvaluateRequest:
+    expression: str
+    inputs: tuple[InputSource, ...] = ()
+    mode: EvalMode = EvalMode.STREAM
+    options: Options = field(default_factory=Options)
+    in_place: bool = False
+    exit_status: bool = False
+    input_format: str | None = None     # resolved format name (None -> from options)
+    output_format: str | None = None
+    unwrap_scalar: bool | None = None
+    split_expression: str = ""          # -s: name a file per result with this expression
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluateResult:
+    output: str | None
+    printed_anything: bool
+    document_count: int
+    warnings: tuple[str, ...]
+    elapsed_seconds: float
+    input_format: str = ""      # actually used (after "auto" was resolved)
+    output_format: str = ""     # actually used
+
+
+@dataclass(frozen=True, slots=True)
+class ExpressionInfo:
+    expression: str
+    valid: bool
+    message: str = ""
+    position: int = -1
+
+
+@dataclass(frozen=True, slots=True)
+class FormatsInfo:
+    input_formats: tuple[str, ...]
+    output_formats: tuple[str, ...]

yaqpy/app/examples.py

@@ -0,0 +1,142 @@
+"""Worked examples for ``--example``, ``--guide-prompt`` and ``--skill-md``.
+
+Every example is *run* when it is shown: what is printed is the output of the real engine, never a
+hand-typed answer. ``expected`` pins that output, and a test compares the two, so an example can
+not go stale without the test noticing.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from yaqpy.app.dto import EvalMode, EvaluateRequest, InputSource
+from yaqpy.app.printer import MemorySink
+from yaqpy.app.recipe_service import RecipeService
+from yaqpy.app.recipe_text import summary_lines
+from yaqpy.app.service import YqService, with_prune
+from yaqpy.options import Options, SecurityPolicy
+
+SHOP_YAML = """\
+server:
+  host: localhost
+  port: 8080
+items:
+  - {name: pen, price: 120, tags: [a, b]}
+  - {name: cap, price: 80, tags: []}
+  - {name: bag, price: 300, tags: [b]}
+"""
+
+OPENAI_REQUEST = """\
+{
+  "model": "gpt-4o",
+  "messages": [
+    {"role": "system", "content": "You are helpful."},
+    {"role": "user", "content": "Hello"},
+    {"role": "assistant", "content": "Hi!"}
+  ],
+  "temperature": 0.7,
+  "max_completion_tokens": 256,
+  "stop": "END",
+  "stream": true
+}
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class Example:
+    title: str
+    input: str
+    input_name: str                 # what the file is called in the command line shown
+    expected: str                   # the output the example must give (pinned by a test)
+    expression: str = ""
+    flags: str = ""                 # shown before the expression, e.g. "-o json"
+    input_format: str = "yaml"
+    output_format: str = "yaml"
+    eval_all: bool = False
+    prune_null: bool = False
+    prune_empty: bool = False
+    recipe: str = ""                # run this bundled recipe instead of an expression
+    expected_notes: tuple[str, ...] = ()
+
+    @property
+    def command(self) -> str:
+        if self.recipe:
+            return " ".join(["yaqpy", "--recipe", self.recipe] + ([self.flags] if self.flags else [])
+                            + [self.input_name])
+        parts = ["yaqpy"]
+        if self.eval_all:
+            parts.append("eval-all")
+        if self.flags:
+            parts.append(self.flags)
+        if self.prune_null:
+            parts.append("--prune-null")
+        if self.prune_empty:
+            parts.append("--prune-empty")
+        if self.expression:
+            parts.append("'" + self.expression + "'")
+        parts.append(self.input_name)
+        return " ".join(parts)
+
+
+EXAMPLES: tuple[Example, ...] = (
+    Example("値を取り出す", SHOP_YAML, "shop.yaml", "8080\n", ".server.port"),
+    Example("条件で絞り込む", SHOP_YAML, "shop.yaml", "pen\nbag\n",
+            ".items[] | select(.price > 100) | .name"),
+    Example("値を更新する（更新したファイルにしたいときは -i を付ける）", SHOP_YAML, "shop.yaml",
+            "host: localhost\nport: 9090\n", ".server.port = 9090 | .server"),
+    Example("JSON に変換して整える", SHOP_YAML, "shop.yaml", '["pen","cap","bag"]\n',
+            ".items | map(.name)", flags="-o json -I 0", output_format="json"),
+    Example("合計を求める（jq の reduce は ireduce）", SHOP_YAML, "shop.yaml", "500\n",
+            ".items | .[] as $i ireduce (0; . + $i.price)"),
+    Example("並べ替える", SHOP_YAML, "shop.yaml", "bag\n",
+            ".items | sort_by(.price) | reverse | .[0].name"),
+    Example("文字列をつなぐ", SHOP_YAML, "shop.yaml", "pen, cap, bag\n",
+            '[.items[].name] | join(", ")'),
+    Example("条件に合う要素だけを更新する（要素の中の値で書き換えるときは |=）", SHOP_YAML, "shop.yaml",
+            "[120,100,300]\n", "(.items[] | select(.price < 100)) |= (.price = 100) | .items | map(.price)",
+            flags="-o json -I 0", output_format="json"),
+    Example("表（CSV）に変える", SHOP_YAML, "shop.yaml", "name,price\npen,120\ncap,80\nbag,300\n",
+            '.items | map(pick(["name", "price"]))', flags="-o csv", output_format="csv"),
+    Example("キーがあるときだけ代入する（select は代入の左辺の先頭に置く）", SHOP_YAML, "shop.yaml",
+            "port: 8080\n", "(select(.server.port != null) | .backup.port) = .server.port | .backup"),
+    Example("2 つの文書を 1 つに重ねる", "a: 1\nb: 1\n---\nb: 2\nc: 3\n", "two.yaml",
+            "a: 1\nb: 2\nc: 3\n", ". as $doc ireduce ({}; . * $doc)", eval_all=True),
+    Example("データの形（JSON Schema）を出す — yaqpy 独自", SHOP_YAML, "shop.yaml",
+            '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object",'
+            '"properties":{"server":{"type":"object","properties":{"host":{"type":"string"},'
+            '"port":{"type":"integer"}},"required":["host","port"]},"items":{"type":"array",'
+            '"items":{"type":"object","properties":{"name":{"type":"string"},"price":{"type":"integer"},'
+            '"tags":{"type":"array","items":{"type":"string"}}},"required":["name","price","tags"]}}},'
+            '"required":["server","items"]}\n',
+            "schema", flags="-o json -I 0", output_format="json"),
+    Example("null と空の入れ物を取り除く — yaqpy 独自", '{"a": null, "b": {"c": null}, "d": 1}\n',
+            "in.json", '{"d":1}\n', "", flags="-o json -I 0", input_format="json",
+            output_format="json", prune_null=True, prune_empty=True),
+    Example("API のリクエストを変換する（レシピ）— yaqpy 独自", OPENAI_REQUEST, "request.json",
+            '{"systemInstruction":{"parts":[{"text":"You are helpful."}]},'
+            '"contents":[{"role":"user","parts":[{"text":"Hello"}]},{"role":"model","parts":[{"text":"Hi!"}]}],'
+            '"generationConfig":{"temperature":0.7,"maxOutputTokens":256,"stopSequences":["END"]}}\n',
+            input_format="json", output_format="json", recipe="openai-to-gemini", flags="-I 0",
+            expected_notes=("dropped .model", "dropped .stream")),
+)
+
+
+def run_example(service: YqService, example: Example) -> tuple[str, list[str]]:
+    """Run an example on the real engine. Returns the output and, for a recipe, the notes it printed."""
+    options = Options(input_format=example.input_format, output_format=example.output_format,
+                      indent=0 if "-I 0" in example.flags else 2, security=SecurityPolicy.strict())
+    if example.recipe:
+        recipes = RecipeService(service)
+        recipe = recipes.load(example.recipe)
+        run = recipes.run(recipe, InputSource("<text>", example.input), options,
+                          input_format="json", output_format="json")
+        return run.output, summary_lines(run)
+    expression = with_prune(example.expression or ".", nulls=example.prune_null,
+                            empties=example.prune_empty) if (example.prune_null or example.prune_empty) \
+        else example.expression
+    request = EvaluateRequest(
+        expression=expression, inputs=(InputSource("<text>", example.input),),
+        mode=EvalMode.ALL if example.eval_all else EvalMode.STREAM, options=options,
+        input_format=example.input_format, output_format=example.output_format)
+    result = service.evaluate(request, MemorySink())
+    return result.output or "", []

yaqpy/app/local.py

@@ -0,0 +1,57 @@
+"""Local implementations of the ports (pathlib / os.environ)."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+import tempfile
+from collections.abc import Mapping
+from pathlib import Path
+
+
+class LocalFileSystem:
+    def read_text(self, path: str) -> str:
+        data = Path(path).read_bytes()
+        return data.decode("utf-8-sig")
+
+    def read_stdin(self) -> str:
+        data = sys.stdin.buffer.read()
+        return data.decode("utf-8-sig")
+
+    def atomic_write(self, path: str, text: str) -> None:
+        target = Path(path)
+        directory = target.parent if str(target.parent) else Path(".")
+        fd, tmp_name = tempfile.mkstemp(prefix=".yaqpy-", suffix=".tmp", dir=str(directory))
+        tmp = Path(tmp_name)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8", newline="") as handle:
+                handle.write(text)
+            if target.exists():
+                try:
+                    shutil.copymode(str(target), str(tmp))
+                except OSError:
+                    pass
+            os.replace(str(tmp), str(target))
+        except BaseException:
+            try:
+                tmp.unlink()
+            except OSError:
+                pass
+            raise
+
+    def write_file(self, path: str, text: str) -> None:
+        """Create (or replace) a file, making its directories first (Go's ``MkdirAll`` + ``Create``)."""
+        parent = Path(path).parent
+        if str(parent) not in ("", "."):
+            parent.mkdir(mode=0o750, parents=True, exist_ok=True)
+        self.atomic_write(path, text)
+
+    def exists_file(self, path: str) -> bool:
+        p = Path(path)
+        return p.exists() and not p.is_dir()
+
+
+class LocalEnvironment:
+    def environ(self) -> Mapping[str, str]:
+        return os.environ

yaqpy/app/ports.py

@@ -0,0 +1,75 @@
+"""Ports: the abstract I/O the application layer needs (design doc 11-2)."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Protocol
+
+from yaqpy.errors import SecurityError
+
+
+class FileSystemPort(Protocol):
+    def read_text(self, path: str) -> str: ...
+    def read_stdin(self) -> str: ...
+    def atomic_write(self, path: str, text: str) -> None: ...
+    def write_file(self, path: str, text: str) -> None: ...
+    def exists_file(self, path: str) -> bool: ...
+
+
+class EnvironmentPort(Protocol):
+    def environ(self) -> Mapping[str, str]: ...
+
+
+class SandboxFileSystem:
+    """Refuses every file access (used by the API service)."""
+
+    def read_text(self, path: str) -> str:
+        raise SecurityError(f"file access is not allowed: {path}", capability="file")
+
+    def read_stdin(self) -> str:
+        raise SecurityError("stdin is not available", capability="file")
+
+    def atomic_write(self, path: str, text: str) -> None:
+        raise SecurityError(f"file access is not allowed: {path}", capability="file")
+
+    def write_file(self, path: str, text: str) -> None:
+        raise SecurityError(f"file access is not allowed: {path}", capability="file")
+
+    def exists_file(self, path: str) -> bool:
+        return False
+
+
+class InMemoryFileSystem:
+    """Test double."""
+
+    def __init__(self, files: dict[str, str] | None = None, stdin: str = "") -> None:
+        self.files = dict(files or {})
+        self.stdin = stdin
+        self.written: dict[str, str] = {}
+
+    def read_text(self, path: str) -> str:
+        try:
+            return self.files[path]
+        except KeyError:
+            raise FileNotFoundError(path) from None
+
+    def read_stdin(self) -> str:
+        return self.stdin
+
+    def atomic_write(self, path: str, text: str) -> None:
+        self.files[path] = text
+        self.written[path] = text
+
+    def write_file(self, path: str, text: str) -> None:
+        self.atomic_write(path, text)
+
+    def exists_file(self, path: str) -> bool:
+        return path in self.files
+
+
+class StaticEnvironment:
+    def __init__(self, values: Mapping[str, str] | None = None) -> None:
+        self._values = dict(values or {})
+
+    def environ(self) -> Mapping[str, str]:
+        return self._values