extended-data 8.0.0__py3-none-any.whl

extended_data/__init__.py

@@ -0,0 +1,105 @@
+"""Extended Data.
+
+This package provides Python utilities for structured data primitives, inputs,
+logging, file processing, and workflow-oriented data operations.
+"""
+
+from __future__ import annotations
+
+from extended_data._version import __version__
+from extended_data.containers import (
+    ExtendedDict,
+    ExtendedList,
+    ExtendedSet,
+    ExtendedString,
+    ExtendedTuple,
+    extend_data,
+    to_builtin,
+)
+from extended_data.inputs import InputProvider, directed_inputs, input_config
+from extended_data.io.base64 import base64_decode, base64_encode
+from extended_data.io.exporters import (
+    make_raw_data_export_safe,
+    wrap_raw_data_for_export,
+)
+from extended_data.io.files import (
+    DataFile,
+    FilePath,
+    clone_repository_to_temp,
+    decode_file,
+    delete_file,
+    file_path_depth,
+    file_path_rel_to_root,
+    get_encoding_for_file_path,
+    get_parent_repository,
+    get_repository_name,
+    get_tld,
+    is_url,
+    match_file_extensions,
+    read_data_file,
+    read_file,
+    resolve_local_path,
+    write_file,
+)
+from extended_data.io.importers import unwrap_raw_data_from_import
+from extended_data.logging import ExitRunError, KeyTransform, Logging
+from extended_data.primitives.formats.errors import DataDecodeError
+from extended_data.workflows import (
+    DATA_TRANSFORM_STEPS,
+    DataWorkflow,
+    StepLike,
+    WorkflowAction,
+    WorkflowResult,
+    WorkflowStep,
+    data_transform_action,
+    list_data_transform_steps,
+)
+
+
+__all__ = [
+    "DATA_TRANSFORM_STEPS",
+    "DataDecodeError",
+    "DataFile",
+    "DataWorkflow",
+    "ExitRunError",
+    "ExtendedDict",
+    "ExtendedList",
+    "ExtendedSet",
+    "ExtendedString",
+    "ExtendedTuple",
+    "FilePath",
+    "InputProvider",
+    "KeyTransform",
+    "Logging",
+    "StepLike",
+    "WorkflowAction",
+    "WorkflowResult",
+    "WorkflowStep",
+    "__version__",
+    "base64_decode",
+    "base64_encode",
+    "clone_repository_to_temp",
+    "data_transform_action",
+    "decode_file",
+    "delete_file",
+    "directed_inputs",
+    "extend_data",
+    "file_path_depth",
+    "file_path_rel_to_root",
+    "get_encoding_for_file_path",
+    "get_parent_repository",
+    "get_repository_name",
+    "get_tld",
+    "input_config",
+    "is_url",
+    "list_data_transform_steps",
+    "make_raw_data_export_safe",
+    "match_file_extensions",
+    "read_data_file",
+    "read_file",
+    "resolve_local_path",
+    "to_builtin",
+    "unwrap_raw_data_from_import",
+    "wrap_raw_data_for_export",
+    "write_file",
+]

extended_data/_version.py

@@ -0,0 +1,16 @@
+"""Package version helpers."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+
+def get_version() -> str:
+    """Return the installed extended-data distribution version."""
+    try:
+        return version("extended-data")
+    except PackageNotFoundError:
+        return "0+unknown"
+
+
+__version__ = get_version()

extended_data/cli.py

@@ -0,0 +1,204 @@
+"""Top-level command line interface for Extended Data."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from collections.abc import Sequence
+from typing import Any, cast
+
+from extended_data.io import DataFile
+from extended_data.primitives.redaction import redact_sensitive_text
+from extended_data.workflows import DataWorkflow, WorkflowResult, list_data_transform_steps
+
+
+OUTPUT_ENCODINGS = ("json", "yaml", "toml", "hcl", "raw")
+
+
+def _write_stdout(message: str) -> None:
+    """Write one CLI output line."""
+    sys.stdout.write(f"{message}\n")
+
+
+def _write_stderr(message: str) -> None:
+    """Write one CLI error line."""
+    sys.stderr.write(f"{redact_sensitive_text(message)}\n")
+
+
+def _decode_artifact(args: argparse.Namespace) -> DataFile:
+    """Decode an inline payload or file path into a DataFile artifact."""
+    value = getattr(args, "value", None)
+    file_path = getattr(args, "file_path", None)
+
+    if value is not None and file_path is not None:
+        raise ValueError("pass either VALUE or --file, not both")
+    if value is None and file_path is None:
+        raise ValueError("pass VALUE or --file")
+    if file_path is not None:
+        return DataFile.read(file_path, suffix=args.suffix)
+    return DataFile.decode(cast(str, value), suffix=args.suffix)
+
+
+def cmd_decode(args: argparse.Namespace) -> int:
+    """Decode structured data and write it through the shared export boundary."""
+    try:
+        artifact = _decode_artifact(args)
+        _write_stdout(artifact.wrap_for_export(allow_encoding=args.output, **_json_format_opts(args)))
+        return 0
+    except Exception as e:
+        _write_stderr(str(e))
+        return 1
+
+
+def cmd_inspect(args: argparse.Namespace) -> int:
+    """Decode structured data and write its DataFile metadata."""
+    try:
+        artifact = _decode_artifact(args)
+        _write_stdout(artifact.metadata.wrap_for_export(allow_encoding=args.output, **_json_format_opts(args)))
+        return 0
+    except Exception as e:
+        _write_stderr(str(e))
+        return 1
+
+
+def _json_format_opts(args: argparse.Namespace) -> dict[str, Any]:
+    """Return common JSON formatting options for CLI export commands."""
+    if args.output == "json" and not args.compact:
+        return {"indent_2": True}
+    return {}
+
+
+def _merge_workflow(args: argparse.Namespace) -> DataWorkflow:
+    """Build a layered merge workflow from CLI arguments."""
+    file_paths = args.file_paths
+    if len(file_paths) < 2:
+        raise ValueError("merge requires at least two files")
+
+    workflow = DataWorkflow.from_file(file_paths[0], suffix=args.suffix)
+    for file_path in file_paths[1:]:
+        workflow = workflow.merge_file(file_path, suffix=args.suffix)
+    return workflow
+
+
+def cmd_merge(args: argparse.Namespace) -> int:
+    """Merge structured files through DataWorkflow and write or print the result."""
+    try:
+        workflow = _merge_workflow(args)
+        result: WorkflowResult
+        if args.write:
+            result = workflow.write(args.write, encoding=args.output, allow_empty=args.allow_empty)
+        else:
+            result = workflow.result()
+        _write_stdout(result.wrap_for_export(allow_encoding=args.output, **_json_format_opts(args)))
+        return 0
+    except Exception as e:
+        _write_stderr(str(e))
+        return 1
+
+
+def _transform_workflow(args: argparse.Namespace) -> DataWorkflow:
+    """Build a workflow that applies named Tier 2 transforms."""
+    steps = args.steps or []
+    if not steps:
+        raise ValueError("transform requires at least one --step")
+
+    return _decode_artifact(args).workflow().transform(*steps)
+
+
+def cmd_transform(args: argparse.Namespace) -> int:
+    """Apply named Tier 2 transforms through DataWorkflow."""
+    try:
+        workflow = _transform_workflow(args)
+        result: WorkflowResult
+        if args.write:
+            result = workflow.write(args.write, encoding=args.output, allow_empty=args.allow_empty)
+        else:
+            result = workflow.result()
+        _write_stdout(result.wrap_for_export(allow_encoding=args.output, **_json_format_opts(args)))
+        return 0
+    except Exception as e:
+        _write_stderr(str(e))
+        return 1
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    """Build the top-level Extended Data argument parser."""
+    parser = argparse.ArgumentParser(
+        prog="extended-data",
+        description="CLI for Extended Data primitives, files, and workflows",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  extended-data decode '{"service": {"name": "api"}}' --suffix json
+  extended-data decode --file config.yaml --output json
+  extended-data inspect --file config.yaml
+  extended-data merge base.yaml env.yaml --output yaml
+  extended-data transform --file payload.json --step reconstruct --step unhump
+        """,
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Commands")
+
+    decode_parser = subparsers.add_parser("decode", help="Decode inline data or a file")
+    decode_parser.add_argument("value", nargs="?", help="Inline payload to decode")
+    decode_parser.add_argument("--file", dest="file_path", help="File path or URL to decode")
+    decode_parser.add_argument("--suffix", help="Input format override")
+    decode_parser.add_argument("--output", choices=OUTPUT_ENCODINGS, default="json", help="Output encoding")
+    decode_parser.add_argument("--compact", action="store_true", help="Compact JSON output")
+    decode_parser.set_defaults(func=cmd_decode)
+
+    inspect_parser = subparsers.add_parser("inspect", help="Decode data and print artifact metadata")
+    inspect_parser.add_argument("value", nargs="?", help="Inline payload to inspect")
+    inspect_parser.add_argument("--file", dest="file_path", help="File path or URL to inspect")
+    inspect_parser.add_argument("--suffix", help="Input format override")
+    inspect_parser.add_argument("--output", choices=OUTPUT_ENCODINGS, default="json", help="Output encoding")
+    inspect_parser.add_argument("--compact", action="store_true", help="Compact JSON output")
+    inspect_parser.set_defaults(func=cmd_inspect)
+
+    merge_parser = subparsers.add_parser("merge", help="Deep merge structured files")
+    merge_parser.add_argument("file_paths", nargs="+", help="Structured files to merge in order")
+    merge_parser.add_argument("--suffix", help="Input format override for all files")
+    merge_parser.add_argument("--output", choices=OUTPUT_ENCODINGS, default="json", help="Output encoding")
+    merge_parser.add_argument("--compact", action="store_true", help="Compact JSON output")
+    merge_parser.add_argument("--write", help="Write merged output to this file")
+    merge_parser.add_argument("--allow-empty", action="store_true", help="Allow writing empty merged output")
+    merge_parser.set_defaults(func=cmd_merge)
+
+    transform_parser = subparsers.add_parser("transform", help="Apply named Extended Data transforms")
+    transform_parser.add_argument("value", nargs="?", help="Inline payload to transform")
+    transform_parser.add_argument("--file", dest="file_path", help="File path or URL to transform")
+    transform_parser.add_argument("--suffix", help="Input format override")
+    transform_parser.add_argument(
+        "--step",
+        dest="steps",
+        action="append",
+        choices=list_data_transform_steps(),
+        help="Transform step to apply in order",
+    )
+    transform_parser.add_argument("--output", choices=OUTPUT_ENCODINGS, default="json", help="Output encoding")
+    transform_parser.add_argument("--compact", action="store_true", help="Compact JSON output")
+    transform_parser.add_argument("--write", help="Write transformed output to this file")
+    transform_parser.add_argument("--allow-empty", action="store_true", help="Allow writing empty transformed output")
+    transform_parser.set_defaults(func=cmd_transform)
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the Extended Data CLI."""
+    args = list(argv) if argv is not None else sys.argv[1:]
+    parser = _build_parser()
+    parsed = parser.parse_args(args)
+
+    if not parsed.command:
+        parser.print_help()
+        return 0
+
+    try:
+        return parsed.func(parsed)
+    except KeyboardInterrupt:
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())

extended_data/containers/__init__.py

@@ -0,0 +1,17 @@
+"""Tier 2 extended container classes."""
+
+from extended_data.containers.factory import extend_data, to_builtin
+from extended_data.containers.mappings import ExtendedDict
+from extended_data.containers.sequences import ExtendedList, ExtendedSet, ExtendedTuple
+from extended_data.containers.strings import ExtendedString
+
+
+__all__ = [
+    "ExtendedDict",
+    "ExtendedList",
+    "ExtendedSet",
+    "ExtendedString",
+    "ExtendedTuple",
+    "extend_data",
+    "to_builtin",
+]

extended_data/containers/factory.py

@@ -0,0 +1,57 @@
+"""Factories for moving between plain data and extended containers."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+from extended_data.containers.mappings import ExtendedDict
+from extended_data.containers.sequences import ExtendedList, ExtendedSet, ExtendedTuple
+from extended_data.containers.strings import ExtendedString
+from extended_data.primitives.formats.yaml import LiteralScalarString, YamlPairs, YamlTagged
+
+
+def extend_data(value: Any) -> Any:
+    """Recursively wrap built-in containers in Extended Data containers."""
+    if isinstance(value, YamlTagged | YamlPairs | LiteralScalarString):
+        return value
+    if isinstance(value, ExtendedString | ExtendedDict | ExtendedList | ExtendedSet | ExtendedTuple):
+        return value
+    if isinstance(value, str):
+        return ExtendedString(value)
+    if isinstance(value, Mapping):
+        return ExtendedDict({key: extend_data(item) for key, item in value.items()})
+    if isinstance(value, list):
+        return ExtendedList(extend_data(item) for item in value)
+    if isinstance(value, tuple):
+        return ExtendedTuple(extend_data(item) for item in value)
+    if isinstance(value, set | frozenset):
+        return ExtendedSet(extend_data(item) for item in value)
+    return value
+
+
+def to_builtin(value: Any) -> Any:
+    """Recursively unwrap Extended Data containers to built-in Python values."""
+    if isinstance(value, YamlTagged | YamlPairs | LiteralScalarString):
+        return value
+    if isinstance(value, ExtendedString):
+        return str(value)
+    if isinstance(value, ExtendedDict):
+        return {to_builtin(key): to_builtin(item) for key, item in value.items()}
+    if isinstance(value, ExtendedList):
+        return [to_builtin(item) for item in value]
+    if isinstance(value, ExtendedTuple):
+        return tuple(to_builtin(item) for item in value)
+    if isinstance(value, ExtendedSet):
+        return {to_builtin(item) for item in value}
+    if isinstance(value, Mapping):
+        return {to_builtin(key): to_builtin(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [to_builtin(item) for item in value]
+    if isinstance(value, tuple):
+        return tuple(to_builtin(item) for item in value)
+    if isinstance(value, set):
+        return {to_builtin(item) for item in value}
+    if isinstance(value, frozenset):
+        return frozenset(to_builtin(item) for item in value)
+    return value

extended_data/containers/mappings.py

@@ -0,0 +1,181 @@
+"""Extended mapping container built on Tier 1 primitives."""
+
+from __future__ import annotations
+
+from collections import UserDict
+from collections.abc import Iterable, Mapping
+from typing import TYPE_CHECKING, Any, Self, overload
+
+
+if TYPE_CHECKING:
+    from _typeshed import SupportsKeysAndGetItem
+
+    from extended_data.containers.sequences import ExtendedList, ExtendedTuple
+
+from extended_data.primitives.mappings import (
+    all_values_from_map,
+    deduplicate_map,
+    deep_merge,
+    filter_map,
+    first_non_empty_value_from_map,
+    flatten_map,
+    unhump_map,
+)
+from extended_data.primitives.splitting import split_dict_by_type
+from extended_data.primitives.state import all_non_empty_in_dict, any_non_empty, yield_non_empty
+from extended_data.primitives.types import reconstruct_special_types
+
+
+class ExtendedDict(UserDict[str, Any]):
+    """Dictionary wrapper with chainable primitive operations."""
+
+    def __init__(self, initialdata: Mapping[str, Any] | None = None, **kwargs: Any) -> None:
+        """Initialize the extended dictionary."""
+        super().__init__()
+        self.update(initialdata or {}, **kwargs)
+
+    def __setitem__(self, key: str, item: Any) -> None:
+        """Set a value while preserving extended nested containers."""
+        from extended_data.containers.factory import extend_data
+
+        self.data[key] = extend_data(item)
+
+    @overload
+    def update(self, other: SupportsKeysAndGetItem[str, Any], /) -> None: ...
+
+    @overload
+    def update(self, other: SupportsKeysAndGetItem[str, Any], /, **kwargs: Any) -> None: ...
+
+    @overload
+    def update(self, other: Iterable[tuple[str, Any]], /) -> None: ...
+
+    @overload
+    def update(self, other: Iterable[tuple[str, Any]], /, **kwargs: Any) -> None: ...
+
+    @overload
+    def update(self, **kwargs: Any) -> None: ...
+
+    def update(self, *args: Any, **kwargs: Any) -> None:  # type: ignore[misc]
+        """Update values while preserving extended nested containers."""
+        if len(args) > 1:
+            msg = f"update expected at most 1 argument, got {len(args)}"
+            raise TypeError(msg)
+
+        if args:
+            other = args[0]
+            if hasattr(other, "items"):
+                for key, value in other.items():
+                    self[key] = value
+            elif hasattr(other, "keys") and hasattr(other, "__getitem__"):
+                keys = other.keys()
+                for key in keys:
+                    self[key] = other[key]
+            else:
+                for key, value in other:
+                    self[key] = value
+
+        for key, value in kwargs.items():
+            self[key] = value
+
+    def setdefault(self, key: str, default: Any = None) -> Any:
+        """Insert a default while returning the promoted stored value."""
+        if key not in self.data:
+            self[key] = default
+        return self.data[key]
+
+    def __ior__(self, other: Any) -> Self:  # type: ignore[override,misc]
+        """Update from a mapping or item iterable while preserving extended containers."""
+        self.update(other)
+        return self
+
+    def deep_merge(self, *mappings: Mapping[str, Any]) -> ExtendedDict:
+        """Return a deeply merged copy."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(deep_merge(to_builtin(self.data), *(to_builtin(mapping) for mapping in mappings)))
+
+    def flatten(self, *, separator: str = ".") -> ExtendedDict:
+        """Return a flattened copy."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(flatten_map(to_builtin(self.data), separator=separator))
+
+    def filter(
+        self,
+        *,
+        allowlist: list[str] | None = None,
+        denylist: list[str] | None = None,
+    ) -> ExtendedTuple[ExtendedDict]:
+        """Return accepted and rejected mapping entries."""
+        from extended_data.containers.factory import extend_data, to_builtin
+        from extended_data.containers.sequences import ExtendedTuple
+
+        accepted, rejected = filter_map(to_builtin(self.data), allowlist=allowlist, denylist=denylist)
+        return ExtendedTuple((extend_data(accepted), extend_data(rejected)))
+
+    def compact(self) -> ExtendedDict:
+        """Return a copy without values considered empty."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(all_non_empty_in_dict(to_builtin(self.data)))
+
+    def deduplicate(self) -> ExtendedDict:
+        """Return a copy with nested duplicate list values removed."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(deduplicate_map(to_builtin(self.data)))
+
+    def unhump(self, *, drop_without_prefix: str | None = None) -> ExtendedDict:
+        """Return a copy with camelCase keys converted to snake_case."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(unhump_map(to_builtin(self.data), drop_without_prefix=drop_without_prefix))
+
+    def all_values(self) -> ExtendedList[Any]:
+        """Return all values from the nested mapping."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(all_values_from_map(to_builtin(self.data)))
+
+    def split_by_type(self, *, primitive_only: bool = False) -> ExtendedDict:
+        """Return mapping entries grouped by value type name."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        grouped = split_dict_by_type(to_builtin(self.data), primitive_only=primitive_only)
+        return extend_data({type_key.__name__: values for type_key, values in grouped.items()})
+
+    def first_non_empty_value(self, *keys: str) -> Any:
+        """Return the first non-empty value for the provided keys."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(first_non_empty_value_from_map(to_builtin(self.data), *keys))
+
+    def first_non_empty_entry(self, *keys: str) -> ExtendedDict:
+        """Return the first non-empty keyed entry for the provided keys."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(any_non_empty(to_builtin(self.data), *keys))
+
+    def non_empty_entries(self, *keys: str) -> ExtendedList[ExtendedDict]:
+        """Return all non-empty keyed entries for the provided keys."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(list(yield_non_empty(to_builtin(self.data), *keys)))
+
+    def reconstruct_special_types(self, *, fail_silently: bool = False) -> ExtendedDict:
+        """Return a copy with string-like special values reconstructed."""
+        from extended_data.containers.factory import extend_data, to_builtin
+
+        return extend_data(reconstruct_special_types(to_builtin(self.data), fail_silently=fail_silently))
+
+    def to_export_safe(self, *, export_to_yaml: bool = False) -> Any:
+        """Return this mapping converted to export-safe primitive data."""
+        from extended_data.io.exporters import make_raw_data_export_safe
+
+        return make_raw_data_export_safe(self.data, export_to_yaml=export_to_yaml)
+
+    def wrap_for_export(self, allow_encoding: bool | str = True, **format_opts: Any) -> str:
+        """Return this mapping wrapped as an encoded export string."""
+        from extended_data.io.exporters import wrap_raw_data_for_export
+
+        return wrap_raw_data_for_export(self.data, allow_encoding=allow_encoding, **format_opts)