json-schema-utils 0.8__py3-none-any.whl

jsutils/inline.py

@@ -0,0 +1,206 @@
+from typing import Any
+import copy
+from urllib.parse import urlsplit
+
+from .utils import JsonSchema, JSUError, log
+from .schemas import Schemas
+from .recurse import recurseSchema
+
+# TODO fix recursion handling
+
+def mergeProperty(schema: JsonSchema, prop: str, value: Any) -> JsonSchema:
+    """Merge an additional property into en existing schema.
+
+    Note: this is in a best effort basis, on failure a backup plan is required.
+    """
+    # handle boolean schema
+    if isinstance(schema, bool):
+        return { prop: value } if schema else False
+    assert isinstance(schema, dict)  # pyright helper
+
+    # log.debug(f"merging {prop} in {schema}")
+
+    # then object
+    if prop in ("$defs"):  # ignore
+        pass
+    elif prop == "enum":
+        if prop in schema:
+            # intersect in order
+            vals = []
+            for v in schema[prop]:  # pyright: ignore
+                if v in value:
+                    vals.append(v)
+            if len(vals) == 0:
+                log.warning("incompatible enum/enum makes schema unsatisfiable")
+                schema = False
+            else:
+                schema[prop] = vals
+        elif "const" in schema:
+            if schema["const"] in value:
+                pass
+            else:
+                log.warning("incompatible enum/const makes schema unsatisfiable")
+                schema = False
+        else:
+            schema[prop] = value
+    elif prop == "const":
+        if prop in schema:
+            if schema[prop] == value:
+                pass
+            else:
+                log.warning("incompatible const/const makes schema unsatisfiable")
+                schema = False
+        elif "enum" in schema:
+            schema[prop] = value
+            if value in schema["enum"]:
+                del schema["enum"]
+            else:
+                log.warning("incompatible const/enum makes schema unsatisfiable")
+                schema = False
+        else:
+            schema[prop] = value
+    elif prop == "required":
+        assert isinstance(value, list)
+        if prop in schema:
+            # append in order and without duplicates
+            for p in value:
+                if p not in schema["required"]:
+                    schema["required"].append(p)  # pyright: ignore
+        else:
+            schema["required"] = value
+    elif prop == "properties":
+        if prop in schema:
+            props = schema[prop]
+            assert isinstance(value, dict) and isinstance(props, dict)
+            for p, s in value.items():
+                if p in props:
+                    if props[p] == s or s is True:
+                        pass
+                    else:
+                        props[p] = { "allOf": [ props[p], s ] }
+                else:
+                    props[p] = s
+        else:
+            schema[prop] = value
+    elif prop in ("allOf", "anyOf", "oneOf"):
+        assert isinstance(value, list)
+        if prop in schema:
+            schema[prop].extend(value)  # pyright: ignore
+        else:
+            schema[prop] = value
+    elif prop in ("title", "$comment"):
+        # best effort
+        if prop not in schema:
+            schema[prop] = value
+    # FIXME: what about "else"?
+    # TODO extend list of supported validations?
+    elif prop in ("type", "$ref", "pattern",
+                  "additionalProperties", "additionalItems", "items",
+                  "minLength", "maxLength", "minProperties", "maxProperties",
+                  "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum",
+                  "minItems", "maxItems", "minContains", "maxContains", "multipleOf",
+                  "uniqueItems"):
+        # allow identical values only (for now)
+        if prop in schema:
+            if schema[prop] == value:
+                pass
+            else:
+                raise JSUError(f"cannot merge prop {prop} distinct values")
+        else:
+            schema[prop] = value
+    else:
+        raise JSUError(f"merging of prop {prop} is not supported (yet)")
+
+    # log.debug(f"result: {schema}")
+    return schema
+
+
+# properties keept at the root while merging
+_KEEP_PROPS = {
+    "$schema", "$id", "$comment", "title", "description", "examples",
+    # containers
+    "$defs", "oneOf", "anyOf", "allOf",
+    # special cases
+    "unevaluatedProperties", "unevaluatedItems",
+    # older version compatibility?
+    "definitions",
+}
+
+
+def mergeSchemas(schema: JsonSchema, refschema: JsonSchema) -> JsonSchema:
+    """Merge two schemas."""
+
+    if isinstance(refschema, bool):
+        return schema if refschema else False
+    elif isinstance(schema, bool):
+        return refschema if schema else False
+    assert isinstance(schema, dict) and isinstance(refschema, dict)
+
+    saved = schema
+    schema = copy.deepcopy(schema)
+
+    try:
+        # best effort
+        for p, v in refschema.items():
+            schema = mergeProperty(schema, p, v)
+
+        # log.debug(f"merged: {schema}")
+
+    except JSUError as e:
+        # backup merge with allOf
+        log.warning(f"merge error: {e}")
+        log.info("merging schemas with allOf")
+        schema = saved
+        separate = {}
+        for p in list(schema.keys()):
+            if p not in _KEEP_PROPS:
+                separate[p] = schema[p]
+                del schema[p]
+        if "allOf" not in schema:
+            schema["allOf"] = []
+        if len(separate) > 0:
+            schema["allOf"].append(separate)  # pyright: ignore
+        if len(refschema) > 0:
+            schema["allOf"].append(refschema)  # pyright: ignore
+
+    return schema
+
+
+def _url(ref):
+    """Extract base URL from full reference."""
+    u = urlsplit(ref)
+    return u.scheme + "://" + u.netloc
+
+
+def inlineRefs(schema: JsonSchema, url: str, schemas: Schemas) -> JsonSchema:
+    """Recursively inline $ref in schema, which is modified."""
+
+    def rwtRef(schema: JsonSchema, path: list[str]) -> JsonSchema:
+
+        # recursion avoidance (FIXME insufficient)
+        spath = "/".join(path)
+        skips = {url + "#" + spath, url + "#/" + spath, url + "#./" + spath}
+
+        while isinstance(schema, dict) and "$ref" in schema:
+            ref = schema["$ref"]
+            assert isinstance(ref, str)
+
+            # (direct) recursion detection
+            if ref in skips:
+                log.info(f"skipping recursive ref: {ref}")
+                break
+
+            # actual substitution
+            sub = schemas.schema(url, ref)
+            del schema["$ref"]
+            if isinstance(sub, dict):
+                schema = mergeSchemas(schema, sub)
+            else:
+                assert isinstance(sub, bool)
+                if not sub:
+                    schema = False
+                # else True is coldly ignored
+
+        return schema
+
+    return recurseSchema(schema, url, rwt=rwtRef)

jsutils/recurse.py

@@ -0,0 +1,90 @@
+from .utils import JsonSchema, log, FilterFun, RewriteFun
+
+
+def _recurseSchema(
+        schema: JsonSchema,
+        url: str,
+        path: list[str],
+        flt: FilterFun,
+        rwt: RewriteFun) -> JsonSchema:
+
+    log.debug(f"recuring at {path}")
+
+    # skip recursion
+    if not flt(schema, path):
+        return schema
+
+    if isinstance(schema, bool):
+        return rwt(schema, path)
+    assert isinstance(schema, dict), f"schema must be a dict: {type(schema).__name__}"
+
+    # list of schemas
+    for prop in ("allOf", "oneOf", "anyOf", "prefixItems"):
+        if prop in schema:
+            subs = schema[prop]
+            assert isinstance(subs, list)
+            schema[prop] = [ _recurseSchema(s, url, path + [prop, str(i)], flt, rwt)  # type: ignore
+                             for i, s in enumerate(subs) ]
+
+    # direct schemas
+    for prop in ("additionalProperties", "unevaluatedProperties", "items",
+                 "not", "if", "then", "else", "contains", "propertyNames",
+                 "unevaluatedItems"):
+        if prop in schema:
+            # handle old items ~ prefixItems
+            if prop == "items" and isinstance(schema["items"], list):
+                schema[prop] = [ _recurseSchema(s, url, path + [prop, str(i)], flt, rwt)
+                    for i, s in enumerate(schema[prop]) ]  # type: ignore
+            else:  # standard case
+                schema[prop] = _recurseSchema(schema[prop],  # type: ignore
+                                              url, path + [prop], flt, rwt)
+
+    # handle values as schemas
+    def recValue(schema, *propnames):
+        for prop in propnames:
+            if prop in schema:
+                props = schema[prop]
+                assert isinstance(props, dict)
+                for p, s in list(props.items()):
+                    props[p] = _recurseSchema(s, url, path + [prop, p], flt, rwt)
+
+    recValue(schema, "properties", "dependentSchemas", "patternProperties")
+
+    # apply rwt
+    schema = rwt(schema, path)
+    assert isinstance(schema, (bool, dict))
+
+    # keep last?!
+    if isinstance(schema, dict):
+        recValue(schema, "$defs", "definitions")
+
+    return schema
+
+
+def recurseSchema(schema: JsonSchema, url: str,
+                  flt: FilterFun = lambda s, p: True,
+                  rwt: RewriteFun = lambda s, p: s) -> JsonSchema:
+    """Generic recursion on a JSON Schema.
+
+    :param schema: schema to consider.
+    :param url: url of schema.
+    :param flt: filter (top-down) function, whether to keep recursing.
+    :param rwt: rewrite (bottom-up) function.
+    """
+    return _recurseSchema(schema, url, [], flt, rwt)
+
+
+def hasDirectRef(schema, url):
+    """Tell whether schema has a $ref."""
+
+    some_ref: bool = False
+
+    def fltHasRef(schema: JsonSchema, path: list[str]) -> bool:
+        nonlocal some_ref
+        if "$defs" not in path and isinstance(schema, dict) and "$ref" in schema:
+            some_ref = True
+        return not some_ref
+
+    recurseSchema(schema, url, flt=fltHasRef)
+
+    return some_ref

jsutils/schemas.py

@@ -0,0 +1,151 @@
+from typing import Callable
+import os.path
+from urllib.parse import urlsplit
+import json
+import requests
+from .utils import JsonSchema, JSUError, log
+from .recurse import recurseSchema
+
+ProcessFun = Callable[[JsonSchema, str], JsonSchema]
+
+
+def _full_url(url: str, ref: str) -> str:
+    """Build full normalized URL from a reference.
+
+    :param url: the url of the schema.
+    :param ref: the reference (relative) url.
+    """
+    if "#" in ref:
+        lurl, lpath = ref.split("#", 1)
+    else:
+        lurl, lpath = ref, ""
+    # build the full url
+    if lurl == "":
+        return url + "#" + lpath
+    elif lurl.startswith("/"):
+        u = urlsplit(url)
+        return u.scheme + "://" + u.netloc + lurl + "#" + lpath
+    else:
+        return lurl + "#" + lpath
+
+
+def _fullURL(schema: JsonSchema, url: str) -> JsonSchema:
+    """Replace relative references with absolute references.
+
+    :param schema: schema to consider.
+    :param url: the url of the schema.
+    """
+
+    # we need full references to avoid ambiguities on inline!
+    def fullref(schema: JsonSchema, _path: list[str]) -> JsonSchema:
+        if isinstance(schema, dict) and "$ref" in schema:
+            ref = schema["$ref"]
+            assert isinstance(ref, str)
+            nref = _full_url(url, ref)
+            log.debug(f"updating {ref} in {url}: {nref}")
+            schema["$ref"] = nref
+        return schema
+
+    return recurseSchema(schema, url, rwt=fullref)
+
+
+class Schemas:
+    """Hold a set of identified schemas and sub-schemas."""
+
+    def __init__(self):
+        # url mapping for local storage: https://schema.psl.eu -> .
+        self._urlmap: dict[str, str] = {}
+        # cache of schema references
+        self._schemas: dict[str, JsonSchema] = {}
+        # additional processing on store
+        self._process: list[ProcessFun] = [_fullURL]
+
+    def addProcess(self, process: ProcessFun):
+        self._process.append(process)
+
+    def addMap(self, url: str, target: str):
+        """Add local mapping for URLs."""
+        self._urlmap[url] = target
+
+    def store(self, url: str, schema: JsonSchema):
+        """Store schema associated to URL."""
+        log.info(f"adding schema {url}")
+
+        if isinstance(schema, dict):
+            for prop in ("$id", "id"):
+                if prop in schema:
+                    if url != schema[prop]:
+                        log.warning(f"{prop}={schema[prop]} / {url}")
+                    # del schema["$id"]  # FIXME?
+
+        # NOTE intermediate to avoid an infinite recursion
+        self._schemas[url] = schema
+
+        # process schema through all filters
+        for process in self._process:
+            schema = process(schema, url)
+
+        # store final processed version
+        self._schemas[url] = schema
+
+    def _load(self, url: str):
+        """Load schema from URL if needed."""
+        log.info(f"loading schema: {url}")
+
+        assert "#" not in url
+
+        # rewrite url for local search
+        path = url
+        for u, t in self._urlmap.items():
+            if path.startswith(u):
+                path = t + path[len(u):]
+                break
+
+        # FIXME what about actual http download?
+        schema: JsonSchema|None
+        if path.startswith("http://") or path.startswith("https://"):
+            schema = requests.get(path).json  # type: ignore
+        else:
+            schema = None
+            for suffix in ("", ".json", ".schema.json"):
+                fn = f"{path}{suffix}"
+                if os.path.isfile(fn):
+                    log.debug(f"loading file: {fn}")
+                    schema = json.load(open(fn))
+                    break
+
+        if schema is None:
+            raise JSUError(f"schema {url} not found")
+
+        self.store(url, schema)
+
+    def _resolve(self, schema: JsonSchema, lpath: str) -> JsonSchema:
+        """Extract sub-schema from schema following path.
+
+        :param schema: schema to consider.
+        :param path: path to consider.
+
+        FIXME this does not handle URL escaping?
+        """
+        for p in lpath.split("/"):
+            if p in (".", ""):
+                pass
+            else:
+                assert isinstance(schema, dict) and p in schema
+                schema = schema[p]  # type: ignore
+
+        return schema
+
+    def schema(self, url: str, ref: str) -> JsonSchema:
+        """Resolve schema reference.
+
+        :param url: url of schema.
+        :param ref: reference to resolve in url.
+        """
+        assert isinstance(ref, str) and len(ref) > 0
+        fref = _full_url(url, ref)
+        assert "#" in fref
+        curl, path = fref.split("#", 1)
+        if curl not in self._schemas:
+            self._load(curl)
+        return self._resolve(self._schemas[curl], path)