unstructured-platform-plugins 0.0.0__py3-none-any.whl

unstructured_platform_plugins/schema/json_schema.py

@@ -0,0 +1,344 @@
+import inspect
+from dataclasses import MISSING, fields, is_dataclass
+from enum import Enum, EnumMeta
+from inspect import Parameter
+from pathlib import Path
+from types import GenericAlias, NoneType, UnionType
+from typing import Any, Optional, Type, Union, _UnionGenericAlias
+
+from pydantic import BaseModel, create_model
+from pydantic.fields import FieldInfo, PydanticUndefined
+from unstructured.ingest.v2.interfaces import FileData
+
+from unstructured_platform_plugins.schema.utils import TypedParameter
+from unstructured_platform_plugins.type_hints import get_type_hints
+
+# https://json-schema.org/understanding-json-schema/reference/type
+types_map: dict[Type, str] = {
+    str: "string",
+    bool: "boolean",
+    int: "integer",
+    float: "number",
+    dict: "object",
+    list: "array",
+}
+
+typed_map_reverse: dict[str, Type] = {v: k for k, v in types_map.items()}
+
+
+def is_generic_alias(val: Any) -> bool:
+    return hasattr(val, "__origin__") and hasattr(val, "__args__")
+
+
+def is_typed_dict(val: Any) -> bool:
+    return (
+        inspect.isclass(val) and dict in inspect.getmro(val) and inspect.getmro(val)[0] is not dict
+    )
+
+
+def type_to_json_schema(t: Type, args: Optional[tuple[Any, ...]] = None) -> dict:
+    resp = {"type": types_map[t]}
+    if t is list and args:
+        list_type = args[0]
+        resp["items"] = to_json_schema(list_type)
+    if t is dict and args:
+        key = args[0]
+        value = args[1]
+        resp["items"] = {
+            "key": to_json_schema(key),
+            "value": to_json_schema(value),
+        }
+    return resp
+
+
+def path_to_json_schema(path: Path) -> dict:
+    return {"type": "string", "is_path": True}
+
+
+def enum_to_json_schema(e: EnumMeta) -> dict:
+    values = [i.value for i in e]
+    value_types = [type(value) for value in values]
+    unique_value_types = list(set(value_types))
+    if len(unique_value_types) > 1:
+        raise ValueError(
+            "enum must have consistent types, found mixes: {}".format(
+                ", ".join([e.__name__ for e in unique_value_types])
+            )
+        )
+    value_types = unique_value_types[0]
+    type_string = types_map[value_types]
+    return {"type": type_string, "enum": values}
+
+
+def generic_alias_to_json_schema(t: GenericAlias) -> dict:
+    origin = t.__origin__
+    if origin is Union:
+        types = t.__args__
+        if len(types) == 1:
+            return to_json_schema(types[0])
+        else:
+            return {"anyOf": [to_json_schema(t) for t in types]}
+    return type_to_json_schema(t=origin, args=t.__args__)
+
+
+def union_type_to_json_schema(t: UnionType) -> dict:
+    types = t.__args__
+    if len(types) == 1:
+        return to_json_schema(types[0])
+    else:
+        return {"anyOf": [to_json_schema(t) for t in types]}
+
+
+def dataclass_to_json_schema(class_or_instance: Any) -> dict:
+    resp = {"type": "object"}
+    is_filedata_instance = isinstance(class_or_instance, FileData)
+    is_filedata_class = class_or_instance is FileData
+    if is_filedata_instance or is_filedata_class:
+        resp["is_file_data"] = True
+    fs = fields(class_or_instance)
+    if not fs:
+        return resp
+    properties = {}
+    required = []
+    type_hints = get_type_hints(class_or_instance)
+    for f in fs:
+        t = type_hints[f.name]
+        f_resp = to_json_schema(t)
+        if f.default is not MISSING:
+            f_resp["default"] = f.default.value if isinstance(f.default, Enum) else f.default
+        else:
+            required.append(f.name)
+        properties[f.name] = f_resp
+    resp["properties"] = properties
+    resp["required"] = required
+    return resp
+
+
+def pydantic_base_model_to_json_schema(model: Type[BaseModel]) -> dict:
+    resp = {"type": "object"}
+    fs: dict[str, FieldInfo] = model.model_fields
+    if not fs:
+        return resp
+    properties = {}
+    required = []
+    for name, f in fs.items():
+        t = f.annotation
+        f_resp = to_json_schema(t)
+        if f.default != PydanticUndefined:
+            f_resp["default"] = f.default
+        else:
+            required.append(name)
+        properties[name] = f_resp
+    resp["properties"] = properties
+    resp["required"] = required
+    return resp
+
+
+def typed_dict_to_json_schem(typed_dict_class) -> dict:
+    resp = {"type": "object"}
+    fs = typed_dict_class.__annotations__
+    if not fs:
+        return resp
+    properties = {}
+    required = []
+    type_hints = get_type_hints(typed_dict_class)
+    for name in fs:
+        t = type_hints[name]
+        f_resp = to_json_schema(t)
+        properties[name] = f_resp
+        required.append(name)
+    resp["properties"] = properties
+    resp["required"] = required
+    return resp
+
+
+def parameter_to_json_schema(parameter: Parameter) -> dict:
+    annotation = parameter.annotation
+    resp = to_json_schema(annotation)
+    if parameter.default != Parameter.empty:
+        resp["default"] = parameter.default
+    return resp
+
+
+def typed_parameter_to_json_schema(parameter: TypedParameter) -> dict:
+    param_type = parameter.param_type
+    resp = to_json_schema(param_type)
+    if parameter.default != Parameter.empty:
+        resp["default"] = parameter.default
+    return resp
+
+
+def to_json_schema(val: Any) -> dict:
+    if val in [None, NoneType]:
+        return {"type": "null"}
+    if val is Any:
+        return {}
+    if isinstance(val, TypedParameter):
+        return typed_parameter_to_json_schema(parameter=val)
+    if isinstance(val, Parameter):
+        return parameter_to_json_schema(parameter=val)
+    if isinstance(val, UnionType):
+        return union_type_to_json_schema(t=val)
+    if isinstance(val, EnumMeta):
+        return enum_to_json_schema(e=val)
+    if is_generic_alias(val=val):
+        return generic_alias_to_json_schema(t=val)
+    if val is Type:
+        return type_to_json_schema(t=val)
+    if is_dataclass(val):
+        return dataclass_to_json_schema(val)
+    if val is Path:
+        return path_to_json_schema(val)
+    if val in types_map:
+        return type_to_json_schema(t=val)
+    if inspect.isclass(val) and issubclass(val, BaseModel):
+        return pydantic_base_model_to_json_schema(model=val)
+    if is_typed_dict(val):
+        return typed_dict_to_json_schem(val)
+    raise ValueError(f"Unsupported type: ({type(val).__name__}) {val}")
+
+
+def run_input_checks(parameters: list[Parameter]):
+    # Any variable positional or keyword args are not allowed (i.e. *args, **kwargs)
+    var_positional = [str(p) for p in parameters if p.kind == Parameter.VAR_POSITIONAL]
+    if var_positional:
+        raise TypeError(
+            "function has variable positional arguments, which are not allowed: {}".format(
+                ", ".join(var_positional)
+            )
+        )
+    var_keywords = [str(p) for p in parameters if p.kind == Parameter.VAR_KEYWORD]
+    if var_keywords:
+        raise TypeError(
+            "function has variable keyword arguments, which are not allowed: {}".format(
+                ", ".join(var_keywords)
+            )
+        )
+
+
+def parameters_to_json_schema(parameters: list[Parameter]) -> dict:
+    if not parameters:
+        return {"type": "null"}
+    run_input_checks(parameters=parameters)
+    resp = {"type": "object"}
+    properties = {}
+    required_fields = []
+    for p in parameters:
+        schema = to_json_schema(val=p)
+        properties[p.name] = schema
+        if p.default == Parameter.empty:
+            required_fields.append(p.name)
+    if required_fields:
+        resp["required"] = required_fields
+    if properties:
+        resp["properties"] = properties
+    return resp
+
+
+def run_output_checks(return_annotation: Any):
+    if is_generic_alias(val=return_annotation):  # noqa: SIM102
+        if return_annotation.__origin__ in [Union, list]:
+            for arg in return_annotation.__args__:
+                run_output_checks(arg)
+            return
+    if is_typed_dict(return_annotation):
+        return
+    if is_dataclass(return_annotation):
+        return
+    try:
+        if inspect.isclass(return_annotation) and issubclass(return_annotation, BaseModel):
+            return
+    except TypeError:
+        pass
+    if return_annotation is None:
+        return
+    if return_annotation is NoneType:
+        return
+    raise ValueError(f"Unsupported response type: {return_annotation}")
+
+
+def response_to_json_schema(return_annotation: Any) -> dict:
+    run_output_checks(return_annotation=return_annotation)
+    return to_json_schema(val=return_annotation)
+
+
+def schema_to_base_model_type(json_type_name, name: str, type_info: dict) -> Type:
+    t = typed_map_reverse[json_type_name]
+    if t is dict and type_info.get("is_file_data", False):
+        return FileData
+    if t is str and type_info.get("is_path", False):
+        return Path
+    if t is dict and "properties" in type_info:
+        t = schema_to_base_model(
+            schema=type_info,
+            name=name,
+        )
+    if t is dict and "items" in type_info and isinstance(type_info["items"], dict):
+        items = type_info["items"]
+        if "key" in items and "value" in items:
+            key_info = items["key"]
+            key_type_name = key_info["type"]
+            key_subtype = schema_to_base_model_type(
+                json_type_name=key_type_name, name=f"{name}_key", type_info=key_info
+            )
+            value_info = items["value"]
+            if not value_info:
+                value_subtype = Any
+            else:
+                value_type_name = items["value"]["type"]
+                value_subtype = schema_to_base_model_type(
+                    json_type_name=value_type_name, name=f"{name}_value", type_info=value_info
+                )
+            t = dict[key_subtype, value_subtype]
+    if t is list and "items" in type_info and isinstance(type_info["items"], dict):
+        items = type_info["items"]
+        item_type_name = items["type"]
+        subtype = schema_to_base_model_type(
+            json_type_name=item_type_name, name=f"{name}_type", type_info=items
+        )
+        t = list[subtype]
+    if "enum" in type_info and isinstance(type_info["enum"], list):
+        enum_content = type_info["enum"]
+        t = Enum(f"{name}_enum", {v: v for v in enum_content})
+    return t
+
+
+def schema_to_base_model(schema: dict, name: str = "reconstructed_model") -> Type[BaseModel]:
+    inputs = {}
+    properties = schema["properties"]
+
+    for k, v in properties.items():
+        optional = False
+        if "anyOf" in v:
+            any_of_entries = v["anyOf"]
+            if "null" in [entry["type"] for entry in any_of_entries]:
+                optional = True
+                any_of_entries = [entry for entry in any_of_entries if entry["type"] != "null"]
+            if len(any_of_entries) > 1:
+                type_info = [
+                    schema_to_base_model_type(
+                        type_info["type"], name=f"{k}_{index}", type_info=type_info
+                    )
+                    for index, type_info in enumerate(any_of_entries)
+                ]
+                # To support python3.10, unpacking not supported
+                # t = Union[*type_info]
+                t = _UnionGenericAlias(Union, tuple(type_info))
+            else:
+                entry_info = any_of_entries[0]
+                json_type_name = entry_info["type"]
+                t = schema_to_base_model_type(
+                    json_type_name=json_type_name, name=k, type_info=entry_info
+                )
+        else:
+            json_type_name = v["type"]
+            t = schema_to_base_model_type(json_type_name=json_type_name, name=k, type_info=v)
+        if optional:
+            t = Optional[t]
+        resp = [t]
+        if "default" in v:
+            resp.append(v["default"])
+        else:
+            resp.append(Ellipsis)
+        inputs[k] = tuple(resp)
+    return create_model(name, **inputs)

unstructured_platform_plugins/schema/model.py

@@ -0,0 +1,101 @@
+import inspect
+from types import UnionType
+from typing import Any, Literal, Optional, Union, get_args
+
+from pydantic import BaseModel, Field, ValidationError
+from typing_extensions import Annotated
+
+AnnotatedType = type(Annotated[str, str])
+
+
+class StringEntrySchema(BaseModel):
+    type: Literal["string"]
+    default: Optional[str] = None
+
+
+class BooleanEntrySchema(BaseModel):
+    type: Literal["boolean"]
+    default: Optional[bool] = None
+
+
+class IntegerEntrySchema(BaseModel):
+    type: Literal["integer"]
+    default: Optional[int] = None
+
+
+class NumberEntrySchema(BaseModel):
+    type: Literal["number"]
+    default: Optional[float] = None
+
+
+class ArrayEntrySchema(BaseModel):
+    type: Literal["array"]
+    items: Optional["AnyEntry"] = None
+
+
+class ObjectEntrySchema(BaseModel):
+    type: Literal["object"]
+    properties: Optional[dict[str, "AnyEntry"]] = None
+    required: Optional[list[str]] = None
+
+
+class NullEntrySchema(BaseModel):
+    type: Literal["null"]
+
+
+class AnyOfEntrySchema(BaseModel):
+    anyOf: list["AnyEntry"]
+
+
+TypedAnyEntry = Union[
+    StringEntrySchema,
+    BooleanEntrySchema,
+    IntegerEntrySchema,
+    ArrayEntrySchema,
+    ObjectEntrySchema,
+    NumberEntrySchema,
+    NullEntrySchema,
+]
+TypedAnyEntryTyping = Annotated[TypedAnyEntry, Field(discriminator="type")]
+AnyEntry = Union[TypedAnyEntryTyping, AnyOfEntrySchema]
+
+
+def is_valid_input_dict(schema: dict) -> bool:
+    # Support both null and object cases
+    try:
+        NullEntrySchema.model_validate(schema, strict=True)
+        return True
+    except ValidationError:
+        pass
+    try:
+        ObjectEntrySchema.model_validate(schema, strict=True)
+        return True
+    except ValidationError:
+        pass
+    return False
+
+
+def decompose_union(tp: Any) -> list[BaseModel]:
+    args = get_args(tp)
+    decomposed_types = []
+    for arg in args:
+        if inspect.isclass(arg) and issubclass(arg, BaseModel):
+            decomposed_types.append(arg)
+        elif isinstance(arg, UnionType):
+            decomposed_types.extend(decompose_union(tp=arg))
+        elif isinstance(arg, AnnotatedType):
+            annotated_args = get_args(arg)
+            type_arg = annotated_args[0]
+            decomposed_types.extend(decompose_union(tp=type_arg))
+    return decomposed_types
+
+
+def is_valid_response_dict(schema: dict) -> bool:
+    decomposed_models = decompose_union(tp=AnyEntry)
+    for tp in decomposed_models:
+        try:
+            tp.model_validate(schema, strict=True)
+            return True
+        except ValidationError:
+            pass
+    return False

unstructured_platform_plugins/schema/usage.py

@@ -0,0 +1,7 @@
+from pydantic import BaseModel
+
+
+# This is a stand in until a supported schema is published external to this repo
+class UsageData(BaseModel):
+    value: int
+    name: str

unstructured_platform_plugins/schema/utils.py

@@ -0,0 +1,31 @@
+import inspect
+from inspect import Parameter, _empty
+from typing import Callable
+
+from unstructured_platform_plugins.type_hints import get_type_hints
+
+
+class TypedParameter(Parameter):
+    def __init__(self, *args, param_type=_empty, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.param_type = param_type
+
+    @classmethod
+    def from_parameter(cls, param: Parameter) -> "TypedParameter":
+        return cls(
+            name=param.name, default=param.default, annotation=param.annotation, kind=param.kind
+        )
+
+
+def get_typed_parameters(fn: Callable) -> list[TypedParameter]:
+    type_hints = get_type_hints(fn)
+    parameters = list(inspect.signature(fn).parameters.values())
+    typed_params = []
+    for p in parameters:
+        typed_param = TypedParameter.from_parameter(param=p)
+        if isinstance(typed_param.annotation, str):
+            typed_param.param_type = type_hints[typed_param.name]
+        else:
+            typed_param.param_type = typed_param.annotation
+        typed_params.append(typed_param)
+    return typed_params

unstructured_platform_plugins/type_hints.py

@@ -0,0 +1,106 @@
+import sys
+from typing import ForwardRef, _allowed_types, _eval_type, _strip_annotations, types
+
+"""
+python3.10 behavior breaks python3.12 behavior, injecting Optional[] into the type
+if the field has a default value set. Below is the 3.12 behavior, pulling the same logic in when
+3.10 is used.
+"""
+
+
+def get_type_hints(obj, globalns=None, localns=None, include_extras=False):
+    """Return type hints for an object.
+
+    This is often the same as obj.__annotations__, but it handles
+    forward references encoded as string literals and recursively replaces all
+    'Annotated[T, ...]' with 'T' (unless 'include_extras=True').
+
+    The argument may be a module, class, method, or function. The annotations
+    are returned as a dictionary. For classes, annotations include also
+    inherited members.
+
+    TypeError is raised if the argument is not of a type that can contain
+    annotations, and an empty dictionary is returned if no annotations are
+    present.
+
+    BEWARE -- the behavior of globalns and localns is counterintuitive
+    (unless you are familiar with how eval() and exec() work).  The
+    search order is locals first, then globals.
+
+    - If no dict arguments are passed, an attempt is made to use the
+      globals from obj (or the respective module's globals for classes),
+      and these are also used as the locals.  If the object does not appear
+      to have globals, an empty dictionary is used.  For classes, the search
+      order is globals first then locals.
+
+    - If one dict argument is passed, it is used for both globals and
+      locals.
+
+    - If two dict arguments are passed, they specify globals and
+      locals, respectively.
+    """
+    if getattr(obj, "__no_type_check__", None):
+        return {}
+    # Classes require a special treatment.
+    if isinstance(obj, type):
+        hints = {}
+        for base in reversed(obj.__mro__):
+            if globalns is None:
+                base_globals = getattr(sys.modules.get(base.__module__, None), "__dict__", {})
+            else:
+                base_globals = globalns
+            ann = base.__dict__.get("__annotations__", {})
+            if isinstance(ann, types.GetSetDescriptorType):
+                ann = {}
+            base_locals = dict(vars(base)) if localns is None else localns
+            if localns is None and globalns is None:
+                # This is surprising, but required.  Before Python 3.10,
+                # get_type_hints only evaluated the globalns of
+                # a class.  To maintain backwards compatibility, we reverse
+                # the globalns and localns order so that eval() looks into
+                # *base_globals* first rather than *base_locals*.
+                # This only affects ForwardRefs.
+                base_globals, base_locals = base_locals, base_globals
+            for name, value in ann.items():
+                if value is None:
+                    value = type(None)
+                if isinstance(value, str):
+                    value = ForwardRef(value, is_argument=False, is_class=True)
+                value = _eval_type(value, base_globals, base_locals)
+                hints[name] = value
+        return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()}
+
+    if globalns is None:
+        if isinstance(obj, types.ModuleType):
+            globalns = obj.__dict__
+        else:
+            nsobj = obj
+            # Find globalns for the unwrapped object.
+            while hasattr(nsobj, "__wrapped__"):
+                nsobj = nsobj.__wrapped__
+            globalns = getattr(nsobj, "__globals__", {})
+        if localns is None:
+            localns = globalns
+    elif localns is None:
+        localns = globalns
+    hints = getattr(obj, "__annotations__", None)
+    if hints is None:
+        # Return empty annotations for something that _could_ have them.
+        if isinstance(obj, _allowed_types):
+            return {}
+        else:
+            raise TypeError(f"{obj!r} is not a module, class, method, " "or function.")
+    hints = dict(hints)
+    for name, value in hints.items():
+        if value is None:
+            value = type(None)
+        if isinstance(value, str):
+            # class-level forward refs were handled above, this must be either
+            # a module-level annotation or a function argument annotation
+            value = ForwardRef(
+                value,
+                is_argument=not isinstance(obj, types.ModuleType),
+                is_class=False,
+            )
+        hints[name] = _eval_type(value, globalns, localns)
+    return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()}

unstructured_platform_plugins/validate_api.py

@@ -0,0 +1,110 @@
+from urllib.parse import urljoin
+
+import click
+from pydantic import HttpUrl
+from requests import Session
+
+from unstructured_platform_plugins.schema.model import is_valid_input_dict
+
+
+class ApiSession(Session):
+    def __init__(self, base_url: HttpUrl = None):
+        super().__init__()
+        self.base_url = str(base_url)
+
+    def request(self, method, url, *args, **kwargs):
+        joined_url = urljoin(self.base_url, url)
+        return super().request(method, joined_url, *args, **kwargs)
+
+    def get_url(self, url: str) -> str:
+        return urljoin(self.base_url, url)
+
+
+class ValidationError(Exception):
+    pass
+
+
+def check_endpoint_exists(api_session: ApiSession, endpoint: str):
+    try:
+        api_session.head(endpoint)
+    except Exception as e:
+        raise ValidationError(
+            f"failed to validate that url exists: {api_session.get_url(url=endpoint)}"
+        ) from e
+
+
+def check_schema_response(api_session: ApiSession):
+    resp = api_session.get("/schema")
+    try:
+        resp.raise_for_status()
+    except Exception as e:
+        raise ValidationError(
+            "failed to validate response from schema "
+            "endpoint {}: {}".format(api_session.get_url(url="/schema"), e)
+        ) from e
+    contents = resp.json()
+    if not is_valid_input_dict(contents):
+        raise ValidationError("schema response don't conform to expected format")
+
+
+def check_id_response(api_session: ApiSession):
+    resp = api_session.get("/id")
+    try:
+        resp.raise_for_status()
+        contents = resp.text
+        assert contents.strip() != ""
+    except Exception as e:
+        raise ValidationError(
+            "failed to validate response from id endpoint "
+            "{}: {}".format(api_session.get_url(url="/id"), e)
+        ) from e
+
+
+def create_report(api_session: ApiSession) -> dict[str, str]:
+    report = {}
+    try:
+        check_endpoint_exists(api_session=api_session, endpoint="/invoke")
+    except ValidationError as e:
+        report["/invoke endpoint existence"] = str(e)
+
+    try:
+        check_endpoint_exists(api_session=api_session, endpoint="/schema")
+
+        try:
+            check_schema_response(api_session=api_session)
+        except ValidationError as e:
+            report["/schema endpoint response"] = str(e)
+
+    except ValidationError as e:
+        report["/schema endpoint existence"] = str(e)
+
+    try:
+        check_endpoint_exists(api_session=api_session, endpoint="/id")
+
+        try:
+            check_id_response(api_session=api_session)
+        except ValidationError as e:
+            report["/id endpoint response"] = str(e)
+    except ValidationError as e:
+        report["/id endpoint existence"] = str(e)
+
+    return report
+
+
+@click.command()
+@click.option("--api-url", type=HttpUrl, required=True, help="API URL to run validation against")
+def validate_api(api_url: HttpUrl) -> None:
+    api_session = ApiSession(base_url=api_url)
+    report = create_report(api_session=api_session)
+    if report:
+        print("Api validation failed:")
+        for k, v in report.items():
+            print(f"{k}: {v}")
+
+        exit(1)
+    print("Api validation successful")
+    exit(0)
+
+
+if __name__ == "__main__":
+    validate_api()