UncountablePythonSDK 0.0.20__py3-none-any.whl → 0.0.22__py3-none-any.whl

pkgs/type_spec/emit_python.py

@@ -21,6 +21,17 @@ __all__: list[str] = [
 """
 END_ALL_EXPORTS = "]\n"
 
+ASYNC_BATCH_TYPE_NAMESPACE = builder.SpecNamespace(name="uncountable.types.async_batch")
+ASYNC_BATCH_REQUEST_PATH_STYPE = builder.SpecTypeDefnStringEnum(
+    namespace=ASYNC_BATCH_TYPE_NAMESPACE, name="AsyncBatchRequestPath"
+)
+ASYNC_BATCH_REQUEST_STYPE = builder.SpecTypeDefnObject(
+    namespace=ASYNC_BATCH_TYPE_NAMESPACE, name="AsyncBatchRequest"
+)
+QUEUED_BATCH_REQUEST_STYPE = builder.SpecTypeDefnObject(
+    namespace=ASYNC_BATCH_TYPE_NAMESPACE, name="QueuedAsyncBatchRequest"
+)
+
 
 @dataclass(kw_only=True)
 class TrackingContext:
@@ -194,6 +205,7 @@ def emit_python(builder: builder.SpecBuilder, *, config: PythonConfig) -> None:
     _emit_api_stubs(builder=builder, config=config)
     _emit_api_argument_lookup(builder=builder, config=config)
     _emit_client_class(spec_builder=builder, config=config)
+    _emit_async_batch_processor(spec_builder=builder, config=config)
 
 
 def _emit_types_imports(*, out: io.StringIO, ctx: Context) -> None:
@@ -353,11 +365,9 @@ def _validate_supports_handler_generation(
 def _emit_endpoint_invocation_docstring(
     ctx: Context,
     endpoint: builder.SpecEndpoint,
-    arguments_type: builder.SpecTypeDefnObject,
+    properties: list[builder.SpecProperty],
 ) -> None:
-    has_argument_desc = arguments_type.properties is not None and any(
-        prop.desc is not None for prop in arguments_type.properties.values()
-    )
+    has_argument_desc = any(prop.desc is not None for prop in properties)
     has_endpoint_desc = endpoint.desc
     if not has_argument_desc and not has_endpoint_desc:
         return
@@ -370,55 +380,39 @@ def _emit_endpoint_invocation_docstring(
         ctx.out.write(f"{endpoint.desc}\n")
         ctx.out.write("\n")
 
-    if arguments_type.properties is not None and has_argument_desc:
-        for prop in arguments_type.properties.values():
+    if has_argument_desc:
+        for prop in properties:
             if prop.desc:
                 ctx.out.write(f"{FULL_INDENT}:param {prop.name}: {prop.desc}\n")
 
     ctx.out.write(f'{FULL_INDENT}"""\n')
 
 
-def _emit_endpoint_invocation_function(
-    ctx: Context, namespace: builder.SpecNamespace
+def _emit_endpoint_invocation_function_signature(
+    ctx: Context,
+    endpoint: builder.SpecEndpoint,
+    arguments_type: builder.SpecTypeDefnObject,
+    data_type: builder.SpecTypeDefnObject,
+    extra_params: list[builder.SpecProperty] | None = None,
 ) -> None:
-    endpoint = namespace.endpoint
-    if endpoint is None:
-        return
-    if not endpoint.is_sdk:
-        return
-
-    arguments_type = namespace.types["Arguments"]
-    data_type = namespace.types["Data"]
+    all_arguments = (
+        list(arguments_type.properties.values())
+        if arguments_type.properties is not None
+        else []
+    ) + (extra_params if extra_params is not None else [])
 
-    arguments_type = _validate_supports_handler_generation(arguments_type, "arguments")
-    data_type = _validate_supports_handler_generation(
-        data_type, "response", supports_inheritance=True
-    )
-
-    endpoint_method_stype = builder.SpecTypeDefnObject(
-        namespace=arguments_type.namespace, name=ENDPOINT_METHOD
-    )
-    endpoint_path_stype = builder.SpecTypeDefnObject(
-        namespace=arguments_type.namespace, name=ENDPOINT_PATH
-    )
-
-    has_arguments = (
-        arguments_type.properties is not None
-        and len(arguments_type.properties.values()) > 0
-    )
     assert endpoint.function is not None
     function_name = endpoint.function.split(".")[-1]
-    ctx.out.write("\n")
     ctx.out.write(
         f"""
     def {function_name}(
         self,\n"""
     )
-    if has_arguments:
+    if len(all_arguments) > 0:
         ctx.out.write(f"{INDENT}{INDENT}*,\n")
-        _emit_type_properties(
+        _emit_properties(
             ctx=ctx,
-            stype=arguments_type,
+            properties=all_arguments,
             num_indent=2,
             separator=",\n",
             class_out=ctx.out,
@@ -426,23 +420,143 @@ def _emit_endpoint_invocation_function(
     ctx.out.write(f"{INDENT}) -> {refer_to(ctx=ctx, stype=data_type)}:")
     ctx.out.write("\n")
 
-    _emit_endpoint_invocation_docstring(
-        ctx=ctx,
-        endpoint=endpoint,
-        arguments_type=arguments_type,
-    )
+    if len(all_arguments) > 0:
+        _emit_endpoint_invocation_docstring(
+            ctx=ctx, endpoint=endpoint, properties=all_arguments
+        )
+
 
-    ctx.out.write(f"{INDENT}{INDENT}args = {refer_to(ctx=ctx, stype=arguments_type)}(")
-    if has_arguments:
-        assert arguments_type.properties is not None
+def _emit_instantiate_type_from_locals(
+    ctx: Context, variable_name: str, variable_type: builder.SpecTypeDefnObject
+) -> None:
+    ctx.out.write(
+        f"{INDENT}{INDENT}{variable_name} = {refer_to(ctx=ctx, stype=variable_type)}("
+    )
+    if variable_type.properties is not None and len(variable_type.properties) > 0:
         ctx.out.write("\n")
-        for prop in arguments_type.properties.values():
+        for prop in variable_type.properties.values():
             ctx.out.write(f"{INDENT}{INDENT}{INDENT}{prop.name}={prop.name},")
             ctx.out.write("\n")
         ctx.out.write(f"{INDENT}{INDENT})")
     else:
         ctx.out.write(")")
 
+
+def _emit_async_batch_invocation_function(
+    ctx: Context, namespace: builder.SpecNamespace
+) -> None:
+    endpoint = namespace.endpoint
+    if endpoint is None:
+        return
+    if endpoint.async_batch_path is None or not endpoint.is_sdk:
+        return
+
+    ctx.out.write("\n")
+    arguments_type = namespace.types["Arguments"]
+    arguments_type = _validate_supports_handler_generation(arguments_type, "arguments")
+    data_type = QUEUED_BATCH_REQUEST_STYPE
+
+    list_str_params: list[builder.SpecType] = []
+    list_str_params.append(
+        builder.SpecTypeGenericParameter(
+            name="str",
+            spec_type_definition=builder.SpecTypeDefnObject(
+                namespace=namespace, name=builder.BaseTypeName.s_string, is_base=True
+            ),
+        )
+    )
+
+    _emit_endpoint_invocation_function_signature(
+        ctx=ctx,
+        endpoint=endpoint,
+        arguments_type=arguments_type,
+        data_type=data_type,
+        extra_params=[
+            builder.SpecProperty(
+                name="depends_on",
+                extant=builder.PropertyExtant.optional,
+                convert_value=builder.PropertyConvertValue.auto,
+                name_case=builder.NameCase.convert,
+                label="depends_on",
+                desc="A list of batch reference keys to process before processing this request",
+                spec_type=builder.SpecTypeInstance(
+                    defn_type=builder.SpecTypeDefnObject(
+                        name=builder.BaseTypeName.s_list,
+                        is_base=True,
+                        namespace=namespace,
+                    ),
+                    parameters=list_str_params,
+                ),
+            )
+        ],
+    )
+
+    # Emit function body
+    _emit_instantiate_type_from_locals(
+        ctx=ctx, variable_name="args", variable_type=arguments_type
+    )
+
+    path = _emit_value(
+        ctx=ctx,
+        stype=ASYNC_BATCH_REQUEST_PATH_STYPE,
+        value=endpoint.async_batch_path,
+    )
+
+    ctx.out.write(
+        f"""
+        json_data = serialize_for_api(args)
+
+        batch_reference = str(uuid.uuid4())
+
+        req = {refer_to(ctx=ctx, stype=ASYNC_BATCH_REQUEST_STYPE)}(
+            path={path},
+            data=json_data,
+            depends_on=depends_on,
+            batch_reference=batch_reference,
+        )
+
+        self._enqueue(req)
+
+        return {refer_to(ctx=ctx, stype=data_type)}(
+            path=req.path,
+            batch_reference=req.batch_reference,
+        )"""
+    )
+
+
+def _emit_endpoint_invocation_function(
+    ctx: Context, namespace: builder.SpecNamespace
+) -> None:
+    endpoint = namespace.endpoint
+    if endpoint is None:
+        return
+    if not endpoint.is_sdk or endpoint.is_beta:
+        return
+
+    ctx.out.write("\n")
+    arguments_type = namespace.types["Arguments"]
+    data_type = namespace.types["Data"]
+    arguments_type = _validate_supports_handler_generation(arguments_type, "arguments")
+    data_type = _validate_supports_handler_generation(
+        data_type, "response", supports_inheritance=True
+    )
+
+    _emit_endpoint_invocation_function_signature(
+        ctx=ctx, endpoint=endpoint, arguments_type=arguments_type, data_type=data_type
+    )
+
+    endpoint_method_stype = builder.SpecTypeDefnObject(
+        namespace=arguments_type.namespace, name=ENDPOINT_METHOD
+    )
+    endpoint_path_stype = builder.SpecTypeDefnObject(
+        namespace=arguments_type.namespace, name=ENDPOINT_PATH
+    )
+
+    # Emit function body
+    _emit_instantiate_type_from_locals(
+        ctx=ctx, variable_name="args", variable_type=arguments_type
+    )
+
     ctx.out.write(
         f"""
         api_request = APIRequest(
@@ -452,7 +566,6 @@ def _emit_endpoint_invocation_function(
         )
         return self.do_request(api_request=api_request, return_type={refer_to(ctx=ctx, stype=data_type)})"""
     )
-    ctx.out.write("\n")
 
 
 def _emit_string_enum(ctx: Context, stype: builder.SpecTypeDefnStringEnum) -> None:
@@ -516,17 +629,32 @@ def _emit_type_properties(
     stype: builder.SpecTypeDefnObject,
     num_indent: int = 1,
     separator: str = "\n",
+) -> EmittedPropertiesMetadata:
+    return _emit_properties(
+        ctx=ctx,
+        class_out=class_out,
+        properties=list((stype.properties or {}).values()),
+        num_indent=num_indent,
+        separator=separator,
+    )
+
+
+def _emit_properties(
+    *,
+    ctx: Context,
+    class_out: io.StringIO,
+    properties: list[builder.SpecProperty],
+    num_indent: int = 1,
+    separator: str = "\n",
 ) -> EmittedPropertiesMetadata:
     unconverted_keys: set[str] = set()
     unconverted_values: set[str] = set()
     to_string_values: set[str] = set()
     parse_require: set[str] = set()
 
-    if stype.properties is not None and len(stype.properties) > 0:
+    if len(properties) > 0:
 
         def write_field(prop: builder.SpecProperty) -> None:
-            # Checked in outer function, MyPy doens't track the check inside here
-            assert isinstance(stype, builder.SpecTypeDefn)
             if prop.name_case == builder.NameCase.preserve:
                 unconverted_keys.add(prop.name)
             py_name = python_field_name(prop.name, prop.name_case)
@@ -559,10 +687,10 @@ def _emit_type_properties(
                 class_out.write(f" = {default}")
             class_out.write(separator)
 
-        for prop in stype.properties.values():
+        for prop in properties:
             if prop.extant == builder.PropertyExtant.required:
                 write_field(prop)
-        for prop in stype.properties.values():
+        for prop in properties:
             if prop.extant != builder.PropertyExtant.required:
                 write_field(prop)
     else:
@@ -1014,6 +1142,64 @@ CLIENT_CLASS_IMPORTS = [
     "from abc import ABC, abstractmethod",
     "from dataclasses import dataclass",
 ]
+ASYNC_BATCH_PROCESSOR_FILENAME = "async_batch_processor"
+ASYNC_BATCH_PROCESSOR_IMPORTS = [
+    "import uuid",
+    "from abc import ABC, abstractmethod",
+    "from dataclasses import dataclass",
+    "from pkgs.serialization_util.serialization_helpers import serialize_for_api",
+]
+
+
+def _emit_async_batch_processor(
+    *, spec_builder: builder.SpecBuilder, config: PythonConfig
+) -> None:
+    if not config.emit_async_batch_processor:
+        return
+
+    async_batch_processor_out = io.StringIO()
+    ctx = Context(
+        out=io.StringIO(), namespace=builder.SpecNamespace("async_batch_processor")
+    )
+
+    for namespace in sorted(
+        spec_builder.namespaces.values(),
+        key=lambda ns: _resolve_namespace_name(ns),
+    ):
+        _emit_async_batch_invocation_function(ctx=ctx, namespace=namespace)
+
+    async_batch_processor_out.write(MODIFY_NOTICE)
+    async_batch_processor_out.write(LINT_HEADER)
+    async_batch_processor_out.write("# ruff: noqa: PLR0904\n")
+
+    _emit_types_imports(out=async_batch_processor_out, ctx=ctx)
+    _emit_namespace_imports(
+        out=async_batch_processor_out,
+        namespaces=ctx.namespaces,
+        from_namespace=None,
+        config=config,
+    )
+
+    async_batch_processor_out.write(
+        f"""{LINE_BREAK.join(ASYNC_BATCH_PROCESSOR_IMPORTS)}
+
+
+class AsyncBatchProcessorBase(ABC):
+    @abstractmethod
+    def _enqueue(self, req: {refer_to(ctx=ctx, stype=ASYNC_BATCH_REQUEST_STYPE)}) -> None:
+        ...
+
+    @abstractmethod
+    def send(self) -> base_t.ObjectId:
+        ..."""
+    )
+    async_batch_processor_out.write(ctx.out.getvalue())
+    async_batch_processor_out.write("\n")
+
+    util.rewrite_file(
+        f"{config.types_output}/{ASYNC_BATCH_PROCESSOR_FILENAME}.py",
+        async_batch_processor_out.getvalue(),
+    )
 
 
 def _emit_client_class(
@@ -1023,7 +1209,7 @@ def _emit_client_class(
         return
 
     client_base_out = io.StringIO()
-    ctx = Context(out=io.StringIO(), namespace=builder.SpecNamespace("client_class"))
+    ctx = Context(out=io.StringIO(), namespace=builder.SpecNamespace("client_base"))
     for namespace in sorted(
         spec_builder.namespaces.values(),
         key=lambda ns: _resolve_namespace_name(ns),
@@ -1038,7 +1224,7 @@ def _emit_client_class(
     _emit_namespace_imports(
         out=client_base_out,
         namespaces=ctx.namespaces,
-        from_namespace=builder.SpecNamespace("client_base"),
+        from_namespace=None,
         config=config,
     )
 
@@ -1059,10 +1245,10 @@ class ClientMethods(ABC):
 
     @abstractmethod
     def do_request(self, *, api_request: APIRequest, return_type: type[DT]) -> DT:
-        ...
-"""
+        ..."""
     )
     client_base_out.write(ctx.out.getvalue())
+    client_base_out.write("\n")
 
     util.rewrite_file(
         f"{config.types_output}/{CLIENT_CLASS_FILENAME}.py", client_base_out.getvalue()

pkgs/type_spec/load_types.py

@@ -1,5 +1,6 @@
 import os
 from collections.abc import Callable
+from io import StringIO
 from typing import Optional
 
 import yaml
@@ -13,11 +14,28 @@ ext_map = {
     ".py": "python",
 }
 
+_DOC_FILE_REFEX = ".*/docs/(examples|guides)/.*yaml"
+_EXAMPLE_FILE_REGEX = ".*/docs/examples/.*yaml"
+_GUIDE_FILE_REGEX = ".*/docs/guides/.*md"
+
 
 def find_and_handle_files(
-    root_folder: str, name_regex: str, handler: Callable[[str, str], None]
+    *,
+    root_folder: str,
+    handler: Callable[[str, str], None],
+    name_regex: str | None = None,
+    not_name_regex: str | None = None,
+    whole_name_regex: str | None = None,
+    not_whole_name_regex: str | None = None,
 ) -> None:
-    for file_name in fs.find(root_folder, name_regex=name_regex, relative=True):
+    for file_name in fs.find(
+        root_folder,
+        name_regex=name_regex,
+        not_name_regex=not_name_regex,
+        whole_name_regex=whole_name_regex,
+        not_whole_name_regex=not_whole_name_regex,
+        relative=True,
+    ):
         with open(os.path.join(root_folder, file_name), encoding="utf-8") as file:
             handler(file_name, file.read())
 
@@ -32,9 +50,16 @@ def load_types(config: Config) -> Optional[SpecBuilder]:
         name, ext = os.path.splitext(by_name)
         handler(ext_map[ext], name, file_content)
 
+    def handle_builder_example_add(file_name: str, file_content: str) -> None:
+        yaml_content = yaml.safe_load(StringIO(file_content))
+        builder.add_example_file(yaml_content)
+
+    def handle_builder_guide_add(file_name: str, file_content: str) -> None:
+        builder.add_guide_file(file_content)
+
     for folder in config.type_spec_types:
         find_and_handle_files(
-            folder,
+            root_folder=folder,
             name_regex=".*\\.(ts|py)\\.part",
             handler=lambda file_name, file_content: handle_builder_add(
                 file_name, file_content, builder.add_part_file
@@ -43,7 +68,7 @@ def load_types(config: Config) -> Optional[SpecBuilder]:
 
     for folder in config.type_spec_types:
         find_and_handle_files(
-            folder,
+            root_folder=folder,
             name_regex=".*\\.(ts|py)\\.prepart",
             handler=lambda file_name, file_content: handle_builder_add(
                 file_name, file_content, builder.add_prepart_file
@@ -64,9 +89,27 @@ def load_types(config: Config) -> Optional[SpecBuilder]:
 
     for folder in config.type_spec_types:
         find_and_handle_files(
-            folder, name_regex=".*\\.yaml", handler=builder_prescan_file
+            root_folder=folder,
+            name_regex=".*\\.yaml",
+            not_whole_name_regex=_DOC_FILE_REFEX,
+            handler=builder_prescan_file,
         )
 
+    if config.open_api is not None:
+        for folder in config.type_spec_types:
+            find_and_handle_files(
+                root_folder=folder,
+                whole_name_regex=_EXAMPLE_FILE_REGEX,
+                handler=handle_builder_example_add,
+            )
+
+        for folder in config.type_spec_types:
+            find_and_handle_files(
+                root_folder=folder,
+                whole_name_regex=_GUIDE_FILE_REGEX,
+                handler=handle_builder_guide_add,
+            )
+
     if not builder.process():
         return None
 

pkgs/type_spec/open_api_util.py

@@ -1,6 +1,5 @@
 from abc import ABC, abstractmethod
 from enum import StrEnum
-from io import UnsupportedOperation
 from typing import Optional
 
 
@@ -16,9 +15,6 @@ class OpenAPIType(ABC):
     def asdict(self) -> dict[str, object]:
         pass
 
-    def asarguments(self) -> dict[str, dict[str, object]]:
-        raise UnsupportedOperation
-
     def add_addl_info(self, emitted: dict[str, object]) -> dict[str, object]:
         if self.description is not None:
             emitted["description"] = self.description
@@ -148,9 +144,6 @@ class OpenAPIFreeFormObjectType(OpenAPIType):
     def asdict(self) -> dict[str, object]:
         return self.add_addl_info({"type": "object"})
 
-    def asarguments(self) -> dict[str, dict[str, object]]:
-        return {}
-
 
 class OpenAPIObjectType(OpenAPIType):
     """
@@ -174,31 +167,30 @@ class OpenAPIObjectType(OpenAPIType):
             self.property_desc = property_desc
         super().__init__(description=description, nullable=nullable)
 
+    def _emit_property_desc(self, property_name: str) -> dict[str, str]:
+        desc = self.property_desc.get(property_name)
+        if desc is None or desc.strip() == "":
+            return {}
+
+        return {"description": desc}
+
     def asdict(self) -> dict[str, object]:
         return self.add_addl_info({
             "type": "object",
+            "required": [
+                property_name
+                for property_name, property_type in self.properties.items()
+                if not property_type.nullable
+            ],
             "properties": {
                 property_name: {
                     **property_type.asdict(),
-                    "description": self.property_desc.get(property_name),
                 }
+                | self._emit_property_desc(property_name)
                 for property_name, property_type in self.properties.items()
             },
         })
 
-    def asarguments(self) -> dict[str, dict[str, object]]:
-        argument_types: dict[str, dict[str, object]] = {}
-        for property_name, property_type in self.properties.items():
-            desc = self.property_desc.get(property_name)
-            argument_types[property_name] = {
-                "name": property_name,
-                "in": "query",
-                "schema": property_type.asdict(),
-                "required": not property_type.nullable,
-                "description": desc or "",
-            }
-        return argument_types
-
 
 class OpenAPIUnionType(OpenAPIType):
     """
@@ -220,12 +212,6 @@ class OpenAPIUnionType(OpenAPIType):
         # TODO: use parents description and nullable
         return {"oneOf": [base_type.asdict() for base_type in self.base_types]}
 
-    def asarguments(self) -> dict[str, dict[str, object]]:
-        # TODO handle inheritence (allOf and refs); need to inline here...
-        # for now skip this endpoint
-
-        return {}
-
 
 class OpenAPIIntersectionType(OpenAPIType):
     """
@@ -246,9 +232,3 @@ class OpenAPIIntersectionType(OpenAPIType):
     def asdict(self) -> dict[str, object]:
         # TODO: use parents description and nullable
         return {"allOf": [base_type.asdict() for base_type in self.base_types]}
-
-    def asarguments(self) -> dict[str, dict[str, object]]:
-        # TODO handle inheritence (allOf and refs); need to inline here...
-        # for now skip this endpoint
-
-        return {}