UncountablePythonSDK 0.0.7__py3-none-any.whl → 0.0.92__py3-none-any.whl

pkgs/type_spec/type_info/emit_type_info.py

@@ -1,19 +1,23 @@
 import copy
 import dataclasses
+import decimal
 import io
 import json
-from typing import Any, Optional, TypeAlias, Union, cast
+from typing import Any, cast
 
-from main.base.types import data_t
-from main.base.types.base import PureJsonValue
+from main.base.types import data_t, type_info_t
+from main.base.types.base_t import PureJsonValue
 from pkgs.argument_parser import CachedParser
-from pkgs.serialization_util import serialize_for_api, serialize_for_storage
+from pkgs.serialization_util import (
+    serialize_for_api,
+    serialize_for_storage,
+)
 
 from .. import builder, util
 from ..emit_typescript_util import MODIFY_NOTICE, ts_name
 from ..value_spec import convert_to_value_spec_type
 
-ext_info_parser = CachedParser(data_t.ExtInfo)
+ext_info_parser = CachedParser(type_info_t.ExtInfo, strict_property_parsing=True)
 
 
 def type_path_of(stype: builder.SpecType) -> object:  # NamePath
@@ -23,7 +27,7 @@ def type_path_of(stype: builder.SpecType) -> object:  # NamePath
     extended scopes, generics, and enum literal values.
     - Scoped Type: [ (namespace-string)..., type-string ]
     - Instance Type: [ "$instance", Scoped-Type-Base, [TypePath-Parameters...] ]
-    - Literal Type: [ "$literal", [ "$value", value ]... ]
+    - Literal Type: [ "$literal", [ "$value", value, value-type-string ]... ]
 
     @return (string-specific, multiple-types)
     """
@@ -34,11 +38,15 @@ def type_path_of(stype: builder.SpecType) -> object:  # NamePath
 
     if isinstance(stype, builder.SpecTypeInstance):
         if stype.defn_type.name == builder.BaseTypeName.s_literal:
-            parts: list[str | list[str | bool]] = ["$literal"]
+            parts: list[object] = ["$literal"]
             for parameter in stype.parameters:
                 assert isinstance(parameter, builder.SpecTypeLiteralWrapper)
                 # This allows expansion to enum literal values later
-                parts.append(["$value", parameter.value])
+                parts.append([
+                    "$value",
+                    parameter.value,
+                    type_path_of(parameter.value_type),
+                ])
             return parts
 
         return [
@@ -65,12 +73,21 @@ def _dict_null_strip(data: dict[str, object]) -> dict[str, object]:
     }
 
 
+class JsonEncoder(json.JSONEncoder):
+    """We have some defaults of special types that we need to emit"""
+
+    def default(self, obj: object) -> object:
+        if isinstance(obj, decimal.Decimal):
+            return str(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
 def emit_type_info(build: builder.SpecBuilder, output: str) -> None:
     type_map = _build_map_all(build)
 
     # sort for stability, indent for smaller diffs
     stripped = _dict_null_strip(dataclasses.asdict(type_map))
-    serial = json.dumps(stripped, sort_keys=True, indent=2)
+    serial = json.dumps(stripped, sort_keys=True, indent=2, cls=JsonEncoder)
     type_map_out = io.StringIO()
     type_map_out.write(MODIFY_NOTICE)
     type_map_out.write(f"export const TYPE_MAP = {serial}")
@@ -117,7 +134,7 @@ class MapStringEnum(MapTypeBase):
     values: dict[str, str]
 
 
-MapType: TypeAlias = Union[MapTypeObject, MapTypeAlias, MapStringEnum]
+type MapType = MapTypeObject | MapTypeAlias | MapStringEnum
 
 
 @dataclasses.dataclass
@@ -148,6 +165,129 @@ def _build_map_all(build: builder.SpecBuilder) -> MapAll:
     return map_all
 
 
+@dataclasses.dataclass(kw_only=True)
+class InheritablePropertyParts:
+    """This uses only the "soft" information for now, things that aren't relevant
+    to the language emitted types. There are some fields that should be inherited
+    at that level, but that needs to be done in builder. When that is done, the
+    "label" and "desc" could probably be removed from this list."""
+
+    label: str | None = None
+    desc: str | None = None
+    ext_info: type_info_t.ExtInfo | None = None
+
+
+def _extract_inheritable_property_parts(
+    stype: builder.SpecTypeDefnObject,
+    prop: builder.SpecProperty,
+) -> InheritablePropertyParts:
+    if not stype.is_base and isinstance(stype.base, builder.SpecTypeDefn):
+        base_prop = (stype.base.properties or {}).get(prop.name)
+        if base_prop is None:
+            base_parts = InheritablePropertyParts()
+        else:
+            base_parts = _extract_inheritable_property_parts(stype.base, base_prop)
+            # Layout should not be inherited, as it'd end up hiding properties in the derived type
+            if base_parts.ext_info is not None:
+                base_parts.ext_info.layout = None
+    else:
+        base_parts = InheritablePropertyParts()
+
+    label = prop.label or base_parts.label
+    desc = prop.desc or base_parts.desc
+    local_ext_info = _parse_ext_info(prop.ext_info)
+    if local_ext_info is None:
+        ext_info = base_parts.ext_info
+    elif base_parts.ext_info is None:
+        ext_info = local_ext_info
+    else:
+        ext_info = dataclasses.replace(
+            local_ext_info,
+            **{
+                field.name: getattr(base_parts.ext_info, field.name)
+                for field in dataclasses.fields(type_info_t.ExtInfo)
+                if getattr(base_parts.ext_info, field.name) is not None
+            },
+        )
+
+    return InheritablePropertyParts(label=label, desc=desc, ext_info=ext_info)
+
+
+ExtInfoLayout = dict[str, set[str]]
+ALL_FIELDS_GROUP = "*all_fields"
+
+
+def _extract_and_validate_layout(
+    stype: builder.SpecTypeDefnObject,
+    ext_info: type_info_t.ExtInfo,
+    base_layout: ExtInfoLayout | None,
+) -> ExtInfoLayout:
+    """
+    Produce a map of groups to fields, for validation.
+    """
+    if ext_info.layout is None:
+        return {}
+    assert stype.properties is not None
+
+    all_fields_group: set[str] = set()
+    layout: ExtInfoLayout = {ALL_FIELDS_GROUP: all_fields_group}
+
+    for group in ext_info.layout.groups:
+        fields = set(group.fields or [])
+        for field in fields:
+            assert field in stype.properties, f"layout-refers-to-missing-field:{field}"
+
+        local_ref_name = None
+        if group.ref_name is not None:
+            assert base_layout is None or base_layout.get(group.ref_name) is None, (
+                f"group-name-duplicate-in-base:{group.ref_name}"
+            )
+            local_ref_name = group.ref_name
+
+        if group.extends:
+            assert base_layout is not None, "missing-base-layout"
+            base_group = base_layout.get(group.extends)
+            assert base_group is not None, f"missing-base-group:{group.extends}"
+            fields.update(base_group)
+            local_ref_name = group.extends
+
+        assert local_ref_name not in layout, f"duplicate-group:{local_ref_name}"
+        if local_ref_name is not None:
+            layout[local_ref_name] = fields
+        all_fields_group.update(fields)
+
+    for group_ref_name in base_layout or {}:
+        assert group_ref_name in layout, f"missing-base-group:{group_ref_name}"
+
+    for prop_ref_name in stype.properties:
+        assert prop_ref_name in all_fields_group, (
+            f"layout-missing-field:{prop_ref_name}"
+        )
+
+    return layout
+
+
+def _validate_type_ext_info(
+    stype: builder.SpecTypeDefnObject,
+) -> tuple[ExtInfoLayout | None, type_info_t.ExtInfo | None]:
+    ext_info = _parse_ext_info(stype.ext_info)
+    if ext_info is None:
+        return None, None
+
+    if ext_info.label_fields is not None:
+        assert stype.properties is not None
+        for name in ext_info.label_fields:
+            prop = stype.properties.get(name)
+            assert prop is not None, f"missing-label-field:{name}"
+
+    if not stype.is_base and isinstance(stype.base, builder.SpecTypeDefnObject):
+        base_layout, _ = _validate_type_ext_info(stype.base)
+    else:
+        base_layout = None
+
+    return _extract_and_validate_layout(stype, ext_info, base_layout), ext_info
+
+
 def _build_map_type(
     build: builder.SpecBuilder, stype: builder.SpecTypeDefn
 ) -> MapType | None:
@@ -158,6 +298,8 @@ def _build_map_type(
         and not stype.is_base
         and stype.base is not None
     ):
+        _, ext_info = _validate_type_ext_info(stype)
+
         properties: dict[str, MapProperty] = {}
         map_type = MapTypeObject(
             type_name=stype.name,
@@ -165,19 +307,22 @@ def _build_map_type(
             properties=properties,
             desc=stype.desc,
             base_type_path=type_path_of(stype.base),
-            ext_info=_convert_ext_info(stype.ext_info),
+            ext_info=serialize_for_api(ext_info),  # type: ignore[arg-type]
         )
 
         if stype.properties is not None:
             for prop in stype.properties.values():
+                parts = _extract_inheritable_property_parts(stype, prop)
+                # Propertis can't have layouts
+                assert parts.ext_info is None or parts.ext_info.layout is None
                 map_property = MapProperty(
                     type_name=prop.name,
-                    label=prop.label,
+                    label=parts.label,
                     api_name=ts_name(prop.name, prop.name_case),
                     extant=prop.extant,
                     type_path=type_path_of(prop.spec_type),
-                    ext_info=_convert_ext_info(prop.ext_info),
-                    desc=prop.desc,
+                    ext_info=serialize_for_api(parts.ext_info),  # type: ignore[arg-type]
+                    desc=parts.desc,
                     default=prop.default,
                 )
                 map_type.properties[prop.name] = map_property
@@ -194,6 +339,19 @@ def _build_map_type(
             discriminator=stype.discriminator,
         )
 
+    if isinstance(stype, builder.SpecTypeDefnUnion):
+        # Emit as a basic alias for now, as the front-end supports only those for now
+        # IMPROVE: We should emit a proper union type and support that
+        backing = stype.get_backing_type()
+        return MapTypeAlias(
+            type_name=stype.name,
+            label=stype.label,
+            desc=stype.desc,
+            alias_type_path=type_path_of(backing),
+            ext_info=_convert_ext_info(stype.ext_info),
+            discriminator=stype.discriminator,
+        )
+
     if isinstance(stype, builder.SpecTypeDefnStringEnum):
         return MapStringEnum(
             type_name=stype.name,
@@ -211,7 +369,7 @@ def _build_map_type(
     return None
 
 
-def _convert_ext_info(in_ext: Any) -> Optional[PureJsonValue]:
+def _parse_ext_info(in_ext: Any) -> type_info_t.ExtInfo | None:
     if in_ext is None:
         return None
     assert isinstance(in_ext, dict)
@@ -229,6 +387,10 @@ def _convert_ext_info(in_ext: Any) -> Optional[PureJsonValue]:
             df["result_type"] = serialize_for_storage(converted)
             mod_ext["data_format"] = df
 
-    parsed = ext_info_parser.parse_storage(mod_ext)
+    return ext_info_parser.parse_storage(mod_ext)
+
+
+def _convert_ext_info(in_ext: Any) -> PureJsonValue | None:
     # we need to convert this to API storage since it'll be used as-is in the UI
+    parsed = _parse_ext_info(in_ext)
     return cast(PureJsonValue, serialize_for_api(parsed))

pkgs/type_spec/util.py

@@ -1,7 +1,7 @@
 import json
 import os
 from dataclasses import dataclass
-from typing import Dict, List, Optional, TypeVar, Union
+from typing import TypeVar, Union
 
 import regex as re
 
@@ -29,11 +29,11 @@ LiteralTypeValue = Union[str, bool]
 class ParsedTypePart:
     name: str
     # An empty list is distinct from None
-    parameters: Optional[List["ParsedTypePath"]] = None
-    literal_value: Optional[LiteralTypeValue] = None
+    parameters: list["ParsedTypePath"] | None = None
+    literal_value: LiteralTypeValue | None = None
 
 
-ParsedTypePath = List[ParsedTypePart]
+ParsedTypePath = list[ParsedTypePart]
 
 
 @dataclass
@@ -48,7 +48,7 @@ def consume_parameter(
 ) -> ConsumedParameter:
     if bits[at] != "'":
         return ConsumedParameter(at=at, part=ParsedTypePart(name=bits[at]))
-    quote_stack: List[str] = []
+    quote_stack: list[str] = []
     at += 1
     while at < len(bits):
         if bits[at] == "'":
@@ -68,7 +68,7 @@ def parse_type_str(type_str: str) -> ParsedTypePath:
     """
     IMPROVE: will not detect all errors yet, focuses on correct cases
     """
-    raw_bits: List[str] = re.split(r"([.<>,'])", type_str)
+    raw_bits: list[str] = re.split(r"([.<>,'])", type_str)
     bits = [
         stripped_bit
         for stripped_bit in (padded_bit.strip() for padded_bit in raw_bits)
@@ -81,7 +81,7 @@ def parse_type_str(type_str: str) -> ParsedTypePath:
     cur_path = result
     cur_path.append(cur_part)
 
-    path_stack: List[ParsedTypePath] = []
+    path_stack: list[ParsedTypePath] = []
 
     at = 1
     while at < len(bits):
@@ -156,19 +156,19 @@ def is_valid_property_name(name: str) -> bool:
     return re_pattern_property_name.match(name) is not None
 
 
-def check_fields(data: Dict[str, T], allowed: List[str]) -> None:
+def check_fields(data: dict[str, T], allowed: list[str]) -> None:
     for key in data:
         if key not in allowed:
-            raise Exception(f"unexpected-field: {key}")
+            raise Exception(f"unexpected-field: {key}. Allowed: {allowed}")
 
 
-def split_any_name(name: str) -> List[str]:
+def split_any_name(name: str) -> list[str]:
     """
     Splits a name on case and underscores.
     myName => [my, name]
     my_name => [my, name]
     """
-    bits: List[str] = re_pattern_split_name.split(name)
+    bits: list[str] = re_pattern_split_name.split(name)
     return [s.lower() for s in filter(lambda x: x is not None and x != "_", bits)]
 
 

pkgs/type_spec/value_spec/__main__.py

@@ -17,12 +17,12 @@ If null is allowed as a legitimate value, such as in conditionals like `if`, the
 """
 
 import sys
-from typing import Type, TypeVar, cast
+from typing import TypeVar, cast
 
 import regex as re
-import yaml
 
 from main.base.types import base_t, value_spec_t
+from pkgs.serialization import yaml
 
 from ..util import parse_type_str, rewrite_file
 from .convert_type import convert_to_value_spec_type
@@ -177,7 +177,7 @@ def main() -> None:
             raise Exception(f"missing-{key}:{get_where()}")
         return cast(base_t.PureJsonValue, x)
 
-    def get_as(node: base_t.PureJsonValue, key: str, type_: Type[TypeT]) -> TypeT:
+    def get_as(node: base_t.PureJsonValue, key: str, type_: type[TypeT]) -> TypeT:
         raw = get(node, key)
         assert isinstance(raw, type_)
 

pkgs/type_spec/value_spec/convert_type.py

@@ -25,10 +25,17 @@ TYPE_MAP = {
     "List": MappedType(base_type=value_spec_t.BaseType.LIST, param_count=1),
     "Optional": MappedType(base_type=value_spec_t.BaseType.OPTIONAL, param_count=1),
     "String": MappedType(base_type=value_spec_t.BaseType.STRING),
-    "Union": MappedType(base_type=value_spec_t.BaseType.UNION, variable_param_count=True),
+    "Union": MappedType(
+        base_type=value_spec_t.BaseType.UNION, variable_param_count=True
+    ),
     # not part of type_spec's types now
     "Symbol": MappedType(base_type=value_spec_t.BaseType.SYMBOL),
     "Any": MappedType(base_type=value_spec_t.BaseType.ANY),
+    "None": MappedType(base_type=value_spec_t.BaseType.NONE),
+    "Tuple": MappedType(
+        base_type=value_spec_t.BaseType.TUPLE, variable_param_count=True
+    ),
+    "Never": MappedType(base_type=value_spec_t.BaseType.NEVER),
 }
 
 

pkgs/type_spec/value_spec/emit_python.py

@@ -22,6 +22,7 @@ def emit_functions(functions: list[value_spec_t.Function]) -> str:
     out.write(
         f"""{MODIFY_NOTICE}
 {LINT_HEADER}
+import datetime
 from typing import cast, Union
 
 from decimal import Decimal
@@ -159,7 +160,9 @@ def _emit_function(function: value_spec_t.Function, indent: str) -> str:
     sub_indent = indent + INDENT
     out.write(f"{_function_symbol_name(function)} = value_spec_t.Function(\n")
     out.write(f"{sub_indent}name={encode_common_string(function.name)},\n")
-    out.write(f"{sub_indent}description={encode_common_string(function.description)},\n")
+    out.write(
+        f"{sub_indent}description={encode_common_string(function.description)},\n"
+    )
     out.write(f"{sub_indent}brief={encode_common_string(function.brief)},\n")
     out.write(
         f"{sub_indent}return_value={_emit_function_return(function.return_value, sub_indent)},\n"
@@ -183,16 +186,22 @@ def _emit_argument(argument: value_spec_t.FunctionArgument, indent: str) -> str:
     out.write("value_spec_t.FunctionArgument(\n")
     out.write(f"{sub_indent}ref_name={encode_common_string(argument.ref_name)},\n")
     out.write(f"{sub_indent}name={encode_common_string(argument.name)},\n")
-    out.write(f"{sub_indent}description={encode_common_string(argument.description)},\n")
+    out.write(
+        f"{sub_indent}description={encode_common_string(argument.description)},\n"
+    )
     out.write(f"{sub_indent}pass_null={str(argument.pass_null)},\n")
-    out.write(f"{sub_indent}extant=value_spec_t.ArgumentExtant.{argument.extant.name},\n")
+    out.write(
+        f"{sub_indent}extant=value_spec_t.ArgumentExtant.{argument.extant.name},\n"
+    )
     out.write(f"{sub_indent}type={_emit_type(argument.type, sub_indent)},\n")
     out.write(f"{indent})")
 
     return out.getvalue()
 
 
-def _emit_function_return(return_value: value_spec_t.FunctionReturn, indent: str) -> str:
+def _emit_function_return(
+    return_value: value_spec_t.FunctionReturn, indent: str
+) -> str:
     out = io.StringIO()
 
     sub_indent = indent + INDENT

uncountable/__init__.py

@@ -1,5 +1,4 @@
-from . import types
-from . import core
+from . import core, types
 
 __all__ = []
 __all__ += types.__all__

uncountable/core/__init__.py

@@ -1,3 +1,13 @@
-from .client import AuthDetailsApiKey, Client
+from .async_batch import AsyncBatchProcessor
+from .client import Client
+from .file_upload import MediaFileUpload, UploadedFile
+from .types import AuthDetailsApiKey, AuthDetailsOAuth
 
-__all__: list[str] = ["AuthDetailsApiKey", "Client"]
+__all__: list[str] = [
+    "AuthDetailsApiKey",
+    "AuthDetailsOAuth",
+    "AsyncBatchProcessor",
+    "Client",
+    "MediaFileUpload",
+    "UploadedFile",
+]

uncountable/core/async_batch.py

@@ -0,0 +1,37 @@
+from uncountable.core.client import Client
+from uncountable.types import async_batch_t, base_t
+from uncountable.types.async_batch import AsyncBatchRequest
+from uncountable.types.async_batch_processor import AsyncBatchProcessorBase
+
+
+class AsyncBatchSubmissionError(Exception):
+    pass
+
+
+class AsyncBatchProcessor(AsyncBatchProcessorBase):
+    _client: Client
+    _queue: list[AsyncBatchRequest]
+    _submitted_job_ids: list[base_t.ObjectId]
+
+    def __init__(self, *, client: Client) -> None:
+        super().__init__()
+        self._client = client
+        self._queue = []
+        self._submitted_job_ids = []
+
+    def _enqueue(self, req: async_batch_t.AsyncBatchRequest) -> None:
+        self._queue.append(req)
+
+    def current_queue_size(self) -> int:
+        return len(self._queue)
+
+    def send(self) -> base_t.ObjectId:
+        if len(self._queue) == 0:
+            raise AsyncBatchSubmissionError("queue is empty")
+        job_id = self._client.execute_batch_load_async(requests=self._queue).job_id
+        self._submitted_job_ids.append(job_id)
+        self._queue = []
+        return job_id
+
+    def get_submitted_job_ids(self) -> list[base_t.ObjectId]:
+        return self._submitted_job_ids