architect-py 5.0.0b1__py3-none-any.whl → 5.0.0b3__py3-none-any.whl

scripts/preprocess_grpc_schema.py

@@ -1,24 +1,21 @@
 import argparse
 import json
+import logging
 import os
 import re
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List
+
+logging.basicConfig(level=logging.WARN)
 
 # ---------------------------------------------------------------------
 # Constants and Regular Expressions
 # ---------------------------------------------------------------------
 
-DECIMAL_RE = re.compile(r'(^\s*)"\$ref": "#/definitions/Decimal"', flags=re.MULTILINE)
-SINGLE_ALL_OF_DECIMAL = re.compile(
-    r'"allOf":\s*\[\s*\{\s*'
-    r'(?P<indent>[ \t]*)"type":\s*"number",\s*'
-    r'(?P=indent)"format":\s*"decimal"\s*'
-    r"\}\s*\]",
-    flags=re.MULTILINE,
-)
-CAMEL_CASE_RE = re.compile(r"(?<!^)(?=[A-Z])")
+SNAKE_CASE_RE = re.compile(r"(?<!^)(?=[A-Z])")
 EXTRACT_REF_RE = re.compile(r'("\$ref":\s*"#/definitions/)([^"]+)(")')
 
+TAG_PATTERN = re.compile(r"<!--\s*py:\s*(.*?)\s*-->")
+
 
 def replace_and_indent(match: re.Match) -> str:
     indent = match.group(1)
@@ -36,18 +33,6 @@ def _apply_type_fixes_to_text(
     """
     Perform string replacements and regex substitutions on the JSON text.
     """
-    replacements = {
-        "uint32": "default",
-        "uint64": "default",
-        '"format": "int"': '"format": "default"',
-        '"format": "partial-date-time"': '"format": "time"',
-    }
-    for old, new in replacements.items():
-        text = text.replace(old, new)
-
-    # Fix Decimal references with proper indenting.
-    text = DECIMAL_RE.sub(replace_and_indent, text)
-    text = SINGLE_ALL_OF_DECIMAL.sub(replace_and_indent, text)
 
     def replace_ref(match: re.Match) -> str:
         prefix, class_title, suffix = match.groups()
@@ -86,15 +71,31 @@ def apply_type_fixes(
 # ---------------------------------------------------------------------
 
 
-def parse_class_description(text: str) -> Tuple[Dict[str, str], str]:
+def clean_metadata_from_description(schema: Dict):
+    text = schema.get("description", "")
+    new_description = TAG_PATTERN.sub("", text).strip()
+
+    if new_description:
+        schema["description"] = new_description
+    else:
+        schema.pop("description", None)
+
+    for definitions in schema.get("definitions", {}).values():
+        definitions["description"] = TAG_PATTERN.sub(
+            "", definitions.get("description", "")
+        ).strip()
+        if not definitions["description"]:
+            definitions.pop("description", None)
+
+
+def parse_class_description(text: str) -> Dict[str, str]:
     """
     Parse metadata from a special comment in the description.
     Expected format: <!-- py: key1=value1, key2=value2 -->
     """
-    pattern = r"<!--\s*py:\s*(.*?)\s*-->"
-    match = re.search(pattern, text, re.DOTALL)
+    match = TAG_PATTERN.search(text)
     if not match:
-        raise ValueError("No valid 'py:' comment found in the text.")
+        return {}
 
     metadata_str = match.group(1)
     metadata: Dict[str, str] = {}
@@ -107,8 +108,7 @@ def parse_class_description(text: str) -> Tuple[Dict[str, str], str]:
         key, value = map(str.strip, pair.split("=", 1))
         metadata[key] = value
 
-    cleaned_text = re.sub(pattern, "", text, flags=re.DOTALL)
-    return metadata, cleaned_text
+    return metadata
 
 
 def correct_flattened_types(schema: Dict[str, Any]) -> None:
@@ -120,6 +120,9 @@ def correct_flattened_types(schema: Dict[str, Any]) -> None:
     while we want 1 type.
 
     Removes the oneOf list and merges common keys and additional properties.
+
+    if the flatten field is not the tag, enum_variant_to_other_required_keys
+    will be in the json but will be empty.
     """
     if "oneOf" not in schema or "required" not in schema:
         return
@@ -127,12 +130,19 @@ def correct_flattened_types(schema: Dict[str, Any]) -> None:
     one_of: List[Dict[str, Any]] = schema.pop("oneOf")
     additional_properties: Dict[str, Any] = {}
 
-    enum_tag: str = ""
     enum_value_to_required: Dict[str, List[str]] = {}
-    enum_tag_title = CAMEL_CASE_RE.sub("_", schema["title"]).lower()
+
+    description = schema.get("description", "")
+    metadata = parse_class_description(description)
+
+    try:
+        field_name, field_title, enum_type_name = metadata["unflatten"].split("/")
+    except KeyError:
+        raise KeyError(f"Missing 'flatten' metadata in {schema['title']}")
+
     enum_tag_property: Dict[str, Any] = {
         "type": "string",
-        "title": f"{enum_tag_title}_type",
+        "title": field_title,
         "enum": [],
     }
 
@@ -144,10 +154,6 @@ def correct_flattened_types(schema: Dict[str, Any]) -> None:
         required = item.get("required", [])
         for key, prop in properties.items():
             if "enum" in prop:
-                if not enum_tag:
-                    enum_tag = key
-                else:
-                    assert enum_tag == key, f"Enum field mismatch in {schema['title']}"
                 [enum_value] = prop["enum"]
                 enum_tag_property["enum"].append(enum_value)
                 enum_value_to_required[enum_value] = required
@@ -159,18 +165,25 @@ def correct_flattened_types(schema: Dict[str, Any]) -> None:
                 else:
                     additional_properties[key] = prop
 
-    if not enum_tag:
+    if not field_name:
         raise ValueError(f"Enum value not found in {schema['title']}")
 
-    sets = [set(group["required"]) for group in one_of]
+    sets: list[set[str]] = [set(group["required"]) for group in one_of]
     common_keys: list[str] = list(set.intersection(*sets)) if sets else []
     common_keys.sort()
     schema["required"].extend(common_keys)
 
+    if tag_field := metadata.get("tag"):
+        schema["tag_field"] = tag_field
+
     schema["properties"].update(additional_properties)
-    schema["properties"][enum_tag] = enum_tag_property
-    schema["enum_tag"] = enum_tag
-    schema["enum_tag_to_other_required_keys"] = enum_value_to_required
+    schema["properties"][field_name] = {
+        "title": field_title,
+        "$ref": f"#/definitions/{enum_type_name}",
+    }
+
+    schema["definitions"][enum_type_name] = enum_tag_property
+    schema["enum_variant_to_other_required_keys"] = enum_value_to_required
 
 
 def correct_variant_types(
@@ -196,20 +209,15 @@ def correct_variant_types(
     }
     """
 
-    one_of_key = "oneOf"
-    if one_of_key not in schema or "required" in schema:
+    if "oneOf" not in schema or "required" in schema:
         return
 
     description = schema.get("description", "")
-    metadata, new_description = parse_class_description(description)
-    if new_description.strip():
-        schema["description"] = new_description.strip()
-    else:
-        schema.pop("description", None)
+    metadata = parse_class_description(description)
 
-    tag_field = metadata["tag"]
+    tag_field: str = metadata["tag"]
     new_one_of: List[Dict[str, Any]] = []
-    for item in schema[one_of_key]:
+    for item in schema["oneOf"]:
         item["required"].remove(tag_field)
         [tag_value] = item["properties"].pop(tag_field)["enum"]
         title = item.pop("title")
@@ -248,7 +256,7 @@ def correct_variant_types(
 
         new_one_of.append(enum_ref)
 
-    schema[one_of_key] = new_one_of
+    schema["oneOf"] = new_one_of
     schema["tag_field"] = tag_field
 
 
@@ -348,24 +356,9 @@ def correct_enums_with_multiple_titles(schema: Dict[str, Any]) -> None:
         }
 
 
-def correct_enums_with_x_enumNames(schema: Dict[str, Any]) -> None:
+def correct_repr_enums_with_x_enumNames(schema: Dict[str, Any]) -> None:
     """
-    Process enums that have x-enumNames in the schema.
-    "FillKind": {
-      "type": "integer",
-      "enum": [
-        0,
-        1,
-        2
-      ],
-      "x-enumNames": [
-        "Normal",
-        "Reversal",
-        "Correction"
-      ]
-    },
-    this should actually be a string enum, the values of the integers actually do not matter
-    the names and values should be x-enumNames
+    This should currently not do anything
     """
     if "definitions" not in schema:
         return
@@ -374,16 +367,20 @@ def correct_enums_with_x_enumNames(schema: Dict[str, Any]) -> None:
     for t, definition in definitions.items():
         if "x-enumNames" not in definition:
             continue
-        assert definition["type"] == "integer"
-        enum_names: list[str] = definition["x-enumNames"]
-        enum_ints: list[int] = definition.pop("enum")
-        definition["old_enum"] = enum_ints
-        if len(enum_names) != len(enum_ints):
-            raise ValueError(
-                f"Enum names and values length mismatch in {t} in {schema['title']}"
-            )
-        definition["enum"] = enum_names
-        definition["type"] = "string"
+        description = definition.get("description", "")
+        metadata = parse_class_description(description)
+
+        if metadata.get("schema") == "repr":
+            assert definition["type"] == "integer"
+            enum_names: list[str] = definition["x-enumNames"]
+            enum_ints: list[int] = definition.pop("enum")
+            definition["old_enum"] = enum_ints
+            if len(enum_names) != len(enum_ints):
+                raise ValueError(
+                    f"Enum names and values length mismatch in {t} in {schema['title']}"
+                )
+            definition["enum"] = enum_names
+            definition["type"] = "string"
 
 
 def correct_enums_with_descriptions(schema: Dict[str, Any]) -> None:
@@ -457,6 +454,8 @@ def correct_null_types_with_constraints(schema: Dict[str, Any]) -> None:
     if "properties" in schema:
         properties = schema["properties"]
         for prop_def in properties.values():
+            if isinstance(prop_def, bool):
+                continue
             if "type" in prop_def and "null" in prop_def["type"]:
                 for constraint in constraints:
                     if constraint in prop_def:
@@ -467,12 +466,56 @@ def correct_null_types_with_constraints(schema: Dict[str, Any]) -> None:
         for definition in definitions.values():
             properties = definition.get("properties", {})
             for prop_def in properties.values():
+                if isinstance(prop_def, bool):
+                    continue
                 if "type" in prop_def and "null" in prop_def["type"]:
                     for constraint in constraints:
                         if constraint in prop_def:
                             prop_def.pop(constraint)
 
 
+def correct_type_from_description(schema: Dict[str, Any]) -> None:
+    """
+    on the rust side, due to
+    derive SerializeDisplay and DeserializeFromStr
+    classes can have different types than normal
+    in the case of OrderId, it is a string
+
+    "OrderId": {
+        "description": "System-unique, persistent order identifiers",
+        "type": "object",
+        "required": [
+          "seqid",
+          "seqno"
+        ],
+        "properties": {
+          "seqid": {
+            "type": "string",
+            "format": "uuid"
+          },
+          "seqno": {
+            "type": "integer",
+            "format": "default",
+            "minimum": 0.0
+          }
+        }
+    },
+    """
+    if "definitions" not in schema:
+        return
+
+    definitions: dict[str, Any] = schema["definitions"]
+    for t, definition in definitions.items():
+        description = definition.get("description", "")
+        metadata = parse_class_description(description)
+
+        new_type = metadata.get("type")
+        if new_type is not None:
+            definition["type"] = new_type
+            definition.pop("required")
+            definition.pop("properties")
+
+
 def process_schema_definitions(
     schema: Dict[str, Any],
     definitions: Dict[str, Any],
@@ -480,20 +523,19 @@ def process_schema_definitions(
 ) -> None:
     """
     Extract and process definitions from a schema.
-    Updates the "Decimal" format and applies enum corrections.
     """
-    if "definitions" not in schema:
-        return
-
-    if "Decimal" in schema["definitions"]:
-        schema["definitions"]["Decimal"]["format"] = "decimal"
-
     correct_enums_with_multiple_titles(schema)
-    correct_enums_with_x_enumNames(schema)
+    # correct_repr_enums_with_x_enumNames(schema)
     correct_enums_with_descriptions(schema)
     correct_variant_types(schema, definitions, type_to_json_file)
     correct_flattened_types(schema)
     correct_null_types_with_constraints(schema)
+    correct_type_from_description(schema)
+
+    clean_metadata_from_description(schema)
+
+    if "definitions" not in schema:
+        return
 
     new_defs: dict[str, Any] = schema.pop("definitions")
     for t, definition in new_defs.items():
@@ -517,8 +559,7 @@ def add_info_to_schema(services: List[Dict[str, Any]]) -> Dict[str, str]:
     """
     type_to_json_file: Dict[str, str] = {}
     for service in services:
-        service_name = service["name"].replace(" ", "_")
-        service["service_name"] = service_name
+        service_name = service["name"]
 
         for rpc in service["rpcs"]:
             req_schema = rpc["request_type"]
@@ -554,7 +595,7 @@ def write_json_file(data: Dict[Any, Any], path: str) -> None:
         json.dump(data, out_file, indent=2)
 
 
-def process_schema(
+def preprocess_rpc_schema(
     schema: Dict[str, Any],
     definitions: Dict[str, Any],
     type_to_json_file: Dict[str, str],
@@ -567,23 +608,26 @@ def process_schema(
     return apply_type_fixes(schema, is_definitions_file, type_to_json_file)
 
 
-def process_service(
+def preprocess_service_schemas(
     service: Dict[str, Any],
     output_dir: str,
     definitions: Dict[str, Any],
     type_to_json_file: Dict[str, str],
 ) -> None:
     """
+    Pre-process each service schema.
     Process each RPC within a service (both request and response) and write the resulting schema files.
     """
-    service_name = service["service_name"]
+    service_name = service["name"]
     service_dir = os.path.join(output_dir, service_name)
     os.makedirs(service_dir, exist_ok=True)
 
     for rpc in service["rpcs"]:
+        logging.debug(f"Processing {service_name} {rpc['type']} RPC: {rpc['route']}")
         for key in ["request_type", "response_type"]:
             schema = rpc[key]
-            processed_schema = process_schema(
+            logging.debug(f"Processing {schema['title']}")
+            processed_schema = preprocess_rpc_schema(
                 schema, definitions, type_to_json_file, False
             )
             schema_title = processed_schema["title"]
@@ -591,20 +635,47 @@ def process_service(
             write_json_file(processed_schema, file_path)
 
 
-def preprocess_json(input_file: str, output_dir: str) -> None:
+def preprocess_schemas(input_file: str, output_dir: str) -> None:
     """
-    Preprocess the gRPC JSON file by splitting each RPC's request and response schemas
-    into separate JSON files and creating a unified definitions file.
+    Rust build generates a JSON file containing RPC definitions and schemas
+    for each service.  The top level object is an array of service definitions.
+
+    We split out each service and each schema within each service into separate
+    JSON schema files for post-processing.  Simple corrections to the schema
+    are made here as well.
     """
     os.makedirs(output_dir, exist_ok=True)
+
     with open(input_file, "r") as f:
-        services = json.load(f)
+        input_s = f.read()
+
+    replacements = {
+        # Suppress UserWarning: format of 'uint32' not understood for 'integer' - using default
+        "uint32": "default",
+        "uint64": "default",
+        # Suppress UserWarning: format of 'int' not understood for 'integer' - using default
+        '"format": "int",': '"format": "default",',
+        '"format": "int"': '"format": "default"',
+        # Manually map partial-date-time format to time format (python str -> datetime.time)
+        '"format": "partial-date-time",': '"format": "time",',
+        '"format": "partial-date-time"': '"format": "time"',
+        # Add decimal format hint to all regex patterns that look like decimals;
+        # drop the original string pattern validation as redundant.
+        '"pattern": "^-?[0-9]+(\\\\.[0-9]+)?$",': '"format": "decimal",',
+        '"pattern": "^-?[0-9]+(\\\\.[0-9]+)?$"': '"format": "decimal"',
+    }
+
+    for old, new in replacements.items():
+        input_s = input_s.replace(old, new)
+
+    services = json.loads(input_s)
 
     type_to_json_file = add_info_to_schema(services)
     definitions: Dict[str, Any] = {}
 
     for service in services:
-        process_service(service, output_dir, definitions, type_to_json_file)
+        logging.debug(f"Processing service {service['name']}")
+        preprocess_service_schemas(service, output_dir, definitions, type_to_json_file)
 
     # we look at the line level to fix definitions
     fixed_definitions = apply_type_fixes(definitions, True, type_to_json_file)
@@ -617,30 +688,20 @@ def preprocess_json(input_file: str, output_dir: str) -> None:
     write_json_file(fixed_definitions, definitions_path)
 
 
-# ---------------------------------------------------------------------
-# Main Entry Point
-# ---------------------------------------------------------------------
-
-
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Process a gRPC JSON schema file.")
+    parser = argparse.ArgumentParser(description="Preprocess gRPC JSON schema file.")
     parser.add_argument(
-        "--architect_dir",
-        type=str,
-        default="~/architect",
-        help="Path to the architect directory containing the api/schema.json file.",
+        "--schema", type=str, required=True, help="JSON schema file to preprocess."
     )
     parser.add_argument(
-        "--output_dir",
+        "--output-dir",
         type=str,
-        default="processed_schema",
-        help="Path to output the extracted schema files.",
+        required=True,
+        help="Output path for preprocessed schema.",
     )
     args = parser.parse_args()
-
-    architect_dir = os.path.expanduser(args.architect_dir)
-    input_file = os.path.join(architect_dir, "api/schema.json")
-    preprocess_json(input_file, args.output_dir)
+    schema_file = os.path.expanduser(args.schema)
+    preprocess_schemas(schema_file, args.output_dir)
 
 
 if __name__ == "__main__":

architect_py/grpc/models/Algo/AlgoOrderForTwapAlgo.py

@@ -1,61 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  Algo/AlgoOrderForTwapAlgo.json
-
-from __future__ import annotations
-
-from datetime import datetime
-from typing import List, Optional
-
-from msgspec import Struct
-
-from .. import definitions
-
-
-class AlgoOrderForTwapAlgo(Struct, omit_defaults=True):
-    account: str
-    algo_name: str
-    algo_order_id: definitions.OrderId
-    create_time: datetime
-    params: definitions.TwapParams
-    state: definitions.AlgoState
-    status: definitions.TwapStatus
-    trader: definitions.UserId
-    display_symbols: Optional[List[str]] = None
-    last_error: Optional[str] = None
-    last_error_time: Optional[datetime] = None
-    parent_order_id: Optional[definitions.OrderId] = None
-
-    # Constructor that takes all field titles as arguments for convenience
-    @classmethod
-    def new(
-        cls,
-        account: str,
-        algo_name: str,
-        algo_order_id: definitions.OrderId,
-        create_time: datetime,
-        params: definitions.TwapParams,
-        state: definitions.AlgoState,
-        status: definitions.TwapStatus,
-        trader: definitions.UserId,
-        display_symbols: Optional[List[str]] = None,
-        last_error: Optional[str] = None,
-        last_error_time: Optional[datetime] = None,
-        parent_order_id: Optional[definitions.OrderId] = None,
-    ):
-        return cls(
-            account,
-            algo_name,
-            algo_order_id,
-            create_time,
-            params,
-            state,
-            status,
-            trader,
-            display_symbols,
-            last_error,
-            last_error_time,
-            parent_order_id,
-        )
-
-    def __str__(self) -> str:
-        return f"AlgoOrderForTwapAlgo(account={self.account},algo_name={self.algo_name},algo_order_id={self.algo_order_id},create_time={self.create_time},params={self.params},state={self.state},status={self.status},trader={self.trader},display_symbols={self.display_symbols},last_error={self.last_error},last_error_time={self.last_error_time},parent_order_id={self.parent_order_id})"

architect_py/grpc/models/Algo/CreateAlgoOrderRequestForTwapAlgo.py

@@ -1,59 +0,0 @@
-# generated by datamodel-codegen:
-#   filename:  Algo/CreateAlgoOrderRequestForTwapAlgo.json
-
-from __future__ import annotations
-from architect_py.grpc.models.Algo.AlgoOrderForTwapAlgo import AlgoOrderForTwapAlgo
-
-from typing import Optional
-
-from msgspec import Struct
-
-from .. import definitions
-
-
-class CreateAlgoOrderRequestForTwapAlgo(Struct, omit_defaults=True):
-    algo_name: str
-    params: definitions.TwapParams
-    account: Optional[str] = None
-    algo_order_id: Optional[definitions.OrderId] = None
-    parent_order_id: Optional[definitions.OrderId] = None
-    trader: Optional[definitions.UserId] = None
-
-    # Constructor that takes all field titles as arguments for convenience
-    @classmethod
-    def new(
-        cls,
-        algo_name: str,
-        params: definitions.TwapParams,
-        account: Optional[str] = None,
-        algo_order_id: Optional[definitions.OrderId] = None,
-        parent_order_id: Optional[definitions.OrderId] = None,
-        trader: Optional[definitions.UserId] = None,
-    ):
-        return cls(
-            algo_name,
-            params,
-            account,
-            algo_order_id,
-            parent_order_id,
-            trader,
-        )
-
-    def __str__(self) -> str:
-        return f"CreateAlgoOrderRequestForTwapAlgo(algo_name={self.algo_name},params={self.params},account={self.account},algo_order_id={self.algo_order_id},parent_order_id={self.parent_order_id},trader={self.trader})"
-
-    @staticmethod
-    def get_response_type():
-        return AlgoOrderForTwapAlgo
-
-    @staticmethod
-    def get_unannotated_response_type():
-        return AlgoOrderForTwapAlgo
-
-    @staticmethod
-    def get_route() -> str:
-        return "/json.architect.Algo/CreateTwapAlgoOrder"
-
-    @staticmethod
-    def get_rpc_method():
-        return "unary"