pyopenapi-gen 2.7.2__py3-none-any.whl

pyopenapi_gen/core/loader/schemas/extractor.py

@@ -0,0 +1,275 @@
+"""Schema extractors for OpenAPI IR transformation.
+
+Provides functions to extract and transform schemas from raw OpenAPI specs.
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+from typing import Any, Mapping
+
+from pyopenapi_gen import IRSchema
+from pyopenapi_gen.core.parsing.context import ParsingContext
+from pyopenapi_gen.core.parsing.schema_parser import _parse_schema
+from pyopenapi_gen.core.utils import NameSanitizer
+
+logger = logging.getLogger(__name__)
+
+
+def build_schemas(raw_schemas: dict[str, Mapping[str, Any]], raw_components: Mapping[str, Any]) -> ParsingContext:
+    """Build all named schemas up front, populating a ParsingContext.
+
+    Contracts:
+        Preconditions:
+            - raw_schemas is a valid dict containing schema definitions
+            - raw_components is a valid mapping containing component definitions
+        Postconditions:
+            - A ParsingContext is returned with all schemas parsed
+            - All schemas in raw_schemas are populated in context.parsed_schemas
+    """
+    if not isinstance(raw_schemas, dict):
+        raise TypeError("raw_schemas must be a dict")
+    if not isinstance(raw_components, Mapping):
+        raise TypeError("raw_components must be a Mapping")
+
+    context = ParsingContext(raw_spec_schemas=raw_schemas, raw_spec_components=raw_components)
+
+    # Build initial IR for all schemas found in components
+    for n, nd in raw_schemas.items():
+        if n not in context.parsed_schemas:
+            _parse_schema(n, nd, context, allow_self_reference=True)
+
+    # Post-condition check
+    if not all(n in context.parsed_schemas for n in raw_schemas):
+        raise RuntimeError("Not all schemas were parsed")
+
+    return context
+
+
+def extract_inline_array_items(schemas: dict[str, IRSchema]) -> dict[str, IRSchema]:
+    """Extract inline array item schemas as unique named schemas and update references.
+
+    Contracts:
+        Preconditions:
+            - schemas is a dict of IRSchema objects
+        Postconditions:
+            - Returns an updated schemas dict with extracted array item types
+            - All array item schemas have proper names
+            - No duplicate schema names are created
+    """
+    if not isinstance(schemas, dict):
+        raise TypeError("schemas must be a dict")
+    if not all(isinstance(s, IRSchema) for s in schemas.values()):
+        raise TypeError("all values must be IRSchema objects")
+
+    # Store original schema count for post-condition validation
+    original_schema_count = len(schemas)
+    original_schemas = set(schemas.keys())
+
+    new_item_schemas = {}
+    for schema_name, schema in list(schemas.items()):
+        # Check properties for array types
+        for prop_name, prop_schema in list(schema.properties.items()):
+            if prop_schema.type == "array" and prop_schema.items and not prop_schema.items.name:
+                # Only extract complex item schemas (objects and arrays), not simple primitives or references
+                items_schema = prop_schema.items
+                # Check if items is a "null" type (malformed schema with no type) - these resolve to Any
+                is_null_type_items = items_schema.type == "null"
+                # Check if items is an empty object (no properties, no composition)
+                is_empty_object = (
+                    items_schema.type == "object"
+                    and not items_schema.properties
+                    and not items_schema.any_of
+                    and not items_schema.one_of
+                    and not items_schema.all_of
+                )
+                is_complex_item = (
+                    not is_null_type_items
+                    and not is_empty_object
+                    and (
+                        items_schema.type == "object"
+                        or items_schema.type == "array"
+                        or items_schema.properties
+                        or items_schema.any_of
+                        or items_schema.one_of
+                        or items_schema.all_of
+                    )
+                )
+
+                if is_complex_item:
+                    # Generate a descriptive name for the item schema using content-aware naming
+                    # For arrays of complex objects, use the pattern: {Parent}{Property}Item
+                    # For arrays in response wrappers (like "data" fields), consider the content type
+                    if prop_name.lower() in ["data", "items", "results", "content"]:
+                        # For generic wrapper properties, try to derive name from the item type or parent
+                        if items_schema.type == "object" and schema_name.endswith("Response"):
+                            # Pattern: MessageBatchResponse.data -> MessageItem
+                            base_name = schema_name.replace("Response", "").replace("List", "")
+                            item_schema_name = f"{base_name}Item"
+                        else:
+                            # Fallback to standard pattern
+                            item_schema_name = (
+                                f"{NameSanitizer.sanitize_class_name(schema_name)}"
+                                f"{NameSanitizer.sanitize_class_name(prop_name)}Item"
+                            )
+                    else:
+                        # Standard pattern for named properties
+                        item_schema_name = (
+                            f"{NameSanitizer.sanitize_class_name(schema_name)}"
+                            f"{NameSanitizer.sanitize_class_name(prop_name)}Item"
+                        )
+
+                    base_item_name = item_schema_name
+                    i = 1
+                    while item_schema_name in schemas or item_schema_name in new_item_schemas:
+                        item_schema_name = f"{base_item_name}{i}"
+                        i += 1
+
+                    # Create a copy of the item schema with a name
+                    items_copy = copy.deepcopy(prop_schema.items)
+                    items_copy.name = item_schema_name
+                    new_item_schemas[item_schema_name] = items_copy
+
+                    # Update the original array schema to reference the named item schema
+                    prop_schema.items.name = item_schema_name
+
+    # Update the schemas dict with the new item schemas
+    schemas.update(new_item_schemas)
+
+    # Post-condition checks
+    if len(schemas) < original_schema_count:
+        raise RuntimeError("Schemas count should not decrease")
+    if not original_schemas.issubset(set(schemas.keys())):
+        raise RuntimeError("Original schemas should still be present")
+
+    return schemas
+
+
+def extract_inline_enums(schemas: dict[str, IRSchema]) -> dict[str, IRSchema]:
+    """Extract inline property enums as unique schemas and update property references.
+
+    Also ensures top-level enum schemas are properly marked for generation.
+
+    Contracts:
+        Preconditions:
+            - schemas is a dict of IRSchema objects
+        Postconditions:
+            - Returns an updated schemas dict with extracted enum types and array item types
+            - All property schemas with enums have proper names
+            - All array item schemas have proper names
+            - No duplicate schema names are created
+            - Top-level enum schemas have generation_name set
+    """
+    if not isinstance(schemas, dict):
+        raise TypeError("schemas must be a dict")
+    if not all(isinstance(s, IRSchema) for s in schemas.values()):
+        raise TypeError("all values must be IRSchema objects")
+
+    # Store original schema count for post-condition validation
+    original_schema_count = len(schemas)
+    original_schemas = set(schemas.keys())
+
+    # First extract array item schemas so they can have enums extracted in the next step
+    schemas = extract_inline_array_items(schemas)
+
+    new_enums = {}
+    for schema_name, schema in list(schemas.items()):
+        # Handle top-level enum schemas (those defined directly in components/schemas)
+        # These are already enums but need generation_name set
+        if schema.enum and schema.type in ["string", "integer", "number"]:
+            # This is a top-level enum schema
+            # Ensure it has generation_name set (will be properly set by emitter later,
+            # but we can set it here to avoid the warning)
+            if not hasattr(schema, "generation_name") or not schema.generation_name:
+                schema.generation_name = schema.name
+                logger.info(
+                    f"Set generation_name for top-level enum schema: {schema_name} with values {schema.enum[:3]}..."
+                )
+            # Mark this as a properly processed enum by ensuring generation_name is set
+            # This serves as the marker that this enum was properly processed
+            logger.debug(f"Marked top-level enum schema: {schema_name}")
+
+        # Extract inline enums from properties
+        for prop_name, prop_schema in list(schema.properties.items()):
+            # Check if this property has an inline enum that needs extraction
+            # An inline enum needs extraction if:
+            # 1. It has enum values defined
+            # 2. The enum doesn't already exist as a separate schema in the schemas dict
+            # Note: After schema parsing, property schemas have 'name' set to the property key
+            # and 'generation_name' set to a sanitised class name, but the enum itself
+            # isn't registered as a separate schema yet.
+            has_inline_enum = prop_schema.enum and prop_schema.type in ["string", "integer", "number"]
+
+            # Check if the enum was already extracted or is a named reference
+            # Case 1: generation_name exists in schemas dict (already extracted)
+            # Case 2: property name itself is a schema reference (e.g., ExistingStatusEnum)
+            enum_already_extracted = (
+                (
+                    prop_schema.generation_name
+                    and prop_schema.generation_name in schemas
+                    and schemas[prop_schema.generation_name].enum
+                )
+                or (
+                    # Property name is an explicit enum reference (class-like name, not property key)
+                    prop_schema.name
+                    and prop_schema.name in schemas
+                    and schemas[prop_schema.name].enum
+                )
+                or (
+                    # Property name looks like an enum class name (not a property key)
+                    # Property keys are typically snake_case, class names are PascalCase
+                    prop_schema.name
+                    and prop_schema.name[0].isupper()
+                    and "_" not in prop_schema.name
+                    and prop_schema.name != prop_name  # Name differs from property key
+                )
+            )
+
+            if has_inline_enum and not enum_already_extracted:
+                # Use property's existing generation_name if set, otherwise create a new name
+                # This keeps naming consistent with what the type resolver already assigned
+                if prop_schema.generation_name:
+                    enum_name = prop_schema.generation_name
+                else:
+                    enum_name = (
+                        f"{NameSanitizer.sanitize_class_name(schema_name)}"
+                        f"{NameSanitizer.sanitize_class_name(prop_name)}Enum"
+                    )
+                base_enum_name = enum_name
+                i = 1
+                while enum_name in schemas or enum_name in new_enums:
+                    enum_name = f"{base_enum_name}{i}"
+                    i += 1
+
+                # Derive module stem from final enum name
+                module_stem = NameSanitizer.sanitize_module_name(enum_name)
+
+                enum_schema = IRSchema(
+                    name=enum_name,
+                    type=prop_schema.type,
+                    enum=copy.deepcopy(prop_schema.enum),
+                    description=prop_schema.description or f"Enum for {schema_name}.{prop_name}",
+                )
+                enum_schema.generation_name = enum_name
+                enum_schema.final_module_stem = module_stem
+                new_enums[enum_name] = enum_schema
+                logger.debug(f"Extracted inline enum from {schema_name}.{prop_name}: {enum_name}")
+
+                # Update the original property to reference the extracted enum
+                prop_schema.name = enum_name
+                prop_schema.type = enum_name  # Make the property reference the enum by name
+                prop_schema.generation_name = enum_name  # Ensure property also has correct generation_name
+                prop_schema.final_module_stem = module_stem  # And module stem
+                prop_schema.enum = None  # Clear the inline enum since it's now extracted
+
+    # Update the schemas dict with the new enums
+    schemas.update(new_enums)
+
+    # Post-condition checks
+    if len(schemas) < original_schema_count:
+        raise RuntimeError("Schemas count should not decrease")
+    if not original_schemas.issubset(set(schemas.keys())):
+        raise RuntimeError("Original schemas should still be present")
+
+    return schemas

pyopenapi_gen/core/pagination.py

@@ -0,0 +1,64 @@
+"""
+Pagination utilities for handling paginated API endpoints.
+
+This module provides functions for working with paginated API responses,
+turning them into convenient async iterators that automatically handle
+fetching subsequent pages.
+"""
+
+from typing import Any, AsyncIterator, Awaitable, Callable
+
+
+def paginate_by_next(
+    fetch_page: Callable[..., Awaitable[dict[str, Any]]],
+    items_key: str = "items",
+    next_key: str = "next",
+    **params: Any,
+) -> AsyncIterator[Any]:
+    """
+    Create an async iterator that yields items from paginated API responses.
+
+    This function creates a paginator that automatically handles fetching
+    subsequent pages of results by using a "next page token" pattern. It calls
+    the provided `fetch_page` function repeatedly with the given parameters,
+    updating the next token parameter between calls.
+
+    Args:
+        fetch_page: Async function to fetch a page of results
+        items_key: The key in the response dict where items are located (default: "items")
+        next_key: The key in the response dict for the next page token (default: "next")
+        **params: Initial parameters to pass to fetch_page
+
+    Returns:
+        An AsyncIterator that yields individual items from all pages
+
+    Example:
+        ```python
+        async def fetch_users_page(page_token=None, limit=100):
+            url = f"/users?limit={limit}"
+            if page_token:
+                url += f"&page_token={page_token}"
+            return await http_client.get(url)
+
+        async for user in paginate_by_next(fetch_users_page,
+                                          items_key="users",
+                                          next_key="page_token",
+                                          limit=50):
+            print(user["name"])
+        ```
+    """
+
+    async def _paginate() -> AsyncIterator[Any]:
+        while True:
+            result = await fetch_page(**params)
+            # result is expected to be a dict
+            # (assumed since fetch_page is typed to return dict[str, Any])
+            items = result.get(items_key, [])
+            for item in items:
+                yield item
+            token = result.get(next_key)
+            if not token:
+                break
+            params[next_key] = token
+
+    return _paginate()

pyopenapi_gen/core/parsing/__init__.py

@@ -0,0 +1,13 @@
+# Initialize the parsing module
+
+# Expose the main schema parsing entry point if desired,
+# otherwise, it remains internal (_parse_schema).
+# from .schema_parser import _parse_schema as parse_openapi_schema_node
+
+# Other parsers can be imported here if they need to be part of the public API
+# of this sub-package, though most are internal helpers for _parse_schema.
+from typing import List
+
+__all__: List[str] = [
+    # "parse_openapi_schema_node", # Example if we were to expose it
+]

pyopenapi_gen/core/parsing/common/__init__.py

@@ -0,0 +1 @@
+# common parsing utilities

pyopenapi_gen/core/parsing/common/ref_resolution/__init__.py

@@ -0,0 +1,9 @@
+"""
+Module for handling schema reference resolution.
+"""
+
+from .resolve_schema_ref import resolve_schema_ref
+
+__all__ = [
+    "resolve_schema_ref",
+]

pyopenapi_gen/core/parsing/common/ref_resolution/helpers/cyclic_properties.py

@@ -0,0 +1,66 @@
+"""
+Helper module for handling cyclic property references in schemas.
+"""
+
+import logging
+from typing import Set
+
+from pyopenapi_gen.ir import IRSchema
+
+from ....context import ParsingContext
+
+logger = logging.getLogger(__name__)
+
+
+def mark_cyclic_property_references(schema_obj: IRSchema, ref_name: str, context: ParsingContext) -> None:
+    """
+    Marks properties in a schema that form cycles as unresolved.
+
+    Args:
+        schema_obj: The schema object to check for cycles
+        ref_name: The name of the schema being referenced
+        context: The parsing context
+
+    Pre-conditions:
+        - schema_obj is a valid IRSchema instance
+        - ref_name is a non-empty string
+        - context is a valid ParsingContext instance
+
+    Post-conditions:
+        - Properties that form cycles are marked as unresolved
+        - Non-cyclic properties remain unchanged
+    """
+    if not schema_obj.properties:
+        return
+
+    visited: Set[str] = set()
+
+    def _check_cycle(prop_name: str) -> bool:
+        if prop_name in visited:
+            return True
+        visited.add(prop_name)
+
+        prop_schema = schema_obj.properties.get(prop_name)
+        if not prop_schema or not prop_schema._refers_to_schema:
+            return False
+
+        if prop_schema._refers_to_schema.name == ref_name:
+            return True
+
+        if prop_schema._refers_to_schema.properties:
+            for nested_prop_name in prop_schema._refers_to_schema.properties:
+                nested_prop = prop_schema._refers_to_schema.properties[nested_prop_name]
+                if nested_prop._refers_to_schema and nested_prop._refers_to_schema.name == ref_name:
+                    return True
+                if _check_cycle(nested_prop_name):
+                    return True
+
+        return False
+
+    # Check each property for cycles
+    for prop_name, prop_schema in schema_obj.properties.items():
+        if _check_cycle(prop_name):
+            prop_schema._from_unresolved_ref = True
+            prop_schema._is_circular_ref = True
+            context.cycle_detected = True
+            logger.debug(f"Cyclic property reference detected: {ref_name}.{prop_name}")

pyopenapi_gen/core/parsing/common/ref_resolution/helpers/direct_cycle.py

@@ -0,0 +1,33 @@
+"""
+Module for handling direct cycle detection.
+"""
+
+import logging
+
+from pyopenapi_gen.ir import IRSchema
+
+from ....context import ParsingContext
+
+logger = logging.getLogger(__name__)
+
+
+def handle_direct_cycle(ref_name: str, context: ParsingContext) -> IRSchema:
+    """
+    Handles a direct cycle in schema references.
+
+    Contracts:
+        Pre-conditions:
+            - ref_name must be a valid schema name
+            - context must be a valid ParsingContext instance
+            - ref_name must exist in context.parsed_schemas
+        Post-conditions:
+            - Returns the existing schema from context.parsed_schemas
+            - The schema's _from_unresolved_ref flag is set to True
+            - The schema's _is_circular_ref flag is set to True (for harmonized cycle detection)
+    """
+    existing_schema = context.parsed_schemas[ref_name]
+    existing_schema._from_unresolved_ref = True
+    existing_schema._is_circular_ref = True  # Harmonize with cycle detection contract
+    context.cycle_detected = True  # Mark cycle in context
+    logger.debug(f"Direct cycle detected for schema '{ref_name}'")
+    return existing_schema

pyopenapi_gen/core/parsing/common/ref_resolution/helpers/existing_schema.py

@@ -0,0 +1,22 @@
+"""
+Module for handling existing schema references.
+"""
+
+from pyopenapi_gen.ir import IRSchema
+
+from ....context import ParsingContext
+
+
+def handle_existing_schema(ref_name: str, context: ParsingContext) -> IRSchema:
+    """
+    Handles an existing schema reference.
+
+    Contracts:
+        Pre-conditions:
+            - ref_name must be a valid schema name
+            - context must be a valid ParsingContext instance
+            - ref_name must exist in context.parsed_schemas
+        Post-conditions:
+            - Returns the existing schema from context.parsed_schemas
+    """
+    return context.parsed_schemas[ref_name]

pyopenapi_gen/core/parsing/common/ref_resolution/helpers/list_response.py

@@ -0,0 +1,54 @@
+"""
+Module for handling ListResponse fallback strategy.
+"""
+
+import logging
+from typing import Any, Callable, Mapping
+
+from pyopenapi_gen.ir import IRSchema
+
+from ....context import ParsingContext
+
+logger = logging.getLogger(__name__)
+
+
+def try_list_response_fallback(
+    ref_name: str,
+    ref_value: str,
+    context: ParsingContext,
+    max_depth: int,
+    parse_fn: Callable[[str | None, Mapping[str, Any] | None, ParsingContext, int], IRSchema],
+) -> IRSchema | None:
+    """
+    Attempts to resolve a reference by treating it as a list of a base type.
+
+    Contracts:
+        Pre-conditions:
+            - ref_name must end with "ListResponse"
+            - parse_fn must be a callable that parses schemas
+            - context must be a valid ParsingContext instance
+        Post-conditions:
+            - If successful, returns an array IRSchema with items of the base type
+            - If unsuccessful, returns None
+            - Successful resolutions are added to context.parsed_schemas
+    """
+    list_response_suffix = "ListResponse"
+    if not ref_name.endswith(list_response_suffix):
+        return None
+
+    base_name = ref_name[: -len(list_response_suffix)]
+    referenced_node_data_fallback = context.raw_spec_schemas.get(base_name)
+
+    if not referenced_node_data_fallback:
+        return None
+
+    item_schema = parse_fn(base_name, referenced_node_data_fallback, context, max_depth)
+    if item_schema._from_unresolved_ref:
+        return None
+
+    warning_msg = f"Resolved $ref: {ref_value} by falling back to LIST of base name '{base_name}'."
+    context.collected_warnings.append(warning_msg)
+
+    resolved_schema = IRSchema(name=ref_name, type="array", items=item_schema)
+    context.parsed_schemas[ref_name] = resolved_schema
+    return resolved_schema

pyopenapi_gen/core/parsing/common/ref_resolution/helpers/missing_ref.py

@@ -0,0 +1,52 @@
+"""
+Module for handling missing schema references.
+"""
+
+import logging
+from typing import Any, Callable, Mapping
+
+from pyopenapi_gen.ir import IRSchema
+
+from ....context import ParsingContext
+from .list_response import try_list_response_fallback
+from .stripped_suffix import try_stripped_suffix_fallback
+
+logger = logging.getLogger(__name__)
+
+
+def handle_missing_ref(
+    ref_value: str,
+    ref_name: str,
+    context: ParsingContext,
+    max_depth: int,
+    parse_fn: Callable[[str | None, Mapping[str, Any] | None, ParsingContext, int], IRSchema],
+) -> IRSchema:
+    """
+    Handles a missing schema reference by attempting fallback strategies.
+
+    Contracts:
+        Pre-conditions:
+            - ref_value must be a valid reference string
+            - ref_name must be a valid schema name
+            - context must be a valid ParsingContext instance
+            - max_depth must be a non-negative integer
+            - parse_fn must be a callable that parses schemas
+        Post-conditions:
+            - Returns a valid IRSchema instance
+            - The schema is registered in context.parsed_schemas
+            - If no fallback succeeds, returns an unresolved schema
+    """
+    # Try ListResponse fallback
+    list_response_schema = try_list_response_fallback(ref_name, ref_value, context, max_depth, parse_fn)
+    if list_response_schema is not None:
+        return list_response_schema
+
+    # Try stripped suffix fallback
+    stripped_schema = try_stripped_suffix_fallback(ref_name, ref_value, context, max_depth, parse_fn)
+    if stripped_schema is not None:
+        return stripped_schema
+
+    # If all fallbacks fail, create an unresolved schema
+    unresolved_schema = IRSchema(name=ref_name, _from_unresolved_ref=True)
+    context.parsed_schemas[ref_name] = unresolved_schema
+    return unresolved_schema

pyopenapi_gen/core/parsing/common/ref_resolution/helpers/new_schema.py

@@ -0,0 +1,50 @@
+"""
+Module for handling new schema references.
+"""
+
+import logging
+from typing import Any, Callable, Mapping
+
+from pyopenapi_gen.ir import IRSchema
+
+from ....context import ParsingContext
+from .cyclic_properties import mark_cyclic_property_references
+
+logger = logging.getLogger(__name__)
+
+
+def parse_new_schema(
+    ref_name: str,
+    node_data: dict[str, Any],
+    context: ParsingContext,
+    max_depth: int,
+    parse_fn: Callable[[str | None, Mapping[str, Any] | None, ParsingContext, int], IRSchema],
+) -> IRSchema:
+    """
+    Parses a new schema from raw data.
+
+    Contracts:
+        Pre-conditions:
+            - ref_name must be a valid schema name not already fully parsed
+            - node_data must contain raw schema definition
+            - parse_fn must be a callable that parses schemas
+            - context must be a valid ParsingContext instance
+        Post-conditions:
+            - Returns a valid parsed IRSchema instance
+            - The schema is registered in context.parsed_schemas
+            - Cyclic property references are marked correctly
+    """
+    # Create stub to prevent infinite recursion during parsing
+    stub_schema = IRSchema(name=ref_name)
+    context.parsed_schemas[ref_name] = stub_schema
+
+    # Parse the actual schema
+    schema_obj = parse_fn(ref_name, node_data, context, max_depth)
+
+    # Update the entry in parsed_schemas with the fully parsed schema
+    context.parsed_schemas[ref_name] = schema_obj
+
+    # Mark any property references involved in cycles
+    mark_cyclic_property_references(schema_obj, ref_name, context)
+
+    return schema_obj