pyxsdata 26.3.0__py3-none-any.whl

pyxsdata/__init__.py

@@ -0,0 +1 @@
+__version__ = "26.3.0"

pyxsdata/__main__.py

@@ -0,0 +1,16 @@
+import sys
+
+
+def main() -> None:
+    """Cli entry point."""
+    try:
+        from pyxsdata.cli import cli
+
+        cli()
+    except ImportError:
+        print('Install cli requirements "pip install xsdata[cli]"')
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

pyxsdata/cli.py

@@ -0,0 +1,169 @@
+import logging
+import platform
+import sys
+import warnings
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+import click
+
+from pyxsdata import __version__
+from pyxsdata.codegen.transformer import ResourceTransformer
+from pyxsdata.logger import logger
+from pyxsdata.models.config import GeneratorConfig, GeneratorOutput
+from pyxsdata.utils.click import URL, LogFormatter, LogHandler, model_options
+from pyxsdata.utils.downloader import Downloader
+from pyxsdata.utils.hooks import load_entry_points
+
+# Load cli plugins
+load_entry_points("pyxsdata.plugins.cli")
+
+# Setup xsdata logger to print records to stdout/stderr
+handler = LogHandler()
+handler.formatter = LogFormatter()
+
+logger.handlers = [handler]
+logger.propagate = False
+
+# Attach the cli handler to the python warnings logger
+py_warnings = logging.getLogger("py.warnings")
+py_warnings.handlers = [handler]
+py_warnings.propagate = False
+
+# Log warnings as well
+logging.captureWarnings(True)
+
+
+@click.group()
+@click.pass_context
+@click.version_option(__version__)
+def cli(ctx: click.Context, **kwargs: Any) -> None:
+    """Pyxsdata command line interface."""
+    logger.setLevel(logging.INFO)
+    formatwarning_orig = warnings.formatwarning
+
+    logger.info(
+        "========= pyxsdata v%s / Python %s / Platform %s =========\n",
+        __version__,
+        platform.python_version(),
+        sys.platform,
+    )
+
+    def format_warning(message: Any, category: Any, *args: Any) -> str:
+        return (
+            f"{category.__name__}: {message}" if category else message
+        )  # pragma: no cover
+
+    def format_warning_restore() -> None:
+        warnings.formatwarning = formatwarning_orig
+
+    warnings.formatwarning = format_warning  # type: ignore
+
+    ctx.call_on_close(format_warning_restore)
+
+
+@cli.command("init-config")
+@click.argument("output", type=click.Path(), default=".pyxsdata.xml")
+def init_config(**kwargs: Any) -> None:
+    """Create or update a configuration file."""
+    file_path = Path(kwargs["output"])
+    if file_path.exists():
+        config = GeneratorConfig.read(file_path)
+        logger.info("Updating configuration file %s", kwargs["output"])
+    else:
+        logger.info("Initializing configuration file %s", kwargs["output"])
+        config = GeneratorConfig.create()
+
+    with file_path.open("w") as fp:
+        config.write(fp, config)
+
+    handler.emit_warnings()
+
+
+@cli.command("download")
+@click.argument("source", type=URL(), required=True)
+@click.option(
+    "-o",
+    "--output",
+    type=click.Path(),
+    default="./",
+    help="Output directory, default cwd",
+)
+def download(source: str, output: str) -> None:
+    """Download a schema or a definition locally with all its dependencies."""
+    downloader = Downloader(output=Path(output).resolve())
+    downloader.wget(source)
+
+    handler.emit_warnings()
+
+
+_SUPPORTED_EXTENSIONS = ("wsdl", "xsd", "dtd", "xml", "json")
+
+
+@cli.command("generate")
+@click.argument("source", required=True)
+@click.option(
+    "-r",
+    "--recursive",
+    is_flag=True,
+    default=False,
+    help="Search files recursively in the source directory",
+)
+@click.option("-c", "--config", default=".pyxsdata.xml", help="Project configuration")
+@click.option("--cache", is_flag=True, default=False, help="Cache sources loading")
+@click.option("--debug", is_flag=True, default=False, help="Show debug messages")
+@click.option(
+    "--extensions",
+    default=",".join(_SUPPORTED_EXTENSIONS),
+    help="Comma-separated list of extensions to filter",
+)
+@model_options(GeneratorOutput)
+def generate(**kwargs: Any) -> None:
+    """Generate code from xsd, dtd, wsdl, xml and json files.
+
+    The input source can be either a filepath, uri or a directory
+    containing xml, json, xsd and wsdl files.
+    """
+    debug = kwargs.pop("debug")
+    if debug:
+        logger.setLevel(logging.DEBUG)
+
+    source = kwargs.pop("source")
+    cache = kwargs.pop("cache")
+    recursive = kwargs.pop("recursive")
+    config_file = Path(kwargs.pop("config")).resolve()
+
+    # Parse the comma-separated extensions string into a tuple
+    extensions_str = kwargs.pop("extensions")
+    extensions = tuple(ext.strip() for ext in extensions_str.split(",") if ext.strip())
+
+    params = {k.replace("__", "."): v for k, v in kwargs.items() if v is not None}
+    config = GeneratorConfig.read(config_file)
+    config.output.update(**params)
+
+    transformer = ResourceTransformer(config=config)
+    uris = sorted(resolve_source(source, recursive=recursive, extensions=extensions))
+    transformer.process(uris, cache=cache)
+
+    handler.emit_warnings()
+
+
+def resolve_source(
+    source: str, recursive: bool, extensions: tuple[str, ...] = _SUPPORTED_EXTENSIONS
+) -> Iterator[str]:
+    """Yields all supported resource URIs."""
+    if source.find("://") > -1 and not source.startswith("file://"):
+        yield source
+    else:
+        path = Path(source).resolve()
+        match = "**/*" if recursive else "*"
+        if path.is_dir():
+            for ext in extensions:
+                yield from (x.as_uri() for x in path.glob(f"{match}.{ext}"))
+        else:  # is a file
+            yield path.as_uri()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    cli()

pyxsdata/codegen/__init__.py

@@ -0,0 +1,6 @@
+from urllib.request import build_opener
+
+from pyxsdata import __version__
+
+opener = build_opener()
+opener.addheaders = [("User-agent", f"xsdata/{__version__}")]

pyxsdata/codegen/container.py

@@ -0,0 +1,291 @@
+from collections.abc import Callable, Iterator
+
+from pyxsdata.codegen.handlers import (
+    AddAttributeSubstitutions,
+    CalculateAttributePaths,
+    CreateCompoundFields,
+    CreateWrapperFields,
+    DesignateClassPackages,
+    DetectCircularReferences,
+    DisambiguateChoices,
+    FilterClasses,
+    FlattenAttributeGroups,
+    FlattenClassExtensions,
+    MergeAttributes,
+    MergeDuplicateClasses,
+    ProcessAttributeTypes,
+    ProcessMixedContentClass,
+    RenameDuplicateAttributes,
+    RenameDuplicateClasses,
+    ResetAttributeSequenceNumbers,
+    ResetAttributeSequences,
+    SanitizeAttributesDefaultValue,
+    SanitizeEnumerationClass,
+    UnnestInnerClasses,
+    UpdateAttributesEffectiveChoice,
+    VacuumInnerClasses,
+    ValidateAttributesOverrides,
+    ValidateReferences,
+)
+from pyxsdata.codegen.mixins import ContainerInterface
+from pyxsdata.codegen.models import Class, Status
+from pyxsdata.codegen.stopwatch import stopwatch
+from pyxsdata.codegen.utils import ClassUtils
+from pyxsdata.codegen.validator import ClassValidator
+from pyxsdata.models.config import GeneratorConfig
+from pyxsdata.utils import collections
+from pyxsdata.utils.constants import return_true
+
+
+class Steps:
+    """Process steps."""
+
+    UNGROUP = 10
+    FLATTEN = 20
+    SANITIZE = 30
+    RESOLVE = 40
+    CLEANUP = 50
+    FINALIZE = 60
+
+
+class ClassContainer(ContainerInterface):
+    """A class list wrapper with an easy access api.
+
+    Args:
+        config: The generator configuration instance
+
+    Attributes:
+        processors: A step-processors mapping
+        data: The class qname map
+        step: The current process step
+    """
+
+    __slots__ = ("processors", "step")
+
+    def __init__(self, config: GeneratorConfig):
+        """Initialize the container and all the class processors.
+
+        The order of the steps and the processors is the secret
+        recipe of the xsdata code generator.
+
+        Args:
+            config: The generator configuration instance
+        """
+        super().__init__(config)
+        self.step: int = 0
+        self.processors: dict[int, list] = {
+            Steps.UNGROUP: [
+                FlattenAttributeGroups(self),
+            ],
+            Steps.FLATTEN: [
+                CalculateAttributePaths(),
+                FlattenClassExtensions(self),
+                SanitizeEnumerationClass(self),
+                UpdateAttributesEffectiveChoice(),
+                UnnestInnerClasses(self),
+                AddAttributeSubstitutions(self),
+                ProcessAttributeTypes(self),
+                MergeAttributes(),
+                ProcessMixedContentClass(),
+            ],
+            Steps.SANITIZE: [
+                ResetAttributeSequences(),
+                RenameDuplicateAttributes(),
+            ],
+            Steps.RESOLVE: [
+                ValidateAttributesOverrides(self),
+            ],
+            Steps.CLEANUP: [
+                VacuumInnerClasses(),
+            ],
+            Steps.FINALIZE: [
+                DetectCircularReferences(self),
+                CreateCompoundFields(self),
+                CreateWrapperFields(self),
+                DisambiguateChoices(self),
+                SanitizeAttributesDefaultValue(self),
+                ResetAttributeSequenceNumbers(self),
+            ],
+        }
+
+    def __iter__(self) -> Iterator[Class]:
+        """Yield an iterator for the class map values."""
+        for items in list(self.data.values()):
+            yield from items
+
+    def find(self, qname: str, condition: Callable = return_true) -> Class | None:
+        """Find class that matches the given qualified name and condition callable.
+
+        Classes are allowed to have the same qualified name, e.g. xsd:Element
+        extending xsd:ComplexType with the same name, you can provide and additional
+        callback to filter the classes like the tag.
+
+        Args:
+            qname: The qualified name of the class
+            condition: A user callable to filter further
+
+        Returns:
+            A class instance or None if no match found.
+        """
+        for row in self.data.get(qname, []):
+            if condition(row):
+                if row.status < self.step:
+                    self.process_class(row, self.step)
+                    return self.find(qname, condition)
+
+                return row
+        return None
+
+    def find_inner(self, source: Class, qname: str) -> Class:
+        """Search by qualified name for a specific inner class or fail.
+
+        Args:
+            source: The source class to search for the inner class
+            qname: The qualified name of the inner class to look up
+
+        Returns:
+            The inner class instance
+
+        Raises:
+            CodeGenerationError: If the inner class is not found.
+        """
+        inner = ClassUtils.find_nested(source, qname)
+        if inner.status < self.step:
+            self.process_class(inner, self.step)
+
+        return inner
+
+    def first(self, qname: str) -> Class:
+        """Return the first class that matches the qualified name.
+
+        Args:
+            qname: The qualified name of the class
+
+        Returns:
+            The first matching class
+
+        Raises:
+            KeyError: If no class matches the qualified name
+        """
+        classes = self.data.get(qname)
+        if not classes:
+            raise KeyError(f"Class {qname} not found")
+
+        return classes[0]
+
+    def process(self) -> None:
+        """Run the processor and filter steps."""
+        self.validate_classes()
+        self.process_classes(Steps.UNGROUP)
+        self.remove_groups()
+        self.process_classes(Steps.FLATTEN)
+        self.filter_classes()
+        self.process_classes(Steps.SANITIZE)
+        self.process_classes(Steps.RESOLVE)
+        self.process_classes(Steps.CLEANUP)
+        self.process_classes(Steps.FINALIZE)
+        self.designate_classes()
+
+    def validate_classes(self) -> None:
+        """Merge redefined classes."""
+        with stopwatch("ClassValidator"):
+            ClassValidator(self).process()
+
+    def process_classes(self, step: int) -> None:
+        """Run the given step processors for all classes.
+
+        Args:
+            step: The step reference number
+        """
+        self.step = step
+        for obj in self:
+            if obj.status < step:
+                self.process_class(obj, step)
+
+    def process_class(self, target: Class, step: int) -> None:
+        """Run the step processors for the given class.
+
+        Process recursively any inner classes as well.
+
+        Args:
+            target: The target class to process
+            step: The step reference number
+        """
+        target.status = Status(step)
+        for processor in self.processors.get(step, []):
+            with stopwatch(processor.__class__.__name__):
+                processor.process(target)
+
+        for inner in target.inner:
+            if inner.status < step:
+                self.process_class(inner, step)
+
+        target.status = Status(step + 1)
+
+    def designate_classes(self) -> None:
+        """Designate the final class names, packages and modules."""
+        designators = [
+            MergeDuplicateClasses(self),
+            RenameDuplicateClasses(self),
+            ValidateReferences(self),
+            DesignateClassPackages(self),
+        ]
+
+        for designator in designators:
+            with stopwatch(designator.__class__.__name__):
+                designator.run()
+
+    def filter_classes(self) -> None:
+        """Filter the classes to be generated."""
+        with stopwatch(FilterClasses.__name__):
+            FilterClasses(self).run()
+
+    def remove_groups(self) -> None:
+        """Remove xs:groups and xs:attributeGroups from the container."""
+        self.set([x for x in iter(self) if not x.is_group])
+
+    def add(self, item: Class) -> None:
+        """Add class item to the container.
+
+        Args:
+            item: The class instance to add
+        """
+        self.data.setdefault(item.qname, []).append(item)
+
+    def remove(self, *items: Class) -> None:
+        """Safely remove classes from the container.
+
+        Args:
+            items: The classes to remove
+        """
+        for item in items:
+            self.data[item.qname] = [
+                c for c in self.data[item.qname] if c.ref != item.ref
+            ]
+
+    def reset(self, item: Class, qname: str) -> None:
+        """Update the given class qualified name.
+
+        Args:
+            item: The target class instance to update
+            qname: The new qualified name of the class
+        """
+        self.data[qname] = [c for c in self.data[qname] if c.ref != item.ref]
+        self.add(item)
+
+    def set(self, items: list[Class]) -> None:
+        """Set the list of classes to the container.
+
+        Args:
+            items: The list of classes
+        """
+        self.data.clear()
+        self.extend(items)
+
+    def extend(self, items: list[Class]) -> None:
+        """Add a list of classes to the container.
+
+        Args:
+            items: The list of class instances to add
+        """
+        collections.apply(items, self.add)

pyxsdata/codegen/exceptions.py

@@ -0,0 +1,23 @@
+from typing import IO, Any
+
+from click import ClickException, echo
+
+
+class CodegenWarning(Warning):
+    """Recovered errors during code generation recovered errors."""
+
+
+class CodegenError(ClickException):
+    """Unexpected state during code generation related errors."""
+
+    def __init__(self, message: str, **kwargs: Any):
+        """Click exception constructor with metadata."""
+        super().__init__(message)
+        self.meta = kwargs
+
+    def show(self, file: IO[Any] | None = None) -> None:
+        """Echo codegen error message and details."""
+        echo("=========")
+        super().show(file)
+        for key, value in self.meta.items():
+            echo(f"{key}: {value}")

pyxsdata/codegen/handlers/__init__.py

@@ -0,0 +1,53 @@
+from .add_attribute_substitutions import AddAttributeSubstitutions
+from .calculate_attribute_paths import CalculateAttributePaths
+from .create_compound_fields import CreateCompoundFields
+from .create_wrapper_fields import CreateWrapperFields
+from .designate_class_packages import DesignateClassPackages
+from .detect_circular_references import DetectCircularReferences
+from .disambiguate_choices import DisambiguateChoices
+from .filter_classes import FilterClasses
+from .flatten_attribute_groups import FlattenAttributeGroups
+from .flatten_class_extensions import FlattenClassExtensions
+from .merge_attributes import MergeAttributes
+from .merge_duplicate_classes import MergeDuplicateClasses
+from .process_attributes_types import ProcessAttributeTypes
+from .process_mixed_content_class import ProcessMixedContentClass
+from .rename_duplicate_attributes import RenameDuplicateAttributes
+from .rename_duplicate_classes import RenameDuplicateClasses
+from .reset_attribute_sequence_numbers import ResetAttributeSequenceNumbers
+from .reset_attribute_sequences import ResetAttributeSequences
+from .sanitize_attributes_default_value import SanitizeAttributesDefaultValue
+from .sanitize_enumeration_class import SanitizeEnumerationClass
+from .unnest_inner_classes import UnnestInnerClasses
+from .update_attributes_effective_choice import UpdateAttributesEffectiveChoice
+from .vacuum_inner_classes import VacuumInnerClasses
+from .validate_attributes_overrides import ValidateAttributesOverrides
+from .validate_references import ValidateReferences
+
+__all__ = [
+    "AddAttributeSubstitutions",
+    "CalculateAttributePaths",
+    "CreateCompoundFields",
+    "CreateWrapperFields",
+    "DesignateClassPackages",
+    "DetectCircularReferences",
+    "DisambiguateChoices",
+    "FilterClasses",
+    "FlattenAttributeGroups",
+    "FlattenClassExtensions",
+    "MergeAttributes",
+    "MergeDuplicateClasses",
+    "ProcessAttributeTypes",
+    "ProcessMixedContentClass",
+    "RenameDuplicateAttributes",
+    "RenameDuplicateClasses",
+    "ResetAttributeSequenceNumbers",
+    "ResetAttributeSequences",
+    "SanitizeAttributesDefaultValue",
+    "SanitizeEnumerationClass",
+    "UnnestInnerClasses",
+    "UpdateAttributesEffectiveChoice",
+    "VacuumInnerClasses",
+    "ValidateAttributesOverrides",
+    "ValidateReferences",
+]

pyxsdata/codegen/handlers/add_attribute_substitutions.py

@@ -0,0 +1,122 @@
+from collections import defaultdict
+
+from pyxsdata.codegen.mixins import ContainerInterface, RelativeHandlerInterface
+from pyxsdata.codegen.models import Attr, AttrType, Class
+from pyxsdata.codegen.utils import ClassUtils
+from pyxsdata.models.enums import Tag
+from pyxsdata.utils import collections
+
+
+class AddAttributeSubstitutions(RelativeHandlerInterface):
+    """Apply substitution attributes to the given class recursively.
+
+    Args:
+        container: The class container instance
+
+    Attributes:
+        substitutions: Mapping of type names to attr values
+    """
+
+    __slots__ = "substitutions"
+
+    def __init__(self, container: ContainerInterface):
+        """Initialize the class."""
+        super().__init__(container)
+        self.substitutions: dict[str, list[Attr]] | None = None
+
+    def process(self, target: Class) -> None:
+        """Process the given class attrs for substitution groups.
+
+        This method will ignore attrs in the class derived from
+        a xs:enumeration, xs:anyType and xs:any. If this is the
+        first time we call the method, build the substitution
+        map.
+
+        Args:
+            target: The target class instance
+        """
+        if self.substitutions is None:
+            self.create_substitutions()
+
+        for attr in target.attrs.copy():
+            if not (attr.is_enumeration or attr.is_wildcard):
+                self.process_attribute(target, attr)
+
+    def process_attribute(self, target: Class, attr: Attr) -> None:
+        """Add substitution attrs that refer to the attr type.
+
+        If the given attr is referenced in substitution groups
+        clone all substitution attrs and place them bellow
+        the original attr. Convert all the attrs of the group
+        to repeatable choice elements.
+
+        Guard against multiple substitutions in case of xs:groups.
+
+        Args:
+            target: The target class instance
+            attr: The source attr instance to check and process
+        """
+        index = target.attrs.index(attr)
+        assert self.substitutions is not None
+
+        for attr_type in attr.types:
+            if attr_type.substituted:
+                continue
+
+            attr_type.substituted = True
+            for substitution in self.substitutions.get(attr_type.qname, []):
+                self.prepare_substituted(attr)
+
+                clone = ClassUtils.clone_attribute(substitution, attr.restrictions)
+                clone.restrictions.min_occurs = 0
+                clone.restrictions.max_occurs = attr.restrictions.max_occurs
+
+                attr.substitution = clone.substitution = attr_type.name
+
+                pos = collections.find(target.attrs, clone)
+                index = pos + 1 if pos > -1 else index
+                target.attrs.insert(index, clone)
+
+                self.process_attribute(target, clone)
+
+    def create_substitutions(self) -> None:
+        """Build the substitutions mapping of type names to attr values.
+
+        The values are simple reference attrs that we can easily
+        clone later on demand.
+        """
+        self.substitutions = defaultdict(list)
+        for obj in self.container:
+            for qname in obj.substitutions:
+                attr = self.create_substitution(obj)
+                self.substitutions[qname].append(attr)
+
+    @classmethod
+    def prepare_substituted(cls, attr: Attr):
+        """Prepare the original attr for substitutions.
+
+        Effectively place the attr inside a xs:choice container
+        with min occurs zero.
+        """
+        attr.restrictions.min_occurs = 0
+        if not attr.restrictions.choice:
+            choice = id(attr)
+            attr.restrictions.choice = choice
+            attr.restrictions.path.append(("c", choice, 1, 1))
+
+    @classmethod
+    def create_substitution(cls, source: Class) -> Attr:
+        """Create a reference attr to the source class qname.
+
+        Args:
+            source: The source class to reference
+
+        Returns:
+            The reference to the source class attr.
+        """
+        return Attr(
+            name=source.name,
+            types=[AttrType(qname=source.qname)],
+            tag=Tag.ELEMENT,
+            namespace=source.namespace,
+        )