linkml 1.9.4rc1__py3-none-any.whl → 1.9.5rc1__py3-none-any.whl

linkml/generators/panderagen/transforms/nested_struct_model_transform.py

@@ -0,0 +1,27 @@
+from .model_transform import ModelTransform
+
+
+class NestedStructModelTransform(ModelTransform):
+    """This class assists in converting a LinkML 'nested struct' inline column
+    into a form that is better for representing in a PolaRS dataframe and
+    validating with a Pandera model.
+    """
+
+    def __init__(self, polars_schema):
+        self.polars_schema = polars_schema
+        """A polars schema representing a nested struct column"""
+
+    def transform(self, linkml_nested_struct):
+        """Transforms a nested struct column.
+        This is a pass-through since nested structs are already in the correct format.
+        """
+        return linkml_nested_struct
+
+    def explode_unnest_dataframe(self, df, column_name):
+        """Unnest for nested struct."""
+        return df.lazy().select(column_name).unnest(column_name).collect()
+
+    @classmethod
+    def prepare_dataframe(cls, data, column_name, nested_cls):
+        """Returns the nested struct column as-is since no transformation needed"""
+        return data.lazyframe.collect()

linkml/generators/panderagen/transforms/simple_dict_model_transform.py

@@ -0,0 +1,86 @@
+import polars as pl
+
+from .model_transform import ModelTransform
+
+
+class SimpleDictModelTransform(ModelTransform):
+    """This class assists in converting a LinkML 'simple dict' inline column
+    into a form that is better for representing in a PolaRS dataframe and
+    validating with a Pandera model.
+    """
+
+    def __init__(self, polars_schema, id_col, other_col):
+        self.polars_schema = polars_schema
+        """A polars schema representing a simple dict column"""
+
+        self.id_col = id_col
+        """The ID column in the sense of a LinkML inline simple dict"""
+
+        self.other_col = other_col
+        """The 'other' column in the sense of a LinkML inline simple dict"""
+
+        self.id_col_type = None
+        self.other_col_type = None
+        self.polars_struct = self._build_polars_struct()
+        """A pl.Struct representing the schema of the other range."""
+
+    def _build_polars_struct_simple(self):
+        """Handles the two column (id, other) form of the simple dict"""
+        self.id_col_type = self.polars_schema.columns[self.id_col].dtype.type
+        self.other_col_type = self.polars_schema.columns[self.other_col].dtype.type
+
+        return pl.Struct({self.id_col: self.id_col_type, self.other_col: self.other_col_type})
+
+    def _build_polars_struct_complex(self):
+        """Handles the non-two-column simple dict cases."""
+        struct_items = {}
+        for k, v in self.polars_schema.columns.items():
+            if v.dtype.type == pl.Object:
+                v.dtype.type = pl.Struct
+            else:
+                struct_items[k] = v.dtype.type
+        return pl.Struct(struct_items)
+
+    def _build_polars_struct(self):
+        if len(self.polars_schema.columns.keys()) == 2:
+            return self._build_polars_struct_simple()
+        else:
+            return self._build_polars_struct_complex()
+
+    def transform(self, linkml_simple_dict):
+        """Converts a simple dict nested column to a list of dicts.
+        { 'A': 1, 'B': 2, ... } -> [{'id': 'other': 1}, {'id': 'B', 'other': 2}, ...]
+        """
+        return self._simple_dict_to_list_of_structs(linkml_simple_dict)
+
+    def _simple_dict_to_list_of_structs(self, linkml_simple_dict):
+        """Converts a simple dict nested column to a list of dicts.
+        { 'A': 1, 'B': 2, ... } -> [{'id': 'other': 1}, {'id': 'B', 'other': 2}, ...]
+
+        An inefficient conversion (relative to native PolaRS operations)
+        from a simple dict form to a dataframe struct column.
+
+        e : dict
+            e is a single row entry in a dataframe column (one cell), which itself is a dict.
+            The value entries of e may also be dicts.
+        """
+        arr = []
+        for id_value, range_value in linkml_simple_dict.items():
+            if isinstance(range_value, dict) and (set(range_value.keys()) <= set(self.polars_schema.columns.keys())):
+                range_dict = range_value
+                range_dict[self.id_col] = id_value
+                for column_key in self.polars_schema.columns.keys():
+                    if column_key not in range_dict:
+                        range_dict[column_key] = None
+            else:
+                range_dict = {self.id_col: id_value, self.other_col: range_value}
+            arr.append(range_dict)
+
+        return arr
+
+    def list_dtype(self):
+        return pl.List(self.polars_struct)
+
+    def explode_unnest_dataframe(self, df, column_name):
+        """Explode and unnest for simple dict."""
+        return df.lazy().explode(column_name).unnest(column_name).collect()

linkml/generators/plantumlgen.py

@@ -37,6 +37,7 @@ class PlantumlGenerator(Generator):
     generatorversion = "0.1.1"
     valid_formats = ["puml", "plantuml", "png", "pdf", "jpg", "json", "svg"]
     visit_all_class_slots = False
+    preserve_names: bool = False
 
     referenced: Optional[set[ClassDefinitionName]] = None  # List of classes that have to be emitted
     generated: Optional[set[ClassDefinitionName]] = None  # List of classes that have been emitted
@@ -99,10 +100,9 @@ class PlantumlGenerator(Generator):
             return plantuml_url
         if directory:
             file_suffix = ".svg" if self.format == "puml" or self.format == "puml" else "." + self.format
-            self.output_file_name = os.path.join(
-                directory,
-                camelcase(sorted(classes)[0] if classes else self.schema.name) + file_suffix,
-            )
+            schema_name = sorted(classes)[0] if classes else self.schema.name
+            filename = schema_name if self.preserve_names else camelcase(schema_name)
+            self.output_file_name = os.path.join(directory, filename + file_suffix)
             resp = requests.get(plantuml_url, stream=True, timeout=REQUESTS_TIMEOUT)
             if resp.ok:
                 with open(self.output_file_name, "wb") as f:
@@ -133,14 +133,14 @@ class PlantumlGenerator(Generator):
             for slot in self.filtered_cls_slots(cn, all_slots=True, filtr=lambda s: s.range not in self.schema.classes):
                 if True or cn in slot.domain_of:
                     mod = self.prop_modifier(cls, slot)
+                    slot_name = (
+                        self.aliased_slot_name(slot)
+                        if self.preserve_names
+                        else underscore(self.aliased_slot_name(slot))
+                    )
+                    range_name = slot.range if self.preserve_names else underscore(slot.range)
                     slot_defs.append(
-                        "    {field} "
-                        + underscore(self.aliased_slot_name(slot))
-                        + mod
-                        + " : "
-                        + underscore(slot.range)
-                        + " "
-                        + self.cardinality(slot)
+                        "    {field} " + slot_name + mod + " : " + range_name + " " + self.cardinality(slot)
                     )
             self.class_generated.add(cn)
         self.referenced.add(cn)
@@ -359,6 +359,12 @@ class PlantumlGenerator(Generator):
     show_default=True,
     help="Print out Kroki URL calls instead of sending the real requests",
 )
+@click.option(
+    "--preserve-names/--normalize-names",
+    default=False,
+    show_default=True,
+    help="Preserve original LinkML names in PlantUML diagram output (e.g., for class names, slot names, file names).",
+)
 @click.version_option(__version__, "-V", "--version")
 def cli(yamlfile, **args):
     """Generate a UML representation of a LinkML model"""

linkml/generators/pydanticgen/pydanticgen.py

@@ -1,4 +1,5 @@
 import inspect
+import keyword
 import logging
 import os
 import re
@@ -94,7 +95,10 @@ DEFAULT_IMPORTS = (
             ObjectImport(name="ConfigDict"),
             ObjectImport(name="Field"),
             ObjectImport(name="RootModel"),
+            ObjectImport(name="SerializationInfo"),
+            ObjectImport(name="SerializerFunctionWrapHandler"),
             ObjectImport(name="field_validator"),
+            ObjectImport(name="model_serializer"),
         ],
     )
 )
@@ -143,6 +147,41 @@ DefinitionType = TypeVar("DefinitionType", bound=Union[SchemaDefinition, ClassDe
 TemplateType = TypeVar("TemplateType", bound=Union[PydanticModule, PydanticClass, PydanticAttribute])
 
 
+def make_valid_python_identifier(name: str) -> str:
+    """
+    Convert a string to a valid Python identifier.
+
+    This is used when slot names contain characters that are not valid in Python
+    identifiers (e.g., '@id', '@type'). The original name can be preserved using
+    Pydantic field aliases.
+
+    Args:
+        name: The original name that may contain invalid characters
+
+    Returns:
+        A valid Python identifier that doesn't start with underscore (Pydantic restriction)
+    """
+    # Replace invalid characters with underscores
+    identifier = re.sub(r"[^a-zA-Z0-9_]", "_", name)
+
+    # Remove leading underscores (Pydantic doesn't allow field names starting with _)
+    identifier = identifier.lstrip("_")
+
+    # Ensure it doesn't start with a number
+    if identifier and identifier[0].isdigit():
+        identifier = f"field_{identifier}"
+
+    # Ensure it's not a keyword
+    if keyword.iskeyword(identifier):
+        identifier = f"{identifier}_"
+
+    # Ensure it's not empty
+    if not identifier:
+        identifier = "field"
+
+    return identifier
+
+
 @dataclass
 class PydanticGenerator(OOCodeGenerator, LifecycleMixin):
     """
@@ -461,7 +500,19 @@ class PydanticGenerator(OOCodeGenerator, LifecycleMixin):
             if getattr(slot, k, None) is not None
         }
         slot_alias = slot.alias if slot.alias else slot.name
-        slot_args["name"] = underscore(slot_alias)
+
+        # Create a valid Python identifier for the field name
+        python_field_name = make_valid_python_identifier(underscore(slot_alias))
+        slot_args["name"] = python_field_name
+
+        # If the original name is different from the Python identifier, set an alias
+        if slot_alias != python_field_name:
+            slot_args["alias"] = slot_alias
+        else:
+            # Remove any existing alias if the names are the same
+            if "alias" in slot_args:
+                del slot_args["alias"]
+
         slot_args["description"] = slot.description.replace('"', '\\"') if slot.description is not None else None
         predef = self.predefined_slot_values.get(camelcase(cls.name), {}).get(slot.name, None)
         if predef is not None:
@@ -1241,7 +1292,7 @@ def cli(
         metadata_mode=meta,
         **args,
     )
-    print(gen.serialize())
+    print(gen.serialize(), end="")
 
 
 if __name__ == "__main__":

linkml/generators/pydanticgen/template.py

@@ -1,13 +1,30 @@
 import sys
-from collections.abc import Generator
 from importlib.util import find_spec
+from keyword import iskeyword
 from typing import Any, ClassVar, Literal, Optional, Union, get_args
 
+try:
+    from typing import Self
+except ImportError:
+    from typing_extensions import Self
+
 from jinja2 import Environment, PackageLoader
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, field_validator, model_validator
 from pydantic.version import VERSION as PYDANTIC_VERSION
 
-from linkml.generators.common.template import TemplateModel
+from linkml.generators.common.template import (
+    ConditionalImport as ConditionalImport_,
+)
+from linkml.generators.common.template import (
+    Import as Import_,
+)
+from linkml.generators.common.template import (
+    Imports as Imports_,
+)
+from linkml.generators.common.template import (
+    ObjectImport,  # noqa: F401
+    TemplateModel,
+)
 from linkml.utils.deprecation import deprecation_warning
 
 try:
@@ -113,9 +130,19 @@ class EnumValue(BaseModel):
     """
 
     label: str
+    alias: Optional[str] = None
     value: str
     description: Optional[str] = None
 
+    @model_validator(mode="after")
+    def alias_python_keywords(self) -> Self:
+        """Mask Python keywords used for `label` by appending `_`"""
+        if iskeyword(self.label):
+            if self.alias is None:
+                self.alias = self.label
+            self.label = self.label + "_"
+        return self
+
 
 class PydanticEnum(PydanticTemplateModel):
     """
@@ -170,6 +197,7 @@ class PydanticAttribute(PydanticTemplateModel):
     meta_exclude: ClassVar[list[str]] = ["from_schema", "owner", "range", "inlined", "inlined_as_list"]
 
     name: str
+    alias: Optional[str] = None
     required: bool = False
     identifier: bool = False
     key: bool = False
@@ -200,8 +228,19 @@ class PydanticAttribute(PydanticTemplateModel):
         elif self.required or self.identifier or self.key:
             return "..."
         else:
+            if self.range and self.range.startswith("Optional[list"):
+                return "[]"
             return "None"
 
+    @model_validator(mode="after")
+    def alias_python_keywords(self) -> Self:
+        """Mask Python keywords used for `name` by appending `_`"""
+        if iskeyword(self.name):
+            if self.alias is None:
+                self.alias = self.name
+            self.name = self.name + "_"
+        return self
+
 
 class PydanticValidator(PydanticAttribute):
     """
@@ -250,18 +289,7 @@ class PydanticClass(PydanticTemplateModel):
         return self.attributes
 
 
-class ObjectImport(BaseModel):
-    """
-    An object to be imported from within a module.
-
-    See :class:`.Import` for examples
-    """
-
-    name: str
-    alias: Optional[str] = None
-
-
-class Import(PydanticTemplateModel):
+class Import(Import_, PydanticTemplateModel):
     """
     A python module, or module and classes to be imported.
 
@@ -292,15 +320,6 @@ class Import(PydanticTemplateModel):
     """
 
     template: ClassVar[str] = "imports.py.jinja"
-    module: str
-    alias: Optional[str] = None
-    objects: Optional[list[ObjectImport]] = None
-    is_schema: bool = False
-    """
-    Whether or not this ``Import`` is importing another schema imported by the main schema --
-    ie. that it is not expected to be provided by the environment, but imported locally from within the package.
-    Used primarily in split schema generation, see :func:`.pydanticgen.generate_split` for example usage.
-    """
 
     @computed_field
     def group(self) -> IMPORT_GROUPS:
@@ -324,72 +343,8 @@ class Import(PydanticTemplateModel):
         else:
             return "thirdparty"
 
-    def merge(self, other: "Import") -> list["Import"]:
-        """
-        Merge one import with another, see :meth:`.Imports` for an example.
-
-        * If module don't match, return both
-        * If one or the other are a :class:`.ConditionalImport`, return both
-        * If modules match, neither contain objects, but the other has an alias, return the other
-        * If modules match, one contains objects but the other doesn't, return both
-        * If modules match, both contain objects, merge the object lists, preferring objects with aliases
-        """
-        # return both if we are orthogonal
-        if self.module != other.module:
-            return [self, other]
-
-        # handle conditionals
-        if isinstance(self, ConditionalImport) and isinstance(other, ConditionalImport):
-            # If our condition is the same, return the newer version
-            if self.condition == other.condition:
-                return [other]
-        if isinstance(self, ConditionalImport) or isinstance(other, ConditionalImport):
-            # we don't have a good way of combining conditionals, just return both
-            return [self, other]
-
-        # handle module vs. object imports
-        elif other.objects is None and self.objects is None:
-            # both are modules, return the other only if it updates the alias
-            if other.alias:
-                return [other]
-            else:
-                return [self]
-        elif other.objects is not None and self.objects is not None:
-            # both are object imports, merge and return
-            alias = self.alias if other.alias is None else other.alias
-            # FIXME: super awkward implementation
-            # keep ours if it has an alias and the other doesn't,
-            # otherwise take the other's version
-            self_objs = {obj.name: obj for obj in self.objects}
-            other_objs = {
-                obj.name: obj for obj in other.objects if obj.name not in self_objs or self_objs[obj.name].alias is None
-            }
-            self_objs.update(other_objs)
-
-            return [
-                Import(
-                    module=self.module,
-                    alias=alias,
-                    objects=list(self_objs.values()),
-                    is_schema=self.is_schema or other.is_schema,
-                )
-            ]
-        else:
-            # one is a module, the other imports objects, keep both
-            return [self, other]
-
-    def sort(self) -> None:
-        """
-        Sort imported objects
-
-        * First by whether the first letter is capitalized or not,
-        * Then alphabetically (by object name rather than alias)
-        """
-        if self.objects:
-            self.objects = sorted(self.objects, key=lambda obj: (obj.name[0].islower(), obj.name))
-
 
-class ConditionalImport(Import):
+class ConditionalImport(ConditionalImport_, PydanticTemplateModel):
     """
     Import that depends on some condition in the environment, common when
     using backported features or straddling dependency versions.
@@ -430,22 +385,13 @@ class ConditionalImport(Import):
     """
 
     template: ClassVar[str] = "conditional_import.py.jinja"
-    condition: str
-    alternative: Import
 
     @computed_field
     def group(self) -> Literal["conditional"]:
         return "conditional"
 
-    def sort(self) -> None:
-        """
-        :meth:`.Import.sort` called for self and :attr:`.alternative`
-        """
-        super().sort()
-        self.alternative.sort()
-
 
-class Imports(PydanticTemplateModel):
+class Imports(Imports_, PydanticTemplateModel):
     """
     Container class for imports that can handle merging!
 
@@ -482,135 +428,6 @@ class Imports(PydanticTemplateModel):
     imports: list[Union[Import, ConditionalImport]] = Field(default_factory=list)
     group_order: tuple[str, ...] = get_args(IMPORT_GROUPS)
     """Order in which to sort imports by their :attr:`.Import.group`"""
-    render_sorted: bool = True
-    """When rendering, render in sorted groups"""
-
-    @classmethod
-    def _merge(
-        cls, imports: list[Union[Import, ConditionalImport]], other: Union[Import, "Imports", list[Import]]
-    ) -> list[Union[Import, ConditionalImport]]:
-        """
-        Add a new import to an existing imports list, handling deduplication and flattening.
-
-        Mutates and returns ``imports``
-
-        Generally will prefer the imports in ``other`` , updating those in ``imports``.
-        If ``other`` ...
-        - doesn't match any ``module`` in ``imports``, add it!
-        - matches a single ``module`` in imports, :meth:`.Import.merge` the object imports
-        - matches multiple ``module``s in imports, then there must have been another
-            :class:`.ConditionalImport` already present, so we :meth:`.Import.merge` the existing
-            :class:`.Import` if it is one, and if it's a :class:`.ConditionalImport` just YOLO
-            and append it since there isn't a principled way to merge them from strings.
-        - is :class:`.Imports`  or a list of :class:`.Import` s, call this recursively for each
-          item.
-
-        Since imports can be merged in an undefined order depending on the generator configuration,
-        default behavior for imports with matching ``module`` is to remove them and append to the
-        end of the imports list (rather than keeping it in the position of the existing
-        :class:`.Import` ). :class:`.ConditionalImports` make it possible to have namespace
-        conflicts, so in imperative import style we assume the most recently added :class:`.Import`
-        is the one that should prevail.
-        """
-        #
-        if isinstance(other, Imports) or (isinstance(other, list) and all([isinstance(i, Import) for i in other])):
-            for i in other:
-                imports = cls._merge(imports, i)
-            return imports
-
-        existing = [i for i in imports if i.module == other.module]
-
-        # if we have nothing importing from this module yet, add it!
-        if len(existing) == 0:
-            imports.append(other)
-        elif len(existing) == 1:
-            imports.remove(existing[0])
-            imports.extend(existing[0].merge(other))
-        else:
-            # we have both a conditional and at least one nonconditional already.
-            # If this is another conditional, we just add it, otherwise, we merge it
-            # with the single nonconditional
-            if isinstance(other, ConditionalImport):
-                imports.append(other)
-            else:
-                for e in existing:
-                    if isinstance(e, Import):
-                        imports.remove(e)
-                        merged = e.merge(other)
-                        imports.extend(merged)
-                        break
-
-        # SPECIAL CASE - __future__ annotations must happen at the top of a file
-        # sort here outside of sort method because our imports are invalid without it,
-        # where calling ``sort`` should be optional.
-        imports = sorted(imports, key=lambda i: i.module == "__future__", reverse=True)
-        return imports
-
-    def __add__(self, other: Union[Import, "Imports", list[Import]]) -> "Imports":
-        imports = self.imports.copy()
-        imports = self._merge(imports, other)
-        return Imports.model_construct(
-            imports=imports, **{k: getattr(self, k, None) for k in self.model_fields if k != "imports"}
-        )
-
-    def __len__(self) -> int:
-        return len(self.imports)
-
-    def __iter__(self) -> Generator[Import, None, None]:
-        yield from self.imports
-
-    def __getitem__(self, item: Union[int, str]) -> Import:
-        if isinstance(item, int):
-            return self.imports[item]
-        elif isinstance(item, str):
-            # the name of the module
-            an_import = [i for i in self.imports if i.module == item]
-            if len(an_import) == 0:
-                raise KeyError(f"No import with module {item} was found.\nWe have: {self.imports}")
-            return an_import[0]
-        else:
-            raise TypeError(f"Can only index with an int or a string as the name of the module,\nGot: {type(item)}")
-
-    def __contains__(self, item: Union[Import, "Imports", list[Import]]) -> bool:
-        """
-        Check if all the objects are imported from the given module(s)
-
-        If the import is a bare module import (ie its :attr:`~.Import.objects` is ``None`` )
-        then we must also have a bare module import in this Imports (because even if
-        we import from the module, unless we import it specifically its name won't be
-        available in the namespace.
-
-        :attr:`.Import.alias` must always match for the same reason.
-        """
-        if isinstance(item, Imports):
-            return all([i in self for i in item.imports])
-        elif isinstance(item, list):
-            return all([i in self for i in item])
-        elif isinstance(item, Import):
-            try:
-                an_import = self[item.module]
-            except KeyError:
-                return False
-            if item.objects is None:
-                return an_import == item
-            else:
-                return all([obj in an_import.objects for obj in item.objects])
-        else:
-            raise TypeError(f"Imports only contains single Import objects or other Imports\nGot: {type(item)}")
-
-    @field_validator("imports", mode="after")
-    @classmethod
-    def imports_are_merged(
-        cls, imports: list[Union[Import, ConditionalImport]]
-    ) -> list[Union[Import, ConditionalImport]]:
-        """
-        When creating from a list of imports, construct model as if we have done so by iteratively
-        constructing with __add__ calls
-        """
-        merged_imports = []
-        for i in imports:
-            merged_imports = cls._merge(merged_imports, i)
-        return merged_imports
 
     @computed_field
     def import_groups(self) -> list[IMPORT_GROUPS]:
@@ -637,11 +454,6 @@ class Imports(PydanticTemplateModel):
             i.sort()
         self.imports = imports
 
-    def render(self, environment: Optional[Environment] = None, black: bool = False) -> str:
-        if self.render_sorted:
-            self.sort()
-        return super().render(environment=environment, black=black)
-
 
 class PydanticModule(PydanticTemplateModel):
     """

linkml/generators/pydanticgen/templates/attribute.py.jinja

@@ -1,4 +1,5 @@
 {{name}}: {{ range }} = Field(default={{ field }}
+{%- if alias %}, alias="{{alias}}"{% endif -%}
 {%- if title != None %}, title="{{title}}"{% endif -%}
 {%- if description %}, description="""{{description}}"""{% endif -%}
 {%- if equals_number != None %}

linkml/generators/pydanticgen/templates/base_model.py.jinja

@@ -1,5 +1,7 @@
 class {{ name }}(BaseModel):
     model_config = ConfigDict(
+        serialize_by_alias = True,
+        validate_by_name = True,
         validate_assignment = True,
         validate_default = True,
         extra = "{{ extra_fields }}",
@@ -11,6 +13,18 @@ class {{ name }}(BaseModel):
     {% for field in fields %}
     {{ field }}
     {% endfor %}
-{% else %}
-    {{ "pass" }}
 {% endif %}
+
+    @model_serializer(mode='wrap', when_used='unless-none')
+    def treat_empty_lists_as_none(
+            self, handler: SerializerFunctionWrapHandler,
+            info: SerializationInfo) -> dict[str, Any]:
+        if info.exclude_none:
+            _instance = self.model_copy()
+            for field, field_info in type(_instance).model_fields.items():
+                if getattr(_instance, field) == [] and not(
+                        field_info.is_required()):
+                    setattr(_instance, field, None)
+        else:
+            _instance = self
+        return handler(_instance, info)

linkml/generators/pydanticgen/templates/imports.py.jinja

@@ -5,7 +5,7 @@ import {{ module }}
 import {{ module }} as {{ alias }}
 {%- else %}
     {% if objects | length == 1 %}
-from {{ module }} import {{ objects[0]['name'] }} {% if objects[0]['alias'] is not none %} as {{ objects[0]['alias'] }} {% endif %}
+from {{ module }} import {{ objects[0]['name'] }}{% if objects[0]['alias'] is not none %} as {{ objects[0]['alias'] }}{% endif %}
     {%- else %}
 from {{ module }} import (
     {% for object in objects %}