linkml 1.9.4rc2__py3-none-any.whl → 1.9.5rc1__py3-none-any.whl

linkml/generators/panderagen/panderagen.py

@@ -14,9 +14,10 @@ from linkml_runtime.utils.formatutils import camelcase
 from linkml_runtime.utils.schemaview import SchemaView
 
 from linkml._version import __version__
-from linkml.generators.oocodegen import OOClass, OOCodeGenerator, OODocument
+from linkml.generators.oocodegen import OOCodeGenerator, OODocument
 
 from .class_generator_mixin import ClassGeneratorMixin
+from .dataframe_class import DataframeClass
 from .enum_generator_mixin import EnumGeneratorMixin
 from .slot_generator_mixin import SlotGeneratorMixin
 
@@ -28,6 +29,7 @@ TYPEMAP = {
     "panderagen_class_based": {
         "xsd:string": "str",
         "xsd:integer": "int",
+        "xsd:int": "int",
         "xsd:float": "float",
         "xsd:double": "float",
         "xsd:boolean": "bool",
@@ -83,6 +85,8 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
 
     @staticmethod
     def make_multivalued(range: str) -> str:
+        if range == "Struct":
+            return "pl.List"
         return f"List[{range}]"
 
     def uri_type_map(self, xsd_uri: str, template: str = None):
@@ -92,15 +96,22 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
         return TYPEMAP[template].get(xsd_uri)
 
     def map_type(self, t: TypeDefinition) -> str:
+        logger.info(f"type_map definition: {t}")
+
+        typ = None
+
         if t.uri:
             typ = self.uri_type_map(t.uri)
-            return typ
+            if typ is None:
+                typ = self.map_type(self.schemaview.get_type(t.typeof))
         elif t.typeof:
             typ = self.map_type(self.schemaview.get_type(t.typeof))
-            return typ
-        else:
+
+        if typ is None:
             raise ValueError(f"{t} cannot be mapped to a type")
 
+        return typ
+
     def load_template(self, template_filename):
         jinja_env = Environment(loader=PackageLoader("linkml.generators.panderagen", self.template_path))
         return jinja_env.get_template(template_filename)
@@ -138,6 +149,7 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
             coerce=self.coerce,
             type_map=TYPEMAP,
             template_path=self.template_path,
+            pandera_validator_code=None,
         )
         return code
 
@@ -156,12 +168,17 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
         for c in self.ordered_classes():
             cn = c.name
             safe_cn = camelcase(cn)
-            ooclass = OOClass(
+            annotations = {}
+            identifier_or_key_slot = self.get_identifier_or_key_slot(cn)
+            if identifier_or_key_slot:
+                annotations["identifier_key_slot"] = identifier_or_key_slot.name
+            ooclass = DataframeClass(
                 name=safe_cn,
                 description=c.description,
                 package=self.package,
                 fields=[],
                 source_class=c,
+                annotations=annotations,
             )
             classes.append(ooclass)
             if c.mixin:

linkml/generators/panderagen/panderagen_class_based/class.jinja2

@@ -3,24 +3,81 @@
   Details at https://pandera.readthedocs.io/en/stable/dataframe_models.html
 -#}
 {%- import 'slots.jinja2' as slot_macros -%}
+{%- import 'custom_checks.jinja2' as custom_checks -%}
 
-{%- macro render_class(cls) %}
-class {{cls.name}}(
-        {%- if cls.is_a -%}
-            {{ cls.is_a }}
-        {%- else -%}
-            pla.DataFrameModel, _LinkmlPanderaValidator
-        {%- endif -%}
-):
-    {%- if cls.source_class.description %}
-    """
-    {{ cls.source_class.description }}
-    """
-    {% endif -%}
+{%- macro render_parent_classes(cls) -%}
+    {%- if cls.is_a -%}
+        {{ cls.is_a }}
+    {%- else -%}
+        pla.DataFrameModel, _LinkmlPanderaValidator
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_nested_ranges(cls) -%}
+{#-
+    Helper class member that references another Pandera class that
+    handles a nested association.
+-#}
+    _NESTED_RANGES = {
+    {%- for field in cls.fields -%}
+        {%- if field.reference_class() %}
+        "{{ field.name }}": "{{ field.reference_class() }}",
+{% endif -%}
+    {%- endfor %}
+    }
+{%- endmacro -%}
+
+{%- macro render_inline_form(cls) -%}
+{#-
+    This is a meta object with information used by the various check methods
+-#}
+    _INLINE_FORM = {
+    {%- for field in cls.fields -%}
+        {%- if field.inline_form() %}
+        "{{ field.name }}": "{{ field.inline_form() }}",
+{% endif -%}
+    {%- endfor %}
+    }
+{%- endmacro -%}
+
+{%- macro render_inline_details(cls) -%}
+{#-
+    This is a meta object with information used by the various check methods
+-#}
+    _INLINE_DETAILS = {
+    {%- for field in cls.fields -%}
+        {%- if field.inline_details() %}
+        "{{ field.name }}": {{ field.inline_details() }},
+{% endif -%}
+    {%- endfor %}
+    }
+{%- endmacro -%}
+
+{%- macro render_slots(cls) -%}
+    {{ slot_macros.id_slot_name_class_variable(cls) }}
     {%- if (cls.fields | length) == 0 %}
     pass
     {% endif -%}
     {%- for field in cls.fields -%}
         {{ slot_macros.render_slot(field) }}
     {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro render_class(cls) %}
+{#-
+    Generates the main structure of a Pandera class.
+    This includes the slots, custom checks,
+    and helper class members with information used by the custom checks.
+-#}
+class {{cls.name}}({{ render_parent_classes(cls) }}):
+    {%- if cls.source_class.description %}
+    """
+    {{ cls.source_class.description }}
+    """
+{% endif -%}
+    {{ render_slots(cls) }}
+    {{ custom_checks.render_custom_checks(cls) }}
+    {{ render_nested_ranges(cls) }}
+    {{ render_inline_form(cls) }}
+    {{ render_inline_details(cls) }}
 {% endmacro -%}

linkml/generators/panderagen/panderagen_class_based/custom_checks.jinja2

@@ -0,0 +1,27 @@
+
+{%- macro nested_checks(field) -%}
+{#-
+    field: DataframeField
+    Render a class method that follows nested structures
+    other than the simple_dict form.
+-#}
+    {%- if field.inline_form() %}
+    @pla.check("{{ field.name }}")
+    def check_nested_struct_{{ field.name }}(cls, data: PolarsData):
+        {% if field.inline_form() == 'inline_list_dict' -%}
+        return cls._check_nested_struct(data)
+        {% elif field.inline_form() == 'simple_dict' -%}
+        return cls._check_simple_dict(data)
+        {% elif field.inline_form() == 'inline_collection_dict' -%}
+        return cls._check_collection_struct(data)
+        {% else -%}
+        return cls._check_nested_list_struct(data)
+        {% endif -%}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_custom_checks(cls) -%}
+    {%- for field in cls.fields -%}
+    {{ nested_checks(field) }}
+    {%- endfor %}
+{%- endmacro -%}

linkml/generators/panderagen/panderagen_class_based/enums.jinja2

@@ -1,7 +1,7 @@
-{%- macro enum_parameter(slot) -%}
-    {%- if slot.annotations['permissible_values']|length > 0 -%}
+{%- macro enum_parameter(field) -%}
+    {%- if field.permissible_values()|length > 0 -%}
         dtype_kwargs={"categories":(
-        {%- for pv in slot.annotations['permissible_values'] -%}
+        {%- for pv in field.permissible_values() -%}
             '{{ pv }}',
         {%- endfor -%}
         )}

linkml/generators/panderagen/panderagen_class_based/pandera.jinja2

@@ -4,10 +4,20 @@
 -#}
 {%- import 'header.jinja2' as header -%}
 {%- import 'class.jinja2' as class_macros -%}
-{%- import 'mixins.jinja2' as mixins -%}
 {{ header }}
 
-{{ mixins }}
+{% if pandera_validator_code %}
+{{ pandera_validator_code }}
+{% else %}
+from linkml.generators.panderagen.linkml_pandera_validator import LinkmlPanderaValidator as _LinkmlPanderaValidator
+{% endif %}
+
+# These are all str for now
+ID_TYPES = {
+{%- for cls in doc.classes %}
+    "{{ cls.name }}": "str",
+{%- endfor %}
+}
 
 {% if metamodel_version %}# metamodel_version: {{metamodel_version}}{% endif %}
 {% if model_version %}# version: {{model_version}}{% endif %}

linkml/generators/panderagen/panderagen_class_based/slots.jinja2

@@ -4,33 +4,35 @@
 -#}
 {%- import 'enums.jinja2' as enum_macros -%}
 
-{%- macro constraint_parameters(field, slot) -%}
+{%- macro constraint_parameters(field) -%}
     {%- if coerce is true -%}coerce=True, {% endif -%}
     {%- if field.default_value is not none -%}default={{ field.default_value }}, {% endif -%}
-    {%- if slot.minimum_value is not none -%}ge={{ slot.minimum_value }}, {% endif -%}
-    {%- if slot.maximum_value is not none -%}le={{ slot.maximum_value }}, {% endif -%}
-    {%- if slot.pattern is not none -%}str_matches=r"{{ slot.pattern }}", {% endif -%}
-    {%- if (slot.required is none or slot.required is false) and slot.identifier is not true -%}nullable=True, {% endif -%}
+    {%- if field.minimum_value() is not none -%}ge={{ field.minimum_value() }}, {% endif -%}
+    {%- if field.maximum_value() is not none -%}le={{ field.maximum_value() }}, {% endif -%}
+    {%- if field.pattern() is not none -%}str_matches=r"{{ field.pattern() }}", {% endif -%}
+    {%- if (field.required() is none or field.required() is false) and field.identifier() is not true -%}nullable=True, {% endif -%}
 {%- endmacro -%}
 
-{%- macro render_slot(field) -%}
-    {%- with slot = field.source_slot %}
+{%- macro id_slot_name_class_variable(cls) %}
+    _id_name : str = {% if cls.identifier_key_slot() %} '{{ cls.identifier_key_slot() }}' {% else %}None{% endif %}
+{%- endmacro -%}
+
+{%- macro render_slot(field) %}
     {{ field.name }}:{{ ' ' }}
-        {%- if (slot.required is none or slot.required is false) and slot.identifier is not true -%}
+        {%- if (field.required() is none or field.required() is false) and (field.identifier() is not true) -%}
             Optional[
         {%- endif -%}
        {{field.range}}
-        {%- if (slot.required is none or slot.required is false) and slot.identifier is not true -%}
-            {{ '] ' }}
+        {%- if (field.required() is none or field.required() is false) and (field.identifier() is not true) -%}
+            {{ ']' }}
         {%- endif -%}
-        = pla.Field(
-            {{- constraint_parameters(field, slot) -}}
-            {{- enum_macros.enum_parameter(slot) -}}
+        {{ ' =' }} pla.Field(
+            {{- constraint_parameters(field) -}}
+            {{- enum_macros.enum_parameter(field) -}}
         )
-        {%- if slot.description %}
+        {%- if field.description() %}
     """
-    {{ slot.description }}
+    {{ field.description() }}
     """
-        {% endif -%}
-    {%- endwith -%}
+    {% endif -%}
 {%- endmacro %}

linkml/generators/panderagen/slot_generator_mixin.py

@@ -1,31 +1,156 @@
 import logging
+from typing import Optional
 
-from linkml.generators.oocodegen import OOField
+from linkml_runtime.linkml_model.meta import ClassDefinitionName, SlotDefinition
+
+from linkml.utils.helpers import get_range_associated_slots
+
+from .dataframe_field import DataframeField
 
 logger = logging.getLogger(__file__)
 
 
 class SlotGeneratorMixin:
+    """
+    Prior to rendering the dataframe schema, this class provides
+    and adapter between the LinkML model and schema view
+    and the rendering engine.
+    """
+
     LINKML_ANY_CURIE = "linkml:Any"
+
+    # constants used to render the schema
+    # these will be moved to a dialect-specific place
     ANY_RANGE_STRING = "Object"
     CLASS_RANGE_STRING = "Struct"
+    SIMPLE_DICT_RANGE_STRING = "Struct"
     ENUM_RANGE_STRING = "Enum"
-    DEFAULT_RANGE_STRING = "str"
 
-    # to be implemented by the class
-    def make_multivalued(self, range: str):
-        raise NotImplementedError("please implement make multivalued in the class")
+    # association form flags used for rendering decisions
+    FORM_INLINED_DICT = "inlined_dict"
+    FORM_INLINED_LIST_DICT = "inlined_list_dict"
+    FORM_INLINED_COLLECTION_DICT = "inline_collection_dict"
+    FORM_INLINED_SIMPLE_DICT = "simple_dict"
+    FORM_MULTIVALUED_FOREIGN_KEY = "list_foreign_key"
+    FORM_FOREIGN_KEY = "foreign_key"
+    FORM_ERROR = "error"
+
+    # When nested inlining is done, the Pandera validator needs a specific range
+    INLINED_FORM_RANGE_PANDERA = {
+        FORM_INLINED_SIMPLE_DICT: SIMPLE_DICT_RANGE_STRING,
+        FORM_INLINED_LIST_DICT: CLASS_RANGE_STRING,
+        FORM_INLINED_COLLECTION_DICT: CLASS_RANGE_STRING,
+        FORM_INLINED_DICT: CLASS_RANGE_STRING,
+        FORM_ERROR: None,
+    }
+
+    def is_multivalued(self, slot):
+        return "multivalued" in slot and slot.multivalued is True
+
+    _INTERNAL_INLINED_FORM = {
+        # INLINED, INLINED_AS_LIST, MULTIVALUED,
+        (False, False, False): FORM_FOREIGN_KEY,
+        (False, False, True): FORM_MULTIVALUED_FOREIGN_KEY,
+        (False, True, False): FORM_INLINED_LIST_DICT,
+        (False, True, True): FORM_INLINED_LIST_DICT,
+        (True, False, False): FORM_INLINED_DICT,
+        (True, False, True): FORM_INLINED_COLLECTION_DICT,
+        (True, None, True): FORM_INLINED_DICT,
+        (True, True, False): FORM_INLINED_DICT,
+        (True, True, True): FORM_INLINED_LIST_DICT,
+    }
+
+    def get_identifier_or_key_slot(self, cn: ClassDefinitionName) -> Optional[SlotDefinition]:
+        sv = self.schemaview
+        id_slot = sv.get_identifier_slot(cn)
+        if id_slot:
+            return id_slot
+        else:
+            for s in sv.class_induced_slots(cn):
+                if s.key:
+                    return s
+            return None
+
+    def calculate_inlined_form(self, slot: SlotDefinition) -> str:
+        is_multivalued = self.is_multivalued(slot)
+        internal_inlined_form_key = ((slot.inlined is True), (slot.inlined_as_list is True), is_multivalued)
+        logger.info(f"Inlined form key: {internal_inlined_form_key}")
+        internal_inlined_form = self._INTERNAL_INLINED_FORM.get(
+            internal_inlined_form_key, SlotGeneratorMixin.FORM_ERROR
+        )
+
+        if internal_inlined_form == SlotGeneratorMixin.FORM_INLINED_COLLECTION_DICT:
+            if self.get_identifier_or_key_slot(slot.range) is None:
+                internal_inlined_form = SlotGeneratorMixin.FORM_INLINED_LIST_DICT
+
+        if self.calculate_simple_dict(slot) is not None:
+            return SlotGeneratorMixin.FORM_INLINED_SIMPLE_DICT
+
+        return internal_inlined_form
+
+    def calculate_simple_dict(self, slot: SlotDefinition):
+        """slot is the container for the simple dict slot"""
+
+        (_, range_simple_dict_value_slot, _) = get_range_associated_slots(self.schemaview, slot.range)
+
+        return range_simple_dict_value_slot
 
-    def handle_none_slot(self, slot, range: str) -> str:
+    def handle_none_slot(self, slot) -> str:
         range = self.schema.default_range  # need to figure this out, set at the beginning?
         if range is None:
-            range = SlotGeneratorMixin.DEFAULT_RANGE_STRING
+            range = "str"
 
         return range
 
     def handle_class_slot(self, slot, range: str) -> str:
-        logger.warning(f"PanderaGen does not support class range slots. Using Struct {slot.name}")
-        return SlotGeneratorMixin.CLASS_RANGE_STRING
+        range_info = self.schemaview.all_classes().get(range)
+
+        if range_info["class_uri"] == SlotGeneratorMixin.LINKML_ANY_CURIE:
+            range = SlotGeneratorMixin.ANY_RANGE_STRING
+        else:
+            inlined_form = self.calculate_inlined_form(slot)
+
+            if inlined_form == SlotGeneratorMixin.FORM_INLINED_COLLECTION_DICT:
+                logger.warning(
+                    f"Slot {slot.name} uses inlined dictionary form,"
+                    "which may be less efficient than inlined as list form with the current implementation."
+                )
+            elif inlined_form == SlotGeneratorMixin.FORM_INLINED_SIMPLE_DICT:
+                logger.warning(
+                    f"Slot {slot.name} uses inlined simple dictionary form. Support is incomplete "
+                    "and performance is less efficient than inlined as list form with the current implementation."
+                )
+
+            if inlined_form in (SlotGeneratorMixin.FORM_MULTIVALUED_FOREIGN_KEY, SlotGeneratorMixin.FORM_FOREIGN_KEY):
+                logger.warning(f"Foreign key not implemented for slot {slot.name}")
+                range = f"ID_TYPES['{self.get_class_name(range)}']"
+            else:
+                # TODO: make these setters
+                slot.annotations["reference_class"] = self.get_class_name(range)
+                slot.annotations["inline_form"] = inlined_form
+
+                range = SlotGeneratorMixin.INLINED_FORM_RANGE_PANDERA[inlined_form]
+
+                if inlined_form == SlotGeneratorMixin.FORM_INLINED_SIMPLE_DICT:
+                    self.set_simple_dict_inline_details_annotation(slot)
+                elif inlined_form in [SlotGeneratorMixin.FORM_INLINED_LIST_DICT]:
+                    range = self.make_multivalued(range)
+
+        return range
+
+    def set_simple_dict_inline_details_annotation(self, slot):
+        """Extra metadata is to help with the simple dict case"""
+        (range_id_slot, range_simple_dict_value_slot, _) = get_range_associated_slots(  # range_required_slots,
+            self.schemaview, slot.range
+        )
+
+        simple_dict_id = range_id_slot.name
+        other_slot = range_simple_dict_value_slot.name
+        slot.annotations["inline_details"] = {"id": simple_dict_id, "other": other_slot}
+
+    def handle_non_inlined_class_slot(self, slot, range: str) -> str:
+        """non-inlined class slots have been temporarily removed but this will be needed to support them"""
+        return f"ID_TYPES['{self.get_class_name(range)}']"
 
     def handle_type_slot(self, slot, range: str) -> str:
         del slot  # unused for now
@@ -43,9 +168,10 @@ class SlotGeneratorMixin:
         return range
 
     def handle_multivalued_slot(self, slot, range: str) -> str:
-        if slot.multivalued:
-            if slot.inlined_as_list and range != SlotGeneratorMixin.CLASS_RANGE_STRING:
-                range = self.make_multivalued(range)
+        if (slot.inlined_as_list is True and self.is_multivalued(slot)) or (
+            slot.inlined is True and slot.inlined_as_list is True and self.is_multivalued(slot)
+        ):
+            range = self.make_multivalued(range)
 
         return range
 
@@ -58,19 +184,20 @@ class SlotGeneratorMixin:
             safe_sn = self.get_slot_name(slot.alias)
 
         if range is None:
-            range = self.handle_none_slot(slot, range)
+            range = self.handle_none_slot(slot)
         elif range in self.schemaview.all_classes():
             range = self.handle_class_slot(slot, range)
         elif range in self.schemaview.all_types():
             range = self.handle_type_slot(slot, range)
+            if self.is_multivalued(slot):
+                range = self.make_multivalued(range)
         elif range in self.schemaview.all_enums():
             range = self.handle_enum_slot(slot, range)
+            range = self.handle_multivalued_slot(slot, range)
         else:
             raise Exception(f"Unknown range {range}")
 
-        range = self.handle_multivalued_slot(slot, range)
-
-        return OOField(
+        return DataframeField(
             name=safe_sn,
             source_slot=slot,
             range=range,

linkml/generators/panderagen/transforms/__init__.py

@@ -0,0 +1,19 @@
+"""Transform classes for LinkML Pandera validation.
+
+This module provides transform classes that convert LinkML inline formats
+into forms suitable for Polars DataFrame validation with Pandera models.
+"""
+
+from .collection_dict_model_transform import CollectionDictModelTransform
+from .list_dict_model_transform import ListDictModelTransform
+from .model_transform import ModelTransform
+from .nested_struct_model_transform import NestedStructModelTransform
+from .simple_dict_model_transform import SimpleDictModelTransform
+
+__all__ = [
+    "ModelTransform",
+    "SimpleDictModelTransform",
+    "CollectionDictModelTransform",
+    "ListDictModelTransform",
+    "NestedStructModelTransform",
+]

linkml/generators/panderagen/transforms/collection_dict_model_transform.py

@@ -0,0 +1,62 @@
+import polars as pl
+
+from .model_transform import ModelTransform
+
+
+class CollectionDictModelTransform(ModelTransform):
+    """This class assists in converting a LinkML 'collection dict' inline column
+    into a form that is better for representing in a PolaRS dataframe and
+    validating with a Pandera model.
+    """
+
+    def __init__(self, polars_schema, id_col):
+        self.polars_schema = polars_schema
+        """A polars schema representing a collection dict column"""
+
+        self.id_col = id_col
+        """The ID column in the sense of a LinkML inline collection dict"""
+
+    def transform(self, linkml_collection_dict):
+        """Converts a collection dict nested column to a list of dicts.
+        { 'A': {...}, 'B': {...}, ... } -> [{'id': 'A', ...}, {'id': 'B', ...}, ...]
+        """
+        return self._collection_dict_to_list_of_structs(linkml_collection_dict)
+
+    def _collection_dict_to_list_of_structs(self, linkml_collection_dict):
+        """Converts a collection dict nested column to a list of dicts.
+        { 'A': {...}, 'B': {...}, ... } -> [{'id': 'A', ...}, {'id': 'B', ...}, ...]
+
+        An inefficient conversion (relative to native PolaRS operations)
+        from a collection dict form to a dataframe struct column.
+
+        linkml_collection_dict : dict
+            A single row entry in a dataframe column (one cell), which itself is a dict.
+            The value entries are dicts that get the key added as an id field.
+        """
+        arr = []
+        for k, v in linkml_collection_dict.items():
+            if k not in v:
+                v[self.id_col] = k
+            arr.append(v)
+        return arr
+
+    @classmethod
+    def prepare_dataframe(cls, data, column_name, nested_cls):
+        """Returns just the collection dict column transformed to an inlined list form
+
+        note that this method uses collect and iter_rows so is very inefficient
+        """
+        id_column = nested_cls.get_id_column_name()
+        polars_schema = nested_cls.to_schema()
+
+        collection_dict_transformer = cls(polars_schema, id_column)
+
+        one_column_df = data.lazyframe.select(pl.col(column_name)).collect()
+
+        list_of_structs = [collection_dict_transformer.transform(e) for [e] in one_column_df.iter_rows()]
+
+        return pl.DataFrame(pl.Series(list_of_structs).alias(column_name))
+
+    def explode_unnest_dataframe(self, df, column_name):
+        """Filter, explode and unnest for collection dict."""
+        return df.lazy().filter(pl.col(column_name).list.len() > 0).explode(column_name).unnest(column_name).collect()

linkml/generators/panderagen/transforms/list_dict_model_transform.py

@@ -0,0 +1,66 @@
+import polars as pl
+
+from .model_transform import ModelTransform
+
+
+class ListDictModelTransform(ModelTransform):
+    """This class assists in converting a LinkML 'list dict' inline column
+    into a form that is better for representing in a PolaRS dataframe and
+    validating with a Pandera model.
+    """
+
+    def __init__(self, polars_schema):
+        self.polars_schema = polars_schema
+        """A polars schema representing a list dict column"""
+
+    def transform(self, linkml_list_dict):
+        """Transforms a list dict nested column.
+        This is a pass-through since list dicts are already in the correct format.
+        """
+        return linkml_list_dict
+
+    @classmethod
+    def unnest_list_struct(cls, column_name: str, df):
+        """Use this in a custom check. Pass the nested model as pandera_model."""
+
+        # fmt: off
+        unnested_column = (
+            df
+            .select(column_name)
+            .filter(pl.col(column_name).list.len() > 0) # see: https://github.com/pola-rs/polars/issues/14381
+            .explode(column_name)
+            .unnest(column_name)
+        )
+        # fmt: on
+
+        return unnested_column
+
+    @classmethod
+    def prepare_dataframe(cls, data, column_name, nested_cls):
+        """Returns just the list dict column transformed to an inlined list form
+
+        note that this method uses collect and iter_rows so is very inefficient
+        """
+        polars_schema = nested_cls.to_schema()
+
+        list_dict_transformer = cls(polars_schema)
+
+        one_column_df = data.lazyframe.select(pl.col(column_name)).collect()
+
+        list_of_structs = [list_dict_transformer.transform(e) for [e] in one_column_df.iter_rows()]
+
+        return pl.DataFrame(pl.Series(list_of_structs).alias(column_name))
+
+    def explode_unnest_dataframe(self, df, column_name, data=None):
+        """Filter, explode and unnest for list dict with struct fallback."""
+        try:
+            return (
+                df.lazy().filter(pl.col(column_name).list.len() > 0).explode(column_name).unnest(column_name).collect()
+            )
+        except (pl.exceptions.PanicException, Exception):
+            if data:
+                from .nested_struct_model_transform import NestedStructModelTransform
+
+                nested_transform = NestedStructModelTransform(self.polars_schema)
+                return nested_transform.explode_unnest_dataframe(data.lazyframe, column_name)
+            raise

linkml/generators/panderagen/transforms/model_transform.py

@@ -0,0 +1,8 @@
+from abc import ABC, abstractmethod
+
+
+class ModelTransform(ABC):
+    @abstractmethod
+    def explode_unnest_dataframe(self, df, column_name):
+        """Abstract method for exploding and unnesting dataframes."""
+        pass