linkml 1.9.4rc2__py3-none-any.whl → 1.9.5rc2__py3-none-any.whl

linkml/generators/panderagen/linkml_pandera_validator.py

@@ -0,0 +1,186 @@
+import inspect
+from functools import wraps
+
+import pandera
+import polars as pl
+from pandera.api.polars.types import PolarsData
+
+from linkml.generators.panderagen.transforms import (
+    CollectionDictModelTransform,
+    ListDictModelTransform,
+    NestedStructModelTransform,
+    SimpleDictModelTransform,
+)
+
+
+def handle_validation_exceptions(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except pl.exceptions.PanicException:
+            data = args[2] if len(args) > 2 else kwargs.get("data")
+            return data.lazyframe.select(pl.lit(False))
+        except pandera.errors.SchemaError as e:
+            raise e
+        except Exception:
+            data = args[2] if len(args) > 2 else kwargs.get("data")
+            return data.lazyframe.select(pl.lit(False))
+
+    return wrapper
+
+
+class LinkmlPanderaValidator:
+    @classmethod
+    def get_id_column_name(cls):
+        return cls._id_name
+
+    @classmethod
+    def _simple_dict_fields(cls, column_name):
+        details = cls._INLINE_DETAILS[column_name]  # <-- THESE ARE GOING ON THE OUTER CLASS
+
+        return (details["id"], details["other"])
+
+    @classmethod
+    def _prepare_simple_dict(cls, data: PolarsData):
+        """Returns just the simple dict column transformed to an inlined list form
+
+        note that this method uses collect and iter_rows so is very inefficient
+        """
+        column_name = data.key
+        polars_schema = cls.get_nested_range(column_name).to_schema()
+        (id_column, other_column) = cls._simple_dict_fields(column_name)
+
+        simple_dict_transformer = SimpleDictModelTransform(polars_schema, id_column, other_column)
+
+        one_column_df = data.lazyframe.select(pl.col(column_name)).collect()
+
+        list_of_structs = [simple_dict_transformer.transform(e) for [e] in one_column_df.iter_rows()]
+
+        return pl.DataFrame(pl.Series(list_of_structs).alias(column_name))
+
+    @classmethod
+    @handle_validation_exceptions
+    def _check_simple_dict(cls, data: PolarsData):
+        """
+        The 'simple dict' format, in which the key serves as a local identifier is not a good match for a PolaRS
+        DataFrame. At present the format is
+        """
+        df = cls._prepare_simple_dict(data)
+
+        column_name = data.key
+
+        polars_schema = cls.get_nested_range(column_name).to_schema()
+        simple_transform = SimpleDictModelTransform(polars_schema, *cls._simple_dict_fields(column_name))
+        df = simple_transform.explode_unnest_dataframe(df, column_name)
+
+        nested_cls = cls.get_nested_range(column_name)
+        nested_cls.validate(df)
+        return data.lazyframe.select(pl.lit(True))
+
+    @classmethod
+    @handle_validation_exceptions
+    def _check_collection_struct(cls, data: PolarsData):
+        column_name = data.key
+        nested_cls = cls.get_nested_range(column_name)
+
+        df = CollectionDictModelTransform.prepare_dataframe(data, column_name, nested_cls)
+
+        collection_transform = CollectionDictModelTransform(nested_cls.to_schema(), nested_cls.get_id_column_name())
+        df = collection_transform.explode_unnest_dataframe(df, column_name)
+
+        nested_cls.validate(df)
+        return data.lazyframe.select(pl.lit(True))
+
+    @classmethod
+    @handle_validation_exceptions
+    def _check_nested_list_struct(cls, data: PolarsData):
+        """Use this in a custom check. Pass the nested model as pandera_model."""
+        column_name = data.key
+        nested_cls = cls.get_nested_range(column_name)
+
+        df = ListDictModelTransform.prepare_dataframe(data, column_name, nested_cls)
+
+        list_transform = ListDictModelTransform(nested_cls.to_schema())
+        df = list_transform.explode_unnest_dataframe(df, column_name, data)
+
+        nested_cls.validate(df)
+        return data.lazyframe.select(pl.lit(True))
+
+    @classmethod
+    @handle_validation_exceptions
+    def _check_nested_struct(cls, data: PolarsData):
+        """Use this in a custom check. Pass the nested model as pandera_model."""
+        column_name = data.key
+        nested_cls = cls.get_nested_range(column_name)
+
+        df = NestedStructModelTransform.prepare_dataframe(data, column_name, nested_cls)
+        nested_transform = NestedStructModelTransform(nested_cls.to_schema())
+        df = nested_transform.explode_unnest_dataframe(df, column_name)
+
+        nested_cls.validate(df)
+        return data.lazyframe.select(pl.lit(True))
+
+    @classmethod
+    def get_nested_range(cls, column_name):
+        """Resolve a nested class range at runtime.
+
+        Nested classes are not stored in the pandera schema,
+        but rather in the _NESTED_RANGES dictionary as strings.
+        """
+        nested_cls_name = cls._NESTED_RANGES[column_name]
+        shared_model_module = inspect.getmodule(cls)
+        nested_cls = getattr(shared_model_module, nested_cls_name)
+
+        return nested_cls
+
+    @classmethod
+    def generate_polars_schema_simple(cls):
+        # This is not nesting or list aware, so needs to be aligned with the other method
+        return pl.Struct({k: v.dtype.type for k, v in cls.to_schema().columns.items()})
+
+    @classmethod
+    def generate_polars_schema(cls, object_to_validate, parser=False) -> dict:
+        """Creates a nested PolaRS schema suitable for loading the object_to_validate.
+        Optional columns that are not present in the data are omitted.
+        This approach is only suitable to enable the test fixtures.
+        """
+        polars_schema = {}
+
+        if isinstance(object_to_validate, list):
+            object_to_validate = object_to_validate[0]
+
+        for column_name, column in cls.to_schema().columns.items():
+            dtype = column.properties["dtype"]
+            required = column.properties["required"]
+
+            if required or column_name in object_to_validate:
+                if dtype.type in [pl.Struct, pl.List]:  # maybe use inline form directly here
+                    inline_form = cls._INLINE_FORM.get(column_name, "not_inline")
+                    if inline_form == "simple_dict":
+                        polars_schema[column_name] = pl.Object  # make this a struct and make the nested non-
+                    elif inline_form == "not_inline":
+                        polars_schema[column_name] = dtype.type
+                    else:
+                        nested_cls = cls.get_nested_range(column_name)
+                        if inline_form == "inlined_dict":
+                            if parser:
+                                nested_schema = nested_cls.generate_polars_schema(
+                                    object_to_validate[column_name], parser
+                                )
+                                polars_schema[column_name] = pl.Struct(nested_schema)
+                            else:
+                                polars_schema[column_name] = pl.Struct
+                        elif inline_form == "inlined_list_dict":
+                            if parser:
+                                nested_schema = nested_cls.generate_polars_schema(
+                                    object_to_validate[column_name], parser
+                                )
+                                polars_schema[column_name] = pl.List(pl.Struct(nested_schema))
+                            else:
+                                # transformed form
+                                polars_schema[column_name] = pl.List
+                else:
+                    polars_schema[column_name] = dtype.type
+
+        return polars_schema

linkml/generators/panderagen/panderagen.py

@@ -14,9 +14,10 @@ from linkml_runtime.utils.formatutils import camelcase
 from linkml_runtime.utils.schemaview import SchemaView
 
 from linkml._version import __version__
-from linkml.generators.oocodegen import OOClass, OOCodeGenerator, OODocument
+from linkml.generators.oocodegen import OOCodeGenerator, OODocument
 
 from .class_generator_mixin import ClassGeneratorMixin
+from .dataframe_class import DataframeClass
 from .enum_generator_mixin import EnumGeneratorMixin
 from .slot_generator_mixin import SlotGeneratorMixin
 
@@ -28,6 +29,7 @@ TYPEMAP = {
     "panderagen_class_based": {
         "xsd:string": "str",
         "xsd:integer": "int",
+        "xsd:int": "int",
         "xsd:float": "float",
         "xsd:double": "float",
         "xsd:boolean": "bool",
@@ -83,6 +85,8 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
 
     @staticmethod
     def make_multivalued(range: str) -> str:
+        if range == "Struct":
+            return "pl.List"
         return f"List[{range}]"
 
     def uri_type_map(self, xsd_uri: str, template: str = None):
@@ -92,15 +96,22 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
         return TYPEMAP[template].get(xsd_uri)
 
     def map_type(self, t: TypeDefinition) -> str:
+        logger.info(f"type_map definition: {t}")
+
+        typ = None
+
         if t.uri:
             typ = self.uri_type_map(t.uri)
-            return typ
+            if typ is None:
+                typ = self.map_type(self.schemaview.get_type(t.typeof))
         elif t.typeof:
             typ = self.map_type(self.schemaview.get_type(t.typeof))
-            return typ
-        else:
+
+        if typ is None:
             raise ValueError(f"{t} cannot be mapped to a type")
 
+        return typ
+
     def load_template(self, template_filename):
         jinja_env = Environment(loader=PackageLoader("linkml.generators.panderagen", self.template_path))
         return jinja_env.get_template(template_filename)
@@ -138,6 +149,7 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
             coerce=self.coerce,
             type_map=TYPEMAP,
             template_path=self.template_path,
+            pandera_validator_code=None,
         )
         return code
 
@@ -156,12 +168,17 @@ class PanderaGenerator(OOCodeGenerator, EnumGeneratorMixin, ClassGeneratorMixin,
         for c in self.ordered_classes():
             cn = c.name
             safe_cn = camelcase(cn)
-            ooclass = OOClass(
+            annotations = {}
+            identifier_or_key_slot = self.get_identifier_or_key_slot(cn)
+            if identifier_or_key_slot:
+                annotations["identifier_key_slot"] = identifier_or_key_slot.name
+            ooclass = DataframeClass(
                 name=safe_cn,
                 description=c.description,
                 package=self.package,
                 fields=[],
                 source_class=c,
+                annotations=annotations,
             )
             classes.append(ooclass)
             if c.mixin:

linkml/generators/panderagen/panderagen_class_based/class.jinja2

@@ -3,24 +3,81 @@
   Details at https://pandera.readthedocs.io/en/stable/dataframe_models.html
 -#}
 {%- import 'slots.jinja2' as slot_macros -%}
+{%- import 'custom_checks.jinja2' as custom_checks -%}
 
-{%- macro render_class(cls) %}
-class {{cls.name}}(
-        {%- if cls.is_a -%}
-            {{ cls.is_a }}
-        {%- else -%}
-            pla.DataFrameModel, _LinkmlPanderaValidator
-        {%- endif -%}
-):
-    {%- if cls.source_class.description %}
-    """
-    {{ cls.source_class.description }}
-    """
-    {% endif -%}
+{%- macro render_parent_classes(cls) -%}
+    {%- if cls.is_a -%}
+        {{ cls.is_a }}
+    {%- else -%}
+        pla.DataFrameModel, _LinkmlPanderaValidator
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_nested_ranges(cls) -%}
+{#-
+    Helper class member that references another Pandera class that
+    handles a nested association.
+-#}
+    _NESTED_RANGES = {
+    {%- for field in cls.fields -%}
+        {%- if field.reference_class() %}
+        "{{ field.name }}": "{{ field.reference_class() }}",
+{% endif -%}
+    {%- endfor %}
+    }
+{%- endmacro -%}
+
+{%- macro render_inline_form(cls) -%}
+{#-
+    This is a meta object with information used by the various check methods
+-#}
+    _INLINE_FORM = {
+    {%- for field in cls.fields -%}
+        {%- if field.inline_form() %}
+        "{{ field.name }}": "{{ field.inline_form() }}",
+{% endif -%}
+    {%- endfor %}
+    }
+{%- endmacro -%}
+
+{%- macro render_inline_details(cls) -%}
+{#-
+    This is a meta object with information used by the various check methods
+-#}
+    _INLINE_DETAILS = {
+    {%- for field in cls.fields -%}
+        {%- if field.inline_details() %}
+        "{{ field.name }}": {{ field.inline_details() }},
+{% endif -%}
+    {%- endfor %}
+    }
+{%- endmacro -%}
+
+{%- macro render_slots(cls) -%}
+    {{ slot_macros.id_slot_name_class_variable(cls) }}
     {%- if (cls.fields | length) == 0 %}
     pass
     {% endif -%}
     {%- for field in cls.fields -%}
         {{ slot_macros.render_slot(field) }}
     {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro render_class(cls) %}
+{#-
+    Generates the main structure of a Pandera class.
+    This includes the slots, custom checks,
+    and helper class members with information used by the custom checks.
+-#}
+class {{cls.name}}({{ render_parent_classes(cls) }}):
+    {%- if cls.source_class.description %}
+    """
+    {{ cls.source_class.description }}
+    """
+{% endif -%}
+    {{ render_slots(cls) }}
+    {{ custom_checks.render_custom_checks(cls) }}
+    {{ render_nested_ranges(cls) }}
+    {{ render_inline_form(cls) }}
+    {{ render_inline_details(cls) }}
 {% endmacro -%}

linkml/generators/panderagen/panderagen_class_based/custom_checks.jinja2

@@ -0,0 +1,27 @@
+
+{%- macro nested_checks(field) -%}
+{#-
+    field: DataframeField
+    Render a class method that follows nested structures
+    other than the simple_dict form.
+-#}
+    {%- if field.inline_form() %}
+    @pla.check("{{ field.name }}")
+    def check_nested_struct_{{ field.name }}(cls, data: PolarsData):
+        {% if field.inline_form() == 'inline_list_dict' -%}
+        return cls._check_nested_struct(data)
+        {% elif field.inline_form() == 'simple_dict' -%}
+        return cls._check_simple_dict(data)
+        {% elif field.inline_form() == 'inline_collection_dict' -%}
+        return cls._check_collection_struct(data)
+        {% else -%}
+        return cls._check_nested_list_struct(data)
+        {% endif -%}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_custom_checks(cls) -%}
+    {%- for field in cls.fields -%}
+    {{ nested_checks(field) }}
+    {%- endfor %}
+{%- endmacro -%}

linkml/generators/panderagen/panderagen_class_based/enums.jinja2

@@ -1,7 +1,7 @@
-{%- macro enum_parameter(slot) -%}
-    {%- if slot.annotations['permissible_values']|length > 0 -%}
+{%- macro enum_parameter(field) -%}
+    {%- if field.permissible_values()|length > 0 -%}
         dtype_kwargs={"categories":(
-        {%- for pv in slot.annotations['permissible_values'] -%}
+        {%- for pv in field.permissible_values() -%}
             '{{ pv }}',
         {%- endfor -%}
         )}

linkml/generators/panderagen/panderagen_class_based/pandera.jinja2

@@ -4,10 +4,20 @@
 -#}
 {%- import 'header.jinja2' as header -%}
 {%- import 'class.jinja2' as class_macros -%}
-{%- import 'mixins.jinja2' as mixins -%}
 {{ header }}
 
-{{ mixins }}
+{% if pandera_validator_code %}
+{{ pandera_validator_code }}
+{% else %}
+from linkml.generators.panderagen.linkml_pandera_validator import LinkmlPanderaValidator as _LinkmlPanderaValidator
+{% endif %}
+
+# These are all str for now
+ID_TYPES = {
+{%- for cls in doc.classes %}
+    "{{ cls.name }}": "str",
+{%- endfor %}
+}
 
 {% if metamodel_version %}# metamodel_version: {{metamodel_version}}{% endif %}
 {% if model_version %}# version: {{model_version}}{% endif %}

linkml/generators/panderagen/panderagen_class_based/slots.jinja2

@@ -4,33 +4,35 @@
 -#}
 {%- import 'enums.jinja2' as enum_macros -%}
 
-{%- macro constraint_parameters(field, slot) -%}
+{%- macro constraint_parameters(field) -%}
     {%- if coerce is true -%}coerce=True, {% endif -%}
     {%- if field.default_value is not none -%}default={{ field.default_value }}, {% endif -%}
-    {%- if slot.minimum_value is not none -%}ge={{ slot.minimum_value }}, {% endif -%}
-    {%- if slot.maximum_value is not none -%}le={{ slot.maximum_value }}, {% endif -%}
-    {%- if slot.pattern is not none -%}str_matches=r"{{ slot.pattern }}", {% endif -%}
-    {%- if (slot.required is none or slot.required is false) and slot.identifier is not true -%}nullable=True, {% endif -%}
+    {%- if field.minimum_value() is not none -%}ge={{ field.minimum_value() }}, {% endif -%}
+    {%- if field.maximum_value() is not none -%}le={{ field.maximum_value() }}, {% endif -%}
+    {%- if field.pattern() is not none -%}str_matches=r"{{ field.pattern() }}", {% endif -%}
+    {%- if (field.required() is none or field.required() is false) and field.identifier() is not true -%}nullable=True, {% endif -%}
 {%- endmacro -%}
 
-{%- macro render_slot(field) -%}
-    {%- with slot = field.source_slot %}
+{%- macro id_slot_name_class_variable(cls) %}
+    _id_name : str = {% if cls.identifier_key_slot() %} '{{ cls.identifier_key_slot() }}' {% else %}None{% endif %}
+{%- endmacro -%}
+
+{%- macro render_slot(field) %}
     {{ field.name }}:{{ ' ' }}
-        {%- if (slot.required is none or slot.required is false) and slot.identifier is not true -%}
+        {%- if (field.required() is none or field.required() is false) and (field.identifier() is not true) -%}
             Optional[
         {%- endif -%}
        {{field.range}}
-        {%- if (slot.required is none or slot.required is false) and slot.identifier is not true -%}
-            {{ '] ' }}
+        {%- if (field.required() is none or field.required() is false) and (field.identifier() is not true) -%}
+            {{ ']' }}
         {%- endif -%}
-        = pla.Field(
-            {{- constraint_parameters(field, slot) -}}
-            {{- enum_macros.enum_parameter(slot) -}}
+        {{ ' =' }} pla.Field(
+            {{- constraint_parameters(field) -}}
+            {{- enum_macros.enum_parameter(field) -}}
         )
-        {%- if slot.description %}
+        {%- if field.description() %}
     """
-    {{ slot.description }}
+    {{ field.description() }}
     """
-        {% endif -%}
-    {%- endwith -%}
+    {% endif -%}
 {%- endmacro %}