python-datamodel 0.10.1__cp313-cp313-win32.whl

datamodel/validation.pyx

@@ -0,0 +1,315 @@
+# cython: language_level=3, embedsignature=True, initializedcheck=False
+# Copyright (C) 2018-present Jesus Lara
+#
+from typing import get_args, get_origin, Union, Optional
+from collections.abc import Callable, Awaitable
+import typing
+import asyncio
+import inspect
+from libcpp cimport bool as bool_t
+from cpython.object cimport PyObject_IsInstance, PyObject_IsSubclass
+from cpython.type cimport PyType_Check
+from enum import Enum
+from decimal import Decimal
+import datetime
+from uuid import UUID
+import asyncpg.pgproto.pgproto as pgproto
+from .types import uint64_min, uint64_max, Text
+from .fields import Field
+from .functions import (
+    is_iterable,
+    is_primitive,
+    is_dataclass,
+    is_function,
+    is_callable,
+    is_empty
+)
+
+
+cdef str valid_int(object field, str name, object value, object _type):
+    # Basic check for integer type.
+    if not isinstance(value, int):
+        return f"Field {name} expected an integer, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_float(object field, str name, object value, object _type):
+    if not isinstance(value, (float, int)):  # sometimes ints are allowed
+        return f"Field {name} expected a float, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_str(object field, str name, object value, object _type):
+    if not isinstance(value, str):
+        return f"Field {name} expected a string, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_uuid(object field, str name, object value, object _type):
+    if not isinstance(value, (UUID, pgproto.UUID)):
+        return f"Field {name} expected a UUID, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_boolean(object field, str name, object value, object _type):
+    if not isinstance(value, bool):
+        return f"Field {name} expected a boolean, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_date(object field, str name, object value, object _type):
+    if not isinstance(value, datetime.date):
+        return f"Field {name} expected a date, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_datetime(object field, str name, object value, object _type):
+    if isinstance(value, datetime.datetime):
+        return None
+    if isinstance(value, datetime.date):
+        return None
+    return f"Field {name} expected a datetime, got {value!r} of type {type(value).__name__}"
+
+cdef str valid_timedelta(object field, str name, object value, object _type):
+    if not isinstance(value, datetime.timedelta):
+        return f"Field {name} expected a timedelta, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_time(object field, str name, object value, object _type):
+    if not isinstance(value, datetime.time):
+        return f"Field {name} expected a time, got {value!r} of type {type(value).__name__}"
+    return None
+
+cdef str valid_decimal(object field, str name, object value, object _type):
+    if not isinstance(value, Decimal):
+        return f"Field {name} expected a Decimal, got {value!r} of type {type(value).__name__}"
+    return None
+
+
+# List of Validators (primitive types):
+validators = {
+    str: valid_str,
+    int: valid_int,
+    float: valid_float,
+    UUID: valid_uuid,
+    pgproto.UUID: valid_uuid,
+    bool: valid_boolean,
+    datetime.date: valid_date,
+    datetime.datetime: valid_datetime,
+    datetime.timedelta: valid_timedelta,
+    datetime.time: valid_time,
+    Decimal: valid_decimal,
+    Text: valid_str
+}
+
+
+cdef inline bint is_enum_class(object annotated_type):
+    cdef int res
+    # First, check if annotated_type is a type
+    if not PyType_Check(annotated_type):
+        return False
+    # Then check if it is a subclass of Enum.
+    res = PyObject_IsSubclass(annotated_type, <object>Enum)
+    if res < 0:
+        # If an error occurred, you might want to raise an exception.
+        raise RuntimeError("Error in PyObject_IsSubclass")
+    return res != 0
+
+cdef bool_t is_instanceof(object value, type annotated_type):
+    if annotated_type.__module__ == 'typing':
+        return True # TODO: validate subscripted generic (typing extensions)
+    elif value in (datetime.date, datetime.time, datetime.datetime):
+        # check if is a pendulum instance:
+        return issubclass(value, (datetime.date, datetime.time, datetime.datetime))
+    else:
+        try:
+            return isinstance(value, annotated_type)
+        except (AttributeError, TypeError, ValueError) as e:
+            raise TypeError(
+                f"{e}"
+            )
+
+cpdef bool_t is_optional_type(object annotated_type):
+    if get_origin(annotated_type) is Union:
+        return type(None) in get_args(annotated_type)
+    return False
+
+cdef object get_primary_key_field(object annotated_type, str name, object field_meta):
+    for f in annotated_type.__dataclass_fields__.values():
+        if name == f.name:
+            return f
+        if field_meta.get('alias') == f.name:
+            return f
+        if f.metadata.get('primary_key', False):
+            return f
+    return None
+
+cpdef dict _validation(
+    object F,
+    str name,
+    object value,
+    object annotated_type,
+    object val_type,
+    str field_type,
+    bint as_objects=False
+):
+    cdef bint _valid = False
+    cdef object field_meta = F.metadata
+    cdef dict error = {}
+
+    if not annotated_type:
+        annotated_type = F.type
+    elif isinstance(annotated_type, Field):
+        annotated_type = annotated_type.type
+
+    if fn := F.metadata.get('validator', None):
+        try:
+            result = fn(F, value, annotated_type, val_type)
+            if result is False:
+                msg = f"Validation failed for *{name}*: {value} with result: {result}"
+                return _create_error(name, value, msg, val_type, annotated_type)
+        except ValueError:
+            raise
+    # check: data type hint
+    # If field_type is known, short-circuit certain checks
+    if F.type == Text:
+        if val_type != str:
+            return _create_error(name, value, f'invalid type for {annotated_type}.{name}, expected {annotated_type}', val_type, annotated_type)
+        return {}
+    # if inspect.isclass(annotated_type) and issubclass(annotated_type, Enum):
+    if is_enum_class(annotated_type):
+        return validate_enum(name, value, annotated_type, val_type)
+    if F.origin is Callable:
+        if not is_callable(value):
+            return _create_error(name, value, f'Invalid function type, expected {annotated_type}', val_type, annotated_type)
+        return {}
+    elif F.origin is Awaitable:
+        if asyncio.iscoroutinefunction(value):
+            return  _create_error(name, value, f"Field '{name}': provided coroutine function is not awaitable; call it to obtain a coroutine object.", val_type, annotated_type)
+        # Otherwise, check if it is awaitable
+        elif not hasattr(value, '__await__'):
+            return _create_error(name, value, f"Field '{name}': provided object is not awaitable; it does not have an '__await__' method. but got {type(value)}.", val_type, annotated_type)
+        return {}
+    elif field_type == 'type':
+        return validate_type(F, name, value, annotated_type, val_type)
+    elif field_type == 'typing' or hasattr(annotated_type, '__module__') and annotated_type.__module__ == 'typing':
+        if F.origin is tuple:
+            # Check if we are in the homogeneous case: Tuple[T, ...]
+            if len(F.args) == 2 and F.args[1] is Ellipsis:
+                for i, elem in enumerate(value):
+                    if not isinstance(elem, F.args[0]):
+                        return _create_error(
+                            f"{name}[{i}]",
+                            elem,
+                            f"Invalid type at index {i}: expected {F.args[0]}",
+                            type(elem), F.args[0]
+                        )
+            else:
+                if len(value) != len(F.args):
+                    return _create_error(name, value, f"Invalid length for {annotated_type}.{name}, expected {len(F.args)} elements", val_type, annotated_type)
+                else:
+                    for i, elem in enumerate(value):
+                        if not isinstance(elem, F.args[i]):
+                            return _create_error(
+                                f"{name}[{i}]",
+                                elem,
+                                f"Invalid type at index {i}: expected {F.args[i]}",
+                                type(elem), F.args[i]
+                            )
+        # Handle Optional Types:
+        elif F.origin is Union and type(None) in F.args:
+            inner_types = [t for t in F.args if t is not type(None)]
+            # If value is None then that is valid:
+            if value is None:
+                return {}
+            # Otherwise check that value is an instance of at least one inner type:
+            for t in inner_types:
+                base_type = get_origin(t) or t
+                if isinstance(value, base_type):
+                    _valid = True
+                    break
+            if not _valid:
+                return _create_error(name, value, f"Invalid type for {annotated_type}.{name}, expected one of {inner_types}", val_type, annotated_type)
+    elif type(annotated_type).__name__ == "ModelMeta":
+        # Check if there's a field in the annotated type that matches the name and type
+        if as_objects:
+            if isinstance(value, annotated_type):
+                # if value is already a Object, no further check needed for columns
+                return {}
+            try:
+                field = annotated_type.get_column(name)
+            except AttributeError as e:
+                return _create_error(name, value, f'{annotated_type} has no column {name}', val_type, annotated_type, e)
+            ftype = field.type
+            if ftype <> val_type:
+                return _create_error(name, value, f"Invalid type for {annotated_type}.{name}, expected {ftype}", val_type, annotated_type)
+        else:
+            # Validate primary key
+            pk_field = get_primary_key_field(annotated_type, name, field_meta)
+            if pk_field:
+                pk_type = pk_field.type
+                if not isinstance(value, pk_type):
+                    return _create_error(
+                        name,
+                        value,
+                        f"Invalid type for {annotated_type}.{pk_field.name}, expected {pk_type}",
+                        val_type,
+                        pk_type
+                    )
+    elif is_optional_type(annotated_type):
+        inner_types = get_args(annotated_type)
+        for t in inner_types:
+            if t is type(None) and value is None:
+                break
+            elif is_instanceof(val_type, t):
+                break
+            elif val_type == t:
+                break
+        else:
+            return _create_error(name, value, f"Invalid type for {annotated_type}.{name}, expected one of {inner_types}", val_type, annotated_type)
+    elif val_type != annotated_type:
+        instance = is_instanceof(value, annotated_type)
+        if not instance:
+            return _create_error(name, value, f"Invalid Instance for {annotated_type}.{name}, expected {annotated_type}", val_type, annotated_type)
+    return error
+
+cdef dict _create_error(str name, object value, object error, object val_type, object annotated_type, object exception = None):
+    return {
+        "field": name,
+        "value": value,
+        "error": error,
+        "value_type": val_type,
+        "annotation": annotated_type,
+        "exception": exception
+    }
+
+# Define a validator function for uint64
+def validate_uint64(value: int) -> None:
+    """Validate uint64 values.
+    """
+    if value < uint64_min or value > uint64_max:
+        raise ValueError(f"{value} is not a valid uint64")
+
+cdef dict validate_type(object F, object name, object value, object annotated_type, object val_type):
+    """
+    Validate that 'value' is of type 'annotated_type'.
+    If 'annotated_type' is a Union, checks if 'value' is an instance of any of the union types.
+    Returns None if valid; otherwise, returns an error object via _create_error.
+    """
+    if not isinstance(value, type):
+        return _create_error(name, value, f'Invalid type for {annotated_type}.{name}, expected a type', val_type, annotated_type)
+    inner_types = get_args(F.args[0])
+    for allowed in inner_types:
+        if value is allowed:
+            return {}
+    expected = ', '.join([str(t) for t in F.args])
+    return _create_error(name, value, f'Invalid type for {annotated_type}.{name}, expected a type of {expected}', val_type, annotated_type)
+
+cdef dict validate_enum(object name, object value, object annotated_type, object val_type):
+    """
+    Validate that 'value' is a valid member of the enum 'annotated_type'.
+    If 'value' is an instance of 'annotated_type', use its .value.
+    Cache the allowed enum values on the enum type to avoid repeated introspection.
+    Returns None if valid; otherwise, returns an error object via _create_error.
+    """
+    cdef object val = value.value if PyObject_IsInstance(value, annotated_type) else value
+    cdef object enum_values = [e.value for e in annotated_type]
+    if val not in enum_values:
+        return _create_error(
+            name, value, f"Invalid value for {annotated_type}.{name}, expected one of {enum_values}", val_type, annotated_type
+        )
+    return {}

datamodel/version.py

@@ -0,0 +1,13 @@
+"""DataModel Meta information."""
+
+
+__title__ = 'python-datamodel'
+__description__ = (
+    'simple library based on python +3.8 to use Dataclass-syntax'
+    'for interacting with Data'
+)
+__version__ = '0.10.1'
+__copyright__ = 'Copyright (c) 2020-2024 Jesus Lara'
+__author__ = 'Jesus Lara'
+__author_email__ = 'jesuslarag@gmail.com'
+__license__ = 'BSD'

examples/nn/examples.py

@@ -0,0 +1,311 @@
+from typing import List, Any
+from datetime import datetime
+import pandas as pd
+from datamodel import BaseModel, Field
+from datamodel.parsers.json import json_decoder
+from datamodel.exceptions import ValidationError
+
+
+client_payload = """
+{
+    "metadata": {
+        "type": "client",
+        "transactionType": "UPSERT",
+        "source": "MainEvent",
+        "client": "global"
+    },
+    "payload": {
+        "client_id": 61,
+        "client_name": "ASSEMBLY",
+        "status": true,
+        "orgid": 71,
+        "org_name": "assembly"
+    }
+}
+"""
+
+formid_payload = """
+{
+    "metadata": {
+        "type": "recapDefinition",
+        "transactionType": "UPSERT",
+        "source": "MainEvent",
+        "client": "assembly"
+    },
+    "payload": {
+        "formid": 7,
+        "form_name": "Assembly Tech Form",
+        "active": true,
+        "created_on": "2024-09-24T07:13:20-05:00",
+        "updated_on": "2024-09-25T12:51:53-05:00",
+        "is_store_stamp": false,
+        "client_id": 61,
+        "client_name": "ASSEMBLY",
+        "orgid": 71
+    }
+}
+"""
+
+metadata_payload = """
+{
+    "metadata": {
+        "type": "form_metadata",
+        "transactionType": "UPSERT",
+        "source": "MainEvent",
+        "client": "assembly"
+    },
+    "payload": {
+        "column_name": "9080",
+        "description": "Were all the Kitchen Suite appliances installed correctly before leaving the store?",
+        "is_active": true,
+        "data_type": "FIELD_MULTISELECT",
+        "formid": 10,
+        "form_name": "Lowe's Call Form",
+        "client_id": 57,
+        "client_name": "HISENSE",
+        "orgid": 106
+    }
+}
+"""
+
+user_payload = """
+{
+    "metadata": {
+        "type": "user",
+        "transactionType": "UPSERT",
+        "source": "MainEvent",
+        "client": "global"
+    },
+    "payload": {
+        "user_id": 2661,
+        "first_name": "M",
+        "last_name": "Rosado",
+        "email": "mrosado@trocglobal.com",
+        "mobile_number": "237-222-3576",
+        "address": "800 S. Douglas Rd",
+        "city": "Coral Gables",
+        "state_name": "FL",
+        "zipcode": "33134",
+        "latitude": 25.763887,
+        "longitude": -80.2567,
+        "username": "mrosado",
+        "role_id": 1,
+        "employee_number": 158,
+        "physical_country": "USA",
+        "role_name": "Global Admin",
+        "is_active": true,
+        "org_name": "assembly",
+        "client_id": [
+            61,
+            54,
+            55,
+            56,
+            60,
+            57,
+            58,
+            62,
+            59,
+            65,
+            63
+        ],
+        "orgid": [
+            71,
+            138,
+            74,
+            69,
+            60,
+            106,
+            137,
+            62,
+            77,
+            3,
+            63
+        ],
+        "client_names": [
+            "ASSEMBLY",
+            "AT&T",
+            "BOSE",
+            "EPSON",
+            "FLEX-ROC",
+            "HISENSE",
+            "POKEMON",
+            "TCT MOBILE",
+            "TRENDMICRO",
+            "TRO MSO",
+            "WORP"
+        ]
+    }
+}
+"""
+
+class Organization(BaseModel):
+    orgid: int = Field(primary_key=True)
+    org_name: str
+    status: bool = Field(required=True, default=True)
+
+    class Meta:
+        name: str = 'organizations'
+        strict: bool = True
+
+def create_organization(
+    name: str,
+    value: Any,
+    obj: Any,
+    parent_data: BaseModel
+) -> Organization:
+    org_name = parent_data.get('org_name', None) if parent_data else None
+    print('Creating organization')
+    args = {
+        name: value,
+        "org_name": org_name,
+        "status": True,
+    }
+    return obj(**args)
+
+BaseModel.register_parser(Organization, create_organization, 'orgid')
+
+class Client(BaseModel):
+    client_id: int = Field(primary_key=True)
+    client_name: str
+    status: bool = Field(required=True, default=True)
+    orgid: Organization = Field(required=False)
+    org_name: str
+
+    class Meta:
+        name: str = 'clients'
+        strict: bool = True
+        as_objects: bool = True
+
+
+class Form(BaseModel):
+    formid: int = Field(primary_key=True)
+    form_name: str = Field(required=True)
+    active: bool = Field(required=False, default=True)
+    created_on: datetime = Field(required=True)
+    updated_on: datetime = Field(required=True)
+    is_store_stamp: bool = Field(required=True)
+    client_id: Client = Field(required=True)
+    client_name: Client = Field(required=True)
+    orgid: Organization = Field(required=True)
+
+    class Meta:
+        name: str = 'forms'
+        strict: bool = True
+
+class FormMetadata(BaseModel):
+    column_name: str = Field(required=True)
+    description: str = Field(required=True)
+    is_active: bool = Field(required=False, default=True)
+    data_type: str = Field(required=True)
+    formid: Form = Field(required=True)
+    form_name: Form = Field(required=True)
+    client_id: Client = Field(required=True)
+    client_name: Client = Field(required=True)
+    orgid: Organization = Field(required=True)
+
+    class Meta:
+        name: str = 'form_metadata'
+        strict: bool = True
+
+class User(BaseModel):
+    user_id: int = Field(primary_key=True)
+    first_name: str = Field(required=True)
+    last_name: str = Field(required=True)
+    email: str = Field(required=True)
+    mobile_number: str = Field(required=True)
+    address: str = Field(required=True)
+    city: str = Field(required=True)
+    state_name: str = Field(required=True)
+    zipcode: str = Field(required=True)
+    latitude: float = Field(required=True)
+    longitude: float = Field(required=True)
+    username: str = Field(required=True)
+    role_id: int = Field(required=True)
+    employee_number: int = Field(required=True)
+    physical_country: str = Field(required=True)
+    role_name: str = Field(required=True)
+    is_active: bool = Field(required=False, default=True)
+    client_id: List[Client] = Field(required=True)
+    orgid: List[Organization] = Field(required=True)
+    org_name: str
+    client_names: List[str] = Field(required=True)
+
+    class Meta:
+        name: str = 'users'
+        strict: bool = True
+        as_objects: bool = False
+
+
+network_ninja_map = {
+    "client": Client,
+    "organization": Organization,
+    "recapDefinition": Form,
+    "form_metadata": FormMetadata,
+    "user": User
+}
+
+def get_client():
+    payload = json_decoder(client_payload)
+    metadata = payload.get("metadata")
+    payload = payload.get("payload")
+    model = network_ninja_map.get(metadata.get("type"))
+    try:
+        client = model(**payload)
+    except ValidationError as e:
+        print(e)
+        print(e.payload)
+    print('CLIENT > ', client)
+    return client
+
+def get_formid():
+    payload = json_decoder(formid_payload)
+    metadata = payload.get("metadata")
+    payload = payload.get("payload")
+    model = network_ninja_map.get(metadata.get("type"))
+    form = None
+    try:
+        form = model(**payload)
+    except ValidationError as e:
+        print(e)
+        print(e.payload)
+    if form:
+        print('FORM ID > ', form)
+        return form
+
+def get_formmetadata():
+    payload = json_decoder(metadata_payload)
+    metadata = payload.get("metadata")
+    payload = payload.get("payload")
+    model = network_ninja_map.get(metadata.get("type"))
+    form_metadata = None
+    try:
+        form_metadata = model(**payload)
+    except ValidationError as e:
+        print(e)
+        print(e.payload)
+    if form_metadata:
+        print('FORM METADATA > ', form_metadata)
+        return form_metadata
+
+def get_user():
+    payload = json_decoder(user_payload)
+    metadata = payload.get("metadata")
+    payload = payload.get("payload")
+    model = network_ninja_map.get(metadata.get("type"))
+    user = None
+    try:
+        user = model(**payload)
+    except ValidationError as e:
+        print(e)
+        print(e.payload)
+    if user:
+        print('USER > ', user)
+        return user
+
+if __name__ == "__main__":
+    # client = get_client()
+    # formid = get_formid()
+    # form_metadata = get_formmetadata()
+    user = get_user()
+    # df = pd.DataFrame([client.to_dict(as_values=True)])
+    # print('DF = ', df)