python-datamodel 0.10.1__cp313-cp313-win32.whl

datamodel/parsers/json.pyx

@@ -0,0 +1,250 @@
+# cython: language_level=3, embedsignature=True, boundscheck=False, wraparound=True, initializedcheck=False
+# Copyright (C) 2018-present Jesus Lara
+#
+"""
+Module for JSON encoding and decoding using orjson with custom type handling.
+
+This module provides the JSONContent class, which extends orjson's functionality to
+support additional types (e.g., Decimal, datetime, custom Enum types, etc.), and a BaseEncoder
+class as a drop-in replacement for json.dumps.
+"""
+import uuid
+from pathlib import PosixPath, PurePath, Path
+from datetime import datetime
+from asyncpg.pgproto import pgproto
+from psycopg2 import Binary
+from cpython cimport PyErr_Clear
+from cpython.object cimport (
+    PyObject_IsInstance,
+    PyObject_IsSubclass,
+    PyObject_TypeCheck,
+    PyObject_HasAttr,
+    PyObject_GetAttr,
+    PyCallable_Check
+)
+from dataclasses import _MISSING_TYPE, MISSING, InitVar
+from typing import Any, Union
+from decimal import Decimal
+from enum import Enum, EnumType
+import orjson
+from ..exceptions cimport ParserError
+from ..fields import Field
+
+
+cdef inline bint is_callable(object obj):
+    cdef int res = PyCallable_Check(obj)
+    # PyCallable_Check normally returns 1 or 0, but if an error occurs,
+    # assume it's not callable.
+    return res != 0
+
+
+cdef inline bint has_attribute(object obj, str name):
+    cdef int result = PyObject_HasAttr(obj, name)
+    if result < 0:
+        PyErr_Clear()  # Clear any error that occurred
+        return False
+    return result != 0
+
+
+cdef inline object get_attribute(object obj, str name):
+    cdef object attr = PyObject_GetAttr(obj, name)
+    if attr is None:
+        PyErr_Clear()  # Clear any error that occurred
+    return attr
+
+
+cdef inline bint is_subclassof(object obj, object cls):
+    cdef int res = PyObject_IsSubclass(obj, cls)
+    if res < 0:
+        PyErr_Clear()  # Clear error if subclass check fails
+        return False
+    return res != 0
+
+
+ORJSON_DEFAULT_OPTIONS = (
+    orjson.OPT_SERIALIZE_NUMPY |
+    orjson.OPT_UTC_Z
+    # orjson.OPT_NON_STR_KEYS
+)
+
+
+cdef class JSONContent:
+    """
+    A basic JSON encoder/decoder using orjson.
+
+    This class provides methods to serialize Python objects to JSON strings and deserialize
+    JSON strings back into Python objects, with custom handling for additional data types.
+    """
+    def __call__(self, object obj, **kwargs):
+        return self.encode(obj, **kwargs)
+
+    def default(self, object obj):
+        if isinstance(obj, Decimal):
+            return float(obj)
+        elif isinstance(obj, datetime):
+            return str(obj)
+        elif has_attribute(obj, "isoformat"):
+            return obj.isoformat()
+        elif isinstance(obj, (PosixPath, PurePath, Path)):
+            return str(obj)
+        elif isinstance(obj, pgproto.UUID):
+            return str(obj)
+        elif isinstance(obj, uuid.UUID):
+            return obj
+        elif has_attribute(obj, "hex"):
+            try:
+                hex_method = get_attribute(obj, "hex")
+                if is_callable(hex_method):
+                    return hex_method()
+            except AttributeError:
+                return obj.hex
+            if isinstance(obj, bytes):
+                return obj.hex()
+            else:
+                return obj.hex
+        elif has_attribute(obj, 'lower'): # asyncPg Range:
+            up = obj.upper
+            if isinstance(up, int):
+                up = up - 1  # discrete representation
+            return [obj.lower, up]
+        elif has_attribute(obj, 'tolist'): # numpy array
+            return obj.tolist()
+        elif isinstance(obj, _MISSING_TYPE):
+            return None
+        elif obj is MISSING:
+            return None
+        elif PyObject_IsInstance(obj, type) and is_subclassof(obj, Enum):
+            return [{'value': e.value, 'name': e.name} for e in obj]
+        elif isinstance(obj, Enum):
+            if has_attribute(obj, 'value'):
+                return obj.value
+            else:
+                return obj.name
+        elif isinstance(obj, Binary):  # Handle bytea column from PostgreSQL
+            return str(obj)  # Convert Binary object to string
+        elif isinstance(obj, Field):
+            if has_attribute(obj, 'to_dict'):
+                return obj.to_dict()
+            return str(obj)
+        elif has_attribute(obj, 'to_dict'):
+            return obj.to_dict()
+        elif has_attribute(obj, 'to_json'):
+            # Return a JSON representation of the object.
+            return orjson.Fragment(obj().encode())
+        elif isinstance(obj, InitVar) or type(obj).__name__ == 'InitVar':
+            # Handle InitVar explicitly
+            return None
+        raise TypeError(
+            f'{obj!r} of Type {type(obj)} is not JSON serializable'
+        )
+
+    def encode(self, object obj, bint naive_utc = True, bint non_str_keys = False, **kwargs) -> str:
+        """
+        Custom default method for handling non-standard JSON serializable types.
+
+        Supported types include Decimal, datetime, UUID, pathlib Paths, numpy arrays,
+        custom Enums, asyncpg Ranges, and more.
+
+        Parameters:
+            obj: The object to convert to a JSON-compatible format.
+            naive_utc (bool, optional): If True, convert datetime objects to naive UTC.
+            non_str_keys (bool, optional): If True, use non-string keys for dictionaries.
+
+        Returns:
+            A JSON-serializable representation of `obj`.
+
+        Raises:
+            TypeError: If `obj` is not JSON serializable.
+        """
+        cdef int opt = ORJSON_DEFAULT_OPTIONS
+        if naive_utc:
+            opt |= orjson.OPT_NAIVE_UTC
+        if non_str_keys:
+            opt |= orjson.OPT_NON_STR_KEYS
+        options = {
+            "default": self.default,
+            "option": opt
+        }
+        options.update(kwargs)
+        try:
+            return orjson.dumps(
+                obj,
+                **options
+            ).decode('utf-8')
+        except orjson.JSONEncodeError as ex:
+            raise ParserError(
+                f"Invalid JSON data: {ex}"
+            )
+
+    dumps = encode
+
+    @classmethod
+    def dump(cls, object obj, **kwargs):
+        """
+        Class method to encode an object using a new JSONContent instance.
+        """
+        return cls().encode(obj, **kwargs)
+
+    def decode(self, object obj):
+        """
+        Decode a JSON string into a Python object.
+
+        Parameters:
+            obj: A JSON string or bytes.
+
+        Returns:
+            The corresponding Python object.
+
+        Raises:
+            ParserError: If the JSON data is invalid.
+        """
+        try:
+            return orjson.loads(
+                obj
+            )
+        except orjson.JSONDecodeError as ex:
+            raise ParserError(
+                f"Invalid JSON data: {ex}"
+            )
+
+    loads = decode
+
+    @classmethod
+    def load(cls, object obj, **kwargs):
+        """
+        Class method to decode JSON data using a new JSONContent instance.
+        """
+        return cls().decode(obj, **kwargs)
+
+cpdef str json_encoder(object obj, bint naive_utc = True, bint non_str_keys = False):
+    """
+    Encode an object to JSON using the default JSONContent encoder.
+    """
+    return JSONContent().dumps(obj, naive_utc=naive_utc, non_str_keys=non_str_keys)
+
+cpdef object json_decoder(object obj):
+    """
+    Decode JSON data using the default JSONContent decoder.
+    """
+    return JSONContent().loads(obj)
+
+
+cpdef object json_fragment(bytes obj):
+    """
+    Return a Orjson Fragment of an already-serialized JSON Document.
+    """
+    return orjson.Fragment(obj)
+
+
+cdef class BaseEncoder:
+    """
+    Encoder replacement for json.dumps but using orjson,
+
+    This is a drop-in replacement for json.dumps using orjson.
+    """
+    def __init__(self, *args, **kwargs):
+        # Filter/adapt JSON arguments to ORJSON ones
+        rjargs = ()
+        rjkwargs = {}
+        encoder = JSONContent(*rjargs, **rjkwargs)
+        self.encode = encoder.__call__

datamodel/profiler.py

@@ -0,0 +1,21 @@
+import cProfile
+import functools
+import pstats
+
+
+def profile(func):
+
+    @functools.wraps(func)
+    def inner(*args, **kwargs):
+        profiler = cProfile.Profile()
+        profiler.enable()
+        try:
+            retval = func(*args, **kwargs)
+        finally:
+            profiler.disable()
+            with open('profile.out', 'w') as profile_file:
+                stats = pstats.Stats(profiler, stream=profile_file)
+                stats.print_stats()
+        return retval
+
+    return inner

datamodel/rs_core/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "rs_core"
+version = "0.1.0"
+edition = "2021"
+authors = ["Jesus Lara <jesuslarag@gmail.com>"]
+description = "Core Implementation for parsing and validating Datamodel Fields."
+license = "MIT"
+repository = "https://github.com/phenobarbital/python-datamodel"
+
+[lib]
+name = "rs_core"
+crate-type = ["cdylib"]
+
+
+[dependencies]
+pyo3 = { version = "0.23.3", features = ["generate-import-lib", "extension-module"] }
+rayon = "1.5.3"

datamodel/rs_core/src/lib.rs

@@ -0,0 +1,294 @@
+use pyo3::prelude::*;
+// use pyo3::exceptions::PyValueError;
+use pyo3::exceptions::PyTypeError;
+use pyo3::wrap_pyfunction;
+use pyo3::types::PyType;
+use pyo3::types::{PyDate, PyDateTime, PyAny, PyDict};
+// use pyo3::PyTypeInfo;
+// use chrono::{Datelike, Timelike, NaiveDate, NaiveTime, NaiveDateTime, DateTime, Utc};
+use rayon::prelude::*;
+// use std::collections::HashMap;
+
+
+#[pyfunction]
+fn validate_datamodel(py: Python<'_>, dataclass_instance: PyObject) -> PyResult<Vec<(String, bool)>> {
+    // Get the class of the instance
+    let dataclass: &PyType = dataclass_instance.as_ref(py).get_type();
+
+    // Get the __dataclass_fields__ attribute from the class
+    let fields_dict: &PyDict = dataclass.getattr("__dataclass_fields__")?.downcast::<PyDict>()?;
+
+    // Validate each field in the main thread
+    let results: Vec<(String, bool)> = fields_dict
+        .items()
+        .iter()
+        .map(|item| {
+            let (key, field) = item.extract::<(String, &PyAny)>().unwrap();
+
+            // Extract information from the dataclass.Field object
+            let field_type = field.getattr("type").unwrap().to_object(py);
+            let value = dataclass_instance.getattr(py, key.as_str()).unwrap();
+
+            let is_valid = match validate_field(py, &field_type, &value) {
+                Ok(result) => result,
+                Err(e) => {
+                    eprintln!("Validation error for field {}: {}", key, e);
+                    false
+                }
+            };
+            (key.to_string(), is_valid)
+        })
+        .collect();
+
+    Ok(results)
+}
+
+fn validate_field(py: Python<'_>, field_type: &PyObject, value: &PyObject) -> PyResult<bool> {
+    // Check if it's a primitive type
+    if let Ok(type_) = field_type.extract::<&PyType>(py) {
+        let type_name = type_.name()?;
+        match type_name {
+            "str" => {
+                return Ok(value.extract::<&str>(py).is_ok());
+            }
+            "int" => {
+                return Ok(value.extract::<i64>(py).is_ok());
+            }
+            "float" => {
+                return Ok(value.extract::<f64>(py).is_ok());
+            }
+            "bool" => {
+                return Ok(value.extract::<bool>(py).is_ok());
+            }
+            "datetime" => {
+                return Ok(value.extract::<&PyDateTime>(py).is_ok());
+            }
+            "date" => {
+                return Ok(value.extract::<&PyDate>(py).is_ok());
+            }
+            _ => {
+                // Not a primitive type, you can either skip validation or return an error
+                // eprintln!("Skipping validation for non-primitive type: {}", type_name);
+                // Ok(true) // Option 1: Skip validation
+                return Err(PyTypeError::new_err(format!(
+                    "Validation for type {} is not implemented yet.",
+                    type_name
+                ))); // Option 2: Return an error
+            }
+        }
+    } else {
+        // Handle the case where field_type is not a PyType (e.g., it's a generic type)
+        eprintln!("Field type is not a PyType: {:?}", field_type);
+        return Err(PyTypeError::new_err(
+            "Field type is not a PyType, cannot validate.",
+        ));
+    }
+}
+
+#[derive(Debug)]
+enum FieldType {
+    Str,
+    Int,
+    Float,
+    Bool,
+    DateTime,
+    Date,
+    Time,
+    // Extend with more types as needed
+}
+
+impl FieldType {
+    /// Convert type name string to FieldType enum
+    fn from_str(type_name: &str) -> Option<Self> {
+        match type_name {
+            "str" => Some(FieldType::Str),
+            "int" => Some(FieldType::Int),
+            "float" => Some(FieldType::Float),
+            "bool" => Some(FieldType::Bool),
+            "datetime.datetime" => Some(FieldType::DateTime),
+            "datetime.date" => Some(FieldType::Date),
+            "datetime.time" => Some(FieldType::Time),
+            _ => None,
+        }
+    }
+
+    /// Parse the string representation into Rust-native types if necessary
+    fn parse(&self, value: &FieldValue) -> bool {
+        match self {
+            FieldType::Str => true, // Already a string
+            FieldType::Int => true, // Already an integer
+            FieldType::Float => true, // Already a float
+            FieldType::Bool => true, // Already a bool
+            FieldType::DateTime => {
+                if let FieldValue::Str(s) = value {
+                    DateTime::parse_from_rfc3339(s).is_ok()
+                } else {
+                    false
+                }
+            },
+            FieldType::Date => {
+                if let FieldValue::Str(s) = value {
+                    NaiveDate::parse_from_str(s, "%Y-%m-%d").is_ok()
+                } else {
+                    false
+                }
+            },
+            FieldType::Time => {
+                if let FieldValue::Str(s) = value {
+                    NaiveTime::parse_from_str(s, "%H:%M:%S").is_ok()
+                } else {
+                    false
+                }
+            },
+            // Implement other parsing as needed
+        }
+    }
+
+    /// Validate the PyObject against the FieldType
+    fn validate(&self, value: &FieldValue) -> bool {
+        match self {
+            FieldType::Str => matches!(value, FieldValue::Str(_)),
+            FieldType::Int => matches!(value, FieldValue::Int(_)),
+            FieldType::Float => matches!(value, FieldValue::Float(_)),
+            FieldType::Bool => matches!(value, FieldValue::Bool(_)),
+            FieldType::DateTime => matches!(value, FieldValue::DateTime(_)),
+            FieldType::Date => matches!(value, FieldValue::Date(_)),
+            FieldType::Time => matches!(value, FieldValue::Time(_)),
+            // Add more validations as needed
+        }
+    }
+}
+
+/// Enum representing the Rust-native value of a field
+#[derive(Debug)]
+enum FieldValue {
+    Str(String),
+    Int(i64),
+    Float(f64),
+    Bool(bool),
+    DateTime(String), // Store as String; parse validation done separately
+    Date(String),
+    Time(String),
+    // Extend with more types as needed
+}
+
+// A Rust struct representing the minimal info we need from each dataclass Field
+#[derive(Debug)]
+struct RustFieldInfo {
+    pub field_name: String,
+    pub field_type: FieldType,
+    pub type_name: String, // Assuming type is always present for simplicity
+    value: FieldValue,
+}
+
+/// Collect the minimal field data we need into native Rust structs
+fn get_field_info(py: Python<'_>, dataclass_instance: &PyObject, fields_dict: &PyDict) -> PyResult<Vec<RustFieldInfo>> {
+    let mut result = Vec::new();
+
+    for (key, field_obj) in fields_dict.iter() {
+        let field_name = key.extract::<String>()?;
+
+        // Extract type name
+        let type_obj = field_obj.getattr("type")?;
+        let type_name = type_obj.extract::<&PyType>()?.name()?.to_string();
+
+        // Convert type name to FieldType enum
+        let field_type = match FieldType::from_str(&type_name) {
+            Some(ft) => ft,
+            None => continue, // Skip unsupported types or handle as needed
+        };
+
+        // Extract value
+        let py_value = dataclass_instance.getattr(py, &field_name[..])?;
+
+        // Convert PyObject to Rust-native FieldValue
+        let value = match field_type {
+            FieldType::Str => {
+                FieldValue::Str(py_value.extract::<String>(py)?)
+            },
+            FieldType::Int => {
+                FieldValue::Int(py_value.extract::<i64>(py)?)
+            },
+            FieldType::Float => {
+                FieldValue::Float(py_value.extract::<f64>(py)?)
+            },
+            FieldType::Bool => {
+                FieldValue::Bool(py_value.extract::<bool>(py)?)
+            },
+            FieldType::DateTime => {
+                let s: String = py_value.extract::<String>(py)?;
+                FieldValue::DateTime(s)
+            },
+            FieldType::Date => {
+                let s: String = py_value.extract::<String>(py)?;
+                FieldValue::Date(s)
+            },
+            FieldType::Time => {
+                let s: String = py_value.extract::<String>(py)?;
+                FieldValue::Time(s)
+            },
+            // Handle other types as needed
+        };
+
+        result.push(RustFieldInfo {
+            field_name,
+            field_type,
+            type_name,
+            value,
+        });
+    }
+
+    Ok(result)
+}
+
+/// A mock-up function that showcases a single iteration over fields
+/// performing these steps:
+/// 1) Handle `default` or `default_factory` if the field is missing
+/// 2) Parse the field's value (e.g. str -> UUID, str -> date, etc.)
+/// 3) Validate the resulting value against the annotated type
+#[pyfunction]
+fn parse_datamodel(py: Python<'_>, dataclass_instance: PyObject) -> PyResult<Vec<(String, bool)>> {
+    // Acquire the GIL using `Python::with_gil`
+    Python::with_gil(|py| {
+        // 1) Get dataclass instance's class
+        let dataclass_type: &PyType = dataclass_instance.as_ref(py).get_type();
+
+        // 2) Get __dataclass_fields__ from the class
+        let fields_dict: &PyDict = dataclass_type
+            .getattr("__dataclass_fields__")?
+            .downcast::<PyDict>()?;
+
+        // 3) Convert Python fields into a native Rust Vec<RustFieldInfo>
+        let field_infos = get_field_info(py, &dataclass_instance, fields_dict)?;
+
+        // 4) Drop the GIL before parallel processing
+        // Note: `Python::with_gil` automatically drops the GIL when the closure ends
+        // Hence, no need to explicitly drop `py` here
+
+        // 5) Perform parallel iteration over `field_infos`
+        let results: Vec<(String, bool)> = field_infos
+            .into_par_iter()
+            .map(|field_info| {
+                // Perform parsing and validation purely in Rust
+                let is_parsed = field_info.field_type.parse(&field_info.value);
+                if !is_parsed {
+                    return (field_info.field_name, false);
+                }
+
+                let is_valid = field_info.field_type.validate(&field_info.value);
+                (field_info.field_name, is_valid)
+            })
+            .collect();
+
+        Ok(results)
+    })
+}
+
+
+/// Python module declaration
+#[pymodule]
+fn rs_core(_py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(validate_datamodel, m)?)?;
+    m.add_function(wrap_pyfunction!(parse_datamodel, m)?)?;
+    Ok(())
+}

datamodel/rs_parsers/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "rs_parsers"
+version = "0.1.0"
+edition = "2021"
+authors = ["Jesus Lara <jesuslarag@gmail.com>"]
+description = "Parallel DataModel Parser and validator using Rust"
+license = "MIT"
+repository = "https://github.com/phenobarbital/python-datamodel"
+
+[lib]
+name = "rs_parsers"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.23.3", features = ["generate-import-lib", "extension-module"] }
+rayon = "1.5.3"
+chrono = "0.4.39"
+speedate = "0.15.0"
+uuid = "1.11.0"
+fastuuid = "0.3.0"
+rust_decimal = "1.36"
+rust_decimal_macros = "1.36"