data-sitter 0.1.4__tar.gz → 0.1.6__tar.gz

data_sitter-0.1.6/PKG-INFO

@@ -0,0 +1,220 @@
+Metadata-Version: 2.4
+Name: data-sitter
+Version: 0.1.6
+Summary: A Python library that reads data contracts and generates Pydantic models for seamless data validation.
+Author-email: Lázaro Pereira Candea <lazaro@candea.es>
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: PyYAML==6.0.2
+Requires-Dist: parse_type==0.6.4
+Requires-Dist: pydantic==2.10.5
+Provides-Extra: dev
+Requires-Dist: pytest==8.3.5; extra == "dev"
+Requires-Dist: pytest-cov==6.0.0; extra == "dev"
+Requires-Dist: pytest-mock==3.14.0; extra == "dev"
+Requires-Dist: twine==6.1.0; extra == "dev"
+Requires-Dist: build==1.2.2.post1; extra == "dev"
+
+# Data-Sitter
+
+![Coverage](./coverage.svg)
+
+## Overview
+
+Data-Sitter is a Python library designed to simplify data validation by converting data contracts into Pydantic models. This allows for easy and efficient validation of structured data, ensuring compliance with predefined rules and constraints.
+
+## Features
+
+- Define structured data contracts in JSON format.
+- Generate Pydantic models automatically from contracts.
+- Enforce validation rules at the field level.
+- Support for rule references within the contract.
+
+## Installation
+
+```sh
+pip install data-sitter
+```
+
+## Development and Deployment
+
+### CI/CD Pipeline
+
+The project uses GitHub Actions for continuous integration and deployment:
+
+1. **Pull Request Checks**
+   - Automatically checks if the version has been bumped in `pyproject.toml`
+   - Fails if the version is the same as in the main branch
+   - Ensures every PR includes a version update
+
+2. **Automatic Releases**
+   - When code is merged to the main branch:
+     - Builds the package
+     - Publishes to PyPI automatically
+   - Uses PyPI API token for secure authentication
+
+To set up the CI/CD pipeline:
+
+1. Create a PyPI API token:
+   - Go to [PyPI Account Settings](https://pypi.org/manage/account/)
+   - Create a new API token with "Upload" scope
+   - Copy the token
+
+2. Add the token to GitHub:
+   - Go to your repository's Settings > Secrets and variables > Actions
+   - Create a new secret named `PYPI_API_TOKEN`
+   - Paste your PyPI API token
+
+### Setting Up Development Environment
+
+To set up a development environment with all the necessary tools, install the package with development dependencies:
+
+```sh
+pip install -e ".[dev]"
+```
+
+This will install:
+- The package in editable mode
+- Testing tools (pytest, pytest-cov, pytest-mock)
+- Build tools (build, twine)
+
+### Building the Package
+
+To build the package, run:
+
+```sh
+python -m build
+```
+
+This will create a `dist` directory containing both a source distribution (`.tar.gz`) and a wheel (`.whl`).
+
+### Deploying to PyPI
+
+To upload to PyPI:
+
+```sh
+twine upload dist/*
+```
+
+You'll be prompted for your PyPI username and password. For security, it's recommended to use an API token instead of your password.
+
+## Usage
+
+### Creating a Pydantic Model from a Contract
+
+To convert a data contract into a Pydantic model, follow these steps:
+
+```python
+from data_sitter import Contract
+
+contract_dict = {
+    "name": "test",
+    "fields": [
+        {
+            "name": "FID",
+            "type": "Integer",
+            "rules": ["Positive"]
+        },
+        {
+            "name": "SECCLASS",
+            "type": "String",
+            "rules": [
+                "Validate Not Null",
+                "Value In ['UNCLASSIFIED', 'CLASSIFIED']",
+            ]
+        }
+    ],
+}
+
+contract = Contract.from_dict(contract_dict)
+pydantic_contract = contract.pydantic_model
+```
+
+### Using Rule References
+
+Data-Sitter allows you to define reusable values in the `values` key and reference them in field rules using `$values.[key]`. For example:
+
+```json
+{
+    "name": "example_contract",
+    "fields": [
+        {
+            "name": "CATEGORY",
+            "type": "String",
+            "rules": ["Value In $values.categories"]
+        },
+        {
+            "name": "NAME",
+            "type": "String",
+            "rules": [
+                "Length Between $values.min_length and $values.max_length"
+            ]
+        }
+
+    ],
+    "values": {"categories": ["A", "B", "C"], "min_length": 5,"max_length": 50}
+}
+```
+
+## Available Rules
+
+The available validation rules can be retrieved programmatically:
+
+```python
+from data_sitter import RuleRegistry
+
+rules = RuleRegistry.get_rules_definition()
+print(rules)
+```
+
+### Rule Definitions
+
+Below are the available rules grouped by field type:
+
+#### Base
+
+- Is not null
+
+#### String - (Inherits from `Base`)
+
+- Is not empty
+- Starts with {prefix:String}
+- Ends with {suffix:String}
+- Is not one of {possible_values:Strings}
+- Is one of {possible_values:Strings}
+- Has length between {min_val:Integer} and {max_val:Integer}
+- Has maximum length {max_len:Integer}
+- Has minimum length {min_len:Integer}
+- Is uppercase
+- Is lowercase
+- Matches regex {pattern:String}
+- Is valid email
+- Is valid URL
+- Has no digits
+
+#### Numeric - (Inherits from `Base`)
+
+- Is not zero
+- Is positive
+- Is negative
+- Is at least {min_val:Number}
+- Is at most {max_val:Number}
+- Is greater than {threshold:Number}
+- Is less than {threshold:Number}
+- Is not between {min_val:Number} and {max_val:Number}
+- Is between {min_val:Number} and {max_val:Number}
+
+#### Integer  - (Inherits from `Numeric`)
+
+#### Float  - (Inherits from `Numeric`)
+
+- Has at most {decimal_places:Integer} decimal places
+
+## Contributing
+
+Contributions are welcome! Feel free to submit issues or pull requests in the [GitHub repository](https://github.com/lcandea/data-sitter).
+
+## License
+
+Data-Sitter is licensed under the MIT License.

{data_sitter-0.1.4 → data_sitter-0.1.6}/README.md

@@ -1,5 +1,7 @@
 # Data-Sitter
 
+![Coverage](./coverage.svg)
+
 ## Overview
 
 Data-Sitter is a Python library designed to simplify data validation by converting data contracts into Pydantic models. This allows for easy and efficient validation of structured data, ensuring compliance with predefined rules and constraints.
@@ -17,6 +19,68 @@ Data-Sitter is a Python library designed to simplify data validation by converti
 pip install data-sitter
 ```
 
+## Development and Deployment
+
+### CI/CD Pipeline
+
+The project uses GitHub Actions for continuous integration and deployment:
+
+1. **Pull Request Checks**
+   - Automatically checks if the version has been bumped in `pyproject.toml`
+   - Fails if the version is the same as in the main branch
+   - Ensures every PR includes a version update
+
+2. **Automatic Releases**
+   - When code is merged to the main branch:
+     - Builds the package
+     - Publishes to PyPI automatically
+   - Uses PyPI API token for secure authentication
+
+To set up the CI/CD pipeline:
+
+1. Create a PyPI API token:
+   - Go to [PyPI Account Settings](https://pypi.org/manage/account/)
+   - Create a new API token with "Upload" scope
+   - Copy the token
+
+2. Add the token to GitHub:
+   - Go to your repository's Settings > Secrets and variables > Actions
+   - Create a new secret named `PYPI_API_TOKEN`
+   - Paste your PyPI API token
+
+### Setting Up Development Environment
+
+To set up a development environment with all the necessary tools, install the package with development dependencies:
+
+```sh
+pip install -e ".[dev]"
+```
+
+This will install:
+- The package in editable mode
+- Testing tools (pytest, pytest-cov, pytest-mock)
+- Build tools (build, twine)
+
+### Building the Package
+
+To build the package, run:
+
+```sh
+python -m build
+```
+
+This will create a `dist` directory containing both a source distribution (`.tar.gz`) and a wheel (`.whl`).
+
+### Deploying to PyPI
+
+To upload to PyPI:
+
+```sh
+twine upload dist/*
+```
+
+You'll be prompted for your PyPI username and password. For security, it's recommended to use an API token instead of your password.
+
 ## Usage
 
 ### Creating a Pydantic Model from a Contract
@@ -30,14 +94,14 @@ contract_dict = {
     "name": "test",
     "fields": [
         {
-            "field_name": "FID",
-            "field_type": "IntegerField",
-            "field_rules": ["Positive"]
+            "name": "FID",
+            "type": "Integer",
+            "rules": ["Positive"]
         },
         {
-            "field_name": "SECCLASS",
-            "field_type": "StringField",
-            "field_rules": [
+            "name": "SECCLASS",
+            "type": "String",
+            "rules": [
                 "Validate Not Null",
                 "Value In ['UNCLASSIFIED', 'CLASSIFIED']",
             ]
@@ -58,14 +122,14 @@ Data-Sitter allows you to define reusable values in the `values` key and referen
     "name": "example_contract",
     "fields": [
         {
-            "field_name": "CATEGORY",
-            "field_type": "StringField",
-            "field_rules": ["Value In $values.categories"]
+            "name": "CATEGORY",
+            "type": "String",
+            "rules": ["Value In $values.categories"]
         },
         {
-            "field_name": "NAME",
-            "field_type": "StringField",
-            "field_rules": [
+            "name": "NAME",
+            "type": "String",
+            "rules": [
                 "Length Between $values.min_length and $values.max_length"
             ]
         }
@@ -90,11 +154,11 @@ print(rules)
 
 Below are the available rules grouped by field type:
 
-#### BaseField
+#### Base
 
 - Is not null
 
-#### StringField - (Inherits from `BaseField`)
+#### String - (Inherits from `Base`)
 
 - Is not empty
 - Starts with {prefix:String}
@@ -111,7 +175,7 @@ Below are the available rules grouped by field type:
 - Is valid URL
 - Has no digits
 
-#### NumericField - (Inherits from `BaseField`)
+#### Numeric - (Inherits from `Base`)
 
 - Is not zero
 - Is positive
@@ -123,9 +187,9 @@ Below are the available rules grouped by field type:
 - Is not between {min_val:Number} and {max_val:Number}
 - Is between {min_val:Number} and {max_val:Number}
 
-#### IntegerField  - (Inherits from `NumericField`)
+#### Integer  - (Inherits from `Numeric`)
 
-#### FloatField  - (Inherits from `NumericField`)
+#### Float  - (Inherits from `Numeric`)
 
 - Has at most {decimal_places:Integer} decimal places
 

{data_sitter-0.1.4 → data_sitter-0.1.6}/data_sitter/Contract.py

@@ -8,7 +8,7 @@ from pydantic import BaseModel
 from .Validation import Validation
 from .field_types import BaseField
 from .FieldResolver import FieldResolver
-from .rules import MatchedRule, RuleRegistry, RuleParser
+from .rules import ProcessedRule, RuleRegistry, RuleParser
 
 
 class ContractWithoutFields(Exception):
@@ -20,9 +20,9 @@ class ContractWithoutName(Exception):
 
 
 class Field(NamedTuple):
-    field_name: str
-    field_type: str
-    field_rules: List[str]
+    name: str
+    type: str
+    rules: List[str]
 
 
 class Contract:
@@ -37,8 +37,8 @@ class Contract:
         self.fields = fields
         self.rule_parser = RuleParser(values)
         self.field_resolvers = {
-            field_type: FieldResolver(RuleRegistry.get_type(field_type), self.rule_parser)
-            for field_type in list({field.field_type for field in self.fields})  # Unique types
+            _type: FieldResolver(RuleRegistry.get_type(_type), self.rule_parser)
+            for _type in list({field.type for field in self.fields})  # Unique types
         }
 
     @classmethod
@@ -66,21 +66,18 @@ class Contract:
     def field_validators(self) -> Dict[str, BaseField]:
         field_validators = {}
         for field in self.fields:
-            field_resolver = self.field_resolvers[field.field_type]
-            field_validators[field.field_name] = field_resolver.get_field_validator(field.field_name, field.field_rules)
+            field_resolver = self.field_resolvers[field.type]
+            field_validators[field.name] = field_resolver.get_field_validator(field.name, field.rules)
         return field_validators
 
     @cached_property
-    def rules(self) -> Dict[str, List[MatchedRule]]:
+    def rules(self) -> Dict[str, List[ProcessedRule]]:
         rules = {}
         for field in self.fields:
-            field_resolver = self.field_resolvers[field.field_type]
-            rules[field.field_name] = field_resolver.get_matched_rules(field.field_rules)
+            field_resolver = self.field_resolvers[field.type]
+            rules[field.name] = field_resolver.get_processed_rules(field.rules)
         return rules
 
-    def model_validate(self, item: dict):
-        return self.pydantic_model.model_validate(item).model_dump()
-
     def validate(self, item: dict) -> Validation:
         return Validation.validate(self.pydantic_model, item)
 
@@ -88,8 +85,8 @@ class Contract:
     def pydantic_model(self) -> BaseModel:
         return type(self.name, (BaseModel,), {
             "__annotations__": {
-                field_name: field_validator.get_annotation()
-                for field_name, field_validator in self.field_validators.items()
+                name: field_validator.get_annotation()
+                for name, field_validator in self.field_validators.items()
             }
         })
 
@@ -99,11 +96,11 @@ class Contract:
             "name": self.name,
             "fields": [
                 {
-                    "field_name": field_name,
-                    "field_type": field_validator.__class__.__name__,
-                    "field_rules": [rule.parsed_rule for rule in self.rules.get(field_name, [])]
+                    "name": name,
+                    "type": field_validator.type_name.value,
+                    "rules": [rule.parsed_rule for rule in self.rules.get(name, [])]
                 }
-                for field_name, field_validator in self.field_validators.items()
+                for name, field_validator in self.field_validators.items()
             ],
             "values": self.rule_parser.values
         }
@@ -119,19 +116,14 @@ class Contract:
             "name": self.name,
             "fields": [
                 {
-                    "field_name": field_name,
-                    "field_type": field_validator.__class__.__name__,
-                    "field_rules": [
-                        {
-                            "rule": rule.field_rule,
-                            "parsed_rule": rule.parsed_rule,
-                            "rule_params": rule.rule_params,
-                            "parsed_values": rule.parsed_values,
-                        }
-                        for rule in self.rules.get(field_name, [])
+                    "name": name,
+                    "type": field_validator.type_name.value,
+                    "rules": [
+                        rule.get_front_end_repr()
+                        for rule in self.rules.get(name, [])
                     ]
                 }
-                for field_name, field_validator in self.field_validators.items()
+                for name, field_validator in self.field_validators.items()
             ],
             "values": self.rule_parser.values
         }

data_sitter-0.1.6/data_sitter/FieldResolver.py

@@ -0,0 +1,62 @@
+from typing import  Dict, List, Type, Union
+
+from .field_types import BaseField
+from .rules import Rule, ProcessedRule, LogicalRule, MatchedRule, RuleRegistry, LogicalOperator
+from .rules.Parser import RuleParser
+
+
+class RuleNotFoundError(Exception):
+    """No matching rule found for the given parsed rule."""
+
+
+class MalformedLogicalRuleError(Exception):
+    """Logical rule structure not recognised."""
+
+
+class FieldResolver:
+    field_class: Type[BaseField]
+    rule_parser: RuleParser
+    rules: List[Rule]
+    _match_rule_cache: Dict[str, MatchedRule]
+
+    def __init__(self, field_class: Type[BaseField], rule_parser: RuleParser) -> None:
+        self.field_class = field_class
+        self.rule_parser = rule_parser
+        self.rules = RuleRegistry.get_rules_for(field_class)
+        self._match_rule_cache = {}
+
+    def get_field_validator(self, name: str, parsed_rules: List[Union[str, dict]]) -> BaseField:
+        field_validator = self.field_class(name)
+        processed_rules = self.get_processed_rules(parsed_rules)
+        validators = [pr.get_validator(field_validator) for pr in processed_rules]
+        field_validator.validators = validators
+        return field_validator
+
+    def get_processed_rules(self, parsed_rules: List[Union[str, dict]]) -> List[ProcessedRule]:
+        processed_rules = []
+        for parsed_rule in parsed_rules:
+            if isinstance(parsed_rule, dict):
+                if len(keys := tuple(parsed_rule)) != 1 or (operator := keys[0]) not in LogicalOperator:
+                    raise MalformedLogicalRuleError()
+                if operator == LogicalOperator.NOT and not isinstance(parsed_rule[operator], list):
+                    parsed_rule = {operator: [parsed_rule[operator]]}  # NOT operator can be a single rule
+                processed_rule = LogicalRule(operator, self.get_processed_rules(parsed_rule[operator]))
+            elif isinstance(parsed_rule, str):
+                processed_rule = self._match_rule(parsed_rule)
+                if not processed_rule:
+                    raise RuleNotFoundError(f"Rule not found for parsed rule: '{parsed_rule}'")
+            else:
+                raise TypeError(f'Parsed Rule type not recognised: {type(parsed_rule)}')
+            processed_rules.append(processed_rule)
+        return processed_rules
+
+    def _match_rule(self, parsed_rule: str) -> MatchedRule:
+        if parsed_rule in self._match_rule_cache:
+            return self._match_rule_cache[parsed_rule]
+
+        for rule in self.rules:
+            matched_rule = self.rule_parser.match(rule, parsed_rule)
+            if matched_rule:
+                self._match_rule_cache[parsed_rule] = matched_rule
+                return matched_rule
+        return None

{data_sitter-0.1.4 → data_sitter-0.1.6}/data_sitter/cli.py

@@ -44,5 +44,5 @@ def main():
     print(f"The file {args.file} pass the contract {args.contract}")
 
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     main()

{data_sitter-0.1.4 → data_sitter-0.1.6}/data_sitter/field_types/BaseField.py

@@ -1,18 +1,24 @@
 from abc import ABC
-from typing import Annotated, List, Optional, Type
+from typing import Annotated, Callable, List, Optional, Type
 
 from pydantic import AfterValidator
+
+from .FieldTypes import FieldTypes
 from ..rules import register_rule, register_field
 
 
-def aggregated_validator(validators: List[callable], is_optional: bool):
-    def _validator(value):
+class NotInitialisedError(Exception):
+    """The field instance is initialised without validators"""
+
+
+def aggregated_validator(validators: List[Callable], is_optional: bool):
+    def validator(value):
         if is_optional and value is None:
             return value
         for validator_func in validators:
             validator_func(value)
         return value
-    return _validator
+    return validator
 
 @register_field
 class BaseField(ABC):
@@ -20,39 +26,42 @@ class BaseField(ABC):
     is_optional: bool
     validators = None
     field_type = None
+    type_name = FieldTypes.BASE
 
     def __init__(self, name: str) -> None:
         self.name = name
         self.is_optional = True
-        self.validators = []
+        self.validators = None
 
     @register_rule("Is not null")
     def validator_not_null(self):
-        def _validator(value):
-            if self.is_optional:
-                return value
+        def validator(value):
             if value is None:
                 raise ValueError("Value cannot be null.")
             return value
 
         self.is_optional = False
-        self.validators.append(_validator)
+        return validator
 
     def validate(self, value):
+        if self.validators is None:
+            raise NotInitialisedError()
         for validator in self.validators:
             validator(value)
 
     def get_annotation(self):
+        if self.validators is None:
+            raise NotInitialisedError()
         field_type = Optional[self.field_type] if self.is_optional else self.field_type
         return Annotated[field_type, AfterValidator(aggregated_validator(self.validators, self.is_optional))]
 
     @classmethod
     def get_parents(cls: Type["BaseField"]) -> List[Type["BaseField"]]:
-        if cls.__name__ == "BaseField":
+        if cls == BaseField:
             return []
-        ancestors = []
+        ancestors = set()
         for base in cls.__bases__:
-            if base.__name__.endswith("Field"):
-                ancestors.append(base)
-                ancestors.extend(base.get_parents())  # It wont break because we have a base case
-        return ancestors
+            if issubclass(base, BaseField):
+                ancestors.add(base)
+                ancestors.update(base.get_parents())
+        return list(ancestors)

data_sitter-0.1.6/data_sitter/field_types/FieldTypes.py

@@ -0,0 +1,9 @@
+from enum import StrEnum
+
+
+class FieldTypes(StrEnum):
+    BASE = "Base"
+    INT = "Integer"
+    FLOAT = "Float"
+    STRING = "String"
+    NUMERIC = "Numeric"

data_sitter-0.1.6/data_sitter/field_types/FloatField.py

@@ -0,0 +1,26 @@
+from .FieldTypes import FieldTypes
+from .NumericField import NumericField
+from ..rules import register_field, register_rule
+from decimal import Decimal
+
+
+@register_field
+class FloatField(NumericField):
+    field_type = float
+    type_name = FieldTypes.FLOAT
+
+
+    @register_rule("Has at most {decimal_places:Integer} decimal places")
+    def validate_max_decimal_places(self, decimal_places: int):
+        def validator(value):
+            decimal_str = str(Decimal(str(value)).normalize())
+            # If no decimal point or only zeros after decimal, it has 0 decimal places
+            if '.' not in decimal_str:
+                decimal_places_count = 0
+            else:
+                decimal_places_count = len(decimal_str.split('.')[1])
+
+            if decimal_places_count > decimal_places:
+                raise ValueError(f"Value must have at most {decimal_places} decimal places.")
+            return value
+        return validator

{data_sitter-0.1.4 → data_sitter-0.1.6}/data_sitter/field_types/IntegerField.py

@@ -1,3 +1,4 @@
+from .FieldTypes import FieldTypes
 from .NumericField import NumericField
 from ..rules import register_field
 
@@ -5,3 +6,4 @@ from ..rules import register_field
 @register_field
 class IntegerField(NumericField):
     field_type = int
+    type_name = FieldTypes.INT