patra-toolkit 0.1.1__py3-none-any.whl

patra_toolkit/__init__.py

@@ -0,0 +1,3 @@
+from .patra_model_card import *
+from .fairlearn_bias import *
+from .shap_xai import *

patra_toolkit/fairlearn_bias.py

@@ -0,0 +1,32 @@
+from fairlearn.metrics import demographic_parity_difference, equalized_odds_difference
+
+
+class BiasAnalyzer:
+    """
+    Provides automated capture of fairness metrics in a given AI workflow.
+    """
+    def __init__(self, dataset, true_labels, predicted_labels, sensitive_feature_name, sensitive_feature_data, model):
+        self.dataset = dataset
+        self.true_labels = true_labels
+        self.predicted_labels = predicted_labels
+        self.sensitive_feature = sensitive_feature_name
+        self.sensitive_data = sensitive_feature_data
+        self.model = model
+
+    def calculate_bias_metrics(self):
+        """
+        Calculate bias metrics.
+
+        Args:
+            None
+
+        Returns:
+            dict: A dictionary of bias
+        """
+        demographic_parity = demographic_parity_difference(self.true_labels, self.predicted_labels,
+                                                           sensitive_features=self.sensitive_data)
+        equal_odds_diff = equalized_odds_difference(self.true_labels, self.predicted_labels,
+                                                    sensitive_features=self.sensitive_data)
+
+        return {"demographic_parity_diff": demographic_parity,
+                "equal_odds_difference": equal_odds_diff}

patra_toolkit/patra_model_card.py

@@ -0,0 +1,376 @@
+import hashlib
+import json
+import os.path
+from dataclasses import dataclass, field
+from json import JSONEncoder
+from typing import List, Optional, Dict
+
+import jsonschema
+import pkg_resources
+import requests
+
+from .fairlearn_bias import BiasAnalyzer
+from .shap_xai import ExplainabilityAnalyser
+
+SCHEMA_JSON = os.path.join(os.path.dirname(__file__), 'schema', 'schema.json')
+
+
+@dataclass
+class Metric:
+    """
+    Data class for storing metric key-value pairs.
+
+    Args:
+        key (str): The name of the metric.
+        value (str): The value of the metric.
+    """
+    key: str
+    value: str
+
+@dataclass
+class AIModel:
+    """
+    Represents and stores AI model metadata and its performance metrics.
+
+    Args:
+        name (str): The name of the model.
+        version (str): The version identifier of the model.
+        description (str): A detailed description of the model.
+        owner (str): The owner of the model.
+        location (str): The file path or URL where the model is stored.
+        license (str): The license under which the model is distributed.
+        framework (str): The framework used to build the model (e.g., TensorFlow, PyTorch).
+        model_type (str): The type of model (e.g., classifier, regressor).
+        test_accuracy (str): The accuracy of the model on a test dataset.
+        model_structure (str): The structure of the model as a dictionary (optional).
+        metrics (str): A dictionary storing performance metrics for the model.
+
+    Example:
+        .. code-block:: python
+
+            ai_model = AIModel(
+                name="Model Name",
+                version="1.0",
+                description="Model description",
+                owner="Model owner",
+                location="Model location",
+                license="Model license",
+                framework="Model framework",
+                model_type="Model type",
+                test_accuracy=0.95,
+                model_structure={},
+                metrics={"accuracy": "0.95"}
+            )
+    """
+    name: str
+    version: str
+    description: str
+    owner: str
+    location: str
+    license: str
+    framework: str
+    model_type: str
+    test_accuracy: float
+    model_structure: Optional[object] = field(default_factory=dict)
+    metrics: Dict[str, str] = field(default_factory=dict)
+
+    def add_metric(self, key: str, value: str) -> None:
+        """
+        Adds a performance metric to the model's metrics.
+
+        Args:
+            key (str): The name of the metric.
+            value (str): The value of the metric.
+
+        Returns:
+            None
+        """
+        self.metrics[key] = value
+
+    def remove_nulls(self, model_structure):
+        """
+        Recursively removes null values from the model structure.
+
+        Args:
+            model_structure (object): The model structure as a dictionary or list.
+
+        Returns:
+            object: Model structure with null values removed.
+        """
+        if isinstance(model_structure, dict):
+            return {k: self.remove_nulls(v) for k, v in model_structure.items() if v is not None}
+        elif isinstance(model_structure, list):
+            return [self.remove_nulls(v) for v in model_structure if v is not None]
+        return model_structure
+
+    def populate_model_structure(self, trained_model):
+        """
+        Populates the `model_structure` attribute from a trained model object.
+
+        Args:
+            trained_model (object): A trained machine learning model object.
+
+        Returns:
+            None
+        """
+        if self.framework == 'tensorflow':
+            json_structure = json.loads(trained_model.to_json())
+            self.model_structure = self.remove_nulls(json_structure)
+        else:
+            self.model_structure = {}
+
+
+@dataclass
+class BiasAnalysis:
+    """
+    Class to store results from bias analysis.
+
+    Args:
+        demographic_parity_difference (float): The difference in demographic parity between groups.
+        equal_odds_difference (float): The difference in equal odds between groups.
+    """
+    demographic_parity_difference: float
+    equal_odds_difference: float
+
+
+@dataclass
+class ExplainabilityAnalysis:
+    """
+    Class to store explainability metrics.
+
+    Args:
+        name (str): Name of the explainability method used.
+        metrics (List[Metric]): List of metrics related to explainability analysis.
+    """
+    name: str
+    metrics: List[Metric] = field(default_factory=list)
+
+
+@dataclass
+class ModelCard:
+    """
+    Represents an AI model card to document model metadata, analyses, and requirements.
+
+    Args:
+        name (str): The name of the model.
+        version (str): The model's version.
+        short_description (str): A brief description of the model.
+        full_description (str): A detailed description of the model.
+        keywords (str): Comma-separated keywords for searchability.
+        author (str): The model's creator or owner.
+        input_type (str): Type of input data (e.g., "Image", "Text").
+        category (str): The category of the model (e.g., "Classification", "Regression").
+        input_data (Optional[str]): Description of the model's input data.
+        output_data (Optional[str]): Description of the model's output data.
+        foundational_model (Optional[str]): Reference to any foundational model used.
+        ai_model (Optional[AIModel]): An instance of `AIModel` containing model details.
+        bias_analysis (Optional[BiasAnalysis]): Instance of `BiasAnalysis` containing bias metrics.
+        xai_analysis (Optional[ExplainabilityAnalysis]): Instance of `ExplainabilityAnalysis` with interpretability metrics.
+        model_requirements (Optional[List[str]]): List of required packages and dependencies.
+        id (Optional[str]): Unique identifier for the model card, generated upon submission.
+
+    Example:
+        .. code-block:: python
+
+            model_card = ModelCard(
+                name="Model Name",
+                version="1.0",
+                short_description="A brief description",
+                full_description="A detailed description of the model's purpose and usage.",
+                keywords="classification, AI, image processing",
+                author="Author Name",
+                input_type="Image",
+                category="Classification",
+                input_data="Images of size 28x28.",
+                output_data="Prediction probabilities for classes.",
+                foundational_model="Base Model Reference",
+                ai_model=AIModel(
+                    name="Model Name",
+                    version="1.0",
+                    description="Detailed model description",
+                    owner="Model owner",
+                    location="Storage location",
+                    license="MIT",
+                    framework="TensorFlow",
+                    model_type="Classifier",
+                    test_accuracy=0.95,
+                    model_structure={},
+                    metrics={"accuracy": "0.95"}
+                ),
+                bias_analysis=BiasAnalysis(
+                    demographic_parity_difference=0.05,
+                    equal_odds_difference=0.1
+                ),
+                xai_analysis=ExplainabilityAnalysis(
+                    name="SHAP",
+                    metrics=[Metric(key="Feature A", value="0.1")]
+                ),
+                model_requirements=["numpy>=1.19.2", "tensorflow>=2.4.1"]
+            )
+    """
+    name: str
+    version: str
+    short_description: str
+    full_description: str
+    keywords: str
+    author: str
+    input_type: str
+    category: str
+    input_data: Optional[str] = ""
+    output_data: Optional[str] = ""
+    foundational_model: Optional[str] = ""
+    ai_model: Optional[AIModel] = None
+    bias_analysis: Optional[BiasAnalysis] = None
+    xai_analysis: Optional[ExplainabilityAnalysis] = None
+    model_requirements: Optional[List] = None
+    id: Optional[str] = field(init=False, default=None)
+
+    def __str__(self):
+        """
+        Returns a JSON string representation of the model card.
+
+        Returns:
+            str: A JSON-formatted string representing the model card.
+        """
+        return json.dumps(self.__dict__, cls=ModelCardJSONEncoder, indent=4, separators=(',', ': '))
+
+    def populate_bias(self, dataset, true_labels, predicted_labels, sensitive_feature_name, sensitive_feature_data, model):
+        """
+        Calculates and stores fairness metrics.
+
+        Args:
+            dataset (object): The dataset used for bias analysis.
+            true_labels (list): The ground truth labels.
+            predicted_labels (list): Model's predictions.
+            sensitive_feature_name (str): The name of the sensitive attribute.
+            sensitive_feature_data (list): Values for the sensitive feature.
+            model (object): The model being analyzed.
+
+        Returns:
+            None
+        """
+        bias_analyzer = BiasAnalyzer(dataset, true_labels, predicted_labels, sensitive_feature_name,
+                                     sensitive_feature_data, model)
+        self.bias_analysis = bias_analyzer.calculate_bias_metrics()
+
+    def populate_xai(self, train_dataset, column_names, model, n_features=10):
+        """
+        Computes and stores feature importance metrics.
+
+        Args:
+            train_dataset (object): Training dataset used in the analysis.
+            column_names (list): Names of the features.
+            model (object): The model being explained.
+            n_features (int, optional): Number of features to analyze. Default is 10.
+
+        Returns:
+            None
+        """
+        xai_analyzer = ExplainabilityAnalyser(train_dataset, column_names, model)
+        self.xai_analysis = xai_analyzer.calculate_xai_features(n_features)
+
+    def populate_requirements(self):
+        """
+        Collects package requirements for the model card, excluding specific dependencies.
+
+        Returns:
+            None
+        """
+        exclude_packages = {"shap", "fairlearn"}
+        installed_packages = pkg_resources.working_set
+        packages_list = sorted(["%s==%s" % (i.key, i.version) for i in installed_packages])
+        self.model_requirements = [pkg for pkg in packages_list if pkg.split("==")[0] not in exclude_packages]
+
+    def validate(self):
+        """
+        Validates the model card against a predefined JSON schema.
+
+        Returns:
+            bool: True if the model card is valid according to the schema, False otherwise.
+        """
+        mc_json = self.__str__()
+        try:
+            with open(SCHEMA_JSON, 'r') as schema_file:
+                schema = json.load(schema_file)
+            jsonschema.validate(instance=json.loads(mc_json), schema=schema)
+            return True
+        except jsonschema.ValidationError as e:
+            print(e.message)
+            return False
+        except Exception as e:
+            print(f"An unexpected error occurred: {e}")
+            return False
+
+    def submit(self, patra_server_url):
+        """
+        Validates and submits the model card to the specified Patra server.
+
+        Args:
+            patra_server_url (str): The Patra server URL where the model card should be submitted.
+
+        Returns:
+            dict: The server's response as a JSON object.
+        """
+        if self.validate():
+            try:
+                self.id = self._get_hash_id(patra_server_url)
+                patra_submit_url = f"{patra_server_url}/upload_mc"
+                headers = {'Content-Type': 'application/json'}
+                response = requests.post(patra_submit_url, json=json.loads(str(self)), headers=headers)
+                response.raise_for_status()
+                return response.json()
+            except requests.exceptions.RequestException as e:
+                print("The Patra Server cannot be reached. Please try again.")
+                return None
+        return {"An error occurred: valid patra_server_url not provided. Unable to upload."}
+
+    def _get_hash_id(self, patra_server_url):
+        """
+        Generates a unique identifier for the model card based on its metadata.
+
+        Args:
+            patra_server_url (str): The Patra server URL used to generate the ID.
+
+        Returns:
+            str: A unique hash identifier for the model card.
+        """
+        combined_string = f"{self.name}:{self.version}:{self.author}"
+        try:
+            if patra_server_url:
+                patra_hash_url = f"{patra_server_url}/get_hash_id"
+                headers = {'Content-Type': 'application/json'}
+                response = requests.get(patra_hash_url, params={"combined_string": combined_string}, headers=headers)
+                response.raise_for_status()
+                return response.json()
+            else:
+                return hashlib.sha256(combined_string.encode()).hexdigest()
+        except requests.exceptions.RequestException as e:
+            print("Could not connect to the Patra Server, generating the ID locally")
+            return hashlib.sha256(combined_string.encode()).hexdigest()
+
+    def save(self, file_location):
+        """
+        Saves the model card as a JSON file to the specified location.
+
+        Args:
+            file_location (str): The path where the model card JSON file will be saved.
+
+        Returns:
+            None
+        """
+        with open(file_location, 'w') as json_file:
+            json_file.write(str(self))
+
+
+class ModelCardJSONEncoder(JSONEncoder):
+    """
+    Custom JSON Encoder for ModelCard to handle complex objects.
+
+    Methods:
+        default: Serializes non-serializable fields.
+    """
+
+    def default(self, obj):
+        if isinstance(obj, (ModelCard, Metric, AIModel, ExplainabilityAnalysis, BiasAnalysis)):
+            return obj.__dict__
+        return super().default(obj)

patra_toolkit/schema/schema.json

@@ -0,0 +1,274 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "Patra Model Card",
+  "type": "object",
+   "commonFields": {
+    "urlOrDoi": {
+      "oneOf": [
+        {
+          "pattern": "^(https?|ftp):\\/\\/[^\\s/$.?#].[\\S]*$",
+          "description": "Must be a valid URL"
+        },
+        {
+          "pattern": "^10\\.\\d{4,9}/[^\\s]+$",
+          "description": "or valid DOI"
+        },
+        {
+          "enum": [""],
+          "description": "or empty"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    }
+  },
+  "properties": {
+    "id": {
+      "type": "string",
+      "description": "The id of the Model Card"
+    },
+    "name": {
+      "type": "string",
+      "description": "The name of the Model Card"
+    },
+    "version": {
+      "type": "string",
+      "description": "The version of the Model Card"
+    },
+    "short_description": {
+      "type": "string",
+      "description": "A brief description of the Model Card"
+    },
+    "full_description": {
+      "type": "string",
+      "description": "A comprehensive description of the Model Card"
+    },
+    "keywords": {
+      "type": "string",
+      "description": "Keywords for the model card"
+    },
+    "author": {
+      "type": "string",
+      "description": "The author or creator of the Model Card"
+    },
+    "input_data": {
+      "$ref": "#/commonFields/urlOrDoi"
+    },
+    "input_type": {
+      "type": "string",
+      "description": "Type of the input data"
+    },
+    "output_data": {
+      "$ref": "#/commonFields/urlOrDoi"
+    },
+    "foundational_model": {
+      "description": "If a foundational model has been used, provide the model ID",
+      "type": "string"
+        },
+   "category": {
+    "type": "string",
+    "enum": [
+        "classification",
+        "regression",
+        "clustering",
+        "anomaly detection",
+        "dimensionality reduction",
+        "reinforcement learning",
+        "natural language processing",
+        "computer vision",
+        "recommendation systems",
+        "time series forecasting",
+        "graph learning",
+        "graph neural networks",
+        "generative modeling",
+        "transfer learning",
+        "self-supervised learning",
+        "semi-supervised learning",
+        "unsupervised learning",
+        "causal inference",
+        "multi-task learning",
+        "metric learning",
+        "density estimation",
+        "multi-label classification",
+        "ranking",
+        "structured prediction",
+        "neural architecture search",
+        "sequence modeling",
+        "embedding learning",
+        "other"
+    ]
+    },
+    "documentation": {
+      "type": "string",
+      "description": "URL for documentation if available"
+    },
+    "ai_model": {
+      "description": "Description of the AI Model",
+      "type": [
+        "object"
+      ],
+      "properties": {
+        "name": {
+          "description": "The name of the model.",
+          "type": "string"
+        },
+        "version": {
+          "type": "string",
+          "description": "The version of the Model"
+        },
+        "description": {
+          "description": "Description of the AI Model.",
+          "type": "string"
+        },
+        "owner": {
+          "type": "string",
+          "description": "The owner of the Model"
+        },
+        "location": {
+          "description": "Downloadable URL of the model",
+          "type": "string"
+        },
+        "license": {
+          "description": "Licence the model",
+          "type": "string"
+        },
+        "model_structure": {
+          "description": "Structure of the model",
+          "type": "object"
+        },
+        "framework": {
+          "type": "string",
+          "enum": [
+            "sklearn",
+            "tensorflow",
+            "pytorch",
+            "other"
+          ]
+        },
+        "model_type": {
+          "type": "string",
+          "enum": [
+            "cnn",
+            "decision_tree",
+            "dnn",
+            "rnn",
+            "svm",
+            "kmeans",
+            "llm",
+            "random_forest",
+            "lstm",
+            "gnn",
+            "other"
+          ]
+        },
+        "test_accuracy": {
+          "description": "Accuracy of the model for the test data",
+          "type": "number"
+        },
+        "model_metrics": {
+          "description": "Metrics of the AI Model",
+          "type": "array",
+          "items": {
+            "$ref": "#/fields/metrics"
+          }
+        }
+      },
+      "required": [
+        "name",
+        "version",
+        "description",
+        "owner",
+        "location",
+        "license",
+        "framework",
+        "test_accuracy",
+        "model_type"
+      ]
+    },
+    "bias_analysis": {
+      "type": ["object", "null"],
+      "properties": {
+        "demographic_parity_diff": {
+          "type": "number"
+        },
+        "equal_odds_difference": {
+          "type": "number"
+        }
+      }
+    },
+    "xai_analysis": {
+      "description": "Explainability analysis of the Model Card.",
+      "type": [
+        "object",
+        "null"
+      ],
+      "properties": {
+        "bias_metrics": {
+          "description": "Metrics of the bias scanner",
+          "type": "array",
+          "items": {
+            "$ref": "#/fields/metrics"
+          }
+        }
+      }
+    },
+"model_requirements": {
+  "type": [
+    "array",
+    "null"
+  ],
+  "items": {
+    "type": "string"
+  }
+}
+  },
+  "required": [
+    "name",
+    "version",
+    "short_description",
+    "full_description",
+    "author",
+    "keywords",
+    "input_data",
+    "input_type",
+    "output_data",
+    "ai_model"
+  ],
+  "fields": {
+    "metrics": {
+      "type": "object",
+      "properties": {
+        "key": {
+          "description": "Name of the metric",
+          "type": "string"
+        },
+        "value": {
+          "description": "The value of the performance metric.",
+          "type": "string"
+        }
+      }
+    }
+  },
+     "commonFields": {
+    "urlOrDoi": {
+      "oneOf": [
+        {
+          "pattern": "^(https?|ftp):\\/\\/[^\\s/$.?#].[\\S]*$",
+          "description": "Must be a valid URL"
+        },
+        {
+          "pattern": "^10\\.\\d{4,9}/[^\\s]+$",
+          "description": "or valid DOI"
+        },
+        {
+          "enum": [""],
+          "description": "or empty"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    }
+  }
+}

patra_toolkit/shap_xai.py

@@ -0,0 +1,37 @@
+import shap
+import pandas as pd
+import re
+
+class ExplainabilityAnalyser:
+    """
+    Provides automated capture of xai information in a given AI workflow.
+    """
+    def __init__(self, train_dataset, column_names, model):
+        self.dataset = train_dataset
+        self.column_names = column_names
+        self.model = model
+
+    def calculate_xai_features(self, n_features=10, pytorch = False):
+        # calculate the shap values
+        if pytorch:
+            explainer = shap.DeepExplainer(self.model, self.dataset)
+        else:
+            explainer = shap.Explainer(self.model, self.dataset)
+
+        values = explainer(self.dataset)
+        shap_values = values.values
+
+        if len(shap_values.shape) == 3:
+            shap_values = abs(shap_values).mean(axis=2)
+
+        feature_importance_df = pd.DataFrame(shap_values, columns=self.column_names).abs().mean(axis=0)
+        top_features = feature_importance_df.sort_values(ascending=False).head(n_features)
+
+        result_dict = {}
+
+        for name, importance in top_features.items():
+
+            filtered_name = re.sub(r'[^a-zA-Z0-9]', '_', name)
+            result_dict[filtered_name] = float(importance)
+
+        return result_dict

patra_toolkit-0.1.1.dist-info/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Indiana University Board of Trustees.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

patra_toolkit-0.1.1.dist-info/METADATA

@@ -0,0 +1,16 @@
+Metadata-Version: 2.1
+Name: patra-toolkit
+Version: 0.1.1
+Summary: Toolkit for semi-automated modelcard creation for AI/ML models.
+Home-page: https://github.com/Data-to-Insight-Center/patra-toolkit.git
+Author: Data to Insight Center
+Author-email: d2i@iu.edu
+License: BSD-3-Clause
+License-File: LICENSE
+Requires-Dist: jsonschema>4.18.5
+Requires-Dist: fairlearn~=0.11.0
+Requires-Dist: shap~=0.46.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: numpy>2.0.0
+Requires-Dist: requests>2.32.2
+

patra_toolkit-0.1.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+patra_toolkit/__init__.py,sha256=Ivigf1Cp-Lweizl81EkDOIoo-otYDsfCZbNjY3IjoJc,86
+patra_toolkit/fairlearn_bias.py,sha256=Zd-bItKO3GpX-5rQWEM3GApLFUitGWzXkl5y8fvu2mY,1264
+patra_toolkit/patra_model_card.py,sha256=OeELQrcT4cX47hblkozrYUwqQnCmyN0ng0C5IiQa7Yc,13828
+patra_toolkit/shap_xai.py,sha256=QnEx39rtO-V_QS3FyL4gHmejyG_S-mf14IpCNWn3Lzs,1206
+patra_toolkit/schema/schema.json,sha256=dW9GkAGWT4FVXWPBBlGkHB40P6O4RGntXxuiPG4Udd4,6384
+tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tests/test_patra_hash_id.py,sha256=AhQROssw1uFWjEG-asBsS4hQpAH7uflYMAzxw-U89S8,4380
+tests/test_patra_inout_data.py,sha256=36EqCE0f1KWJsRs2oxlLQ_9Ubtp8k6eAtOtA3rjn54o,3628
+tests/test_patra_mc.py,sha256=ucUU5zLu-Fl1PpohDF0j9uSyMkSVkVNoNojrrldp0nY,5553
+patra_toolkit-0.1.1.dist-info/LICENSE,sha256=mvKzUSZOUwnUYJ-kO0fY0JHMNzkSSXML5n4WED8plpc,1524
+patra_toolkit-0.1.1.dist-info/METADATA,sha256=nU5ZSsmNuFtRURCfTbaj1RQ8s_UIbId5KB0HazVhUNE,483
+patra_toolkit-0.1.1.dist-info/WHEEL,sha256=a7TGlA-5DaHMRrarXjVbQagU3Man_dCnGIWMJr5kRWo,91
+patra_toolkit-0.1.1.dist-info/top_level.txt,sha256=bAHB9O6o1rDnP-Cbo3EOXY7wRsi6HS5X5wi9jMuCRaQ,20
+patra_toolkit-0.1.1.dist-info/RECORD,,

patra_toolkit-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.4.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

patra_toolkit-0.1.1.dist-info/top_level.txt

@@ -0,0 +1,2 @@
+patra_toolkit
+tests

tests/test_patra_hash_id.py

@@ -0,0 +1,116 @@
+import hashlib
+import os
+
+import unittest
+from unittest.mock import patch, MagicMock
+import requests
+
+from patra_toolkit import ModelCard, AIModel, BiasAnalysis, ExplainabilityAnalysis, Metric
+
+SCHEMA_JSON = os.path.join(os.path.dirname(__file__), os.pardir,
+                           'patra_toolkit/schema/schema.json')
+
+
+class ModelCardTestCase2(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.aimodel = AIModel(
+            name="DenseNet",
+            version="v0.1",
+            description="DenseNet CNN model",
+            owner="PyTorch",
+            location="pytorch.org",
+            license="testLicence",
+            framework="tensorflow",
+            model_type="cnn",
+            test_accuracy=0.89
+        )
+        self.bias_analysis = BiasAnalysis(0.1, 0.2)
+        self.xai_analysis = ExplainabilityAnalysis("XAI Test 2", [Metric("xai1", 0.5), Metric("xai2", 0.8)])
+        self.mc_save_location = "./test_mc.json"
+
+    @patch('requests.get')
+    def test_get_hash_id_success(self, mock_get):
+        """Test that id is set correctly when server responds successfully."""
+        # Mock the server response to return a hash ID
+        mock_get.return_value = MagicMock(status_code=200)
+        mock_get.return_value.json.return_value = "e6c22bdf9fd3c164a2a9a083fb56fca9328f6ca30f7dcd2ebfc140a7d6f02149"
+
+        model_card = ModelCard(
+            name="icicle-camera-traps",
+            version="0.1",
+            short_description="Camera Traps CNN inference model card",
+            full_description="Camera Traps CNN full descr inference model card",
+            keywords="cnn, pytorch, icicle",
+            author="Joe",
+            input_data="",
+            input_type="image",
+            output_data="",
+            category="classification",
+            ai_model=self.aimodel,
+            bias_analysis=self.bias_analysis,
+            xai_analysis=self.xai_analysis
+        )
+
+        model_card.id = model_card._get_hash_id("http://127.0.0.1:5002")
+        self.assertEqual(model_card.id, "e6c22bdf9fd3c164a2a9a083fb56fca9328f6ca30f7dcd2ebfc140a7d6f02149")
+        print("Success case id:", model_card.id)
+
+    @patch('requests.get')
+    def test_get_hash_id_server_down(self, mock_get):
+        """Test hash generation when server is down."""
+        # Simulate a server failure
+        mock_get.side_effect = requests.exceptions.RequestException("Server is down")
+
+        model_card = ModelCard(
+            name="icicle-camera-traps",
+            version="0.1",
+            short_description="Camera Traps CNN inference model card",
+            full_description="Camera Traps CNN full descr inference model card",
+            keywords="cnn, pytorch, icicle",
+            author="Joe",
+            input_data="",
+            input_type="image",
+            output_data="",
+            category="classification",
+            ai_model=self.aimodel,
+            bias_analysis=self.bias_analysis,
+            xai_analysis=self.xai_analysis
+        )
+        model_card.id = model_card._get_hash_id("http://127.0.0.1:5002")
+
+        self.assertIsNotNone(model_card.id)
+        print("Server down case id:", model_card.id)
+
+    @patch('requests.get')
+    def test_generate_hash_without_base_url(self, mock_get):
+        """Test hash generation when no base URL is provided."""
+
+        mock_get.assert_not_called()
+
+        model_card = ModelCard(
+            name="icicle-camera-traps",
+            version="0.1",
+            short_description="Camera Traps CNN inference model card",
+            full_description="Camera Traps CNN full descr inference model card",
+            keywords="cnn, pytorch, icicle",
+            author="Joe",
+            input_data="",
+            input_type="image",
+            output_data="",
+            category="classification",
+            ai_model=self.aimodel,
+            bias_analysis=self.bias_analysis,
+            xai_analysis=self.xai_analysis
+        )
+
+        model_card.id = model_card._get_hash_id(None)
+        # Generate the expected hash value
+        combined_string = f"{model_card.name}:{model_card.version}:{model_card.author}"
+        expected_hash = hashlib.sha256(combined_string.encode()).hexdigest()
+
+        self.assertEqual(model_card.id, expected_hash)
+        print("Generated hash id without base_url:", model_card.id)
+
+if __name__ == '__main__':
+        unittest.main()

tests/test_patra_inout_data.py

@@ -0,0 +1,112 @@
+import os
+import unittest
+
+from patra_toolkit import ModelCard, AIModel, BiasAnalysis, ExplainabilityAnalysis, Metric
+
+SCHEMA_JSON = os.path.join(os.path.dirname(__file__), os.pardir,
+                           'patra_toolkit/schema/schema.json')
+
+
+class ModelCardTestCase2(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.aimodel = AIModel(
+            name="DenseNet",
+            version="v0.1",
+            description="DenseNet CNN model",
+            owner="PyTorch",
+            location="pytorch.org",
+            license="testLicence",
+            framework="tensorflow",
+            model_type="cnn",
+            test_accuracy=0.89
+        )
+        self.bias_analysis = BiasAnalysis(0.1, 0.2)
+        self.xai_analysis = ExplainabilityAnalysis("XAI Test 2", [Metric("xai1", 0.5), Metric("xai2", 0.8)])
+
+        self.mc = ModelCard(
+            name="icicle-camera-traps",
+            version="0.1",
+            short_description="Camera Traps CNN inference model card",
+            full_description="Camera Traps CNN full descr inference model card",
+            keywords="cnn, pytorch, icicle",
+            author="Joe",
+            input_data="",
+            input_type="image",
+            output_data="",
+            category="classification",
+            ai_model=self.aimodel,
+            bias_analysis=self.bias_analysis,
+            xai_analysis=self.xai_analysis
+        )
+        self.mc_save_location = "./test_mc.json"
+
+
+    def test_input_data_empty(self):
+        """
+        This function tests whether the input data is empty.
+        """
+        is_valid = self.mc.validate()
+        self.assertTrue(is_valid)
+
+    def test_input_data_doi(self):
+        """
+        This function tests whether the input data is doi.
+        """
+        self.mc.input_data = "10.5281/zenodo.11179653"
+        is_valid = self.mc.validate()
+        self.assertTrue(is_valid)
+
+    def test_input_data_url(self):
+        """
+        This function tests whether the input data is url.
+        """
+        self.mc.input_data = "https://archive.ics.uci.edu/dataset/2/adult"
+        is_valid = self.mc.validate()
+        self.assertTrue(is_valid)
+
+    def test_input_data_string(self):
+        """
+        This function tests whether the input data is string.
+        """
+        self.mc.input_data = "cifar10"
+        is_valid = self.mc.validate()
+        self.assertFalse(is_valid)
+
+    def test_output_data_empty(self):
+        """
+        This function tests whether the output data is empty.
+        """
+        self.mc.input_data = "10.5281/zenodo.11179653"
+        is_valid = self.mc.validate()
+        self.assertTrue(is_valid)
+
+    def test_output_data_doi(self):
+        """
+        This function tests whether the output data is doi.
+        """
+        self.mc.input_data = "10.5281/zenodo.11179653"
+        self.mc.output_data = "10.5281/zenodo.11179653"
+        is_valid = self.mc.validate()
+        self.assertTrue(is_valid)
+
+    def test_output_data_url(self):
+        """
+        This function tests whether the output data is url.
+        """
+        self.mc.input_data = "10.5281/zenodo.11179653"
+        self.mc.output_data = "https://archive.ics.uci.edu/dataset/2/adult"
+        is_valid = self.mc.validate()
+        self.assertTrue(is_valid)
+
+    def test_output_data_string(self):
+        """
+        This function tests whether the output data is string.
+        """
+        self.mc.input_data = "10.5281/zenodo.11179653"
+        self.mc.output_data = "hcifar10"
+        is_valid = self.mc.validate()
+        self.assertFalse(is_valid)
+
+if __name__ == '__main__':
+        unittest.main()

tests/test_patra_mc.py

@@ -0,0 +1,159 @@
+import os
+import json
+import unittest
+
+from jsonschema import validate
+
+from patra_toolkit import ModelCard, AIModel, BiasAnalysis, ExplainabilityAnalysis, Metric, ModelCardJSONEncoder
+
+SCHEMA_JSON = os.path.join(os.path.dirname(__file__), os.pardir,
+                           'patra_toolkit/schema/schema.json')
+
+
+class ModelCardTestCase(unittest.TestCase):
+
+    def setUp(self) -> None:
+
+        self.aimodel = AIModel(
+            name="DenseNet",
+            version="v0.1",
+            description="DenseNet CNN model",
+            owner="PyTorch",
+            location="pytorch.org",
+            license="testLicence",
+            framework="tensorflow",
+            model_type="cnn",
+            test_accuracy=0.89
+        )
+        self.bias_analysis = BiasAnalysis(0.1, 0.2)
+        self.xai_analysis = ExplainabilityAnalysis("XAI Test 2", [Metric("xai1", 0.5), Metric("xai2", 0.8)])
+
+        self.mc = ModelCard(
+            name="icicle-camera-traps",
+            version="0.1",
+            short_description="Camera Traps CNN inference model card",
+            full_description="Camera Traps CNN full descr inference model card",
+            keywords="cnn, pytorch, icicle",
+            author="Joe",
+            input_data="https://archive.ics.uci.edu/dataset/2/adult",
+            input_type="image",
+            output_data="https://archive.ics.uci.edu/dataset/2/adult",
+            category="classification",
+            ai_model=self.aimodel,
+            bias_analysis=self.bias_analysis,
+            xai_analysis=self.xai_analysis
+        )
+        self.mc_save_location = "./test_mc.json"
+
+    def test_json_serialization(self):
+        """
+        This tests the serialization of the Model Card object.
+        :return:
+        """
+        # Convert the dataclass object to JSON string using the custom encoder
+        mc_json = json.dumps(self.mc, cls=ModelCardJSONEncoder, indent=2)
+
+        # Parse the JSON string back to a dictionary
+        parsed_mc = json.loads(mc_json)
+
+        self.assertEqual(parsed_mc['name'], "icicle-camera-traps")
+        self.assertEqual(parsed_mc['ai_model']['name'], "DenseNet")
+        self.assertEqual(parsed_mc['bias_analysis']['demographic_parity_difference'], 0.1)
+
+    def test_schema_compatibility(self):
+        """
+        This tests whether the python classes are inline with the provided model card schema.
+        :return:
+        """
+        # Convert the dataclass object to JSON string using the custom encoder
+        mc_json = json.dumps(self.mc, cls=ModelCardJSONEncoder, indent=4)
+        with open(SCHEMA_JSON, 'r') as schema_file:
+            schema = json.load(schema_file)
+        try:
+            validate(json.loads(mc_json), schema)
+        except:
+            self.fail("Validation Error!")
+
+    def test_validate(self):
+        """
+        This tests the validate function.
+        :return:
+        """
+        is_valid = self.mc.validate()
+        self.assertTrue(is_valid)
+
+    def test_framework_enum(self):
+        """
+        This tests the validate function.
+        :return:
+        """
+        mc = ModelCard(
+            name="icicle-camera-traps",
+            version="0.1",
+            short_description="Camera Traps CNN inference model card",
+            full_description="Camera Traps CNN full descr inference model card",
+            keywords="cnn, pytorch, icicle",
+            author="Joe",
+            input_data="https://archive.ics.uci.edu/dataset/2/adult",
+            input_type="image",
+            output_data="https://archive.ics.uci.edu/dataset/2/adult",
+            category="classification"
+        )
+        aimodel = AIModel(
+            name="DenseNet",
+            version="v0.1",
+            description="DenseNet CNN model",
+            owner="PyTorch",
+            location="pytorch.org",
+            license="testLicence",
+            framework="keras",
+            model_type="dnn",
+            test_accuracy=0.89
+        )
+        bias_analysis = BiasAnalysis(0.1, 0.2)
+        xai_analysis = ExplainabilityAnalysis("XAI Test 2", [Metric("xai1", 0.5), Metric("xai2", 0.8)])
+
+        mc.ai_model = aimodel
+        mc.bias_analysis = bias_analysis
+
+        is_valid = mc.validate()
+        self.assertFalse(is_valid)
+
+    def test_json_conversion(self):
+        """
+        This tests data insertion via converting it to json, converting back to python.
+        :return:
+        """
+
+        # Convert the dataclass object to JSON string using the custom encoder
+        mc_json = json.dumps(self.mc, cls=ModelCardJSONEncoder, indent=4)
+
+        # convert the json string back to python model
+        mc_converted = json.loads(mc_json)
+
+        self.assertEqual(mc_converted['input_data'], "https://archive.ics.uci.edu/dataset/2/adult")
+
+    def test_json_save(self):
+        """
+        This tests if the MC save as json works
+        :return:
+        """
+        self.mc.save(self.mc_save_location)
+
+        # file existing verification
+        self.assertTrue(os.path.exists(self.mc_save_location), "File not found")
+
+        # Verify the content in the file is same as the saved data
+        with open(self.mc_save_location, 'r') as json_file:
+            saved_data = json.load(json_file)
+
+        self.assertEqual(saved_data['name'], self.mc.name, "Saved JSON data doesn't match the original Model Card")
+
+    def tearDown(self):
+        # Removing the saved file for the model card
+        if os.path.exists(self.mc_save_location):
+            os.remove(self.mc_save_location)
+
+
+if __name__ == '__main__':
+    unittest.main()