systemlink-cli 1.3.1__py3-none-any.whl

slcli/examples/spec-compliance-notebooks/notebooks/SpecfileExtractionAndIngestion.ipynb

@@ -0,0 +1,912 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import math\n",
+    "import os\n",
+    "from enum import Enum\n",
+    "from http import HTTPStatus\n",
+    "from typing import Callable\n",
+    "\n",
+    "import backoff\n",
+    "import pandas as pd\n",
+    "import requests\n",
+    "import scrapbook as sb\n",
+    "from pandas import DataFrame, ExcelFile\n",
+    "from requests import Response\n",
+    "from requests.exceptions import HTTPError\n",
+    "from requests.packages.urllib3.exceptions import InsecureRequestWarning\n",
+    "\n",
+    "requests.packages.urllib3.disable_warnings(InsecureRequestWarning)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "api_key = os.getenv(\"SYSTEMLINK_API_KEY\")\n",
+    "systemlink_uri = os.getenv(\"SYSTEMLINK_HTTP_URI\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### HTTP Status codes and constants\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MAX_HTTP_RETRIES = 6\n",
+    "TIMEOUT_IN_SECONDS = 60\n",
+    "CREATE_SPECS_BATCH_SIZE = 1000\n",
+    "MAXIMUM_SPEC_LIMIT = 10000\n",
+    "VERIFY_SSL_CERTIFICATE = False\n",
+    "HTTP_RETRY_CODES = [\n",
+    "    HTTPStatus.TOO_MANY_REQUESTS,\n",
+    "    HTTPStatus.INTERNAL_SERVER_ERROR,\n",
+    "    HTTPStatus.BAD_GATEWAY,\n",
+    "    HTTPStatus.SERVICE_UNAVAILABLE,\n",
+    "    HTTPStatus.GATEWAY_TIMEOUT\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### HTTP Routes\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class HttpRouteConstants:\n",
+    "    TM_BASE_ROUTE = \"/nitestmonitor\"\n",
+    "    POST_QUERY_PRODUCTS = \"/v2/query-products\"\n",
+    "    FILE_BASE_ROUTE = \"/nifile\"\n",
+    "    AVAILABLE_FILES_ROUTE = \"/v1/service-groups/Default/files\"\n",
+    "    SPECS_BASE_ROUTE = \"/nispec/v1\"\n",
+    "    CREATE_SPECS_ROUTE = \"/specs\"\n",
+    "    DELETE_SPECS_ROUTE = \"/delete-specs\"\n",
+    "    QUERY_SPECS_ROUTE = \"/query-specs\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Error Messages\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class ErrorMessages:\n",
+    "    EMPTY_SPEC_ID = \"Spec ID cannot be empty\"\n",
+    "    EMPTY_TYPE = \"Spec Type cannot be empty\"\n",
+    "    EMPTY_PRODUCT_ID = \"Product ID cannot be empty\"\n",
+    "    MULTIPLE_NOTEBOOKS_SELECTED = \"This notebook is designed to operate on one file at a time\"\n",
+    "    WRONG_FILE_EXTENSION = \"File extension is not xlsx\"\n",
+    "    MAXIMUM_SPEC_INGESTION_LIMIT = (f\"Notebook only supports extraction and ingestion of {MAXIMUM_SPEC_LIMIT} specs\")\n",
+    "    CONDITION_EXTRACTION_ERROR = \"Error when extracting condition\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Input Parameters\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "papermill": {
+     "parameters": {
+      "file_ids": [],
+      "product_ids": []
+     }
+    },
+    "systemlink": {
+     "namespaces": [],
+     "parameters": [
+      {
+       "display_name": "file_ids",
+       "id": "file_ids",
+       "type": "string[]"
+      }
+     ],
+     "version": 2
+    },
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "file_ids = [\"\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### API URL's\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "delete_spec_url = f\"{systemlink_uri}{HttpRouteConstants.SPECS_BASE_ROUTE}{HttpRouteConstants.DELETE_SPECS_ROUTE}\"\n",
+    "create_spec_url = f\"{systemlink_uri}{HttpRouteConstants.SPECS_BASE_ROUTE}{HttpRouteConstants.CREATE_SPECS_ROUTE}\"\n",
+    "query_spec_url = f\"{systemlink_uri}{HttpRouteConstants.SPECS_BASE_ROUTE}{HttpRouteConstants.QUERY_SPECS_ROUTE}\"\n",
+    "query_products_url = f\"{systemlink_uri}{HttpRouteConstants.TM_BASE_ROUTE}{HttpRouteConstants.POST_QUERY_PRODUCTS}\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### API Utility functions\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@backoff.on_exception(backoff.expo, HTTPError, max_tries=MAX_HTTP_RETRIES, giveup=lambda e: e.response.status_code not in HTTP_RETRY_CODES)\n",
+    "def retry_request(callable_function: Callable) -> Response:\n",
+    "    response = callable_function()\n",
+    "    response.raise_for_status()\n",
+    "\n",
+    "    return response\n",
+    "\n",
+    "\n",
+    "def create_get_request(url: str, headers: object = {}) -> Response:\n",
+    "    default_headers = {'x-ni-api-key': api_key}\n",
+    "    headers = {**default_headers, **headers}\n",
+    "\n",
+    "    return retry_request(lambda: requests.get(url, headers=headers, verify=VERIFY_SSL_CERTIFICATE, timeout=TIMEOUT_IN_SECONDS))\n",
+    "\n",
+    "\n",
+    "def create_post_request(url: str, body, headers: object = {}) -> Response:\n",
+    "    default_headers = {\n",
+    "        \"accept\": \"application/json\",\n",
+    "        \"Content-Type\": \"application/json\",\n",
+    "        \"x-ni-api-key\": api_key,\n",
+    "    }\n",
+    "    headers = {**default_headers, **headers}\n",
+    "\n",
+    "    return retry_request(lambda: requests.post(url, data=body, headers=headers, verify=VERIFY_SSL_CERTIFICATE, timeout=TIMEOUT_IN_SECONDS))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Constants"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "TEMPLATE_SHEET = \"SpecTemplate\"\n",
+    "SPEC_FILE_MAPPING = {\n",
+    "    \"Spec ID\": \"specId\",\n",
+    "    \"Category\": \"category\",\n",
+    "    \"Block\": \"block\",\n",
+    "    \"Spec Symbol\": \"symbol\",\n",
+    "    \"Spec Name\": \"name\",\n",
+    "    \"Min\": \"min\",\n",
+    "    \"Typical\": \"typical\",\n",
+    "    \"Max\": \"max\",\n",
+    "    \"Unit\": \"unit\",\n",
+    "}\n",
+    "\n",
+    "\n",
+    "class ColumnTypes(Enum):\n",
+    "    STD = \"STD\"\n",
+    "    COND = \"COND\"\n",
+    "    INF = \"INF\"\n",
+    "\n",
+    "\n",
+    "class SpecDataKeys:\n",
+    "    SPEC_ID = \"specId\"\n",
+    "    PRODUCT_ID = \"productId\"\n",
+    "    NAME = \"name\"\n",
+    "    CATEGORY = \"category\"\n",
+    "    TYPE = \"type\"\n",
+    "    SYMBOL = \"symbol\"\n",
+    "    BLOCK = \"block\"\n",
+    "    UNIT = \"unit\"\n",
+    "    LIMIT = \"limit\"\n",
+    "    CONDITIONS = \"conditions\"\n",
+    "    WORKSPACE = \"workspace\"\n",
+    "    PROPERTIES = \"properties\"\n",
+    "\n",
+    "\n",
+    "class SpecLimitKeys:\n",
+    "    MIN = \"min\"\n",
+    "    MAX = \"max\"\n",
+    "    TYPICAL = \"typical\"\n",
+    "\n",
+    "\n",
+    "class SpecConditionKeys:\n",
+    "    CONDITION_TYPE = \"conditionType\"\n",
+    "    NAME = \"name\"\n",
+    "    VALUE = \"value\"\n",
+    "    DISCRETE = \"discrete\"\n",
+    "    RANGE = \"range\"\n",
+    "    MIN = \"min\"\n",
+    "    MAX = \"max\"\n",
+    "    UNIT = \"unit\"\n",
+    "\n",
+    "\n",
+    "class SpecConditionTypeValues:\n",
+    "    STRING = \"STRING\"\n",
+    "    NUMERIC = \"NUMERIC\"\n",
+    "\n",
+    "\n",
+    "class SpecType:\n",
+    "    PARAMETRIC = \"parametric\"\n",
+    "    FUNCTIONAL = \"functional\"\n",
+    "\n",
+    "\n",
+    "class ApiBodyKeys:\n",
+    "    PRODUCT_IDS = \"productIds\"\n",
+    "    TAKE = \"take\"\n",
+    "    IDS = \"ids\"\n",
+    "    SPECS = \"specs\"\n",
+    "    FILTER = \"filter\"\n",
+    "    FILE_IDS = \"fileIds\"\n",
+    "    CONTAINS = \"Contains\"\n",
+    "\n",
+    "\n",
+    "class ApiResponseKeys:\n",
+    "    ID = \"id\"\n",
+    "    ERROR = \"error\"\n",
+    "    SPEC_ID = \"specId\"\n",
+    "    SPECS = \"specs\"\n",
+    "    CREATED_SPECS = \"createdSpecs\"\n",
+    "    AVAILABLE_FILES = \"availableFiles\"\n",
+    "    WORKSPACE = \"workspace\"\n",
+    "    PRODUCTS = \"products\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Get File content and write into a file\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_file_content(file_id: str):\n",
+    "    download_file_url = f\"{systemlink_uri}{HttpRouteConstants.FILE_BASE_ROUTE}{HttpRouteConstants.AVAILABLE_FILES_ROUTE}/{file_id}/data\"\n",
+    "\n",
+    "    headers = {'X-NI-API-KEY': api_key}\n",
+    "    download_resp = create_get_request(\n",
+    "        download_file_url,\n",
+    "        headers\n",
+    "    )\n",
+    "\n",
+    "    download_resp.raise_for_status()\n",
+    "\n",
+    "    return download_resp"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def write_file_content_into_file(response: Response | None):\n",
+    "    filename = response.headers['content-disposition'].split('\"')[1::-1][0]\n",
+    "    file_extension = response.headers['content-disposition'].split('\"')[1::-1][0].split(\".\")[-1]\n",
+    "\n",
+    "    if (file_extension != 'xlsx'):\n",
+    "        raise Exception(ErrorMessages.WRONG_FILE_EXTENSION)\n",
+    "\n",
+    "    with open(filename, 'wb') as file:\n",
+    "        file.write(response.content)\n",
+    "\n",
+    "    return filename"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if len(file_ids) != 1:\n",
+    "    raise Exception(ErrorMessages.MULTIPLE_NOTEBOOKS_SELECTED)\n",
+    "\n",
+    "file_id = file_ids[0]\n",
+    "\n",
+    "file_content_response = get_file_content(file_id)\n",
+    "\n",
+    "filename = write_file_content_into_file(file_content_response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Get Workspace ID for the file\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_workspace_id(file_id: str):\n",
+    "    get_file_properties_url = f\"{systemlink_uri}{HttpRouteConstants.FILE_BASE_ROUTE}{HttpRouteConstants.AVAILABLE_FILES_ROUTE}?id={file_id}\"\n",
+    "    resp_json = create_get_request(get_file_properties_url)\n",
+    "    resp = resp_json.json()\n",
+    "    workspace_id = resp[ApiResponseKeys.AVAILABLE_FILES][0][ApiResponseKeys.WORKSPACE]\n",
+    "    return str(workspace_id)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Get Product ID from File ID\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_product_id_for_file(file_id: str):\n",
+    "    body = {ApiBodyKeys.FILTER: f'{ApiBodyKeys.FILE_IDS}.{ApiBodyKeys.CONTAINS}(\"{file_id}\")'}\n",
+    "    payload = json.dumps(body)\n",
+    "    resp_json = create_post_request(query_products_url, payload)\n",
+    "    resp = resp_json.json()\n",
+    "    product_id = resp[ApiResponseKeys.PRODUCTS][0][ApiResponseKeys.ID]\n",
+    "    return str(product_id)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "product_id = get_product_id_for_file(file_id)\n",
+    "workspace_id = get_workspace_id(file_id)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Spec extraction utility functions\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def process_condition_values(input_string: str, unit: str | None, sheet_name: str, column, row):\n",
+    "    try:\n",
+    "        if input_string is not None:\n",
+    "            input_string = input_string.strip()\n",
+    "\n",
+    "        if not input_string:\n",
+    "            return None\n",
+    "\n",
+    "        if input_string.startswith('[') and input_string.endswith(']'):\n",
+    "            input_string = input_string[1:-1]\n",
+    "            discrete = []\n",
+    "            range_min = None\n",
+    "            range_max = None\n",
+    "\n",
+    "            if '..' in input_string:\n",
+    "                values = input_string.split('..')\n",
+    "\n",
+    "                if (values[0] == '' or values[-1] == ''):\n",
+    "                    if (values[0] == ''):\n",
+    "                        range_max = float(values[-1])\n",
+    "                    elif (values[-1] == ''):\n",
+    "                        range_min = float(values[0])\n",
+    "                    if (len(values) >= 3):\n",
+    "                        for index, value in enumerate(values):\n",
+    "                            if index != 0 and index != len(values)-1:\n",
+    "                                discrete.append(float(value))\n",
+    "                else:\n",
+    "                    range_min = float(values[0])\n",
+    "                    range_max = float(values[-1])\n",
+    "\n",
+    "                    if (len(values) >= 3):\n",
+    "                        for index, value in enumerate(values):\n",
+    "                            if index != 0 and index != len(values)-1:\n",
+    "                                discrete.append(float(value))\n",
+    "                return {\n",
+    "                    SpecConditionKeys.CONDITION_TYPE: SpecConditionTypeValues.NUMERIC,\n",
+    "                    SpecConditionKeys.DISCRETE: discrete,\n",
+    "                    SpecConditionKeys.RANGE: [\n",
+    "                        {\n",
+    "                            SpecConditionKeys.MIN: range_min,\n",
+    "                            SpecConditionKeys.MAX: range_max\n",
+    "                        }\n",
+    "                    ],\n",
+    "                    SpecConditionKeys.UNIT: unit\n",
+    "                }\n",
+    "\n",
+    "            else:\n",
+    "                if ',' in input_string:\n",
+    "                    discrete = [float(value)\n",
+    "                                for value in input_string.split(',')]\n",
+    "                else:\n",
+    "                    discrete = [float(input_string)]\n",
+    "                return {\n",
+    "                    SpecConditionKeys.CONDITION_TYPE: SpecConditionTypeValues.NUMERIC,\n",
+    "                    SpecConditionKeys.DISCRETE: discrete,\n",
+    "                    SpecConditionKeys.UNIT: unit\n",
+    "                }\n",
+    "        elif (not (input_string.startswith('[') and input_string.endswith(']'))):\n",
+    "            discrete = [str(value).strip() for value in input_string.split(',')]\n",
+    "            return {\n",
+    "                SpecConditionKeys.CONDITION_TYPE: SpecConditionTypeValues.STRING,\n",
+    "                SpecConditionKeys.DISCRETE: discrete\n",
+    "            }\n",
+    "        else:\n",
+    "            return None\n",
+    "    except Exception:\n",
+    "        raise SystemExit(\n",
+    "            f\"{ErrorMessages.CONDITION_EXTRACTION_ERROR} in {sheet_name}, {column}, row {row}\")\n",
+    "\n",
+    "\n",
+    "def extract_condition_unit(input_string: str):\n",
+    "    start = input_string.find('(')\n",
+    "    end = input_string.find(')')\n",
+    "    if start != -1 and end != -1 and start < end:\n",
+    "        return input_string[start+1:end]\n",
+    "    else:\n",
+    "        return None\n",
+    "\n",
+    "\n",
+    "def validate_column_types(df_type: pd.DataFrame, sheet_name: str):\n",
+    "    for idx, col in enumerate(df_type.columns):\n",
+    "        if (idx == 0):\n",
+    "            continue\n",
+    "        if isinstance(df_type.at[0, col], str):\n",
+    "            col_value = df_type.at[0, col].upper()\n",
+    "        else:\n",
+    "            raise Exception(\n",
+    "                f\"Column '{col}' Name cannot be empty in {sheet_name}\")\n",
+    "        if col_value not in [column_types.value for column_types in ColumnTypes]:\n",
+    "            raise Exception(f\"Column '{col}' has an invalid value: '{col_value}'. \"\n",
+    "                            f\"Allowed values are: {', '.join(column_type.value for column_type in ColumnTypes)}\")\n",
+    "\n",
+    "    for idx, col in enumerate(df_type.columns):\n",
+    "        col_value = df_type.at[1, col]\n",
+    "        if (idx == 0):\n",
+    "            continue\n",
+    "        if isinstance(df_type.at[1, col], str):\n",
+    "            if (col_value in dict.fromkeys(SPEC_FILE_MAPPING) and df_type.at[0, col].upper() != ColumnTypes.STD.value):\n",
+    "                raise Exception(f\"Column '{col_value}' should be {ColumnTypes.STD.value} type, \"\n",
+    "                                f\"instead it is {df_type.at[0, col].upper()} type in {sheet_name}\")\n",
+    "            if (col_value not in dict.fromkeys(SPEC_FILE_MAPPING) and df_type.at[0, col].upper() == ColumnTypes.STD.value):\n",
+    "                raise Exception(f\"Column '{col}' has an invalid value: '{col_value}' in {sheet_name}. \"\n",
+    "                                f\"Allowed values for STD columns are: {', '.join(SPEC_FILE_MAPPING.keys())}\")\n",
+    "        else:\n",
+    "            raise Exception(\n",
+    "                f\"Column '{col}' Name cannot be empty in {sheet_name}\")\n",
+    "\n",
+    "\n",
+    "def seperate_column_types(df_type: DataFrame):\n",
+    "    \"\"\"Separate columns by type and return their header names for use with pandas usecols.\"\"\"\n",
+    "    standard_columns = [\n",
+    "        df_type.iloc[1, idx] for idx, col in enumerate(df_type.columns) if df_type.at[0, col].upper() == ColumnTypes.STD.value]\n",
+    "    condition_columns = [\n",
+    "        df_type.iloc[1, idx] for idx, col in enumerate(df_type.columns) if df_type.at[0, col].upper() == ColumnTypes.COND.value]\n",
+    "    property_columns = [\n",
+    "        df_type.iloc[1, idx] for idx, col in enumerate(df_type.columns) if df_type.at[0, col].upper() == ColumnTypes.INF.value]\n",
+    "\n",
+    "    return standard_columns, condition_columns, property_columns\n",
+    "\n",
+    "\n",
+    "def extract_column_names(df_type: DataFrame, column_names: list[str]):\n",
+    "    \"\"\"Extract unique column names from the list.\"\"\"\n",
+    "    column_names = list(dict.fromkeys(column_names))\n",
+    "    return column_names\n",
+    "\n",
+    "\n",
+    "def construct_dataframe_for_column_type(excel_file: ExcelFile, sheet_name: str, column_info, column_type: str):\n",
+    "    \"\"\"Construct dataframe for a specific column type using actual column header names.\"\"\"\n",
+    "    column_names = column_info[sheet_name][f\"{column_type}\"]\n",
+    "    if not column_names:\n",
+    "        # Return empty dataframe if no columns of this type\n",
+    "        return pd.DataFrame()\n",
+    "    \n",
+    "    # Use actual column names (which become the headers when header=3)\n",
+    "    data_frame = excel_file.parse(\n",
+    "        sheet_name,\n",
+    "        header=3,\n",
+    "        usecols=column_names,\n",
+    "        dtype=str,\n",
+    "        keep_default_na=False\n",
+    "    ).rename(columns=SPEC_FILE_MAPPING).replace('', None)\n",
+    "\n",
+    "    return data_frame\n",
+    "\n",
+    "\n",
+    "def seperate_columns(excel_file: ExcelFile, sheet_names: list):\n",
+    "    column_info = {}\n",
+    "    for sheet_name in sheet_names:\n",
+    "        data_frame = excel_file.parse(sheet_name, skiprows=2, nrows=2, header=None, dtype=object)\n",
+    "        standard_columns, condition_columns, property_columns = seperate_column_types(data_frame)\n",
+    "\n",
+    "        column_info.update({\n",
+    "            sheet_name: {\n",
+    "                ColumnTypes.STD.value: standard_columns,\n",
+    "                ColumnTypes.COND.value: condition_columns,\n",
+    "                ColumnTypes.INF.value: property_columns\n",
+    "            }\n",
+    "        })\n",
+    "\n",
+    "    return column_info, standard_columns, condition_columns, property_columns\n",
+    "\n",
+    "\n",
+    "def validate_spec_count(excel_file: ExcelFile, sheet_names: list):\n",
+    "    total_specs = 0\n",
+    "\n",
+    "    for sheet_name in sheet_names:\n",
+    "        data_frame = excel_file.parse(sheet_name, header=3)\n",
+    "        total_specs += len(data_frame)\n",
+    "\n",
+    "    if (total_specs > 10000):\n",
+    "        raise Exception(f\"{ErrorMessages.MAXIMUM_SPEC_INGESTION_LIMIT}, total specs in the excel file is {total_specs}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_base_spec_data(df_type: DataFrame, sheet_name: str, category: str, type: str):\n",
+    "    specs = []\n",
+    "    for idx, spec_row in enumerate(df_type.to_dict(\"records\")):\n",
+    "        if not product_id:\n",
+    "            raise Exception(ErrorMessages.EMPTY_PRODUCT_ID)\n",
+    "        if not spec_row.get(SpecDataKeys.SPEC_ID):\n",
+    "            raise Exception(\n",
+    "                f\"{ErrorMessages.EMPTY_SPEC_ID} in {sheet_name} in row {idx}\")\n",
+    "        if isinstance(type, str):\n",
+    "            pass\n",
+    "        elif (math.isnan(type)):\n",
+    "            raise Exception(f\"{ErrorMessages.EMPTY_TYPE} in {sheet_name}\")\n",
+    "        spec_data = {\n",
+    "            SpecDataKeys.PRODUCT_ID: product_id,\n",
+    "            SpecDataKeys.SPEC_ID: spec_row.get(SpecDataKeys.SPEC_ID),\n",
+    "            SpecDataKeys.NAME: spec_row.get(SpecDataKeys.NAME) if isinstance(spec_row.get(SpecDataKeys.NAME), str) else None,\n",
+    "            SpecDataKeys.CATEGORY: category if isinstance(category, str) else None,\n",
+    "            SpecDataKeys.TYPE: type,\n",
+    "            SpecDataKeys.SYMBOL: spec_row.get(SpecDataKeys.SYMBOL) if isinstance(spec_row.get(SpecDataKeys.SYMBOL), str) else None,\n",
+    "            SpecDataKeys.BLOCK: spec_row.get(SpecDataKeys.BLOCK) if isinstance(spec_row.get(SpecDataKeys.BLOCK), str) else None,\n",
+    "            SpecDataKeys.UNIT: spec_row.get(SpecDataKeys.UNIT) if isinstance(spec_row.get(SpecDataKeys.UNIT), str) else None,\n",
+    "            SpecDataKeys.CONDITIONS: [],\n",
+    "            SpecDataKeys.WORKSPACE: workspace_id if workspace_id else None\n",
+    "        }\n",
+    "        if type.lower() == SpecType.PARAMETRIC:\n",
+    "            try:\n",
+    "                spec_data[SpecDataKeys.LIMIT] = {\n",
+    "                    SpecLimitKeys.MIN: float(spec_row.get(SpecLimitKeys.MIN)) if spec_row.get(SpecLimitKeys.MIN) else None,\n",
+    "                    SpecLimitKeys.MAX: float(spec_row.get(SpecLimitKeys.MAX)) if spec_row.get(SpecLimitKeys.MAX) else None,\n",
+    "                    SpecLimitKeys.TYPICAL: float(spec_row.get(SpecLimitKeys.TYPICAL)) if spec_row.get(SpecLimitKeys.TYPICAL) else None\n",
+    "                }\n",
+    "            except ValueError as e:\n",
+    "                raise Exception(\n",
+    "                    f\"Invalid numeric value in {sheet_name} row {idx + 1}. \"\n",
+    "                    f\"Min, Max, and Typical columns must contain valid numbers. \"\n",
+    "                    f\"Error: {str(e)}\")\n",
+    "        specs.append(spec_data)\n",
+    "    return specs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_spec_conditions(df_type: DataFrame, column_names: list, sheet_name: str, specs: list, index: int):\n",
+    "    for idx, row in enumerate(df_type.to_dict(\"records\")):\n",
+    "        for column in column_names:\n",
+    "            if (row.get(column)):\n",
+    "                condition = {}\n",
+    "                condition[SpecConditionKeys.NAME] = column.split('(')[0].strip()\n",
+    "                unit = extract_condition_unit(column)\n",
+    "                condition[SpecConditionKeys.VALUE] = process_condition_values(\n",
+    "                    row[str(column)], unit, sheet_name, column, idx)\n",
+    "                specs[index][SpecDataKeys.CONDITIONS].append(condition)\n",
+    "        index = index + 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_spec_properties(df_type: DataFrame, column_names: list, specs: list, index: int):\n",
+    "    for row in df_type.to_dict(\"records\"):\n",
+    "        property = {column: row.get(str(column)) for column in column_names if row.get(str(column)) is not None}\n",
+    "        if property:\n",
+    "            specs[index][SpecDataKeys.PROPERTIES] = property\n",
+    "            index = index + 1"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### SCM Template to SLE Format extraction logic\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def generate_spec_data():\n",
+    "\n",
+    "    xl = pd.ExcelFile(filename)\n",
+    "    sheet_names = [\n",
+    "        sheet_name for sheet_name in xl.sheet_names if sheet_name != TEMPLATE_SHEET\n",
+    "    ]\n",
+    "    column_info = {}\n",
+    "    specs = []\n",
+    "    index = 0\n",
+    "\n",
+    "    validate_spec_count(xl, sheet_names)\n",
+    "\n",
+    "    # Construct dataframe and generate specs\n",
+    "    for sheet_name in sheet_names:\n",
+    "\n",
+    "        df_type = xl.parse(sheet_name, skiprows=2, nrows=2, header=None, dtype=object)\n",
+    "\n",
+    "        if (df_type.empty):\n",
+    "            continue\n",
+    "\n",
+    "        validate_column_types(df_type, sheet_name)\n",
+    "\n",
+    "        standard_columns, condition_columns, property_columns = seperate_column_types(df_type)\n",
+    "\n",
+    "        column_info.update({\n",
+    "            sheet_name: {\n",
+    "                ColumnTypes.STD.value: standard_columns,\n",
+    "                ColumnTypes.COND.value: condition_columns,\n",
+    "                ColumnTypes.INF.value: property_columns\n",
+    "            }\n",
+    "        }\n",
+    "        )\n",
+    "        all_condition_column_names = extract_column_names(df_type, condition_columns)\n",
+    "        all_properties_column_names = extract_column_names(df_type, property_columns)\n",
+    "\n",
+    "        df_meta = xl.parse(sheet_name, nrows=2, usecols=\"B\", header=None)\n",
+    "        category = df_meta.values[0][0]\n",
+    "        type = df_meta.values[1][0]\n",
+    "\n",
+    "        df_standard = construct_dataframe_for_column_type(xl, sheet_name, column_info, ColumnTypes.STD.value)\n",
+    "        df_condition = construct_dataframe_for_column_type(xl, sheet_name, column_info, ColumnTypes.COND.value)\n",
+    "        df_properties = construct_dataframe_for_column_type(xl, sheet_name, column_info, ColumnTypes.INF.value)\n",
+    "\n",
+    "        specs.extend(generate_base_spec_data(df_standard, sheet_name, category, type))\n",
+    "\n",
+    "        generate_spec_conditions(df_condition, all_condition_column_names, sheet_name, specs, index)\n",
+    "\n",
+    "        generate_spec_properties(df_properties, all_properties_column_names, specs, index)\n",
+    "\n",
+    "        index += len(df_standard)\n",
+    "\n",
+    "    return specs"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Query Specs\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def query_specs(product_ids: str):\n",
+    "    body = json.dumps({\n",
+    "        ApiBodyKeys.PRODUCT_IDS: [\n",
+    "            product_ids\n",
+    "        ],\n",
+    "        ApiBodyKeys.TAKE: MAXIMUM_SPEC_LIMIT\n",
+    "    }, indent=4)\n",
+    "    response = create_post_request(query_spec_url, body)\n",
+    "    response = response.json()\n",
+    "    return response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Delete Specs\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def delete_specs(query_spec_ids: list):\n",
+    "    body = json.dumps({\n",
+    "        ApiBodyKeys.IDS: query_spec_ids\n",
+    "    }, indent=4)\n",
+    "    response = create_post_request(delete_spec_url, body)\n",
+    "    return response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create Specs\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_specs(spec_data: list):\n",
+    "    complete_response = []\n",
+    "    for i in range(0, len(spec_data), CREATE_SPECS_BATCH_SIZE):\n",
+    "        specs_batch = spec_data[i:i+CREATE_SPECS_BATCH_SIZE]\n",
+    "        payload = json.dumps({\n",
+    "            ApiBodyKeys.SPECS: specs_batch\n",
+    "        }, indent=4)\n",
+    "        response = create_post_request(create_spec_url, payload)\n",
+    "        complete_response.append(response.json())\n",
+    "    return complete_response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "spec_data = generate_spec_data()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Call API's and process responses\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "error_messages = []\n",
+    "query_spec_ids = []\n",
+    "created_spec_ids = []\n",
+    "\n",
+    "query_spec_response = query_specs(product_id)\n",
+    "\n",
+    "query_spec_ids = [spec[ApiResponseKeys.ID]\n",
+    "                  for spec in query_spec_response.get(ApiResponseKeys.SPECS, [])]\n",
+    "\n",
+    "delete_spec_response = delete_specs(query_spec_ids) if len(query_spec_ids) > 0 else None\n",
+    "create_spec_response = create_specs(spec_data)\n",
+    "\n",
+    "error_messages = [response[ApiResponseKeys.ERROR] for response in create_spec_response\n",
+    "                  if ApiResponseKeys.ERROR in response]\n",
+    "\n",
+    "if error_messages:\n",
+    "    error = True\n",
+    "\n",
+    "created_spec_ids = [spec[ApiResponseKeys.SPEC_ID] for response in create_spec_response\n",
+    "                    for spec in response.get(ApiResponseKeys.CREATED_SPECS, [])]\n",
+    "\n",
+    "created_spec_ids_df = pd.DataFrame(\n",
+    "    {'Created Spec Id': created_spec_ids}).to_dict(orient='records')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Print the execution result\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if not error_messages:\n",
+    "    sb.glue(\"Result\", \"All specs uploaded successfully\")\n",
+    "    sb.glue(\"Created Specs\", created_spec_ids_df)\n",
+    "else:\n",
+    "    sb.glue(\"Error\", error_messages)\n",
+    "    if created_spec_ids:\n",
+    "        sb.glue(\"Created Specs\", created_spec_ids_df)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}