sm_data_ml_utils 0.1.0__tar.gz

sm_data_ml_utils-0.1.0/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022-2023
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

sm_data_ml_utils-0.1.0/PKG-INFO

@@ -0,0 +1,116 @@
+Metadata-Version: 2.1
+Name: sm_data_ml_utils
+Version: 0.1.0
+Summary: Common Python tools and utilities for ML work
+License: MIT
+Author: Shuming Peh
+Author-email: shuming.peh@gmail.com
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: PyYAML (==6.0)
+Requires-Dist: Werkzeug (>=3.0.3,<4.0.0)
+Requires-Dist: aiobotocore (>=2.8.0,<3.0.0)
+Requires-Dist: appdirs (==1.4.4)
+Requires-Dist: attrs (>=22.2.0,<23.0.0)
+Requires-Dist: black (>=22.6.0,<23.0.0)
+Requires-Dist: boto3 (>=1.33.5,<2.0.0)
+Requires-Dist: botocore (>=1.34.15,<2.0.0)
+Requires-Dist: certifi (>=2023.7.22,<2024.0.0)
+Requires-Dist: cfgv (==3.2.0)
+Requires-Dist: coverage (==5.4)
+Requires-Dist: databricks-sql-connector (>=3.3.0,<4.0.0)
+Requires-Dist: distlib (>=0.3.8,<0.4.0)
+Requires-Dist: filelock (>=3.14.0,<4.0.0)
+Requires-Dist: flake8 (>=4.0.1,<5.0.0)
+Requires-Dist: identify (==1.5.13)
+Requires-Dist: iniconfig (==1.1.1)
+Requires-Dist: isort (>=5.10.1,<6.0.0)
+Requires-Dist: joblib (==1.3.2)
+Requires-Dist: mccabe (==0.6.1)
+Requires-Dist: mlflow (==2.22.2)
+Requires-Dist: mock (>=4.0.3,<5.0.0)
+Requires-Dist: moto (>=4.2.7,<5.0.0)
+Requires-Dist: mypy-extensions (==0.4.3)
+Requires-Dist: nodeenv (>=1.5.0,<2.0.0)
+Requires-Dist: numpy (==1.23.5)
+Requires-Dist: packaging (>=23.1,<24.0)
+Requires-Dist: pandas (==1.4.4)
+Requires-Dist: pluggy (==1.5.0)
+Requires-Dist: polling (==0.3.2)
+Requires-Dist: py (>=1.11.0,<2.0.0)
+Requires-Dist: pyarrow (>=14.0.0,<15.0.0)
+Requires-Dist: pydantic (==1.10.24)
+Requires-Dist: pyparsing (==2.4.7)
+Requires-Dist: pytest (>=8.3.3,<9.0.0)
+Requires-Dist: pytest-cov (>=3.0.0,<4.0.0)
+Requires-Dist: pytest-custom-exit-code (==0.3.0)
+Requires-Dist: regex (>=2024.9.11,<2025.0.0)
+Requires-Dist: requests (>=2.32.0,<3.0.0)
+Requires-Dist: responses (==0.23.1)
+Requires-Dist: s3fs (>=2023.10.0,<2024.0.0)
+Requires-Dist: six (==1.15.0)
+Requires-Dist: toml (==0.10.2)
+Requires-Dist: torch (==2.3.1)
+Requires-Dist: typed-ast (>=1.5.3,<2.0.0)
+Requires-Dist: typing-extensions (>=4.4.0,<5.0.0)
+Description-Content-Type: text/markdown
+
+# data-ml-utils
+A utility python package that covers the common libraries we use
+
+## Installation
+This is an open source library hosted on pypi. Run the following command to install the library.
+```
+pip install data-ml-utils --upgrade
+```
+
+## Documentation
+Head over to https://data-ml-utils.readthedocs.io/en/latest/index.html# to read our library documentation
+
+## Feature
+### Pyathena client initialisation
+Almost one liner
+```python
+import os
+from data_ml_utils.pyathena_client.client import PyAthenaClient
+
+os.environ["AWS_ACCESS_KEY_ID"] = "xxx"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "xxx" # pragma: allowlist secret
+os.environ["S3_BUCKET"] = "xxx"
+
+pyathena_client = PyAthenaClient()
+```
+![Pyathena client initialisation](docs/_static/initialise_pyathena_client.png)
+
+### Pyathena query
+Almost one liner
+```python
+query = """
+    SELECT
+        *
+    FROM
+        dev.example_pyathena_client_table
+    LIMIT 10
+"""
+
+df_raw = pyathena_client.query_as_pandas(final_query=query)
+```
+![Pyathena query](docs/_static/query_pyathena_client.png)
+
+### MLflow utils
+Visit [link](https://data-ml-utils.readthedocs.io/en/latest/index.html#mlflow-utils)
+
+### More to Come
+* You suggest, raise a feature request issue and we will review!
+
+## Tutorials
+### Pyathena
+There is a jupyter notebook to show how to use the package utility package for `pyathena`: [notebook](tutorials/[TUTO]%20pyathena.ipynb)
+
+### MLflow utils
+There is a jupyter notebook to show how to use the package utility package for `mlflow_databricks`: [notebook](tutorials/[TUTO]%20mlflow_databricks.ipynb)
+

sm_data_ml_utils-0.1.0/README.md

@@ -0,0 +1,54 @@
+# data-ml-utils
+A utility python package that covers the common libraries we use
+
+## Installation
+This is an open source library hosted on pypi. Run the following command to install the library.
+```
+pip install data-ml-utils --upgrade
+```
+
+## Documentation
+Head over to https://data-ml-utils.readthedocs.io/en/latest/index.html# to read our library documentation
+
+## Feature
+### Pyathena client initialisation
+Almost one liner
+```python
+import os
+from data_ml_utils.pyathena_client.client import PyAthenaClient
+
+os.environ["AWS_ACCESS_KEY_ID"] = "xxx"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "xxx" # pragma: allowlist secret
+os.environ["S3_BUCKET"] = "xxx"
+
+pyathena_client = PyAthenaClient()
+```
+![Pyathena client initialisation](docs/_static/initialise_pyathena_client.png)
+
+### Pyathena query
+Almost one liner
+```python
+query = """
+    SELECT
+        *
+    FROM
+        dev.example_pyathena_client_table
+    LIMIT 10
+"""
+
+df_raw = pyathena_client.query_as_pandas(final_query=query)
+```
+![Pyathena query](docs/_static/query_pyathena_client.png)
+
+### MLflow utils
+Visit [link](https://data-ml-utils.readthedocs.io/en/latest/index.html#mlflow-utils)
+
+### More to Come
+* You suggest, raise a feature request issue and we will review!
+
+## Tutorials
+### Pyathena
+There is a jupyter notebook to show how to use the package utility package for `pyathena`: [notebook](tutorials/[TUTO]%20pyathena.ipynb)
+
+### MLflow utils
+There is a jupyter notebook to show how to use the package utility package for `mlflow_databricks`: [notebook](tutorials/[TUTO]%20mlflow_databricks.ipynb)

sm_data_ml_utils-0.1.0/pyproject.toml

@@ -0,0 +1,60 @@
+[tool.poetry]
+name = "sm_data_ml_utils"
+version = "0.1.0"
+description = "Common Python tools and utilities for ML work"
+authors = ["Shuming Peh <shuming.peh@gmail.com>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.10,<3.13"
+aiobotocore = "^2.8.0"
+appdirs = "1.4.4"
+attrs = "^22.2.0"
+black = "^22.6.0"
+boto3 = "^1.33.5"
+botocore = "^1.34.15"
+certifi = "^2023.7.22"
+cfgv = "3.2.0"
+coverage = "5.4"
+databricks-sql-connector = "^3.3.0"
+distlib = "^0.3.8"
+filelock = "^3.14.0"
+flake8 = "^4.0.1"
+identify = "1.5.13"
+iniconfig = "1.1.1"
+isort = "^5.10.1"
+joblib= "1.3.2"
+mccabe = "0.6.1"
+mlflow = "2.22.2"
+mock = "^4.0.3"
+moto = "^4.2.7"
+mypy-extensions = "0.4.3"
+nodeenv = "^1.5.0"
+numpy = "1.23.5"
+packaging = "^23.1"
+pandas = "1.4.4"
+pluggy = "1.5.0"
+polling = "0.3.2"
+py = "^1.11.0"
+pyarrow = "^14.0.0"
+pydantic = "1.10.24"
+pyparsing = "2.4.7"
+pytest = "^8.3.3"
+pytest-cov = "^3.0.0"
+pytest-custom-exit-code = "0.3.0"
+PyYAML = "6.0"
+regex = "^2024.9.11"
+requests = "^2.32.0"
+responses = "0.23.1"
+s3fs = "^2023.10.0"
+six = "1.15.0"
+toml = "0.10.2"
+torch = "2.3.1"
+typed-ast = "^1.5.3"
+typing-extensions = "^4.4.0"
+Werkzeug = "^3.0.3"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

sm_data_ml_utils-0.1.0/sm_data_ml_utils/core/config.py

@@ -0,0 +1,25 @@
+from typing import Dict
+
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+
+    AWS_DEFAULT_REGION: str
+    AWS_DEFAULT_REGION = "ap-southeast-2"
+    MODEL_SERVING_WORKLOAD_SIZE: str
+    MODEL_SERVING_WORKLOAD_SIZE = "Medium"
+    MODEL_SERVING_SCALE_TO_ZERO: str
+    MODEL_SERVING_SCALE_TO_ZERO = "True"
+    model_func_dict: Dict
+    model_func_dict = {
+        "sk_model": ["sklearn", "code_paths"],
+        "xgb_model": ["xgboost", "code_paths"],
+        "lgb_model": ["lightgbm", "code_paths"],
+        "keras_model": ["keras", "code_paths"],
+        "pytorch_model": ["pytorch", "code_paths"],
+        "python_model": ["pyfunc", "code_path"],
+    }
+
+
+settings = Settings()

sm_data_ml_utils-0.1.0/sm_data_ml_utils/core/databricks_utils.py

@@ -0,0 +1,134 @@
+import datetime
+import importlib
+import pathlib
+from typing import Callable
+from typing import Dict
+from typing import List
+
+import yaml
+
+
+def read_sql(file_path: str) -> str:
+    """
+    reads sql file and return string
+
+    Parameters
+    ----------
+    file_path : str
+        file path of sql file
+
+    Returns
+    -------
+    str
+        query string of sql file
+    """
+    return pathlib.Path(file_path).read_text()
+
+def load_yaml(path: str) -> Dict:
+    """
+    function that loads the settings from the yaml file
+
+    Parameters
+    ----------
+    path: str
+        path to the yaml file
+
+    Returns
+    -------
+    Dict
+        returns a dictionary with the settings
+    """
+    with open(path) as f:
+        settings = yaml.safe_load(f)
+    return settings
+
+
+def get_test_date(datetime_provided: datetime.datetime, days_difference: int) -> int:
+    """
+    function that returns a lookback date from specified days ago
+
+    Parameters
+    ----------
+    datetime_provided: datetime.datetime
+        datetime input, most likely datetime.datetime.now()
+    days_difference: int
+        number days ago we should look back
+
+    Returns
+    -------
+    int
+        returns a date dim key, yyyyMMdd
+    """
+    return int(
+        (datetime_provided - datetime.timedelta(days=days_difference)).strftime(
+            "%Y%m%d"
+        )
+    )
+
+
+def get_function_to_load(function_dict: Dict, file_format: str) -> Callable:
+    """
+    function that retrieves the library function callable
+
+    Parameters
+    ----------
+    function_dict: Dict
+        dictionary that contains the function mapping
+    file_format: str
+        name of file format, pkl, parquet or delta table
+
+    Returns
+    -------
+    Callable
+        function that we intend to use
+    """
+    return getattr(
+        importlib.import_module(function_dict[file_format][0]),
+        function_dict[file_format][1],
+    )
+
+
+def get_target_stage_for_env(env: str) -> str:
+    """
+    function to get corresponding target stage based on running environment
+
+    Parameters
+    ----------
+    env: str
+        running environment "dev", "staging" or "prod"
+
+    Returns
+    -------
+    str
+        target stage
+        "Staging" for "dev" and "staging" env
+        "Production" for "prod" env
+    """
+
+    if not (isinstance(env, str)) or (env.lower() not in ["dev", "staging", "prod"]):
+        raise ValueError("Invalid environment")
+
+    return "Staging" if env.lower() in {"dev", "staging"} else "Production"
+
+
+def get_date_intervals_model_drift(
+    date_int: int,
+) -> List:
+    """
+    function to list of date intervals for model drift
+
+    Parameters
+    ----------
+    date_int: int
+        start date of monitoring in YYYYmmdd
+
+    Returns
+    -------
+    List
+        list of date intervals for model drift monitoring
+    """
+    return_list = []
+    for i in range(1, 5):
+        return_list.append(f"{date_int}_hour_pair_{i}")
+
+    return return_list

sm_data_ml_utils-0.1.0/sm_data_ml_utils/databricks_client/client.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import os
+
+import databricks
+import pandas as pd
+from databricks.sql import connect
+
+
+class DatabricksSQLClient:
+    """
+    Class that handles queries from databricks sql connector
+    Main purpose is to create a connection we can query from databricks sql warehouse
+    """
+
+    def __init__(self):
+        self.engine = self._connect()
+
+    def _connect(self) -> databricks.sql.client.Connection:
+        """
+        create a databricks sql connection
+
+        Returns
+        -------
+        databricks.sql.client.Connection
+            databricks sql warehouse connection engine
+        """
+
+        return connect(
+            server_hostname=os.environ["DATABRICKS_HOST"],
+            http_path=os.environ["DATABRICKS_SQL_PATH"],
+            access_token=os.environ["DATABRICKS_TOKEN"],
+        )
+
+    def query_as_pandas(self, final_query: str) -> pd.DataFrame:
+        """
+        query databricks sqls with databricks connection and store them into pandas
+
+        Parameters
+        ----------
+        final_query : str
+            query to run
+
+        Returns
+        -------
+        pd.DataFrame
+            return of pandas dataframe
+        """
+
+        _cursor = self.engine.cursor()
+        return_values = _cursor.execute(final_query).fetchall()
+
+        return pd.DataFrame(
+            return_values, columns=[col[0] for col in _cursor.description]
+        )