vh-pytoolbox 0.1.0__tar.gz

vh_pytoolbox-0.1.0/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024  Vinayak Hegde
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vh_pytoolbox-0.1.0/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.4
+Name: vh-pytoolbox
+Version: 0.1.0
+Summary: Reusable Python code for different projects
+Author-email: Vinayak Hegde <vinayakvhegde@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2024  Vinayak Hegde
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Utilities
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: msal>=1.25.0
+Requires-Dist: azure-identity>=1.15.0
+Provides-Extra: dev
+Requires-Dist: build>=1.2.2.post1; extra == "dev"
+Requires-Dist: ruff>=0.8.4; extra == "dev"
+Requires-Dist: setuptools>=75.6.0; extra == "dev"
+Requires-Dist: wheel>=0.45.1; extra == "dev"
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Dynamic: license-file
+
+# vh-pytoolbox
+
+My own re-usable Python packages.
+
+## Description
+
+This package provides various utilities for handling file I/O, string processing, CSV manipulation, and logging setup. It is designed to streamline common tasks in data processing and analysis.
+
+## Features
+
+- **Path Initialization**: Functions to initialize and manage project paths.
+- **File I/O**: Utilities for saving and loading JSON metadata.
+- **String Processing**: Functions to convert strings to snake_case and translate text based on mappings.
+- **CSV Handling**: Functions to remove columns from CSV files and transform text columns.
+- **Logging Setup**: Easy setup for logging configuration.
+
+## Installation
+
+You can install the package using pip:
+
+```sh
+pip install git+https://github.com/vinay/vh-repo-python-package.git

vh_pytoolbox-0.1.0/README.md

@@ -0,0 +1,22 @@
+# vh-pytoolbox
+
+My own re-usable Python packages.
+
+## Description
+
+This package provides various utilities for handling file I/O, string processing, CSV manipulation, and logging setup. It is designed to streamline common tasks in data processing and analysis.
+
+## Features
+
+- **Path Initialization**: Functions to initialize and manage project paths.
+- **File I/O**: Utilities for saving and loading JSON metadata.
+- **String Processing**: Functions to convert strings to snake_case and translate text based on mappings.
+- **CSV Handling**: Functions to remove columns from CSV files and transform text columns.
+- **Logging Setup**: Easy setup for logging configuration.
+
+## Installation
+
+You can install the package using pip:
+
+```sh
+pip install git+https://github.com/vinay/vh-repo-python-package.git

vh_pytoolbox-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[project]
+name = "vh-pytoolbox"
+version = "0.1.0"
+description = "Reusable Python code for different projects"
+readme = "README.md"
+license = {file = "LICENSE.txt"}
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Utilities",
+]
+requires-python = ">=3.13"
+authors = [
+    {name = "Vinayak Hegde", email = "vinayakvhegde@gmail.com"},
+]
+dependencies = [
+    "pandas>=2.2.3",
+    "pyyaml>=6.0",
+    "requests>=2.31.0",
+    "msal>=1.25.0",
+    "azure-identity>=1.15.0",
+]
+optional-dependencies = { dev = ["build>=1.2.2.post1", "ruff>=0.8.4", "setuptools>=75.6.0", "wheel>=0.45.1", "pytest>=8.0.0"] }
+
+[build-system]
+requires = ["setuptools>=75.6.0", "wheel>=0.45.1"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.ruff]
+line-length = 88
+target-version = "py313"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "C90", "I", "N", "UP", "YTT", "S", "BLE", "FBT", "B", "A", "COM", "C4", "DTZ", "T10", "DJ", "EM", "EXE", "FA", "ISC", "ICN", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET", "SLF", "SLOT", "SIM", "TID", "TCH", "INT", "ARG", "PTH", "ERA", "PD", "PGH", "PL", "TRY", "FLY", "NPY", "AIR", "PERF", "FURB", "LOG", "RUF"]
+ignore = ["S101", "COM812", "ISC001"]  # Allow assert, trailing comma, etc.
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101"]  # Allow assert in tests

vh_pytoolbox-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

vh_pytoolbox-0.1.0/src/vh_pytoolbox/__init__.py

@@ -0,0 +1 @@
+from . import api_handler, cli, data_transformations, iohandler, utils

vh_pytoolbox-0.1.0/src/vh_pytoolbox/api_handler.py

@@ -0,0 +1,333 @@
+"""
+API handling utilities for Microsoft Graph and Power BI Admin APIs.
+"""
+
+import time
+from collections import defaultdict
+from typing import Any, Dict, Optional
+
+import msal
+import requests
+from azure.identity import ClientSecretCredential
+
+from .utils.logging_setup import setup_logger
+
+
+class MSGraphApi:
+    """
+    A helper class to handle Microsoft Graph API
+    using client credential authentication and automatic pagination.
+    """
+
+    SCOPE = ["https://graph.microsoft.com/.default"]
+    AUTH_URL = "https://login.microsoftonline.com"
+
+    def __init__(
+        self,
+        tenant_id: str,
+        client_id: str,
+        client_secret: str,
+        logger: Optional[Any] = None,
+    ) -> None:
+        self.tenant_id = tenant_id
+        self.client_id = client_id
+        self.client_secret = client_secret
+
+        self.app = msal.ConfidentialClientApplication(
+            client_id=self.client_id,
+            client_credential=self.client_secret,
+            authority=f"{self.AUTH_URL}/{self.tenant_id}",
+        )
+
+        self.logger = logger or setup_logger()
+        if self.logger:
+            self.logger.info("MS GraphApi initialized.")
+
+    def _build_headers(self) -> Dict[str, str]:
+        token = self.app.acquire_token_for_client(scopes=self.SCOPE)["access_token"]
+        return {
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "ConsistencyLevel": "eventual",
+        }
+
+    def get_all_pages(self, url: str) -> Dict[str, Any]:
+        """
+        Standard pagination using @odata.nextLink.
+        """
+        headers = self._build_headers()
+        session = requests.Session()
+        session.headers.update(headers)
+
+        aggregated_items: list = []
+        retryable = {429, 500, 502, 503, 504}
+        next_url = url
+        attempts = 0
+
+        while next_url:
+            response = session.get(next_url)
+
+            if response.status_code in retryable:
+                retry_after = response.headers.get("Retry-After")
+                delay = (
+                    int(retry_after)
+                    if retry_after and retry_after.isdigit()
+                    else min(2**attempts, 60)
+                )
+                if self.logger:
+                    self.logger.warning(
+                        f"{response.status_code} retrying in {delay}s ..."
+                    )
+                time.sleep(delay)
+                attempts += 1
+                if attempts > 5:
+                    raise Exception(f"Too many retries for URL: {next_url}")
+                continue
+
+            if response.status_code != 200:
+                msg = f"GET failed ({response.status_code}): {response.text}"
+                if self.logger:
+                    self.logger.error(msg)
+                raise Exception(msg)
+
+            attempts = 0
+            data = response.json()
+
+            # Append only 'value' items if present and is a list
+            page_items = data.get("value", [])
+            if isinstance(page_items, list) and page_items:
+                aggregated_items.extend(page_items)
+
+            # Track last @odata.nextLink
+            delta_link = data.get("@odata.deltaLink", None)
+            next_url = data.get("@odata.nextLink")
+
+        if self.logger:
+            self.logger.info(
+                f"All pages retrieved successfully. Total items: "
+                f"{len(aggregated_items)}"
+            )
+
+        return {"value": aggregated_items, "delta_link": delta_link}
+
+
+class PowerBIAdminApi:
+    """
+    A helper class for interacting with the Microsoft Power BI Admin APIs
+    using a client credential (service principal) authentication flow.
+    """
+
+    # Class-level constants
+    SCOPE: str = "https://analysis.windows.net/powerbi/api/.default"
+    BASE_URL: str = "https://api.powerbi.com/v1.0/myorg"
+
+    def __init__(
+        self,
+        tenant_id: str,
+        client_id: str,
+        client_secret: str,
+        logger: Optional[Any] = None,
+    ) -> None:
+        """
+        Initialize PowerBI admin helper class.
+
+        :param tenant_id: Azure AD tenant ID
+        :param client_id: Service principal client ID
+        :param client_secret: Service principal client secret
+        :param logger: Optional logger instance
+        """
+        self.credential = ClientSecretCredential(
+            tenant_id=tenant_id, client_id=client_id, client_secret=client_secret
+        )
+
+        self.logger = logger or setup_logger()
+        self.logger.info("PowerBIAdminApi initialized.")
+
+    # ------------------------------------------------------
+    # SINGLE: BUILD HEADERS (GETS TOKEN + FORMS HEADERS)
+    # ------------------------------------------------------
+    def _build_headers(self) -> Dict[str, str]:
+        """
+        Build HTTP request headers with a fresh OAuth2 bearer token.
+
+        :return: A dictionary with Authorization and Content-Type headers.
+        """
+        token = self.credential.get_token(self.SCOPE).token
+        return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+
+    # ------------------------------------------------------
+    # GET REQUEST
+    # ------------------------------------------------------
+    def get_response(self, url: str) -> Dict[str, Any]:
+        """
+        Send a GET request with full pagination and retry handling.
+        Returns a merged JSON response containing all pages.
+
+        :param url: Full request URL
+        :return: JSON response as dict
+        """
+        headers = self._build_headers()
+        resp = requests.get(url, headers=headers)
+
+        if resp.status_code != 200:
+            self.logger.error(f"GET failed ({resp.status_code}): {resp.text}")
+            raise Exception(resp.text)
+
+        data = resp.json()
+        return data
+
+    # ------------------------------------------------------
+    # GET REQUEST - Paginated
+    # ------------------------------------------------------
+    def get_paginated_response(self, url: str) -> Dict[str, Any]:
+        """
+        GET request with full pagination + retry handling.
+        Supports only Power BI REST API pagination via continuationUri.
+        """
+
+        headers = self._build_headers()
+        session = requests.Session()
+        session.headers.update(headers)
+
+        next_url = url
+        attempt = 0
+
+        aggregated = defaultdict(list)
+        retryable = {429, 500, 502, 503, 504}
+
+        while next_url:
+            response = session.get(next_url)
+
+            # Retry for transient errors (429, 5xx)
+            if response.status_code in retryable:
+                retry_after = response.headers.get("Retry-After")
+                delay = (
+                    int(retry_after)
+                    if retry_after and retry_after.isdigit()
+                    else min(2**attempts, 60)
+                )
+
+                self.logger.warning(
+                    f"{response.status_code} received. Retrying in {delay}s… "
+                    f"({attempt+1}/5)"
+                )
+                time.sleep(delay)
+
+                attempt += 1
+                if attempt > 5:
+                    raise Exception(f"Too many retries for URL: {next_url}")
+                continue
+
+            # Hard failure
+            if response.status_code != 200:
+                self.logger.error(
+                    f"GET failed ({response.status_code}): {response.text}"
+                )
+                raise Exception(response.text)
+
+            # Success
+            attempt = 0
+            data = response.json()
+
+            # Merge keys (list keys extended, non-list keys stored last)
+            for key, value in data.items():
+                if isinstance(value, list):
+                    aggregated[key].extend(value)
+                else:
+                    aggregated[key] = value
+
+            # Pagination handling
+            if data.get("continuationUri", None):
+                next_url = data.get("continuationUri")
+            else:
+                next_url = None  # No more pages
+
+        self.logger.info("✅ All pages collected successfully.")
+        return dict(aggregated)
+
+    # ------------------------------------------------------
+    # POST REQUEST – START WORKSPACE SCAN
+    # ------------------------------------------------------
+    def post_request(self, url: str, body: Dict[str, Any]) -> str:
+        """
+        Send a POST request and extract scanId from response.
+
+        :param url: POST URL
+        :param body: JSON request body
+        :return: scanId value
+        """
+        headers = self._build_headers()
+        resp = requests.post(url, json=body, headers=headers)
+
+        if resp.status_code not in (200, 202):
+            self.logger.error(f"POST failed ({resp.status_code}): {resp.text}")
+            raise Exception(resp.text)
+
+        data = resp.json()
+        scan_id = data.get("id") or data.get("scanId")
+
+        if not scan_id:
+            self.logger.error("scanId missing from response.")
+            raise Exception("scanId missing")
+
+        self.logger.info(f"Scan started. scanId={scan_id}")
+        return scan_id
+
+    # ------------------------------------------------------
+    # POLL STATUS
+    # ------------------------------------------------------
+    def poll_scan_status(self, url: str, scan_id: str) -> Dict[str, Any]:
+        """
+        Poll the scan status until completion.
+
+        :param url: Scan status URL
+        :param scan_id: Scan identifier
+        :return: JSON response with final status
+        """
+        poll_interval = 10
+        max_attempts = 60
+        headers = self._build_headers()
+        self.logger.info(f"Polling status for scanId={scan_id}")
+
+        url = f"{url}/{scan_id}"
+        for attempt in range(1, max_attempts + 1):
+            resp = requests.get(url, headers=headers)
+
+            if resp.status_code != 200:
+                self.logger.error(f"Polling failed ({resp.status_code}): {resp.text}")
+                raise Exception(resp.text)
+
+            data = resp.json()
+            status = data.get("status")
+            self.logger.info(f"Attempt {attempt}: status={status}")
+
+            if status in ("Succeeded", "Failed", "Unknown"):
+                return data
+
+            time.sleep(poll_interval)
+
+        raise TimeoutError("Scan polling timed out.")
+
+    # ------------------------------------------------------
+    # GET SCAN RESULT
+    # ------------------------------------------------------
+    def get_scan_result(self, url: str, scan_id: str) -> Dict[str, Any]:
+        """
+        Retrieve the final scan result.
+
+        :param url: Scan result URL
+        :param scan_id: Scan identifier
+        :return: JSON scan result
+        """
+        headers = self._build_headers()
+        self.logger.info(f"Fetching scanResult for scanId={scan_id}")
+        url = f"{url}/{scan_id}"
+        resp = requests.get(url, headers=headers)
+
+        if resp.status_code != 200:
+            self.logger.error(f"Result fetch failed ({resp.status_code}): {resp.text}")
+            raise Exception(resp.text)
+
+        data = resp.json()
+        self.logger.info("Scan result retrieved successfully.")
+        return data

vh_pytoolbox-0.1.0/src/vh_pytoolbox/cli.py

@@ -0,0 +1,66 @@
+"""Simple command-line entry point for dataset anonymization."""
+
+import argparse
+from pathlib import Path
+
+from .data_transformations import anonymization
+from .iohandler.file_io import save_metadata_to_json
+from .utils.config_loader import load_yaml_config
+from .utils.logging_setup import setup_logger
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Anonymize CSV datasets according to a YAML configuration."
+    )
+    parser.add_argument(
+        "--config", "-c", required=True, help="Path to the YAML config file."
+    )
+    parser.add_argument(
+        "--raw", "-r", required=True, help="Path to raw data folder to process."
+    )
+    parser.add_argument(
+        "--out", "-o", default=None, help="Path to output (anonymized) folder."
+    )
+    parser.add_argument("--log", help="Path to log file (default: console).")
+
+    args = parser.parse_args()
+
+    logger = setup_logger(log_file=args.log)
+
+    try:
+        config = load_yaml_config(args.config)
+    except Exception as e:
+        logger.error(f"Failed to load config from {args.config}: {e}")
+        return
+
+    raw_folder = Path(args.raw)
+    if args.out:
+        anonymized_folder = Path(args.out)
+    else:
+        anonymized_folder = raw_folder.parent / "anonymized"
+
+    anonymized_folder.mkdir(parents=True, exist_ok=True)
+
+    logger.info(f"Starting anonymization. raw={raw_folder} out={anonymized_folder}")
+    try:
+        file_meta, col_meta = anonymization.anonymize_source_files(
+            raw_folder=str(raw_folder),
+            anonymized_folder=str(anonymized_folder),
+            config=config,
+            logger=logger,
+        )
+
+        # Save metadata
+        file_meta_path = anonymized_folder / "file_metadata.json"
+        col_meta_path = anonymized_folder / "column_metadata.json"
+        save_metadata_to_json(
+            file_meta, col_meta, str(file_meta_path), str(col_meta_path)
+        )
+        logger.info("Finished anonymization.")
+    except Exception as e:
+        logger.error(f"Anonymization failed: {e}")
+
+
+if __name__ == "__main__":
+    main()

vh_pytoolbox-0.1.0/src/vh_pytoolbox/data_transformations/__init__.py

@@ -0,0 +1 @@
+from . import anonymization