SCUDO 0.1.0__tar.gz

scudo-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: SCUDO
+Version: 0.1.0
+Summary: Shared Core Utilities for DataHub Operations
+Author: CNIC
+Requires-Python: >=3.12
+Requires-Dist: pandas
+Requires-Dist: adbc-driver-manager
+Requires-Dist: adbc-driver-flightsql

scudo-0.1.0/README.md

@@ -0,0 +1,4 @@
+# SCUDO - Shared Core Utilities for DataHub Operations
+
+
+

scudo-0.1.0/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "SCUDO"
+version = "0.1.0"
+description = "Shared Core Utilities for DataHub Operations"
+authors = [{ name = "CNIC" }]
+requires-python = ">=3.12"
+
+dependencies = [
+    "pandas",
+    "adbc-driver-manager",
+    "adbc-driver-flightsql"
+]
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]

scudo-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

scudo-0.1.0/src/SCUDO.egg-info/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: SCUDO
+Version: 0.1.0
+Summary: Shared Core Utilities for DataHub Operations
+Author: CNIC
+Requires-Python: >=3.12
+Requires-Dist: pandas
+Requires-Dist: adbc-driver-manager
+Requires-Dist: adbc-driver-flightsql

scudo-0.1.0/src/SCUDO.egg-info/SOURCES.txt

@@ -0,0 +1,19 @@
+README.md
+pyproject.toml
+src/SCUDO.egg-info/PKG-INFO
+src/SCUDO.egg-info/SOURCES.txt
+src/SCUDO.egg-info/dependency_links.txt
+src/SCUDO.egg-info/requires.txt
+src/SCUDO.egg-info/top_level.txt
+src/cudo/__init__.py
+src/cudo/doris/__init__.py
+src/cudo/doris/client.py
+src/cudo/doris/pymysql.py
+src/cudo/minio/__init__.py
+src/cudo/minio/minio.py
+src/cudo/spark/__init__.py
+src/cudo/spark/spark.py
+src/cudo/utils/__init__.py
+src/cudo/utils/clinic.py
+src/cudo/utils/logging.py
+src/cudo/utils/vcf.py

scudo-0.1.0/src/SCUDO.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

scudo-0.1.0/src/SCUDO.egg-info/requires.txt

@@ -0,0 +1,3 @@
+pandas
+adbc-driver-manager
+adbc-driver-flightsql

scudo-0.1.0/src/SCUDO.egg-info/top_level.txt

@@ -0,0 +1 @@
+cudo

scudo-0.1.0/src/cudo/doris/client.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+# Doris Flight SQL Client (ADBC)
+
+Python client for interacting with Apache Doris using Arrow Flight SQL via ADBC.
+
+This class provides:
+- Simple connection handling
+- Query execution
+- Optimized DataFrame fetching (`fetch_df`)
+- Optional Arrow-based and partition-based methods
+- Timing and logging utilities
+
+## Requirements
+
+```bash
+pip install adbc_driver_manager adbc_driver_flightsql pandas
+```
+
+## Example
+    client = DorisFlightSQLClient(
+        uri="grpc://127.0.0.1:9090",
+        username="root",
+        password=""
+    )
+
+    df = client.query("SELECT * FROM my_table LIMIT 1000;")
+    print(df.head())
+
+    client.close()
+'''
+
+import adbc_driver_manager
+import adbc_driver_flightsql.dbapi as flight_sql
+import pandas as pd
+from datetime import datetime
+from typing import Optional
+
+
+
+class DorisFlightSQLClient:
+    '''
+    Apache Doris Flight SQL client using ADBC (DB-API 2.0).
+    This client wraps the ADBC Flight SQL driver and provides optimized
+    methods for querying data into pandas DataFrames.
+
+    Parameters
+    ----------
+    uri : str
+        Flight SQL endpoint (e.g., "grpc://127.0.0.1:9090").
+
+    username : str
+        Database username.
+
+    password : str
+        Database password.
+
+    Attributes
+    ----------
+    conn : flight_sql.Connection
+        Active database connection.
+
+    cursor : flight_sql.Cursor
+        Cursor used for executing queries.
+
+    Notes
+    -----
+    - Uses `fetch_df()` by default for best performance.
+    - Compatible with Apache Doris Arrow Flight SQL service.
+    '''
+
+    def __init__(self, uri: str, username: str, password: str) -> None:
+        self.uri = uri
+        self.db_kwargs = {
+            adbc_driver_manager.DatabaseOptions.USERNAME.value: username,
+            adbc_driver_manager.DatabaseOptions.PASSWORD.value: password,
+        }
+
+        self.conn: Optional[flight_sql.Connection] = None
+        self.cursor: Optional[flight_sql.Cursor] = None
+
+        self._connect()
+
+
+    # ------------------------------------------------------------------
+    # Connection Handling
+    # ------------------------------------------------------------------
+
+    def _connect(self) -> None:        
+        '''
+        Establish a connection and create a cursor.
+        '''
+        try:
+            self.conn = flight_sql.connect(uri=self.uri, db_kwargs=self.db_kwargs)
+            self.cursor = self.conn.cursor()
+        except Exception as e:
+            raise RuntimeError("Connection failed") from e
+
+    def close(self) -> None:
+        '''
+        Close cursor and connection.
+        '''
+        try:
+            if self.cursor:
+                self.cursor.close()
+        finally:
+            if self.conn:
+                self.conn.close()
+
+
+    # ------------------------------------------------------------------
+    # Core Query Method
+    # ------------------------------------------------------------------
+
+    def query(self, sql: str, params: dict | None = None) -> pd.DataFrame:
+        '''
+        Execute a SQL query and return a pandas DataFrame.
+
+        This method uses `fetch_df()` (ADBC optimized path),
+        which is faster than converting from Arrow manually.
+
+        Parameters
+        ----------
+        sql : str
+            SQL query string.
+
+        params : dict, optional
+            Parameters to safely inject into SQL.
+
+        Returns
+        -------
+        pandas.DataFrame
+            Query result as a DataFrame.
+
+        Example
+        -------
+        >>> df = client.query("SELECT * FROM table LIMIT 100;")
+        
+        '''
+        if not self.cursor:
+            raise RuntimeError("Cursor is not initialized.")
+
+        sql = self._render_sql(sql, params=params)
+
+        start_time = datetime.now()
+
+        try:
+            self.cursor.execute(sql)
+            df = self.cursor.fetch_df()
+        except Exception as e:
+            raise RuntimeError("Query execution failed") from e
+
+        duration = datetime.now() - start_time
+
+        print("\n##################")
+        print(f"Query executed in: {duration}")
+        print(df.info(memory_usage="deep"))
+
+        return df
+
+
+    # ------------------------------------------------------------------
+    # Alternative Methods
+    # ------------------------------------------------------------------
+
+    def query_arrow(self, sql: str, params: dict | None = None) -> pd.DataFrame:
+        '''
+        Execute query using Arrow fetch, then convert to pandas.
+
+        Slower than `query()` but useful for debugging or Arrow workflows.
+
+        Parameters
+        ----------
+        sql : str
+            SQL query.
+
+        params : dict, optional
+            Parameters to safely inject into SQL.
+
+        Returns
+        -------
+        pandas.DataFrame
+        '''
+        sql = self._render_sql(sql, params=params)
+
+        start_time = datetime.now()
+
+        try:
+            self.cursor.execute(sql)
+            arrow_data = self.cursor.fetchallarrow()
+            df = arrow_data.to_pandas()
+        except Exception as e:
+            raise RuntimeError("Arrow query execution failed") from e
+
+        duration = datetime.now() - start_time
+
+        print("\n##################")
+        print(f"Arrow fetch in: {duration}, bytes: {arrow_data.nbytes}")
+        print(df.info(memory_usage="deep"))
+
+        return df
+
+
+    def query_partition(self, sql: str, params: dict | None = None) -> pd.DataFrame:
+        '''
+        Execute query using partitioned reads.
+
+        Useful for parallel processing of large datasets.
+
+        Parameters
+        ----------
+        sql : str
+            SQL query.
+            
+        params : dict, optional
+            Parameters to safely inject into SQL.
+
+        Returns
+        -------
+        pandas.DataFrame
+        '''
+        sql = self._render_sql(sql, params=params)
+
+        start_time = datetime.now()
+
+        try:
+            partitions, schema = self.cursor.adbc_execute_partitions(sql)
+            # Example: read first partition (extendable to parallel processing)
+            self.cursor.adbc_read_partition(partitions[0])
+            arrow_data = self.cursor.fetchallarrow()
+            df = arrow_data.to_pandas()
+        except Exception as e:
+            raise RuntimeError("Partition query execution failed") from e
+
+        duration = datetime.now() - start_time
+
+        print("\n##################")
+        print(f"Partitions: {len(partitions)}, time: {duration}")
+        print(df.info(memory_usage="deep"))
+
+        return df
+
+
+    # ------------------------------------------------------------------
+    # Utility Methods
+    # ------------------------------------------------------------------
+
+    def _render_sql(self, sql: str, params: dict | None = None) -> str:
+        '''
+        Render SQL using simple parameter substitution.
+
+        Parameters
+        ----------
+        sql : str
+            SQL query with `{param}` placeholders.
+            
+        params : dict, optional
+            Dictionary of parameters.
+
+        Returns
+        -------
+        str
+            Rendered SQL string.
+        '''
+        if params:
+            safe_params = {}
+            for k, v in params.items():
+                if isinstance(v, (int, float)):
+                    safe_params[k] = v
+                elif isinstance(v, str):
+                    safe_params[k] = "'" + v.replace("'", "''") + "'"
+                else:
+                    raise ValueError(f"Unsafe parameter type for {k}: {type(v)}")
+        try:
+            return sql.format(**safe_params)
+        except KeyError as e:
+            raise ValueError(f"Missing SQL parameter: {e}") from e
+
+
+    def execute(self, sql: str) -> None:
+        '''
+        Execute a SQL statement without returning results.
+
+        Useful for DDL or INSERT operations.
+
+        Parameters
+        ----------
+        sql : str
+        '''
+        if not self.cursor:
+            raise RuntimeError("Cursor is not initialized.")
+
+        print(f"\n### Executing SQL ###\n{sql}")
+        self.cursor.execute(sql)
+
+
+    def __enter__(self):
+        ''' Context manager entry.'''
+        return self
+
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        ''' Context manager exit (auto-close).'''
+        if exc_type:
+            print(f"[DorisFlightSQLClient] Exception: {exc_val}")
+        self.close()
+        return False  # always propagate errors
+