src-connectors 0.1.0__py3-none-any.whl

src_connectors/__init__.py

@@ -0,0 +1,18 @@
+import sys
+from typing import Any
+
+__all__ = ["SQLServerConnector", "SparkConnector"]
+
+
+def __getattr__(name: str) -> Any:
+    if name == "SQLServerConnector":
+        from src_connectors.src_sqlserver.connector import SQLServerConnector
+
+        return SQLServerConnector
+
+    if name == "SparkConnector":
+        from src_connectors.src_spark.connector import SparkConnector
+
+        return SparkConnector
+
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")

src_connectors/src_base/__init__.py

@@ -0,0 +1,3 @@
+from .connector import BaseConnector
+
+__all__ = ["BaseConnector"]

src_connectors/src_base/connector.py

@@ -0,0 +1,31 @@
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+class BaseConnector(ABC):
+    """
+    Abstract base class for database connectors.
+    """
+
+    @abstractmethod
+    def connect(self) -> Any:
+        """Establish a connection to the database."""
+        pass
+
+    @abstractmethod
+    def close(self) -> None:
+        """Close the database connection."""
+        pass
+
+    @abstractmethod
+    def execute_query(
+        self,
+        query: str,
+        params: Optional[tuple] = None,
+        stream: bool = False,
+        batch_size: int = 1000,
+        output_type: str = "list",
+        **kwargs: Any
+    ) -> Any:
+        """Execute a query and return the results."""
+        pass
+

src_connectors/src_spark/__init__.py

@@ -0,0 +1,6 @@
+from src_connectors.src_spark.connector import SparkConnector
+from src_connectors.src_spark.configs.core_config import SparkCoreConfig
+from src_connectors.src_spark.configs.iceberg import SparkIcebergConfig
+from src_connectors.src_spark.configs.jdbc import SparkJdbcConfig
+
+__all__ = ["SparkConnector", "SparkCoreConfig", "SparkIcebergConfig", "SparkJdbcConfig"]

src_connectors/src_spark/configs/__init__.py

@@ -0,0 +1,6 @@
+from src_connectors.src_spark.configs.core_config import SparkCoreConfig
+from src_connectors.src_spark.configs.iceberg import SparkIcebergConfig
+from src_connectors.src_spark.configs.jdbc import SparkJdbcConfig
+from src_connectors.src_spark.configs.base import SparkBaseComponent
+
+__all__ = ["SparkCoreConfig", "SparkIcebergConfig", "SparkJdbcConfig", "SparkBaseComponent"]

src_connectors/src_spark/configs/base.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict
+
+class SparkBaseComponent(ABC):
+    """
+    Abstract base class for Spark components.
+
+    This class defines the interface for components that require
+    specific Spark configurations and external packages.
+    """
+
+    @abstractmethod
+    def get_spark_config(self) -> Dict:
+        """
+        Return Spark configuration settings.
+
+        Returns:
+            dict: A dictionary of Spark configuration keys and values.
+        """
+        pass
+
+    @abstractmethod
+    def get_required_spark_packages(self) -> List:
+        """
+        Return a list of required Spark packages.
+
+        Returns:
+            list: A list of Maven coordinates or package names.
+        """
+        pass

src_connectors/src_spark/configs/core_config.py

@@ -0,0 +1,138 @@
+from typing import Dict, List, Any, Optional
+from pydantic import BaseModel, Field
+
+from src_connectors.src_spark.configs.base import SparkBaseComponent
+from variables.spark import SparkVariables
+
+# Load environment-based defaults
+_spark_defaults = SparkVariables.get_spark_base_config()
+
+
+class SparkCoreConfig(BaseModel, SparkBaseComponent):
+    """
+    Configuration model for core Apache Spark engine settings.
+
+    This class manages Spark core parameters including resource allocation (memory, cores),
+    cluster connectivity, and runtime dependencies. Defaults are initialized from
+    environment variables.
+    """
+
+    # Cluster & Application Settings
+    spark_master: str = Field(
+        default=_spark_defaults.get("SPARK_MASTER") or "local[*]",
+        description="The Spark master URL to connect to."
+    )
+    spark_driver_host: str = Field(
+        default=_spark_defaults.get("SPARK_DRIVER_HOST") or "127.0.0.1",
+        description="Hostname or IP address for the driver to listen on."
+    )
+    spark_driver_bind_address: str = Field(
+        default=_spark_defaults.get("SPARK_DRIVER_BIND_ADDRESS") or "127.0.0.1",
+        description="Address to bind to for the driver."
+    )
+    spark_app_name: str = Field(
+        default=_spark_defaults.get("SPARK_APP_NAME") or "SparkApp",
+        description="The name of the Spark application."
+    )
+
+    # Resource Management
+    spark_executor_memory: str = Field(
+        default=_spark_defaults.get("SPARK_EXECUTOR_MEMORY") or "2g",
+        description="Amount of memory to use per executor process."
+    )
+    spark_executor_cores: int = Field(
+        default=int(_spark_defaults.get("SPARK_EXECUTOR_CORES") or 2),
+        description="The number of cores to use on each executor."
+    )
+    spark_driver_memory: str = Field(
+        default=_spark_defaults.get("SPARK_DRIVER_MEMORY") or "4g",
+        description="Amount of memory to use for the driver process."
+    )
+    spark_driver_cores: int = Field(
+        default=int(_spark_defaults.get("SPARK_DRIVER_CORES") or 2),
+        description="Number of cores to use for the driver process."
+    )
+
+    # Dependencies & Runtime
+    spark_jars: Optional[str] = Field(
+        default=_spark_defaults.get("SPARK_LOCAL_JARS") or "",
+        description="Comma-separated local jar paths."
+    )
+    spark_jars_packages: Optional[str] = Field(
+        default=_spark_defaults.get("SPARK_JARS_PACKAGES") or "",
+        description="Comma-separated Maven coordinates of jars to include."
+    )
+    spark_jars_ivy: str = Field(
+        default=_spark_defaults.get("SPARK_JARS_IVY") or "/tmp/.ivy2",
+        description="Path to local Ivy repository for dependency resolution."
+    )
+
+    # Environment & Versioning
+    spark_version: str = Field(
+        default=_spark_defaults.get("SPARK_VERSION") or "3.0.0",
+        description="Target Spark version."
+    )
+    spark_minor_version: str = Field(
+        default=_spark_defaults.get("SPARK_MINOR_VERSION") or "3.0",
+        description="Target Spark minor version."
+    )
+    spark_local_dir: str = Field(
+        default=_spark_defaults.get("SPARK_LOCAL_DIR") or "/tmp/spark",
+        description="Directory for Spark 'scratch' space (map outputs, RDD spills)."
+    )
+
+    def get_spark_config(self) -> Dict[str, Any]:
+        """
+        Generates a dictionary of Spark configuration properties.
+
+        Returns:
+            Dict[str, Any]: Flat dictionary of 'spark.*' configuration keys and values.
+        """
+        config = {
+            "spark.master": self.spark_master,
+            "spark.app.name": self.spark_app_name,
+            "spark.executor.memory": self.spark_executor_memory,
+            "spark.executor.cores": self.spark_executor_cores,
+            "spark.driver.memory": self.spark_driver_memory,
+            "spark.driver.cores": self.spark_driver_cores,
+            "spark.jars.ivy": self.spark_jars_ivy,
+            "spark.local.dir": self.spark_local_dir,
+        }
+
+        # Handle Driver Host logic:
+        # Force 127.0.0.1 ONLY if running in local mode to avoid RPC errors on dev machines.
+        # In cluster mode (K8s/YARN), we let Spark/Cluster Manager handle it or use provided host.
+        is_local = self.spark_master.startswith("local")
+        if is_local:
+            config["spark.driver.host"] = self.spark_driver_host
+            config["spark.driver.bindAddress"] = self.spark_driver_bind_address
+        else:
+            # In cluster mode, only apply if explicitly provided (not default 127.0.0.1)
+            if self.spark_driver_host != "127.0.0.1":
+                config["spark.driver.host"] = self.spark_driver_host
+            if self.spark_driver_bind_address != "127.0.0.1":
+                config["spark.driver.bindAddress"] = self.spark_driver_bind_address
+
+        # Handle Jars and Packages logic priority
+        if self.spark_jars:
+            config["spark.jars"] = self.spark_jars
+        elif self.spark_jars_packages:
+            config["spark.jars.packages"] = self.spark_jars_packages
+
+        return config
+
+    def get_required_spark_packages(self) -> List[str]:
+        """
+        Returns a list of required Spark packages.
+        If local jars are provided, packages are usually skipped to avoid conflicts.
+
+        Returns:
+            List[str]: A list of cleaned Maven package coordinates.
+        """
+        if self.spark_jars:
+            return []
+
+        if not self.spark_jars_packages:
+            return []
+
+        return [pkg.strip() for pkg in self.spark_jars_packages.split(",") if pkg.strip()]

src_connectors/src_spark/configs/iceberg.py

@@ -0,0 +1,205 @@
+from typing import Dict, Any, Optional, List, Union
+from pydantic import BaseModel, Field, ConfigDict
+
+from src_connectors.src_spark.configs.base import SparkBaseComponent
+from variables.spark import SparkVariables
+
+# Load environment-based defaults from centralized variables
+_iceberg_defaults = SparkVariables.get_iceberg_config()
+_minio_defaults = SparkVariables.get_minio_config()
+_spark_defaults = SparkVariables.get_spark_base_config()
+
+
+class SparkIcebergConfig(BaseModel, SparkBaseComponent):
+    """
+    Configuration model for Apache Iceberg integration with Spark.
+
+    This class manages Iceberg catalog settings, including connection URIs,
+    authentication credentials, and warehouse locations. It constructs the
+    necessary 'spark.sql.catalog' properties required by the Iceberg Spark runtime.
+    """
+    model_config = ConfigDict(extra='allow')
+
+    iceberg_catalog_uri: str = Field(
+        default=_iceberg_defaults.get("SPARK_ICEBERG_CATALOG_URI") or "",
+        description="The URI for the Iceberg catalog (REST/Hive/Glue endpoint)."
+    )
+    iceberg_catalog_type: str = Field(
+        default=_iceberg_defaults.get("SPARK_ICEBERG_CATALOG_TYPE") or "hadoop",
+        description="The type of Iceberg catalog (e.g., 'rest', 'hive', 'hadoop')."
+    )
+    iceberg_oauth2_server_uri: str = Field(
+        default=_iceberg_defaults.get("SPARK_ICEBERG_OAUTH2_SERVER_URI") or "",
+        description="OAuth2 server URI for REST catalog authentication."
+    )
+    iceberg_credential: str = Field(
+        default=_iceberg_defaults.get("SPARK_ICEBERG_CREDENTIAL") or "",
+        description="Credentials for Iceberg catalog (e.g., client_id:client_secret)."
+    )
+    iceberg_scope: str = Field(
+        default=_iceberg_defaults.get("SPARK_ICEBERG_SCOPE") or "",
+        description="OAuth2 scope for catalog authentication."
+    )
+    iceberg_warehouse: Union[str, List[str]] = Field(
+        default=_iceberg_defaults.get("SPARK_ICEBERG_WAREHOUSE") or "",
+        description="The warehouse location/name used for the Spark SQL catalog prefix."
+    )
+    iceberg_version: str = Field(
+        default=_iceberg_defaults.get("SPARK_ICEBERG_VERSION") or "1.4.2",
+        description="The Version of Iceberg runtime packages to load."
+    )
+    spark_jars: Optional[str] = Field(
+        default=_spark_defaults.get("SPARK_LOCAL_JARS") or "",
+        description="Local jar file paths to include in the Spark session."
+    )
+    spark_jars_packages: Optional[str] = Field(
+        default=_spark_defaults.get("SPARK_JARS_PACKAGES") or "",
+        description="Override for Spark dependency packages."
+    )
+
+    # Storage Settings (MinIO / S3)
+    spark_minio_access_key: str = Field(
+        default=_minio_defaults.get("SPARK_MINIO_ACCESS_KEY_ID") or "",
+        description="Access key for MinIO or S3 storage."
+    )
+    spark_minio_secret_key: str = Field(
+        default=_minio_defaults.get("SPARK_MINIO_SECRET_ACCESS_KEY") or "",
+        description="Secret key for MinIO or S3 storage."
+    )
+    spark_minio_endpoint: str = Field(
+        default=_minio_defaults.get("SPARK_MINIO_ENDPOINT_URL") or "",
+        description="Endpoint URL for MinIO or S3 storage (e.g., http://localhost:9000)."
+    )
+
+    def get_spark_config(self) -> Dict[str, Any]:
+        """
+        Generates Spark configuration for the Iceberg catalog.
+
+        Includes SQL extensions, catalog-specific properties, and Hadoop S3A settings
+        for cloud storage connectivity.
+
+        Returns:
+            Dict[str, Any]: A flat dictionary of Spark configuration properties.
+        """
+        config = {
+            "spark.sql.extensions": "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions",
+        }
+
+        # Determine warehouses to configure (handles single string or list)
+        warehouses = [self.iceberg_warehouse] if isinstance(self.iceberg_warehouse, str) else self.iceberg_warehouse
+
+        for wh in warehouses:
+            # Using warehouse name as the catalog identifier prefix
+            prefix = f"spark.sql.catalog.{wh}"
+            config[prefix] = "org.apache.iceberg.spark.SparkCatalog"
+
+            # For non-REST catalogs, type can be used as shorthand (hive, hadoop, etc.)
+            # For REST catalogs, catalog-impl is preferred and setting both causes conflicts.
+            if self.iceberg_catalog_type != "rest":
+                config[f"{prefix}.type"] = self.iceberg_catalog_type
+
+            # Catalog URI and basic I/O settings
+            if self.iceberg_catalog_uri:
+                config[f"{prefix}.uri"] = self.iceberg_catalog_uri
+                config[f"{prefix}.classloader.isolation.enabled"] = "false"
+
+                # Use S3FileIO if S3/MinIO is likely being used
+                if any(s in wh.lower() or s in self.iceberg_catalog_uri.lower() for s in ["s3", "minio"]):
+                    config[f"{prefix}.io-impl"] = "org.apache.iceberg.aws.s3.S3FileIO"
+
+            if wh:
+                config[f"{prefix}.warehouse"] = wh
+
+            # REST Catalog specialized authentication settings
+            if self.iceberg_catalog_type == "rest":
+                self._apply_rest_catalog_config(config, prefix)
+
+        # Handle Spark Jars and Packages logic: Priority to local jars
+        self._apply_dependency_config(config)
+
+        # Apply Hadoop S3A storage configurations if credentials are provided
+        if self.spark_minio_access_key and self.spark_minio_secret_key:
+            config.update(self._get_s3_hadoop_config())
+
+        return config
+
+    def _apply_rest_catalog_config(self, config: Dict[str, Any], prefix: str) -> None:
+        """Applies REST catalog specific authentication and refresh settings."""
+        config[f"{prefix}.catalog-impl"] = "org.apache.iceberg.rest.RESTCatalog"
+        if self.iceberg_oauth2_server_uri:
+            config[f"{prefix}.oauth2-server-uri"] = self.iceberg_oauth2_server_uri
+        if self.iceberg_credential:
+            config[f"{prefix}.credential"] = self.iceberg_credential
+        if self.iceberg_scope:
+            config[f"{prefix}.scope"] = self.iceberg_scope
+
+        config[f"{prefix}.rest.auth.type"] = "oauth2"
+        config[f"{prefix}.token-refresh-enabled"] = "true"
+        config[f"{prefix}.token-refresh-min-validity-time"] = "60s"
+
+    def _apply_dependency_config(self, config: Dict[str, Any]) -> None:
+        """Handles the logic for Spark JARs and Packages with local priority."""
+        if self.spark_jars:
+            config["spark.jars"] = self.spark_jars
+        else:
+            if self.spark_jars_packages:
+                config["spark.jars.packages"] = self.spark_jars_packages
+            else:
+                spark_minor_version = _spark_defaults.get("SPARK_MINOR_VERSION", "3.0")
+                config["spark.jars.packages"] = (
+                    f"org.apache.iceberg:iceberg-spark-runtime-{spark_minor_version}_2.12:{self.iceberg_version},"
+                    f"org.apache.iceberg:iceberg-azure-bundle:{self.iceberg_version},"
+                    f"org.apache.iceberg:iceberg-aws-bundle:{self.iceberg_version},"
+                    f"org.apache.iceberg:iceberg-gcp-bundle:{self.iceberg_version}"
+                )
+
+    def _get_s3_hadoop_config(self) -> Dict[str, str]:
+        """
+        Generates Hadoop S3A configurations for MinIO/S3 compatible storage.
+
+        Returns:
+            Dict[str, str]: Hadoop filesystem properties.
+        """
+        hadoop_conf = {
+            "spark.hadoop.fs.s3a.access.key": self.spark_minio_access_key,
+            "spark.hadoop.fs.s3a.secret.key": self.spark_minio_secret_key,
+            "spark.hadoop.fs.s3a.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem",
+            "spark.hadoop.fs.s3a.path.style.access": "true",
+            "spark.hadoop.fs.s3a.fast.upload": "true",
+            "spark.hadoop.fs.s3a.connection.ssl.enabled": "false" if "http://" in self.spark_minio_endpoint.lower() else "true"
+        }
+
+        if self.spark_minio_endpoint:
+            hadoop_conf["spark.hadoop.fs.s3a.endpoint"] = self.spark_minio_endpoint
+
+        # When keys are provided, we use the SimpleAWSCredentialsProvider
+        hadoop_conf["spark.hadoop.fs.s3a.aws.credentials.provider"] = (
+            "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
+        )
+
+        # Compatibility with libraries using 's3' instead of 's3a'
+        hadoop_conf["spark.hadoop.fs.s3.impl"] = "org.apache.hadoop.fs.s3a.S3AFileSystem"
+
+        return hadoop_conf
+
+    def get_required_spark_packages(self) -> List[str]:
+        """
+        Returns a list of required Spark packages for Iceberg.
+
+        Returns:
+            List[str]: A list of Maven coordinates for Iceberg and related bundles.
+        """
+        if self.spark_jars:
+            return []
+
+        if self.spark_jars_packages:
+            return [pkg.strip() for pkg in self.spark_jars_packages.split(",") if pkg.strip()]
+
+        # Default Iceberg packages if nothing is provided
+        spark_minor_version = _spark_defaults.get("SPARK_MINOR_VERSION", "3.0")
+        return [
+            f"org.apache.iceberg:iceberg-spark-runtime-{spark_minor_version}_2.12:{self.iceberg_version}",
+            f"org.apache.iceberg:iceberg-azure-bundle:{self.iceberg_version}",
+            f"org.apache.iceberg:iceberg-aws-bundle:{self.iceberg_version}",
+            f"org.apache.iceberg:iceberg-gcp-bundle:{self.iceberg_version}"
+        ]

src_connectors/src_spark/configs/jdbc.py

@@ -0,0 +1,81 @@
+from typing import Dict, Any, List
+from pydantic import BaseModel, Field, ConfigDict
+
+from src_connectors.src_spark.configs.base import SparkBaseComponent
+from variables.spark import SparkVariables
+
+# Load environment-based defaults from centralized variables
+_jdbc_defaults = SparkVariables.get_jdbc_config()
+
+
+class SparkJdbcConfig(BaseModel, SparkBaseComponent):
+    """
+    Configuration model for Spark JDBC connections.
+
+    This class manages JDBC connectivity parameters, including driver details,
+    connection URIs, and authentication credentials. It provides both descriptive
+    metadata and functional options for Spark data source operations.
+    """
+    model_config = ConfigDict(extra='allow')
+
+    jdbc_url: str = Field(
+        default=_jdbc_defaults.get("SPARK_JDBC_URL") or "",
+        description="The JDBC connection URL (e.g., 'jdbc:postgresql://localhost:5432/db')."
+    )
+    jdbc_driver: str = Field(
+        default=_jdbc_defaults.get("SPARK_JDBC_DRIVER") or "",
+        description="The fully qualified class name of the JDBC driver (e.g., 'org.postgresql.Driver')."
+    )
+    jdbc_username: str = Field(
+        default=_jdbc_defaults.get("SPARK_JDBC_USERNAME") or "",
+        description="The database username for the JDBC connection."
+    )
+    jdbc_password: str = Field(
+        default=_jdbc_defaults.get("SPARK_JDBC_PASSWORD") or "",
+        description="The database password for the JDBC connection."
+    )
+
+    def get_spark_config(self) -> Dict[str, Any]:
+        """
+        Returns a descriptive overview of the JDBC configuration.
+
+        Security Note: Sensitive credentials (password) are masked in the output.
+        Note: JDBC parameters are typically utilized in .read.format("jdbc").options(...)
+        rather than being part of the global SparkConf.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing JDBC connection metadata.
+        """
+        return {
+            "spark_jdbc_url": self.jdbc_url,
+            "spark_jdbc_driver": self.jdbc_driver,
+            "spark_jdbc_user": self.jdbc_username,
+            "spark_jdbc_password_provided": bool(self.jdbc_password)
+        }
+
+    def get_jdbc_options(self) -> Dict[str, str]:
+        """
+        Generates connection options for Spark's JDBC data source.
+
+        Suitable for use with spark.read.format("jdbc").options(**config.get_jdbc_options()).load().
+
+        Returns:
+            Dict[str, str]: A dictionary of key-value pairs representing JDBC options.
+        """
+        options = {
+            "url": self.jdbc_url,
+            "driver": self.jdbc_driver,
+            "user": self.jdbc_username,
+            "password": self.jdbc_password,
+        }
+        # Filter out empty options and ensure all values are strings for Spark compatibility
+        return {k: str(v) for k, v in options.items() if v}
+
+    def get_required_spark_packages(self) -> List[str]:
+        """
+        Returns a list of required Spark packages for JDBC.
+
+        Returns:
+            List[str]: Usually an empty list as JDBC drivers are managed via JARs.
+        """
+        return []