satisfactoscript 0.1.0__py3-none-any.whl

satisfactoscript/core/core.py

@@ -0,0 +1,696 @@
+"""
+Core Engine Module. Handles the execution flow: Schema Parsing -> Processing -> Writing.
+"""
+
+from pyspark.sql import functions as F
+from pyspark.sql import Window
+from pyspark.sql import SparkSession
+from functools import reduce
+import yaml
+from dotenv import load_dotenv
+import os
+
+# Conditional import for DBUtils, with a mock for local testing
+try:
+    from pyspark.dbutils import DBUtils
+except ImportError:
+    print("⚠️ [Warning] pyspark.dbutils not found. Creating a mock for local testing.")
+    class MockDBUtils:
+        def __init__(self, spark_session=None):
+            self.widgets = self.Widgets()
+
+        class Widgets:
+            def get(self, name):
+                # Return a default or empty value for local tests
+                return ""
+            def __getattr__(self, name):
+                # To avoid errors on other widget calls
+                return lambda *args, **kwargs: None
+    
+    DBUtils = MockDBUtils
+
+
+from satisfactoscript.core.registry import RuleRegistry
+
+# ==============================================================================
+# INTERNAL HELPERS (Essential for backward compatibility)
+# ==============================================================================
+
+def _apply_operation(c, op_str):
+    """
+    Parses a string operation from JSON and applies it to a PySpark column.
+    Handles: cast, lit, col, upper, when/otherwise, split, substring.
+    
+    Args:
+        c (Column): The current PySpark Column object.
+        op_str (str): The string representing the operation to apply.
+        
+    Returns:
+        Column: The transformed PySpark Column object.
+    """
+    def smart_lit(val_str):
+        try:
+            if "." in val_str: return F.lit(float(val_str))
+            return F.lit(int(val_str))
+        except ValueError: return F.lit(val_str)
+
+    # --- 1. CONDITIONS (WHEN) ---
+    if op_str.startswith("when:"):
+        cond = op_str.split(":", 1)[1]
+        if cond == "is_not_null": return c.isNotNull()
+        if cond == "is_null": return c.isNull()
+        if cond.startswith("like:"): return c.like(cond.split(":", 1)[1])
+        if cond.startswith("notlike:"): return ~c.like(cond.split(":", 1)[1])
+        if cond.startswith("contains:"):
+            val = cond.split(":", 1)[1]
+            return c.like(f"%{val}%")
+        if cond.startswith("notcontains:"):
+            val = cond.split(":", 1)[1]
+            return ~c.like(f"%{val}%")
+        if cond.startswith("eq:") or cond == "=": return c == cond.split(":", 1)[1]
+        if cond.startswith("ne:"): return c != cond.split(":", 1)[1]
+        return c
+
+    # --- 2. ACTIONS ---
+    clean_op = op_str
+    if op_str.startswith("then:"): clean_op = op_str.split(":", 1)[1]
+    if op_str.startswith("else:"): clean_op = op_str.split(":", 1)[1]
+
+    if clean_op.startswith("lit:"): return smart_lit(clean_op.split(":", 1)[1])
+    if clean_op.startswith("expr:"): return F.expr(clean_op.split(":", 1)[1])
+    if clean_op.startswith("col:"): return F.col(f"`{clean_op.split(':', 1)[1]}`")
+    if clean_op == "col": return c
+    if clean_op.startswith("coalesce:"): return F.coalesce(c, smart_lit(clean_op.split(":", 1)[1]))
+    if clean_op.startswith("cast:"):
+        t = clean_op.split(":", 1)[1].lower()
+        return c.cast("timestamp") if t in ["datetime", "timestamp"] else c.cast(t)
+    if clean_op.startswith("split:"):
+        sep, idx = clean_op.split(":", 1)[1].split(",")
+        return F.split(c, sep).getItem(int(idx))
+    if clean_op.startswith("substring:"):
+        start, length = clean_op.split(":", 1)[1].split(",")
+        return F.substring(c, int(start), int(length))
+    if clean_op == "trim": return F.trim(c)
+    if clean_op == "upper": return F.upper(c)
+    
+    return c
+
+
+def _build_filter_expression(filter_list):
+    """
+    Converts a list of filter dictionaries (from JSON) into a PySpark Condition Column.
+    
+    Args:
+        filter_list (list of dict): List containing filter definitions.
+                                    Example: [{'column': 'age', 'operator': 'gt', 'value': 18}]
+                                    
+    Returns:
+        Column: A PySpark boolean Column expression.
+    """
+
+    def build_condition(rule):
+        col_name, op, val = rule["column"], rule["operator"].lower(), rule.get("value")
+        c = F.col(f"`{col_name}`")
+
+        if op == "is_not_null": return c.isNotNull()
+        if op == "is_null": return c.isNull()
+
+        if op in ["in", "not_in"]:
+            val_list = val.split(';') if isinstance(val, str) else val
+            return c.isin(val_list) if op == "in" else ~c.isin(val_list)
+
+        # --- DÉBUT DES AJOUTS POUR CONTAINS / NOTCONTAINS ---
+        if op == "contains": return c.like(f"%{val}%")
+        if op == "notcontains": return ~c.like(f"%{val}%")
+        # --- FIN DES AJOUTS ---
+
+        op_map = {"eq": "=", "ne": "!=", "gt": ">", "lt": "<", "gte": ">=", "lte": "<="}
+        clean_op = op_map.get(op, op)
+
+        if op == "sql": return F.expr(val)
+
+        return F.expr(f"`{col_name}` {clean_op} '{val}'")
+
+    if not filter_list:
+        return F.lit(True)
+
+    return reduce(lambda a, b: a & b, [build_condition(r) for r in filter_list])
+
+# ==============================================================================
+# ENGINE CLASS
+# ==============================================================================
+
+class SatisfactoEngine:
+    """
+    Main orchestration engine for SatisfactoScript.
+    Handles environment setup, Spark session initialization, and pipeline execution.
+    """
+    def __init__(self, spark=None, config_path=None):
+        """
+        Initializes the SatisfactoEngine.
+        
+        Args:
+            spark (SparkSession, optional): An existing SparkSession. If None, the engine
+                                            will attempt to auto-detect or create one.
+            config_path (str, optional): Path to the config.yaml file. If None, the engine
+                                         will search for it in parent directories.
+        """
+
+        # ======================================================================
+        # 0. CHARGEMENT DES VARIABLES D'ENVIRONNEMENT (.env)
+        # ======================================================================
+        current_file_path = os.path.abspath(__file__)
+        project_root = os.path.dirname(os.path.dirname(current_file_path))
+        env_path = os.path.join(project_root, ".env")
+
+        if os.path.exists(env_path):
+            load_dotenv(env_path)
+            print(" 🔒 [Init] Sécurité : Variables d'environnement chargées depuis .env")
+        else:
+            print(" ⚠️ [Init] Aucun fichier .env trouvé à la racine.")
+
+        # ======================================================================
+        # 1. AUTO-DÉTECTION DE SPARK (Compatible Web & PyCharm/Local)
+        # ======================================================================
+        if spark:
+            self.spark = spark
+        else:
+            # 1. Tente de récupérer la session active (Standard Databricks Web)
+            self.spark = SparkSession.getActiveSession()
+
+            # 2. Si aucune session active, on en crée une
+            if not self.spark:
+                print("   [Init] Spark session not detected. Creating/Getting one...")
+                try:
+                    # A. Tentative Databricks Connect (Pour IDE Local : PyCharm, VS Code)
+                    from databricks.connect import DatabricksSession
+                    self.spark = DatabricksSession.builder.getOrCreate()
+                    print("   [Init] 🚀 Remote DatabricksSession (Databricks Connect) initialized.")
+                except ImportError:
+                    # B. Fallback PySpark Open-Source / Ancien cluster
+                    self.spark = SparkSession.builder.getOrCreate()
+                    print("   [Init] 📦 Standard SparkSession initialized.")
+
+        if not self.spark:
+            raise RuntimeError("CRITICAL: Failed to initialize Spark Session.")
+
+        self.dbutils = DBUtils(self.spark)
+
+        # ======================================================================
+        # 2. AUTO-DÉTECTION DE LA CONFIG
+        # ======================================================================
+        if config_path:
+            self._load_config_from_yaml(config_path)
+        else:
+            # Recherche automatique basée sur l'emplacement de ce fichier core.py
+            current_file_path = os.path.abspath(__file__)
+            framework_dir = os.path.dirname(current_file_path)
+            project_root = os.path.dirname(framework_dir)
+
+            # Liste des candidats
+            candidates = [
+                os.path.join(project_root, "config.yaml"),       # ../config.yaml (Standard)
+                os.path.join(os.getcwd(), "config.yaml"),        # ./config.yaml
+                os.path.join(os.getcwd(), "../..", "config.yaml")   # ../config.yaml
+            ]
+
+            found = False
+            for path in candidates:
+                if os.path.exists(path):
+                    self._load_config_from_yaml(path)
+                    found = True
+                    break
+
+            if not found:
+                raise FileNotFoundError(
+                    f"CRITICAL: 'config.yaml' not found. Searched in: {candidates}. "
+                    "Please check file location or pass 'config_path' explicitly."
+                )
+
+        # ======================================================================
+        # 3. DÉTECTION UTILISATEUR & SANDBOX
+        # ======================================================================
+        self.current_user = self._get_clean_username()
+        self.is_job_execution = self._is_running_as_job()
+
+        self.schema_suffix = ""
+        # Sécurité par défaut : on considère que ce n'est pas la prod sauf si explicitement écrit
+        is_prod_env = self.config.get("environments", {}).get(self.env.lower(), {}).get("is_production", False)
+
+        # On active la Sandbox SEULEMENT si : PAS Prod ET PAS Job
+        if not is_prod_env and not self.is_job_execution:
+            print(f"🔧 [Interactive Mode] Sandbox enabled for : {self.current_user}")
+            self.schema_suffix = f"_{self.current_user}"
+        else:
+            if self.is_job_execution:
+                print("⚙︝ [Mode Job] Exécution automatisée détectée. Sandbox désactivée.")
+            if is_prod_env:
+                print("🔒 [Mode Prod] Environnement de production. Sandbox désactivée.")
+
+        print(f"✅ Framework Ready. Env: {self.env} | DB: {self.db} | Suffix: '{self.schema_suffix}'")
+
+    def _load_config_from_yaml(self, config_path):
+        """
+        Loads configuration from YAML and determines the active environment.
+        Strictly enforces file existence.
+        
+        Args:
+            config_path (str): The absolute path to the configuration YAML file.
+            
+        Raises:
+            ValueError: If the file does not exist or cannot be parsed.
+        """
+        print(f"   [Config] Loading configuration from: {config_path}")
+
+        if not os.path.exists(config_path):
+             raise ValueError(f"CRITICAL ❌ Config file not found at: {config_path}")
+
+        try:
+            with open(config_path, 'r') as f:
+                self.config = yaml.safe_load(f)
+        except Exception as e:
+            raise ValueError(f"CRITICAL ❌ Error parsing YAML: {e}")
+
+        # Auto-detect environment based on priority list
+        self._auto_detect_environment()
+
+    def _auto_detect_environment(self):
+        """
+        Iterates through 'priority_check' list from YAML.
+        Tries to verify if the catalog exists/is accessible.
+        Sets self.env and self.db to the first matching environment.
+        """
+        print("   [Config] Auto-detecting active environment...")
+
+        priority_list = self.config.get("priority_check", [])
+        environments = self.config.get("environments", {})
+
+        if not priority_list or not environments:
+             raise ValueError("CRITICAL ❌ Invalid Config: 'priority_check' or 'environments' missing.")
+
+        detected_env = None
+
+        for env_name in priority_list:
+            env_config = environments.get(env_name)
+            if not env_config:
+                continue
+
+            catalog = env_config.get("catalog")
+            print(f"      -> Checking access to catalog: '{catalog}' ({env_name})...")
+
+            try:
+                # Test simple d'accès au catalogue
+                # Note: spark.sql("USE catalog") change le contexte global,
+                # on préfère juste vérifier l'existence via une requête légère ou listCatalogs si dispo.
+                # Ici on tente un simple show databases dans ce catalogue.
+                self.spark.sql(f"SHOW SCHEMAS IN `{catalog}`").limit(1).collect()
+
+                print(f"      ✅ Success: Connected to '{catalog}'.")
+                detected_env = env_name
+                self.env = env_name.upper()
+                self.db = catalog
+                break # On s'arrête au premier qui marche
+            except Exception as e:
+                print(f"      ❌ Failed: Cannot access '{catalog}'. Moving to next priority.")
+
+        if not detected_env:
+            raise ValueError("CRITICAL ❌ Could not connect to ANY defined environment catalog.")
+
+    def _is_running_as_job(self):
+        """
+        Détecte si le script est lancé par un Job Databricks.
+        Vérifie le paramètre de Job (Widget) en priorité, puis l'environnement, puis les tags.
+        
+        Returns:
+            bool: True if running as a Databricks Job, False otherwise.
+        """
+        # 1. Vérification du Paramètre de Job (Task parameter / Widget)
+        try:
+            run_mode_param = self.dbutils.widgets.get("SATISFACTO_RUN_MODE")
+            # On met en .lower() au cas où la valeur passée soit "JOB" ou "job"
+            if run_mode_param and run_mode_param.lower() == "job":
+                return True
+        except Exception:
+            # Le widget n'existe pas (mode interactif normal), on ignore l'erreur
+            pass
+
+        # 2. Vérification de la variable d'environnement (Au cas où)
+        if os.environ.get("SATISFACTO_RUN_MODE", "").lower() == "job":
+            return True
+
+        # 3. Vérification des tags du contexte Databricks (Fallback classique)
+        try:
+            context = self.dbutils.notebook.entry_point.getDbutils().notebook().getContext()
+            job_id = context.tags().get("jobId")
+            if job_id is not None and str(job_id) != "None":
+                return True
+        except Exception:
+            pass
+
+        return False
+
+    def _get_clean_username(self):
+        """
+        Récupère le user courant (ex: 'julien_hou').
+        
+        Returns:
+            str: The cleaned username of the current execution context.
+        """
+        try:
+            # Priorité à Spark SQL (compatible VS Code & Notebooks)
+            rows = self.spark.sql("SELECT current_user()").collect()
+            if rows:
+                email = rows[0][0]
+                return email.split('@')[0].replace('.', '_').replace('-', '_').lower()
+        except:
+            pass
+
+        # Fallback Tags
+        try:
+            context = self.dbutils.notebook.entry_point.getDbutils().notebook().getContext()
+            email = context.tags().apply('user')
+            return email.split('@')[0].replace('.', '_').replace('-', '_').lower()
+        except:
+            return "unknown_user"
+
+
+    def get_target_schema(self, base_layer: str) -> str:
+        """
+        Retourne le nom complet du schéma cible en gérant la sandbox.
+        Ex en interactif Dev : 'silver' -> 'silver_hqhoujul'
+        Ex en Job ou en Prod : 'silver' -> 'silver'
+        
+        Args:
+            base_layer (str): The base target schema name (e.g., 'bronze', 'silver').
+            
+        Returns:
+            str: The target schema name with sandbox suffix appended if applicable.
+        """
+        if not self.schema_suffix:
+            return base_layer
+
+        # Sécurité : on évite d'ajouter le suffixe s'il est déjà présent
+        if base_layer.endswith(self.schema_suffix):
+            return base_layer
+
+        return f"{base_layer}{self.schema_suffix}"
+
+    def _drop_table_if_exists(self, fqn):
+        """
+        Drops a table using SQL directly if it exists.
+        
+        Args:
+            fqn (str): The Fully Qualified Name of the table.
+        """
+        print(f"     -> [Cleanup] Dropping table if exists: {fqn}")
+        self.spark.sql(f"DROP TABLE IF EXISTS {fqn}")
+
+    def _write_dataframe(self, df, fqn, label):
+        """
+        Writes a DataFrame to a Delta table, overwriting the schema.
+        
+        Args:
+            df (DataFrame): The PySpark DataFrame to write.
+            fqn (str): The Fully Qualified Name of the target table.
+            label (str): A label to display in the logging output.
+        """
+        if df.isEmpty():
+            print(f"     -> [Write] WARNING: Dataframe for {label} is empty. Skipping write.")
+            return
+        
+        print(f"     -> [Write] Writing data to: {fqn} ...")
+        df.write.format("delta").mode("overwrite").option("overwriteSchema", "true").saveAsTable(fqn)
+        print(f"     -> [Success] Table {label} written successfully.")
+
+    def _apply_business_rules(self, df, rules_list):
+        """
+        Sequentially applies a list of registered Business Rules to a DataFrame.
+        
+        Args:
+            df (DataFrame): The input DataFrame.
+            rules_list (list[str]): List of rule names registered in RuleRegistry.
+            
+        Returns:
+            DataFrame: The DataFrame after applying all business rules.
+        """
+        for rule_name in rules_list:
+            print(f"   -> [Rule] Applying: {rule_name}")
+            rule_func = RuleRegistry.get_rule(rule_name)
+            df = rule_func(df)
+        return df
+
+    def get_select_expressions(self, field_list):
+        """
+        Parses the 'select_final' or 'fields' block from JSON Schema.
+        Supports complex operations via _apply_operation helper.
+        
+        Args:
+            field_list (list of list): A list representing field definitions.
+                                       Each element should be [source_col, target_col, operations].
+                                       
+        Returns:
+            list[Column]: A list of PySpark Column expressions ready to be used in select().
+        """
+        select_exprs = []
+        i = 0
+        while i < len(field_list):
+            config = field_list[i]
+            source_col = config[0]
+            target_col = config[1]
+            operations = config[2] if len(config) > 2 else []
+            
+            c = F.col(f"`{source_col}`") if source_col else None
+            
+            # Support complex conditional logic (when/then/else chains)
+            if operations and len(operations) > 0 and operations[0].startswith("when:"):
+                # Scenario 1: Compact format in one list ["when:..", "then:..", "else:.."]
+                if len(operations) >= 3: 
+                     c = F.when(_apply_operation(c, operations[0]), 
+                                _apply_operation(c, operations[1])).otherwise(_apply_operation(c, operations[2]))
+                # Scenario 2: Multi-row format (handled by logic upstream or simplified here)
+            else:
+                # Linear transformations
+                for op_str in operations:
+                    c = _apply_operation(c, op_str)
+            
+            if c is not None: 
+                select_exprs.append(c.alias(target_col))
+            i += 1
+        return select_exprs
+
+    def process_schema(self, schema_dict, dataframes_in=None):
+        """
+        Main engine logic: Translates a schema dictionary into PySpark operations.
+        Steps: Loaders/Sources -> Preprocess -> Joins -> Business Rules -> Select Final.
+        
+        Args:
+            schema_dict (dict): The declarative pipeline dictionary.
+            dataframes_in (dict, optional): A dictionary of pre-loaded DataFrames to use.
+                                            Keys should map to table aliases.
+                                            
+        Returns:
+            DataFrame: The fully processed PySpark DataFrame.
+            
+        Raises:
+            ValueError: If no tables can be loaded to start processing.
+        """
+        print(" -> [Process] Processing schema logic...")
+        dfs = dataframes_in.copy() if dataframes_in else {}
+        
+        # 1. LOADERS & SOURCES
+        for t in schema_dict.get("tables", []):
+            alias = t.get("alias", t["name"])
+            
+            # CAS A : Table déjà fournie en entrée
+            if t["name"] in dfs:
+                if alias != t["name"]:
+                    dfs[alias] = dfs[t["name"]]
+            
+            # CAS B : Table non chargée, on doit la charger
+            else:
+                print(f"    -> [Load] Loading table/source: {t['name']}")
+                
+                if t.get("source_type") == "loader":
+                    loader_func = RuleRegistry.get_loader(t["function_name"])
+                    df = loader_func(self.spark, self.config, **t.get("arguments", {}))
+                else:
+                    df = self.spark.table(t["name"])
+                
+                # Apply Filters
+                if "filter" in t:
+                    df = df.filter(_build_filter_expression(t["filter"]))
+
+                # Apply Preprocess
+                if "preprocess" in t and "qualify" in t["preprocess"]:
+                    p = t["preprocess"]["qualify"]
+                    partition_cols = [F.col(c) for c in p["partition_by"]]
+                    order_cols = [F.col(o["field"]).desc() if o["order"] == "desc" else F.col(o["field"]).asc() for o in p["order_by"]]
+                    
+                    window_spec = Window.partitionBy(*partition_cols).orderBy(*order_cols)
+                    df = df.withColumn("_rn", F.row_number().over(window_spec)) \
+                           .filter(F.col("_rn") == 1).drop("_rn")
+
+                dfs[alias] = df
+            
+            # Field Selection at source
+            if "fields" in t:
+                dfs[alias] = dfs[alias].select(self.get_select_expressions(t["fields"]))
+
+        # 2. JOINS
+        # Ensure we have a base table to start joining
+        if not dfs:
+            raise ValueError("No tables loaded to process.")
+
+        base_alias = schema_dict.get("join", [{}])[0].get("table_from") if "join" in schema_dict else list(dfs.keys())[0]
+
+        if base_alias not in dfs:
+             # Fallback to first available if specific base not found
+             base_alias = list(dfs.keys())[0]
+
+        df_main = dfs[base_alias]
+        
+        if "join" in schema_dict:
+            for j in schema_dict["join"]:
+                print(f"    -> [Join] {j.get('type', 'left').upper()} JOIN {j['table_from']} -> {j['table_to']}")
+                
+                on_l = j["on_from"] if isinstance(j["on_from"], list) else [j["on_from"]]
+                on_r = j["on_to"] if isinstance(j["on_to"], list) else [j["on_to"]]
+                
+                cond = reduce(lambda x, y: x & y, [F.col(f"l.{l}") == F.col(f"r.{r}") for l, r in zip(on_l, on_r)])
+                
+                df_to = dfs[j["table_to"]]
+                cols_to_select = ["l.*"] + [F.col(f"r.{c}") for c in df_to.columns if c not in on_r]
+                
+                df_main = df_main.alias("l").join(df_to.alias("r"), cond, j.get("type", "left")) \
+                                 .select(*cols_to_select)
+
+        # 3. BUSINESS RULES
+        if "business_rules" in schema_dict:
+            df_main = self._apply_business_rules(df_main, schema_dict["business_rules"])
+        
+        # 4. SELECT FINAL
+        if "select_final" in schema_dict:
+            df_main = df_main.select(self.get_select_expressions(schema_dict["select_final"]))
+
+        return df_main
+
+    def run_split_to_org(self, schema_dict, org_list, target_layer, target_base_name, split_column="sales_org_code"):
+        """
+        Executes a pattern where the processed DataFrame is split based on an organization
+        column and written to multiple Delta tables.
+        
+        Args:
+            schema_dict (dict): The pipeline dictionary schema.
+            org_list (list of dict): List of organizations containing 'org_code' and 'label'.
+            target_layer (str): The target database layer (e.g., 'silver').
+            target_base_name (str): The base name for the target tables.
+            split_column (str): The column used to split the data.
+        """
+        print(f"--- Executing Pattern: split_to_org (Base: {target_base_name}) ---")
+        df_full = self.process_schema(schema_dict)
+        actual_schema = self.get_target_schema(target_layer)
+        df_full.cache()
+        
+        for org in org_list:
+            fqn = f"`{self.db}`.`{actual_schema}`.`{target_base_name}_{org['label']}`"
+            self._drop_table_if_exists(fqn)
+            print(f"  -> Processing Org: {org['label']}")
+            df_org = df_full.filter(F.col(split_column) == org["org_code"])
+            self._write_dataframe(df_org, fqn, org["label"])
+            
+        df_full.unpersist()
+        print("--- Pattern 'split_to_org' completed. ---")
+
+    def run_follow_schema(self, schema_dict, target_layer, target_table_name):
+        """
+        Standard execution pattern: Process a schema and write to a single target table.
+        
+        Args:
+            schema_dict (dict): The pipeline dictionary schema.
+            target_layer (str): The target database layer (e.g., 'gold').
+            target_table_name (str): The name of the target table to create/overwrite.
+        """
+        actual_schema = self.get_target_schema(target_layer)
+        fqn = f"`{self.db}`.`{actual_schema}`.`{target_table_name}`"
+        print(f"--- Executing Pattern: follow_schema (Target: {target_table_name}) ---")
+        self._drop_table_if_exists(fqn)
+        df = self.process_schema(schema_dict)
+        self._write_dataframe(df, fqn, target_table_name)
+        print("--- Pattern 'follow_schema' completed. ---")
+    
+    def run_unify_and_process(self, schema_dict, org_list, source_layer, target_layer, target_table_name, unified_source_base_names, unified_temp_view_key):
+        """
+        Executes a pattern where data from multiple organizations is first unified (unioned)
+        and then passed through the schema processing.
+        
+        Args:
+            schema_dict (dict): The pipeline dictionary schema.
+            org_list (list of dict): List of organizations defining expected source tables.
+            source_layer (str): The database layer containing the source tables.
+            target_layer (str): The database layer to write the output to.
+            target_table_name (str): The name of the target output table.
+            unified_source_base_names (list of str): Base names of the source tables to unify.
+            unified_temp_view_key (str): The alias/key to use for the unified DataFrame
+                                         in the schema logic.
+                                         
+        Raises:
+            ValueError: If no source tables can be found to unify.
+        """
+        actual_schema_source = self.get_target_schema(source_layer)
+        actual_schema_target = self.get_target_schema(target_layer)
+        fqn = f"`{self.db}`.`{actual_schema_target}`.`{target_table_name}`"
+        print(f"--- Executing Pattern: unify_and_process (Target: {target_table_name}) ---")
+        self._drop_table_if_exists(fqn)
+        
+        list_of_dfs = []
+        for base in unified_source_base_names:
+            for org in org_list:
+                source_fqn = f"`{self.db}`.`{actual_schema_source}`.`{base}_{org['label']}`"
+                try:
+                    list_of_dfs.append(self.spark.table(source_fqn))
+                except Exception:
+                    print(f"     - WARNING: Missing table {source_fqn}")
+
+        if not list_of_dfs: raise ValueError("No sources found.")
+        
+        print(f" -> [Union] Merging {len(list_of_dfs)} tables...")
+        unioned_df = reduce(lambda x, y: x.unionByName(y, allowMissingColumns=True), list_of_dfs).dropDuplicates()
+        
+        input_dfs = {unified_temp_view_key: unioned_df}
+        
+        df_final = self.process_schema(schema_dict, dataframes_in=input_dfs)
+        self._write_dataframe(df_final, fqn, target_table_name)
+        print("--- Pattern 'unify_and_process' completed. ---")
+
+    def optimize_table(self, target_layer, target_table_name, zorder_cols=None):
+        """
+        Exécute la commande OPTIMIZE sur une table Delta, avec optionnellement un ZORDER BY.
+        Gère automatiquement la résolution de la sandbox pour la couche cible.
+
+        Args:
+            target_layer (str): La couche cible de base (ex: "silver")
+            target_table_name (str): Le nom de la table (ex: "fact_sales")
+            zorder_cols (list, optional): Liste des colonnes pour le ZORDER BY (ex: ["date", "country"])
+        """
+        # 1. Résolution de la couche (gère la sandbox automatiquement)
+        resolved_layer = self.get_target_schema(target_layer)
+        full_target_fqn = f"`{self.db}`.`{resolved_layer}`.`{target_table_name}`"
+
+        print(f"--- Executing Pattern: optimize (Target: {target_table_name}) ---")
+        print(f"     -> [Optimize] Target FQN: {full_target_fqn}")
+
+        try:
+            # 2. Construction et exécution de la requête SQL
+            if zorder_cols and len(zorder_cols) > 0:
+                zorder_str = ", ".join(zorder_cols)
+                print(f"     -> [Optimize] Applying ZORDER BY ({zorder_str})...")
+                self.spark.sql(f"OPTIMIZE {full_target_fqn} ZORDER BY ({zorder_str})")
+            else:
+                print(f"     -> [Optimize] Running standard OPTIMIZE...")
+                self.spark.sql(f"OPTIMIZE {full_target_fqn}")
+
+            print(f"     -> [Success] Table {target_table_name} optimized successfully.")
+
+        except Exception as e:
+            print(f"     -> ❌ [Error] Optimization failed on {full_target_fqn}: {e}")
+            raise e