informatica-python 1.6.0__tar.gz → 1.7.0__tar.gz

{informatica_python-1.6.0 → informatica_python-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.6.0
+Version: 1.7.0
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.6.0 → informatica_python-1.7.0}/informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.6.0"
+__version__ = "1.7.0"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

{informatica_python-1.6.0 → informatica_python-1.7.0}/informatica_python/generators/helper_gen.py

@@ -1063,10 +1063,15 @@ def _add_expression_helpers(lines):
     lines.append("")
     lines.append("")
     lines.append("def percentile_val(value, pct):")
-    lines.append('    """Informatica PERCENTILE equivalent."""')
+    lines.append('    """Informatica PERCENTILE equivalent (row-level fallback)."""')
     lines.append("    return value")
     lines.append("")
     lines.append("")
+    lines.append("def percentile_df(df, col, pct=0.5):")
+    lines.append('    """Informatica PERCENTILE equivalent (DataFrame-level)."""')
+    lines.append("    return df[col].quantile(pct)")
+    lines.append("")
+    lines.append("")
     lines.append("def first_val(*args):")
     lines.append('    """Informatica FIRST equivalent."""')
     lines.append("    for a in args:")
@@ -1085,20 +1090,38 @@ def _add_expression_helpers(lines):
     lines.append("")
     lines.append("")
     lines.append("def moving_avg(value, window=3):")
-    lines.append('    """Informatica MOVINGAVG equivalent."""')
+    lines.append('    """Informatica MOVINGAVG equivalent (row-level fallback)."""')
     lines.append("    return value")
     lines.append("")
     lines.append("")
+    lines.append("def moving_avg_df(df, col, window=3):")
+    lines.append('    """Informatica MOVINGAVG equivalent (DataFrame-level).')
+    lines.append('    Returns a rolling mean over the specified window size."""')
+    lines.append("    return df[col].rolling(window=window, min_periods=1).mean()")
+    lines.append("")
+    lines.append("")
     lines.append("def moving_sum(value, window=3):")
-    lines.append('    """Informatica MOVINGSUM equivalent."""')
+    lines.append('    """Informatica MOVINGSUM equivalent (row-level fallback)."""')
     lines.append("    return value")
     lines.append("")
     lines.append("")
+    lines.append("def moving_sum_df(df, col, window=3):")
+    lines.append('    """Informatica MOVINGSUM equivalent (DataFrame-level).')
+    lines.append('    Returns a rolling sum over the specified window size."""')
+    lines.append("    return df[col].rolling(window=window, min_periods=1).sum()")
+    lines.append("")
+    lines.append("")
     lines.append("def cume(value):")
-    lines.append('    """Informatica CUME equivalent."""')
+    lines.append('    """Informatica CUME equivalent (row-level fallback)."""')
     lines.append("    return value")
     lines.append("")
     lines.append("")
+    lines.append("def cume_df(df, col):")
+    lines.append('    """Informatica CUME equivalent (DataFrame-level).')
+    lines.append('    Returns the cumulative (expanding) sum of the column."""')
+    lines.append("    return df[col].expanding(min_periods=1).sum()")
+    lines.append("")
+    lines.append("")
     lines.append("def set_count_variable(var_name, value=1):")
     lines.append('    """Informatica SETCOUNTVARIABLE equivalent."""')
     lines.append("    return set_variable(var_name, value)")
@@ -1150,6 +1173,211 @@ def _add_expression_helpers(lines):
     lines.append("")
     lines.append("")
 
+    _add_stored_procedure_functions(lines)
+    _add_update_strategy_functions(lines)
+    _add_state_persistence_functions(lines)
+
+
+def _add_stored_procedure_functions(lines):
+    lines.append("# ============================================================")
+    lines.append("# Stored Procedure Execution")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("def call_stored_procedure(config, proc_name, params=None, connection_name='default', output_params=None):")
+    lines.append('    """')
+    lines.append("    Execute a stored procedure and return results.")
+    lines.append("    params: dict of {param_name: value} for input parameters")
+    lines.append("    output_params: list of output parameter names to capture")
+    lines.append("    Returns: (result_df, output_values) where output_values is a dict")
+    lines.append('    """')
+    lines.append("    if params is None:")
+    lines.append("        params = {}")
+    lines.append("    if output_params is None:")
+    lines.append("        output_params = []")
+    lines.append("")
+    lines.append("    conn = get_db_connection(config, connection_name)")
+    lines.append("    conn_config = config.get('connections', {}).get(connection_name, {})")
+    lines.append("    db_type = conn_config.get('type', 'mssql')")
+    lines.append("")
+    lines.append("    try:")
+    lines.append("        cursor = conn.cursor()")
+    lines.append("        logger.info(f'Calling stored procedure: {proc_name} with params: {params}')")
+    lines.append("")
+    lines.append("        if db_type == 'oracle':")
+    lines.append("            param_list = list(params.values())")
+    lines.append("            for out_p in output_params:")
+    lines.append("                import cx_Oracle")
+    lines.append("                param_list.append(cursor.var(cx_Oracle.STRING))")
+    lines.append("            result = cursor.callproc(proc_name, param_list)")
+    lines.append("            out_values = {}")
+    lines.append("            for i, out_p in enumerate(output_params):")
+    lines.append("                out_values[out_p] = result[len(params) + i].getvalue()")
+    lines.append("        elif db_type == 'mssql':")
+    lines.append("            param_placeholders = ', '.join(['?'] * len(params))")
+    lines.append("            if output_params:")
+    lines.append("                out_decl = '; '.join(f'@{p} OUTPUT' for p in output_params)")
+    lines.append("                sql = f'EXEC {proc_name} {param_placeholders}'")
+    lines.append("                if out_decl:")
+    lines.append("                    sql += f', {out_decl}'")
+    lines.append("            else:")
+    lines.append("                sql = f'EXEC {proc_name} {param_placeholders}'")
+    lines.append("            cursor.execute(sql, list(params.values()))")
+    lines.append("            out_values = {}")
+    lines.append("        else:")
+    lines.append("            param_placeholders = ', '.join(['%s'] * len(params))")
+    lines.append("            cursor.execute(f'CALL {proc_name}({param_placeholders})', list(params.values()))")
+    lines.append("            out_values = {}")
+    lines.append("")
+    lines.append("        try:")
+    lines.append("            columns = [desc[0] for desc in cursor.description] if cursor.description else []")
+    lines.append("            rows = cursor.fetchall()")
+    lines.append("            result_df = pd.DataFrame(rows, columns=columns) if columns else pd.DataFrame()")
+    lines.append("        except Exception:")
+    lines.append("            result_df = pd.DataFrame()")
+    lines.append("")
+    lines.append("        conn.commit()")
+    lines.append("        logger.info(f'Stored procedure {proc_name} returned {len(result_df)} rows')")
+    lines.append("        return result_df, out_values")
+    lines.append("")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.error(f'Stored procedure {proc_name} failed: {e}')")
+    lines.append("        conn.rollback()")
+    lines.append("        raise")
+    lines.append("    finally:")
+    lines.append("        conn.close()")
+    lines.append("")
+    lines.append("")
+
+
+def _add_update_strategy_functions(lines):
+    lines.append("# ============================================================")
+    lines.append("# Update Strategy Target Operations")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("def write_with_update_strategy(config, df, table_name, connection_name='default', key_columns=None, schema=None):")
+    lines.append('    """')
+    lines.append("    Write DataFrame to database using row-level update strategy.")
+    lines.append("    Expects a '_update_strategy' column with values: INSERT, UPDATE, DELETE, REJECT.")
+    lines.append("    key_columns: list of primary key columns for UPDATE/DELETE operations.")
+    lines.append('    """')
+    lines.append("    if '_update_strategy' not in df.columns:")
+    lines.append("        logger.warning('No _update_strategy column found — falling back to bulk INSERT')")
+    lines.append("        write_to_db(config, df, table_name, connection_name, schema=schema)")
+    lines.append("        return")
+    lines.append("")
+    lines.append("    conn = get_db_connection(config, connection_name)")
+    lines.append("    conn_config = config.get('connections', {}).get(connection_name, {})")
+    lines.append("    if schema is None:")
+    lines.append("        schema = conn_config.get('schema', 'dbo')")
+    lines.append("    qualified_table = f'{schema}.{table_name}' if schema else table_name")
+    lines.append("")
+    lines.append("    data_cols = [c for c in df.columns if c != '_update_strategy']")
+    lines.append("    if key_columns is None:")
+    lines.append("        key_columns = data_cols[:1]")
+    lines.append("        logger.warning(f'No key columns specified for update strategy — using first column: {key_columns}')")
+    lines.append("")
+    lines.append("    try:")
+    lines.append("        cursor = conn.cursor()")
+    lines.append("")
+    lines.append("        df_insert = df[df['_update_strategy'] == 'INSERT']")
+    lines.append("        df_update = df[df['_update_strategy'] == 'UPDATE']")
+    lines.append("        df_delete = df[df['_update_strategy'] == 'DELETE']")
+    lines.append("        df_reject = df[df['_update_strategy'] == 'REJECT']")
+    lines.append("")
+    lines.append("        if len(df_reject) > 0:")
+    lines.append("            logger.warning(f'Rejecting {len(df_reject)} rows for target {table_name}')")
+    lines.append("")
+    lines.append("        if len(df_insert) > 0:")
+    lines.append("            logger.info(f'Inserting {len(df_insert)} rows into {qualified_table}')")
+    lines.append("            df_insert[data_cols].to_sql(table_name, conn, schema=schema, if_exists='append', index=False)")
+    lines.append("")
+    lines.append("        if len(df_update) > 0:")
+    lines.append("            non_key_cols = [c for c in data_cols if c not in key_columns]")
+    lines.append("            set_clause = ', '.join(f'{c} = ?' for c in non_key_cols)")
+    lines.append("            where_clause = ' AND '.join(f'{c} = ?' for c in key_columns)")
+    lines.append("            update_sql = f'UPDATE {qualified_table} SET {set_clause} WHERE {where_clause}'")
+    lines.append("            logger.info(f'Updating {len(df_update)} rows in {qualified_table}')")
+    lines.append("            for _, row in df_update.iterrows():")
+    lines.append("                values = [row[c] for c in non_key_cols] + [row[c] for c in key_columns]")
+    lines.append("                cursor.execute(update_sql, values)")
+    lines.append("")
+    lines.append("        if len(df_delete) > 0:")
+    lines.append("            where_clause = ' AND '.join(f'{c} = ?' for c in key_columns)")
+    lines.append("            delete_sql = f'DELETE FROM {qualified_table} WHERE {where_clause}'")
+    lines.append("            logger.info(f'Deleting {len(df_delete)} rows from {qualified_table}')")
+    lines.append("            for _, row in df_delete.iterrows():")
+    lines.append("                values = [row[c] for c in key_columns]")
+    lines.append("                cursor.execute(delete_sql, values)")
+    lines.append("")
+    lines.append("        conn.commit()")
+    lines.append("        total = len(df_insert) + len(df_update) + len(df_delete)")
+    lines.append("        logger.info(f'Update strategy complete for {qualified_table}: {len(df_insert)} inserts, {len(df_update)} updates, {len(df_delete)} deletes, {len(df_reject)} rejects')")
+    lines.append("")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.error(f'Update strategy write error for {qualified_table}: {e}')")
+    lines.append("        conn.rollback()")
+    lines.append("        raise")
+    lines.append("    finally:")
+    lines.append("        conn.close()")
+    lines.append("")
+    lines.append("")
+
+
+def _add_state_persistence_functions(lines):
+    lines.append("# ============================================================")
+    lines.append("# State Persistence (Persistent Variables)")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("_persistent_state = {}")
+    lines.append("")
+    lines.append("")
+    lines.append("def load_persistent_state(state_file='persistent_state.json'):")
+    lines.append('    """Load persistent variable state from JSON file."""')
+    lines.append("    import json")
+    lines.append("    global _persistent_state")
+    lines.append("    if os.path.exists(state_file):")
+    lines.append("        try:")
+    lines.append("            with open(state_file, 'r') as f:")
+    lines.append("                _persistent_state = json.load(f)")
+    lines.append("            logger.info(f'Loaded persistent state from {state_file} ({len(_persistent_state)} scopes)')")
+    lines.append("        except Exception as e:")
+    lines.append("            logger.warning(f'Could not load persistent state from {state_file}: {e}')")
+    lines.append("            _persistent_state = {}")
+    lines.append("    else:")
+    lines.append("        logger.info(f'No persistent state file found at {state_file} — starting fresh')")
+    lines.append("        _persistent_state = {}")
+    lines.append("    return _persistent_state")
+    lines.append("")
+    lines.append("")
+    lines.append("def save_persistent_state(state_file='persistent_state.json'):")
+    lines.append('    """Save persistent variable state to JSON file."""')
+    lines.append("    import json")
+    lines.append("    try:")
+    lines.append("        with open(state_file, 'w') as f:")
+    lines.append("            json.dump(_persistent_state, f, indent=2, default=str)")
+    lines.append("        logger.info(f'Saved persistent state to {state_file}')")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'Could not save persistent state to {state_file}: {e}')")
+    lines.append("")
+    lines.append("")
+    lines.append("def get_persistent_variable(scope, var_name, default=None):")
+    lines.append('    """Get a persistent variable value from state."""')
+    lines.append("    scope_state = _persistent_state.get(scope, {})")
+    lines.append("    return scope_state.get(var_name, default)")
+    lines.append("")
+    lines.append("")
+    lines.append("def set_persistent_variable(scope, var_name, value):")
+    lines.append('    """Set a persistent variable value in state."""')
+    lines.append("    if scope not in _persistent_state:")
+    lines.append("        _persistent_state[scope] = {}")
+    lines.append("    _persistent_state[scope][var_name] = value")
+    lines.append("    return value")
+    lines.append("")
+    lines.append("")
+
 
 def _add_utility_functions(lines):
     lines.append("# ============================================================")

{informatica_python-1.6.0 → informatica_python-1.7.0}/informatica_python/generators/mapping_gen.py

@@ -253,6 +253,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append(f"    start_time = log_mapping_start('{mapping.name}')")
     lines.append("")
 
+    has_persistent_vars = False
     if mapping.variables:
         lines.append("    # Mapping Variables")
         for var in mapping.variables:
@@ -260,7 +261,11 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
             default = var.default_value or "''"
             if var.datatype.lower() in ("integer", "bigint", "int"):
                 default = var.default_value or "0"
-            lines.append(f"    {safe_var} = {default}")
+            if getattr(var, 'is_persistent', 'NO').upper() == 'YES':
+                has_persistent_vars = True
+                lines.append(f"    {safe_var} = get_persistent_variable('{mapping.name}', '{safe_var}', {default})")
+            else:
+                lines.append(f"    {safe_var} = {default}")
         lines.append("")
 
     if session_overrides:
@@ -323,7 +328,6 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append("    _parser = _ap.ArgumentParser()")
     lines.append("    _parser.add_argument('--param-file', default=None)")
     lines.append("    _parser.add_argument('--config', default='config.yml')")
-    lines.append("    _parser.add_argument('--validate-casts', action='store_true', help='Log data quality warnings on type coercion')")
     lines.append("    _args = _parser.parse_args()")
     lines.append("    config = load_config(_args.config, param_file=_args.param_file)")
     lines.append(f"    run_{_safe_name(mapping.name)}(config)")
@@ -979,10 +983,30 @@ def _gen_update_strategy(lines, tx, tx_safe, input_df, source_dfs):
         if attr.name == "Update Strategy Expression":
             strategy_expr = attr.value
     strategy_map = {"0": "INSERT", "1": "UPDATE", "2": "DELETE", "3": "REJECT"}
-    strategy_name = strategy_map.get(strategy_expr, f"EXPR({strategy_expr})")
-    lines.append(f"    # Update Strategy: {strategy_name}")
+
+    lines.append(f"    # Update Strategy: {tx.name}")
     lines.append(f"    df_{tx_safe} = {input_df}.copy()")
-    lines.append(f"    df_{tx_safe}['_update_strategy'] = '{strategy_name}'")
+
+    if strategy_expr in strategy_map:
+        strategy_name = strategy_map[strategy_expr]
+        lines.append(f"    df_{tx_safe}['_update_strategy'] = '{strategy_name}'")
+    else:
+        dd_map = {
+            "DD_INSERT": "INSERT", "DD_UPDATE": "UPDATE",
+            "DD_DELETE": "DELETE", "DD_REJECT": "REJECT",
+        }
+        expr = strategy_expr
+        for dd_const, label in dd_map.items():
+            expr = expr.replace(dd_const, f"'{label}'")
+        try:
+            converted = convert_expression(expr)
+            lines.append(f"    # Original expression: {strategy_expr}")
+            lines.append(f"    def _resolve_strategy(row):")
+            lines.append(f"        return {converted}")
+            lines.append(f"    df_{tx_safe}['_update_strategy'] = df_{tx_safe}.apply(_resolve_strategy, axis=1)")
+        except Exception:
+            lines.append(f"    # Could not parse strategy expression: {strategy_expr}")
+            lines.append(f"    df_{tx_safe}['_update_strategy'] = 'INSERT'")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -1187,13 +1211,47 @@ def _gen_custom_transform(lines, tx, tx_safe, input_df, input_sources, source_df
 
 def _gen_stored_proc(lines, tx, tx_safe, input_df, source_dfs):
     proc_name = ""
+    conn_name = "default"
     for attr in tx.attributes:
         if attr.name in ("Stored Procedure Name", "sp name"):
             proc_name = attr.value
+        elif attr.name in ("Connection Name", "connection_name"):
+            conn_name = attr.value or "default"
+
+    input_params = []
+    output_params = []
+    for fld in tx.fields:
+        pt = (fld.porttype or "").upper()
+        if "INPUT" in pt and "OUTPUT" not in pt:
+            input_params.append(fld.name)
+        elif "OUTPUT" in pt:
+            output_params.append(fld.name)
 
     lines.append(f"    # Stored Procedure: {proc_name or tx.name}")
-    lines.append(f"    # TODO: Execute stored procedure and capture results")
-    lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+
+    if input_params:
+        param_dict_items = ", ".join(f"'{p}': {input_df}['{p}'].iloc[0] if '{p}' in {input_df}.columns else None" for p in input_params)
+        lines.append(f"    _sp_params_{tx_safe} = {{{param_dict_items}}}")
+    else:
+        lines.append(f"    _sp_params_{tx_safe} = {{}}")
+
+    if output_params:
+        out_list = repr(output_params)
+        lines.append(f"    _sp_out_names_{tx_safe} = {out_list}")
+    else:
+        lines.append(f"    _sp_out_names_{tx_safe} = []")
+
+    lines.append(f"    df_{tx_safe}, _sp_out_vals_{tx_safe} = call_stored_procedure(")
+    lines.append(f"        config, '{proc_name or tx.name}', params=_sp_params_{tx_safe},")
+    lines.append(f"        connection_name='{conn_name}', output_params=_sp_out_names_{tx_safe})")
+
+    if output_params:
+        lines.append(f"    if df_{tx_safe}.empty and _sp_out_vals_{tx_safe}:")
+        lines.append(f"        df_{tx_safe} = {input_df}.copy()")
+        for op in output_params:
+            lines.append(f"        if '{op}' in _sp_out_vals_{tx_safe}:")
+            lines.append(f"            df_{tx_safe}['{op}'] = _sp_out_vals_{tx_safe}['{op}']")
+
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -1266,7 +1324,14 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
             lines.append(f"    write_file(df_target_{tgt_safe}, _tgt_path_{tgt_safe}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
     elif tgt_def.database_type and tgt_def.database_type != "Flat File":
         conn_label = tgt_conn or "target"
-        lines.append(f"    write_to_db(config, df_target_{tgt_safe}, '{tgt_def.name}', '{conn_label}')")
+        lines.append(f"    if '_update_strategy' in df_target_{tgt_safe}.columns:")
+        key_cols = [f.name for f in tgt_def.fields if getattr(f, 'keytype', 'NOT A KEY') == 'PRIMARY KEY'] or None
+        if key_cols:
+            lines.append(f"        write_with_update_strategy(config, df_target_{tgt_safe}, '{tgt_def.name}', '{conn_label}', key_columns={key_cols})")
+        else:
+            lines.append(f"        write_with_update_strategy(config, df_target_{tgt_safe}, '{tgt_def.name}', '{conn_label}')")
+        lines.append(f"    else:")
+        lines.append(f"        write_to_db(config, df_target_{tgt_safe}, '{tgt_def.name}', '{conn_label}')")
     elif tgt_def.flatfile:
         _emit_flatfile_write(lines, tgt_safe, tgt_def)
     else:

{informatica_python-1.6.0 → informatica_python-1.7.0}/informatica_python/generators/workflow_gen.py

@@ -22,7 +22,7 @@ def generate_workflow_code(folder: FolderDef) -> str:
     lines.append("import sys")
     lines.append("import logging")
     lines.append("from datetime import datetime")
-    lines.append("from helper_functions import load_config, logger")
+    lines.append("from helper_functions import load_config, logger, load_persistent_state, save_persistent_state, get_persistent_variable, set_persistent_variable")
     lines.append("")
 
     for i, mapping in enumerate(folder.mappings, 1):
@@ -124,12 +124,24 @@ def _generate_workflow_function(lines, wf: WorkflowDef, folder: FolderDef, workl
     lines.append(f"    failed_tasks = []")
     lines.append("")
 
+    has_persistent = any(
+        getattr(v, 'is_persistent', 'NO').upper() == 'YES'
+        for v in (wf.variables or [])
+    )
+    if has_persistent:
+        lines.append("    # Load persistent variable state from previous runs")
+        lines.append("    load_persistent_state()")
+        lines.append("")
+
     if wf.variables:
         lines.append("    # Workflow Variables")
         for var in wf.variables:
             var_name = _safe_name(var.name.replace("$$", ""))
             default = var.default_value or "''"
-            lines.append(f"    {var_name} = {default}")
+            if getattr(var, 'is_persistent', 'NO').upper() == 'YES':
+                lines.append(f"    {var_name} = get_persistent_variable('{wf.name}', '{var_name}', {default})")
+            else:
+                lines.append(f"    {var_name} = {default}")
         lines.append("")
 
     execution_order = _get_task_execution_order(wf)
@@ -146,6 +158,15 @@ def _generate_workflow_function(lines, wf: WorkflowDef, folder: FolderDef, workl
     for task in execution_order:
         _emit_task_code(lines, task, mapping_name_map, session_to_mapping, wf, worklets or [])
 
+    if has_persistent:
+        lines.append("    # Save persistent variable state for next run")
+        for var in wf.variables:
+            if getattr(var, 'is_persistent', 'NO').upper() == 'YES':
+                var_name = _safe_name(var.name.replace("$$", ""))
+                lines.append(f"    set_persistent_variable('{wf.name}', '{var_name}', {var_name})")
+        lines.append("    save_persistent_state()")
+        lines.append("")
+
     lines.append(f"    elapsed = (datetime.now() - wf_start).total_seconds()")
     lines.append(f"    if success:")
     lines.append(f"        logger.info(f'=== Workflow {wf.name} completed successfully in {{elapsed:.2f}}s ===')")

{informatica_python-1.6.0 → informatica_python-1.7.0}/informatica_python/utils/expression_converter.py

@@ -472,7 +472,8 @@ def detect_sql_dialect(sql_text):
 
     if "GETDATE()" in sql_upper or "ISNULL(" in sql_upper or "TOP " in sql_upper:
         return "mssql"
-    if "NVL(" in sql_upper or "SYSDATE" in sql_upper or "ROWNUM" in sql_upper:
+    if ("NVL(" in sql_upper or "SYSDATE" in sql_upper or "ROWNUM" in sql_upper
+            or "DECODE(" in sql_upper or "(+)" in sql_upper or "SYSTIMESTAMP" in sql_upper):
         return "oracle"
     if "NOW()" in sql_upper or "COALESCE(" in sql_upper:
         return "postgresql"

{informatica_python-1.6.0 → informatica_python-1.7.0}/informatica_python/utils/sql_dialect.py

@@ -88,9 +88,17 @@ def _convert_decode(sql):
 def _split_args(s):
     args = []
     depth = 0
+    in_quote = None
     current = []
     for ch in s:
-        if ch == '(':
+        if in_quote:
+            current.append(ch)
+            if ch == in_quote:
+                in_quote = None
+        elif ch in ("'", '"'):
+            in_quote = ch
+            current.append(ch)
+        elif ch == '(':
             depth += 1
             current.append(ch)
         elif ch == ')':
@@ -166,6 +174,7 @@ def translate_sql(sql, source_dialect="auto", target_dialect="ansi"):
             translated = pattern.sub(replacement, translated)
 
     elif source_dialect in ("generic", "postgresql"):
+        translated = _convert_decode(translated)
         for pattern, replacement in ORACLE_TO_ANSI:
             translated = pattern.sub(replacement, translated)
         for pattern, replacement in MSSQL_TO_ANSI:

{informatica_python-1.6.0 → informatica_python-1.7.0}/informatica_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.6.0
+Version: 1.7.0
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.6.0 → informatica_python-1.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "informatica-python"
-version = "1.6.0"
+version = "1.7.0"
 description = "Convert Informatica PowerCenter workflow XML to Python/PySpark code"
 readme = "README.md"
 license = {text = "MIT"}

{informatica_python-1.6.0 → informatica_python-1.7.0}/tests/test_integration.py

@@ -603,6 +603,20 @@ class TestSQLDialectTranslation:
         assert "COALESCE" in result
         assert "CURRENT_TIMESTAMP" in result
 
+    def test_decode_with_quoted_commas(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT DECODE(col, 'A,B', 1, 0) FROM T", source_dialect="oracle")
+        assert "CASE" in result
+        assert "'A,B'" in result
+
+    def test_dialect_detects_decode(self):
+        from informatica_python.utils.expression_converter import detect_sql_dialect
+        assert detect_sql_dialect("SELECT DECODE(X, 1, 'A', 'B') FROM T") == "oracle"
+
+    def test_dialect_detects_outer_join(self):
+        from informatica_python.utils.expression_converter import detect_sql_dialect
+        assert detect_sql_dialect("SELECT * FROM a, b WHERE a.id = b.id(+)") == "oracle"
+
     def test_no_change_for_clean_sql(self):
         from informatica_python.utils.sql_dialect import translate_sql
         sql = "SELECT * FROM employees WHERE id = 1"
@@ -813,3 +827,286 @@ class TestDataQualityValidation:
                 pass
         help_text = f.getvalue()
         assert "--validate-casts" in help_text or "validate_casts" in help_text
+
+
+class TestWindowAnalyticFunctions:
+    def test_moving_avg_df_generated(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def moving_avg_df(df, col, window=3):" in code
+        assert ".rolling(window=window, min_periods=1).mean()" in code
+
+    def test_moving_sum_df_generated(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def moving_sum_df(df, col, window=3):" in code
+        assert ".rolling(window=window, min_periods=1).sum()" in code
+
+    def test_cume_df_generated(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def cume_df(df, col):" in code
+        assert ".expanding(min_periods=1).sum()" in code
+
+    def test_percentile_df_generated(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def percentile_df(df, col, pct=0.5):" in code
+        assert ".quantile(pct)" in code
+
+    def test_row_level_fallbacks_still_exist(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def moving_avg(value, window=3):" in code
+        assert "def moving_sum(value, window=3):" in code
+        assert "def cume(value):" in code
+        assert "def percentile_val(value, pct):" in code
+
+    def test_window_functions_execute(self):
+        import pandas as pd
+        df = pd.DataFrame({"val": [10, 20, 30, 40, 50]})
+        rolling_mean = df["val"].rolling(window=3, min_periods=1).mean()
+        assert rolling_mean.iloc[0] == 10.0
+        assert rolling_mean.iloc[2] == 20.0
+        rolling_sum = df["val"].rolling(window=3, min_periods=1).sum()
+        assert rolling_sum.iloc[2] == 60.0
+        cume_sum = df["val"].expanding(min_periods=1).sum()
+        assert cume_sum.iloc[4] == 150.0
+
+
+class TestUpdateStrategy:
+    def test_static_insert_strategy(self):
+        from informatica_python.models import TransformationDef, TableAttribute
+        from informatica_python.generators.mapping_gen import _gen_update_strategy
+        tx = TransformationDef(name="UPD_INSERT", type="Update Strategy",
+                               attributes=[TableAttribute(name="Update Strategy Expression", value="0")])
+        lines = []
+        source_dfs = {}
+        _gen_update_strategy(lines, tx, "upd_insert", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "'_update_strategy'] = 'INSERT'" in code
+        assert "upd_insert" in source_dfs.get("UPD_INSERT", "")
+
+    def test_static_update_strategy(self):
+        from informatica_python.models import TransformationDef, TableAttribute
+        from informatica_python.generators.mapping_gen import _gen_update_strategy
+        tx = TransformationDef(name="UPD_UPDATE", type="Update Strategy",
+                               attributes=[TableAttribute(name="Update Strategy Expression", value="1")])
+        lines = []
+        source_dfs = {}
+        _gen_update_strategy(lines, tx, "upd_update", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "'_update_strategy'] = 'UPDATE'" in code
+
+    def test_static_delete_strategy(self):
+        from informatica_python.models import TransformationDef, TableAttribute
+        from informatica_python.generators.mapping_gen import _gen_update_strategy
+        tx = TransformationDef(name="UPD_DEL", type="Update Strategy",
+                               attributes=[TableAttribute(name="Update Strategy Expression", value="2")])
+        lines = []
+        source_dfs = {}
+        _gen_update_strategy(lines, tx, "upd_del", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "'_update_strategy'] = 'DELETE'" in code
+
+    def test_dd_constant_expression(self):
+        from informatica_python.models import TransformationDef, TableAttribute
+        from informatica_python.generators.mapping_gen import _gen_update_strategy
+        tx = TransformationDef(name="UPD_EXPR", type="Update Strategy",
+                               attributes=[TableAttribute(name="Update Strategy Expression", value="DD_UPDATE")])
+        lines = []
+        source_dfs = {}
+        _gen_update_strategy(lines, tx, "upd_expr", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "_update_strategy" in code
+        assert "UPD_EXPR" in source_dfs
+
+    def test_target_write_routes_strategy(self):
+        from informatica_python.models import TargetDef, FieldDef
+        from informatica_python.generators.mapping_gen import _generate_target_write
+        tgt = TargetDef(name="TGT_DB", database_type="Oracle",
+                        fields=[FieldDef(name="ID", datatype="integer", keytype="PRIMARY KEY"),
+                                FieldDef(name="VAL", datatype="string")])
+        lines = []
+        source_dfs = {"SRC": "df_src"}
+        connector_graph = {"to": {"TGT_DB": []}, "from": {}}
+        _generate_target_write(lines, "TGT_DB", tgt, connector_graph, source_dfs, {}, {})
+        code = "\n".join(lines)
+        assert "write_with_update_strategy" in code
+        assert "write_to_db" in code
+        assert "_update_strategy" in code
+
+    def test_update_strategy_helper_generated(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def write_with_update_strategy(" in code
+        assert "df_insert" in code
+        assert "df_update" in code
+        assert "df_delete" in code
+        assert "df_reject" in code
+        assert "INSERT" in code
+        assert "UPDATE" in code
+        assert "DELETE" in code
+
+
+class TestStoredProcedure:
+    def test_stored_proc_basic(self):
+        from informatica_python.models import TransformationDef, TableAttribute
+        from informatica_python.generators.mapping_gen import _gen_stored_proc
+        tx = TransformationDef(name="SP_GET_DATA", type="Stored Procedure",
+                               attributes=[TableAttribute(name="Stored Procedure Name", value="usp_get_data")])
+        lines = []
+        source_dfs = {}
+        _gen_stored_proc(lines, tx, "sp_get_data", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "call_stored_procedure" in code
+        assert "usp_get_data" in code
+        assert "SP_GET_DATA" in source_dfs
+
+    def test_stored_proc_with_input_params(self):
+        from informatica_python.models import TransformationDef, TableAttribute, FieldDef
+        from informatica_python.generators.mapping_gen import _gen_stored_proc
+        tx = TransformationDef(name="SP_LOOKUP", type="Stored Procedure",
+                               attributes=[TableAttribute(name="Stored Procedure Name", value="usp_lookup")],
+                               fields=[FieldDef(name="CUSTOMER_ID", datatype="integer", porttype="INPUT"),
+                                       FieldDef(name="RESULT", datatype="string", porttype="OUTPUT")])
+        lines = []
+        source_dfs = {}
+        _gen_stored_proc(lines, tx, "sp_lookup", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "CUSTOMER_ID" in code
+        assert "RESULT" in code
+        assert "_sp_out_names_" in code
+        assert "call_stored_procedure" in code
+
+    def test_stored_proc_with_output_params(self):
+        from informatica_python.models import TransformationDef, TableAttribute, FieldDef
+        from informatica_python.generators.mapping_gen import _gen_stored_proc
+        tx = TransformationDef(name="SP_OUT", type="Stored Procedure",
+                               attributes=[TableAttribute(name="Stored Procedure Name", value="usp_out")],
+                               fields=[FieldDef(name="RET_VAL", datatype="string", porttype="OUTPUT")])
+        lines = []
+        source_dfs = {}
+        _gen_stored_proc(lines, tx, "sp_out", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "['RET_VAL']" in code
+        assert "_sp_out_vals_" in code
+        assert "if df_sp_out.empty" in code
+
+    def test_stored_proc_helper_generated(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def call_stored_procedure(" in code
+        assert "cursor.callproc" in code
+        assert "EXEC" in code
+        assert "CALL" in code
+
+    def test_stored_proc_connection_override(self):
+        from informatica_python.models import TransformationDef, TableAttribute
+        from informatica_python.generators.mapping_gen import _gen_stored_proc
+        tx = TransformationDef(name="SP_CONN", type="Stored Procedure",
+                               attributes=[
+                                   TableAttribute(name="Stored Procedure Name", value="usp_custom"),
+                                   TableAttribute(name="Connection Name", value="oracle_prod"),
+                               ])
+        lines = []
+        source_dfs = {}
+        _gen_stored_proc(lines, tx, "sp_conn", "df_input", source_dfs)
+        code = "\n".join(lines)
+        assert "oracle_prod" in code
+
+
+class TestStatePersistence:
+    def test_persistent_state_helpers_generated(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "def load_persistent_state(" in code
+        assert "def save_persistent_state(" in code
+        assert "def get_persistent_variable(" in code
+        assert "def set_persistent_variable(" in code
+        assert "_persistent_state" in code
+        assert "persistent_state.json" in code
+
+    def test_workflow_loads_persistent_state(self):
+        from informatica_python.models import FolderDef, WorkflowDef, WorkflowVariable
+        from informatica_python.generators.workflow_gen import generate_workflow_code
+        wf = WorkflowDef(name="wf_test",
+                         variables=[WorkflowVariable(name="$$RUN_COUNT", datatype="integer",
+                                                     default_value="0", is_persistent="YES")])
+        folder = FolderDef(name="TestFolder", workflows=[wf])
+        code = generate_workflow_code(folder)
+        assert "load_persistent_state()" in code
+        assert "get_persistent_variable" in code
+        assert "save_persistent_state()" in code
+        assert "set_persistent_variable" in code
+
+    def test_workflow_no_persist_when_not_needed(self):
+        from informatica_python.models import FolderDef, WorkflowDef, WorkflowVariable
+        from informatica_python.generators.workflow_gen import generate_workflow_code
+        wf = WorkflowDef(name="wf_test",
+                         variables=[WorkflowVariable(name="$$TEMP_VAR", datatype="string",
+                                                     default_value="''", is_persistent="NO")])
+        folder = FolderDef(name="TestFolder", workflows=[wf])
+        code = generate_workflow_code(folder)
+        assert "load_persistent_state" not in code or "import" in code
+        assert "save_persistent_state" not in code or "import" in code
+
+    def test_mapping_persistent_variables(self):
+        from informatica_python.models import (
+            FolderDef, MappingDef, MappingVariable, SourceDef, FieldDef,
+            TransformationDef, ConnectorDef, InstanceDef
+        )
+        from informatica_python.generators.mapping_gen import generate_mapping_code
+        mapping = MappingDef(
+            name="m_persist_test",
+            variables=[
+                MappingVariable(name="$$LAST_ID", datatype="integer",
+                                default_value="0", is_persistent="YES"),
+                MappingVariable(name="$$TEMP", datatype="string",
+                                default_value="''", is_persistent="NO"),
+            ],
+            sources=[],
+            targets=[],
+            transformations=[],
+            connectors=[],
+            instances=[],
+        )
+        folder = FolderDef(name="TestFolder", mappings=[mapping])
+        code = generate_mapping_code(mapping, folder, "pandas", 1)
+        assert "get_persistent_variable('m_persist_test', 'last_id'" in code
+        assert "temp = ''" in code
+
+    def test_workflow_persistent_imports(self):
+        from informatica_python.models import FolderDef, WorkflowDef, WorkflowVariable
+        from informatica_python.generators.workflow_gen import generate_workflow_code
+        wf = WorkflowDef(name="wf_test",
+                         variables=[WorkflowVariable(name="$$COUNT", is_persistent="YES")])
+        folder = FolderDef(name="TestFolder", workflows=[wf])
+        code = generate_workflow_code(folder)
+        assert "load_persistent_state" in code
+        assert "save_persistent_state" in code
+
+    def test_state_file_json_format(self):
+        from informatica_python.models import FolderDef
+        from informatica_python.generators.helper_gen import generate_helper_functions
+        folder = FolderDef(name="TestFolder")
+        code = generate_helper_functions(folder)
+        assert "json.load" in code
+        assert "json.dump" in code
+        assert "persistent_state.json" in code