informatica-python 1.9.4__tar.gz → 1.9.6__tar.gz

{informatica_python-1.9.4 → informatica_python-1.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.9.4
+Version: 1.9.6
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.9.4 → informatica_python-1.9.6}/informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.9.4"
+__version__ = "1.9.6"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

{informatica_python-1.9.4 → informatica_python-1.9.6}/informatica_python/generators/helper_gen.py

@@ -11,6 +11,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     lines.append("import os")
+    lines.append("import re")
     lines.append("import sys")
     lines.append("import logging")
     lines.append("import yaml")
@@ -58,6 +59,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     _add_param_file_functions(lines)
+    _add_env_resolution(lines)
     _add_db_functions(lines, data_lib)
     _add_file_functions(lines, data_lib)
     _add_expression_helpers(lines)
@@ -121,23 +123,143 @@ def _add_param_file_functions(lines):
     lines.append("")
 
 
+def _add_env_resolution(lines):
+    lines.append("# ============================================================")
+    lines.append("# Environment Variable Resolution")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("def resolve_env(value, config=None):")
+    lines.append('    """')
+    lines.append("    Resolve ${VAR} placeholders in a string.")
+    lines.append("    Lookup order: OS environment variable -> config connections/variables -> literal.")
+    lines.append('    """')
+    lines.append("    if not isinstance(value, str):")
+    lines.append("        return value")
+    lines.append("    def _replace(m):")
+    lines.append("        var = m.group(1)")
+    lines.append("        env_val = os.environ.get(var)")
+    lines.append("        if env_val is not None:")
+    lines.append("            return env_val")
+    lines.append("        if config:")
+    lines.append("            for section in ('variables', 'connections', 'params'):")
+    lines.append("                sect = config.get(section, {})")
+    lines.append("                if isinstance(sect, dict) and var in sect:")
+    lines.append("                    v = sect[var]")
+    lines.append("                    return str(v) if not isinstance(v, dict) else str(v.get('default_value', ''))")
+    lines.append("        return m.group(0)")
+    lines.append(r"    return re.sub(r'\$\{(\w+)\}', _replace, value)")
+    lines.append("")
+    lines.append("")
+    lines.append("def rename_with_duplicates(df, col_mapping):")
+    lines.append('    """')
+    lines.append("    Rename DataFrame columns supporting one-source-to-many-target mapping.")
+    lines.append("    col_mapping is {target_col: source_col}.")
+    lines.append("    When multiple target cols map to the same source col, we duplicate the column.")
+    lines.append('    """')
+    lines.append("    result = df.copy()")
+    lines.append("    from collections import Counter")
+    lines.append("    src_counts = Counter(col_mapping.values())")
+    lines.append("    simple_rename = {}")
+    lines.append("    for tgt, src in col_mapping.items():")
+    lines.append("        if src_counts[src] == 1 and src in result.columns:")
+    lines.append("            simple_rename[src] = tgt")
+    lines.append("        elif src in result.columns:")
+    lines.append("            result[tgt] = result[src].copy()")
+    lines.append("    if simple_rename:")
+    lines.append("        result = result.rename(columns=simple_rename)")
+    lines.append("    return result")
+    lines.append("")
+    lines.append("")
+
+    lines.append("def resolve_builtin_variable(var_name, mapping_name='', session_name='', folder_name=''):")
+    lines.append('    """Resolve Informatica built-in variables like $PMMappingName, $PMSessionName."""')
+    lines.append("    builtins = {")
+    lines.append("        'PMMappingName': mapping_name,")
+    lines.append("        'PMSessionName': session_name,")
+    lines.append("        'PMFolderName': folder_name,")
+    lines.append("        'PMWorkflowName': os.environ.get('INFA_VAR_PMWorkflowName', ''),")
+    lines.append("        'PMWorkflowRunId': os.environ.get('INFA_VAR_PMWorkflowRunId', '0'),")
+    lines.append("        'PMSessionRunId': os.environ.get('INFA_VAR_PMSessionRunId', '0'),")
+    lines.append("        'PMIntegrationServiceName': os.environ.get('INFA_VAR_PMIntegrationServiceName', ''),")
+    lines.append("        'PMRepositoryServiceName': os.environ.get('INFA_VAR_PMRepositoryServiceName', ''),")
+    lines.append("        'PMSourceDBConnection': os.environ.get('INFA_VAR_PMSourceDBConnection', ''),")
+    lines.append("        'PMTargetDBConnection': os.environ.get('INFA_VAR_PMTargetDBConnection', ''),")
+    lines.append("    }")
+    lines.append("    clean = var_name.lstrip('$').lstrip('PM')")
+    lines.append("    for key, val in builtins.items():")
+    lines.append("        if key.lower() == ('PM' + clean).lower() or key.lower() == var_name.lstrip('$').lower():")
+    lines.append("            return val")
+    lines.append("    return os.environ.get(f'INFA_VAR_{var_name.lstrip(\"$\")}', '')")
+    lines.append("")
+    lines.append("")
+
+
 def _add_db_functions(lines, data_lib):
     lines.append("# ============================================================")
     lines.append("# Database Operations")
     lines.append("# ============================================================")
     lines.append("")
     lines.append("")
+    lines.append("_engine_cache = {}")
+    lines.append("")
+    lines.append("")
     lines.append("def get_db_connection(config, connection_name='default'):")
-    lines.append('    """Create database connection from config."""')
+    lines.append('    """')
+    lines.append("    Create database connection from config.")
+    lines.append("    Prefers SQLAlchemy engine (with connection pooling) over raw drivers.")
+    lines.append("    Config values support ${VAR} env-var placeholders via resolve_env().")
+    lines.append('    """')
     lines.append("    conn_config = config.get('connections', {}).get(connection_name, {})")
-    lines.append("    db_type = conn_config.get('type', 'mssql')")
-    lines.append("    host = conn_config.get('host', 'localhost')")
-    lines.append("    port = conn_config.get('port', 1433)")
-    lines.append("    database = conn_config.get('database', '')")
-    lines.append("    username = conn_config.get('username', '')")
-    lines.append("    password = conn_config.get('password', '')")
-    lines.append("    schema = conn_config.get('schema', 'dbo')")
+    lines.append("    db_type = resolve_env(conn_config.get('type', 'mssql'), config)")
+    lines.append("    host = resolve_env(conn_config.get('host', 'localhost'), config)")
+    lines.append("    port = resolve_env(conn_config.get('port', 1433), config)")
+    lines.append("    database = resolve_env(conn_config.get('database', ''), config)")
+    lines.append("    username = resolve_env(conn_config.get('username', ''), config)")
+    lines.append("    password = resolve_env(conn_config.get('password', ''), config)")
+    lines.append("    schema = resolve_env(conn_config.get('schema', 'dbo'), config)")
+    lines.append("")
+    lines.append("    cache_key = f'{db_type}://{username}@{host}:{port}/{database}'")
+    lines.append("    if cache_key in _engine_cache:")
+    lines.append("        return _engine_cache[cache_key].connect()")
+    lines.append("")
+    lines.append("    try:")
+    lines.append("        from sqlalchemy import create_engine")
+    lines.append("        url = _build_sqlalchemy_url(db_type, host, port, database, username, password)")
+    lines.append("        if url:")
+    lines.append("            engine = create_engine(url, pool_pre_ping=True, pool_size=5)")
+    lines.append("            _engine_cache[cache_key] = engine")
+    lines.append("            return engine.connect()")
+    lines.append("    except ImportError:")
+    lines.append("        logger.info('SQLAlchemy not available, falling back to raw drivers')")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'SQLAlchemy connection failed: {e}, falling back to raw drivers')")
+    lines.append("")
+    lines.append("    return _get_raw_connection(db_type, host, port, database, username, password, conn_config)")
     lines.append("")
+    lines.append("")
+    lines.append("def _build_sqlalchemy_url(db_type, host, port, database, username, password):")
+    lines.append("    from urllib.parse import quote_plus")
+    lines.append("    pw = quote_plus(str(password)) if password else ''")
+    lines.append("    if db_type == 'mssql':")
+    lines.append("        try:")
+    lines.append("            import pyodbc")
+    lines.append("            conn_str = quote_plus(")
+    lines.append("                f'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={host},{port};DATABASE={database};UID={username};PWD={password}'")
+    lines.append("            )")
+    lines.append("            return f'mssql+pyodbc:///?odbc_connect={conn_str}'")
+    lines.append("        except ImportError:")
+    lines.append("            return f'mssql+pymssql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        return f'postgresql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        return f'oracle+cx_oracle://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'mysql':")
+    lines.append("        return f'mysql+pymysql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    return None")
+    lines.append("")
+    lines.append("")
+    lines.append("def _get_raw_connection(db_type, host, port, database, username, password, conn_config):")
     lines.append("    if db_type == 'mssql':")
     lines.append("        try:")
     lines.append("            import pyodbc")
@@ -156,39 +278,19 @@ def _add_db_functions(lines, data_lib):
     lines.append("            return pymssql.connect(server=host, port=int(port), database=database, user=username, password=password)")
     lines.append("        except ImportError:")
     lines.append("            pass")
-    lines.append("        try:")
-    lines.append("            from sqlalchemy import create_engine")
-    lines.append("            engine = create_engine(f'mssql+pymssql://{username}:{password}@{host}:{port}/{database}')")
-    lines.append("            return engine.connect()")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'postgresql':")
-    lines.append("        try:")
-    lines.append("            import psycopg2")
-    lines.append("            return psycopg2.connect(")
-    lines.append("                host=host, port=port, dbname=database,")
-    lines.append("                user=username, password=password")
-    lines.append("            )")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'oracle':")
-    lines.append("        try:")
-    lines.append("            import cx_Oracle")
-    lines.append("            dsn = cx_Oracle.makedsn(host, port, service_name=database)")
-    lines.append("            return cx_Oracle.connect(username, password, dsn)")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        import psycopg2")
+    lines.append("        return psycopg2.connect(host=host, port=port, dbname=database, user=username, password=password)")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        import cx_Oracle")
+    lines.append("        dsn = cx_Oracle.makedsn(host, port, service_name=database)")
+    lines.append("        return cx_Oracle.connect(username, password, dsn)")
     lines.append("")
     lines.append("    jdbc_url = conn_config.get('jdbc_url', '')")
     lines.append("    if jdbc_url:")
-    lines.append("        try:")
-    lines.append("            import jaydebeapi")
-    lines.append("            driver = conn_config.get('jdbc_driver', '')")
-    lines.append("            return jaydebeapi.connect(driver, jdbc_url, [username, password])")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("        import jaydebeapi")
+    lines.append("        driver = conn_config.get('jdbc_driver', '')")
+    lines.append("        return jaydebeapi.connect(driver, jdbc_url, [username, password])")
     lines.append("")
     lines.append("    raise ConnectionError(f'Cannot create connection for type: {db_type}')")
     lines.append("")
@@ -203,6 +305,15 @@ def _add_db_functions(lines, data_lib):
     else:
         read_func = "pd.read_sql"
 
+    lines.append("def _safe_close(conn):")
+    lines.append('    """Close connection safely — handles both SQLAlchemy and raw connections."""')
+    lines.append("    try:")
+    lines.append("        if hasattr(conn, 'close'):")
+    lines.append("            conn.close()")
+    lines.append("    except Exception:")
+    lines.append("        pass")
+    lines.append("")
+    lines.append("")
     lines.append("def read_from_db(config, query, connection_name='default'):")
     lines.append('    """Read data from database using SQL query."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
@@ -220,7 +331,7 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB read error on {{connection_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def write_to_db(config, df, table_name, connection_name='default', if_exists='append', schema=None):")
@@ -242,23 +353,31 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB write error to {{schema}}.{{table_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def execute_sql(config, sql, connection_name='default'):")
     lines.append('    """Execute a SQL statement (INSERT, UPDATE, DELETE, DDL)."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
     lines.append("    try:")
-    lines.append("        cursor = conn.cursor()")
-    lines.append("        cursor.execute(sql)")
-    lines.append("        conn.commit()")
+    lines.append("        if hasattr(conn, 'dialect'):")
+    lines.append("            from sqlalchemy import text")
+    lines.append("            conn.execute(text(sql))")
+    lines.append("            conn.commit()")
+    lines.append("        else:")
+    lines.append("            cursor = conn.cursor()")
+    lines.append("            cursor.execute(sql)")
+    lines.append("            conn.commit()")
     lines.append("        logger.info(f'Executed SQL on {{connection_name}}')")
     lines.append("    except Exception as e:")
     lines.append("        logger.error(f'SQL execution error: {{e}}')")
-    lines.append("        conn.rollback()")
+    lines.append("        try:")
+    lines.append("            conn.rollback()")
+    lines.append("        except Exception:")
+    lines.append("            pass")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
 
@@ -1150,10 +1269,60 @@ def _add_expression_helpers(lines):
     lines.append("    raise SystemExit(message)")
     lines.append("")
     lines.append("")
-    lines.append("def lookup_func(table, condition, *fields):")
-    lines.append('    """Placeholder for Informatica LOOKUP function."""')
-    lines.append("    logger.warning(f'LOOKUP called for table {table} - implement in mapping-specific code')")
-    lines.append("    return None")
+    lines.append("_lookup_cache = {}")
+    lines.append("")
+    lines.append("")
+    lines.append("def lookup_func(table, condition, *fields, config=None, connection_name='default'):")
+    lines.append('    """')
+    lines.append("    Informatica unconnected LOOKUP function.")
+    lines.append("    Loads and caches the lookup table, then filters by condition.")
+    lines.append("    Returns the first matching value of the first return field, or None.")
+    lines.append('    """')
+    lines.append("    global _lookup_cache")
+    lines.append("    if table not in _lookup_cache:")
+    lines.append("        if config is not None:")
+    lines.append("            try:")
+    lines.append("                lkp_conn = connection_name")
+    lines.append("                conns = config.get('connections', {})")
+    lines.append("                for cname, cval in conns.items():")
+    lines.append("                    if isinstance(cval, dict) and cval.get('connection_name', '') == table:")
+    lines.append("                        lkp_conn = cname")
+    lines.append("                        break")
+    lines.append("                df_lkp = read_from_db(config, f'SELECT * FROM {table}', lkp_conn)")
+    lines.append("                _lookup_cache[table] = df_lkp")
+    lines.append("                logger.info(f'Cached lookup table {table}: {len(df_lkp)} rows')")
+    lines.append("            except Exception as e:")
+    lines.append("                logger.warning(f'Could not load lookup table {table}: {e}')")
+    lines.append("                _lookup_cache[table] = None")
+    lines.append("        else:")
+    lines.append("            logger.warning(f'LOOKUP called for {table} without config - returning None')")
+    lines.append("            return None")
+    lines.append("    df_lkp = _lookup_cache.get(table)")
+    lines.append("    if df_lkp is None or df_lkp.empty:")
+    lines.append("        return None")
+    lines.append("    try:")
+    lines.append("        if callable(condition):")
+    lines.append("            matches = df_lkp[condition(df_lkp)]")
+    lines.append("        elif isinstance(condition, str) and '=' in condition:")
+    lines.append("            col, _, val = condition.partition('=')")
+    lines.append("            col = col.strip()")
+    lines.append("            val = val.strip().strip(\"'\")")
+    lines.append("            if col in df_lkp.columns:")
+    lines.append("                matches = df_lkp[df_lkp[col].astype(str) == str(val)]")
+    lines.append("            else:")
+    lines.append("                return None")
+    lines.append("        else:")
+    lines.append("            return None")
+    lines.append("        if matches.empty:")
+    lines.append("            return None")
+    lines.append("        if fields:")
+    lines.append("            field = str(fields[0]).strip()")
+    lines.append("            if field in matches.columns:")
+    lines.append("                return matches.iloc[0][field]")
+    lines.append("        return matches.iloc[0].to_dict()")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'LOOKUP error on {table}: {e}')")
+    lines.append("        return None")
     lines.append("")
     lines.append("")
     lines.append("_param_store = {}")

{informatica_python-1.9.4 → informatica_python-1.9.6}/informatica_python/generators/mapping_gen.py

@@ -316,7 +316,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
                      if t.type in ("Source Qualifier", "Application Source Qualifier")]
     if sq_transforms:
         for sq in sq_transforms:
-            _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides)
+            _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides, mapping_name=mapping.name, folder_name=folder.name)
     else:
         for src_name, src_def in source_map.items():
             safe = _safe_name(src_name)
@@ -347,7 +347,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     for tx in processing_order:
         if tx.type in ("Source Qualifier", "Application Source Qualifier"):
             continue
-        _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib)
+        _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib, mapping_name=mapping.name, folder_name=folder.name)
 
     for tgt_name, tgt_def in target_map.items():
         _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides, validate_casts=validate_casts)
@@ -419,9 +419,10 @@ def _safe_name(name):
     return safe.lower()
 
 
-def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
+def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    ", mapping_name="", session_name="", folder_name=""):
     import re
     params = re.findall(r'\$\$(\w+)', sql_text)
+    pm_vars = re.findall(r'\$(PM\w+)', sql_text)
     lines.append(f"{indent}{sql_var_name} = '''")
     for sql_line in sql_text.strip().split("\n"):
         lines.append(f"{indent}{sql_line}")
@@ -433,6 +434,13 @@ def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
                 continue
             seen.add(p)
             lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('$${p}', str(get_param(config, '{p}')))")
+    if pm_vars:
+        seen_pm = set()
+        for pm in pm_vars:
+            if pm in seen_pm:
+                continue
+            seen_pm.add(pm)
+            lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('${pm}', str(resolve_builtin_variable('{pm}', mapping_name='{mapping_name}', session_name='{session_name}', folder_name='{folder_name}')))")
 
 
 def _flatfile_config_dict(ff):
@@ -618,7 +626,7 @@ def _get_processing_order(transformations, connector_graph, sq_transforms):
     return ordered
 
 
-def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides=None):
+def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides=None, mapping_name="", folder_name=""):
     sq_safe = _safe_name(sq.name)
     sql_override = ""
     pre_sql = ""
@@ -657,7 +665,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
     if not connected_sources:
         sq_src_name = sq.name[3:] if sq.name.upper().startswith("SQ_") else sq.name
         if sql_override:
-            _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override)
+            _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override, mapping_name=mapping_name, folder_name=folder_name)
             lines.append(f"    df_{sq_safe} = read_from_db(config, sql_{sq_safe}, 'default')")
         else:
             lines.append(f"    df_{sq_safe} = read_file(config.get('sources', {{}}).get('{sq_src_name}', {{}}).get('file_path', '{sq_src_name}'),")
@@ -668,7 +676,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
         sq_override = (session_overrides or {}).get(sq.name, {}) or (session_overrides or {}).get(src_name, {})
         conn_name = sq_override.get("connection_name") or (_safe_name(src_def.db_name) if src_def.db_name else "default")
 
-        _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override)
+        _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override, mapping_name=mapping_name, folder_name=folder_name)
         lines.append(f"    df_{sq_safe} = read_from_db(config, sql_{sq_safe}, '{conn_name}')")
     elif len(connected_sources) == 1:
         src_name = next(iter(connected_sources))
@@ -710,7 +718,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
     lines.append("")
 
 
-def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib="pandas"):
+def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib="pandas", mapping_name="", folder_name=""):
     tx_safe = _safe_name(tx.name)
     tx_type = tx.type.lower().strip()
 
@@ -757,7 +765,7 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type in ("joiner",):
         _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph, data_lib)
     elif tx_type in ("lookup procedure", "lookup"):
-        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph, data_lib)
+        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph, data_lib, mapping_name=mapping_name, folder_name=folder_name)
     elif tx_type == "router":
         _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("union",):
@@ -777,7 +785,7 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type in ("java",):
         _gen_java_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("sql",):
-        _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs, mapping_name=mapping_name, folder_name=folder_name)
     else:
         lines.append(f"    # TODO: Unsupported transformation type '{tx.type}' - passing through")
         copy_expr = lib_copy(data_lib, input_df)
@@ -982,7 +990,7 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph=None, data_lib="pandas"):
+def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph=None, data_lib="pandas", mapping_name="", folder_name=""):
     lookup_table = ""
     lookup_sql = ""
     lookup_condition = ""
@@ -1019,7 +1027,7 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_gr
 
     lines.append(f"    # Lookup: {lookup_table or tx.name}")
     if lookup_sql:
-        _emit_sql_with_params(lines, f"lkp_sql_{tx_safe}", lookup_sql)
+        _emit_sql_with_params(lines, f"lkp_sql_{tx_safe}", lookup_sql, mapping_name=mapping_name, folder_name=folder_name)
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, lkp_sql_{tx_safe}, 'default')")
     elif lookup_table:
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, 'SELECT * FROM {lookup_table}', 'default')")
@@ -1415,14 +1423,14 @@ def _gen_java_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs):
+def _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs, mapping_name="", folder_name=""):
     sql_query = ""
     for attr in tx.attributes:
         if attr.name == "Sql Query" and attr.value:
             sql_query = convert_sql_expression(attr.value)
     lines.append(f"    # SQL Transformation: {tx.name}")
     if sql_query:
-        _emit_sql_with_params(lines, f"sql_{tx_safe}", sql_query)
+        _emit_sql_with_params(lines, f"sql_{tx_safe}", sql_query, mapping_name=mapping_name, folder_name=folder_name)
         lines.append(f"    df_{tx_safe} = read_from_db(config, sql_{tx_safe}, 'default')")
     else:
         lines.append(f"    df_{tx_safe} = {input_df}.copy()")
@@ -1461,7 +1469,7 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     if col_mapping:
         lines.append(f"    # Column mapping: source -> target")
         lines.append(f"    target_columns_{tgt_safe} = {col_mapping}")
-        lines.append(f"    df_target_{tgt_safe} = {input_df}.rename(columns={{v: k for k, v in target_columns_{tgt_safe}.items()}})")
+        lines.append(f"    df_target_{tgt_safe} = rename_with_duplicates({input_df}, target_columns_{tgt_safe})")
         target_cols = [f.name for f in tgt_def.fields] if tgt_def.fields else None
         if target_cols:
             lines.append(f"    # Select only target columns")

{informatica_python-1.9.4 → informatica_python-1.9.6}/informatica_python/utils/expression_converter.py

@@ -867,7 +867,7 @@ def _vec_recursive(expr, df_var):
             if v.startswith("'") and v.endswith("'"):
                 vec_parts.append(v)
             else:
-                vec_parts.append(f'{v}.astype(str)')
+                vec_parts.append(f'{v}.fillna(\'\').astype(str)')
         return " + ".join(vec_parts)
 
     for func_name in sorted(INFA_FUNC_MAP.keys(), key=lambda x: -len(x)):

{informatica_python-1.9.4 → informatica_python-1.9.6}/informatica_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.9.4
+Version: 1.9.6
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.9.4 → informatica_python-1.9.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "informatica-python"
-version = "1.9.4"
+version = "1.9.6"
 description = "Convert Informatica PowerCenter workflow XML to Python/PySpark code"
 readme = "README.md"
 license = {text = "MIT"}

{informatica_python-1.9.4 → informatica_python-1.9.6}/tests/test_integration.py

@@ -2485,3 +2485,229 @@ class TestLookupWarning(unittest.TestCase):
                     break
         finally:
             shutil.rmtree(tmpdir)
+
+
+class TestRenameWithDuplicates(unittest.TestCase):
+
+    def test_helper_contains_rename_with_duplicates(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def rename_with_duplicates(" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_target_uses_rename_with_duplicates(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            for fn in os.listdir(tmpdir):
+                if fn.startswith("mapping_") and fn.endswith(".py"):
+                    with open(os.path.join(tmpdir, fn)) as f:
+                        code = f.read()
+                    if "target_columns_" in code:
+                        assert "rename_with_duplicates(" in code, \
+                            "Target rename should use rename_with_duplicates"
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestResolveEnv(unittest.TestCase):
+
+    def test_helper_contains_resolve_env(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def resolve_env(" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_contains_resolve_builtin_variable(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def resolve_builtin_variable(" in code
+            assert "PMMappingName" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestGetDbConnectionSQLAlchemy(unittest.TestCase):
+
+    def test_helper_sqlalchemy_primary(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            sa_pos = code.index("create_engine")
+            pyodbc_pos = code.index("pyodbc")
+            assert sa_pos < pyodbc_pos, "SQLAlchemy should be tried before raw pyodbc"
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_engine_cache(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "_engine_cache" in code
+            assert "pool_pre_ping=True" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_safe_close(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def _safe_close(" in code
+            assert "_safe_close(conn)" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_resolve_env_in_db(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "resolve_env(" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestLookupFuncImpl(unittest.TestCase):
+
+    def test_helper_lookup_func_full_impl(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "_lookup_cache" in code
+            assert "def lookup_func(" in code
+            assert "config=None" in code
+            assert "read_from_db" in code.split("def lookup_func")[1]
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestNullSafeConcat(unittest.TestCase):
+
+    def test_concat_fillna(self):
+        result = convert_expression_vectorized("A || B", "df")
+        assert ".fillna('')" in result, f"Concat should use fillna, got: {result}"
+        assert ".astype(str)" in result
+
+    def test_concat_literal_no_fillna(self):
+        result = convert_expression_vectorized("A || '-' || B", "df")
+        assert "'-'" in result
+        parts = result.split(" + ")
+        for part in parts:
+            if part.strip().startswith("'") and part.strip().endswith("'"):
+                assert ".fillna" not in part
+            else:
+                assert ".fillna('')" in part
+
+    def test_concat_three_fields_all_fillna(self):
+        result = convert_expression_vectorized("X || Y || Z", "df")
+        assert result.count(".fillna('')") == 3
+
+
+class TestPMVariableHandling(unittest.TestCase):
+
+    PM_VAR_XML = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE POWERMART SYSTEM "powrmart.dtd">
+<POWERMART CREATION_DATE="01/01/2025" REPOSITORY_VERSION="1">
+<REPOSITORY NAME="repo" VERSION="1" CODEPAGE="UTF-8" DATABASETYPE="Oracle">
+<FOLDER NAME="TEST_FOLDER" OWNER="admin">
+  <SOURCE NAME="SRC_PM" DATABASETYPE="Microsoft SQL Server" DBDNAME="TestDB" OWNERNAME="dbo">
+    <SOURCEFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NOTNULL" KEYTYPE="PRIMARY KEY" FIELDNUMBER="1"/>
+  </SOURCE>
+  <TARGET NAME="TGT_PM" DATABASETYPE="Microsoft SQL Server">
+    <TARGETFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NOTNULL" KEYTYPE="PRIMARY KEY" FIELDNUMBER="1"/>
+  </TARGET>
+  <MAPPING NAME="m_pm_vars" ISVALID="YES">
+    <TRANSFORMATION NAME="SQ_SRC_PM" TYPE="Source Qualifier" REUSABLE="NO">
+      <TRANSFORMFIELD NAME="ID" DATATYPE="integer" PORTTYPE="INPUT/OUTPUT" PRECISION="10" SCALE="0"/>
+      <TABLEATTRIBUTE NAME="Sql Query" VALUE="SELECT ID FROM dbo.SRC_PM WHERE mapping_name = &apos;$PMMappingName&apos;"/>
+    </TRANSFORMATION>
+    <INSTANCE NAME="SQ_SRC_PM" TRANSFORMATION_NAME="SQ_SRC_PM" TYPE="Source Qualifier"/>
+    <INSTANCE NAME="SRC_PM" TRANSFORMATION_NAME="SRC_PM" TYPE="Source Definition"/>
+    <INSTANCE NAME="TGT_PM" TRANSFORMATION_NAME="TGT_PM" TYPE="Target Definition"/>
+    <CONNECTOR FROMINSTANCE="SRC_PM" FROMFIELD="ID" TOINSTANCE="SQ_SRC_PM" TOFIELD="ID" FROMINSTANCETYPE="Source Definition" TOINSTANCETYPE="Source Qualifier"/>
+    <CONNECTOR FROMINSTANCE="SQ_SRC_PM" FROMFIELD="ID" TOINSTANCE="TGT_PM" TOFIELD="ID" FROMINSTANCETYPE="Source Qualifier" TOINSTANCETYPE="Target Definition"/>
+  </MAPPING>
+  <CONFIG NAME="default_session_config"/>
+  <WORKFLOW NAME="wf_pm_vars" ISVALID="YES">
+    <SESSION NAME="s_pm_vars" ISVALID="YES" MAPPINGNAME="m_pm_vars"/>
+  </WORKFLOW>
+</FOLDER>
+</REPOSITORY>
+</POWERMART>'''
+
+    def test_pm_variable_resolved_in_sql(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(self.PM_VAR_XML, output_dir=tmpdir)
+            for fn in os.listdir(tmpdir):
+                if fn.startswith("mapping_") and fn.endswith(".py"):
+                    with open(os.path.join(tmpdir, fn)) as f:
+                        code = f.read()
+                    if "$PMMappingName" in code:
+                        assert "resolve_builtin_variable" in code, \
+                            "SQL with $PMMappingName should call resolve_builtin_variable"
+                        assert "mapping_name='m_pm_vars'" in code, \
+                            "resolve_builtin_variable should receive actual mapping name"
+                        break
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestExecuteSqlAlchemy(unittest.TestCase):
+
+    def test_execute_sql_handles_sqlalchemy(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            exec_block = code.split("def execute_sql(")[1]
+            assert "text(sql)" in exec_block
+            assert "dialect" in exec_block, "Should check for dialect attribute to detect SQLAlchemy"
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestImportRe(unittest.TestCase):
+
+    def test_helper_imports_re(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "import re" in code
+        finally:
+            shutil.rmtree(tmpdir)