informatica-python 1.9.4__py3-none-any.whl → 1.9.6__py3-none-any.whl

informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.9.4"
+__version__ = "1.9.6"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

informatica_python/generators/helper_gen.py

@@ -11,6 +11,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     lines.append("import os")
+    lines.append("import re")
     lines.append("import sys")
     lines.append("import logging")
     lines.append("import yaml")
@@ -58,6 +59,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     _add_param_file_functions(lines)
+    _add_env_resolution(lines)
     _add_db_functions(lines, data_lib)
     _add_file_functions(lines, data_lib)
     _add_expression_helpers(lines)
@@ -121,23 +123,143 @@ def _add_param_file_functions(lines):
     lines.append("")
 
 
+def _add_env_resolution(lines):
+    lines.append("# ============================================================")
+    lines.append("# Environment Variable Resolution")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("def resolve_env(value, config=None):")
+    lines.append('    """')
+    lines.append("    Resolve ${VAR} placeholders in a string.")
+    lines.append("    Lookup order: OS environment variable -> config connections/variables -> literal.")
+    lines.append('    """')
+    lines.append("    if not isinstance(value, str):")
+    lines.append("        return value")
+    lines.append("    def _replace(m):")
+    lines.append("        var = m.group(1)")
+    lines.append("        env_val = os.environ.get(var)")
+    lines.append("        if env_val is not None:")
+    lines.append("            return env_val")
+    lines.append("        if config:")
+    lines.append("            for section in ('variables', 'connections', 'params'):")
+    lines.append("                sect = config.get(section, {})")
+    lines.append("                if isinstance(sect, dict) and var in sect:")
+    lines.append("                    v = sect[var]")
+    lines.append("                    return str(v) if not isinstance(v, dict) else str(v.get('default_value', ''))")
+    lines.append("        return m.group(0)")
+    lines.append(r"    return re.sub(r'\$\{(\w+)\}', _replace, value)")
+    lines.append("")
+    lines.append("")
+    lines.append("def rename_with_duplicates(df, col_mapping):")
+    lines.append('    """')
+    lines.append("    Rename DataFrame columns supporting one-source-to-many-target mapping.")
+    lines.append("    col_mapping is {target_col: source_col}.")
+    lines.append("    When multiple target cols map to the same source col, we duplicate the column.")
+    lines.append('    """')
+    lines.append("    result = df.copy()")
+    lines.append("    from collections import Counter")
+    lines.append("    src_counts = Counter(col_mapping.values())")
+    lines.append("    simple_rename = {}")
+    lines.append("    for tgt, src in col_mapping.items():")
+    lines.append("        if src_counts[src] == 1 and src in result.columns:")
+    lines.append("            simple_rename[src] = tgt")
+    lines.append("        elif src in result.columns:")
+    lines.append("            result[tgt] = result[src].copy()")
+    lines.append("    if simple_rename:")
+    lines.append("        result = result.rename(columns=simple_rename)")
+    lines.append("    return result")
+    lines.append("")
+    lines.append("")
+
+    lines.append("def resolve_builtin_variable(var_name, mapping_name='', session_name='', folder_name=''):")
+    lines.append('    """Resolve Informatica built-in variables like $PMMappingName, $PMSessionName."""')
+    lines.append("    builtins = {")
+    lines.append("        'PMMappingName': mapping_name,")
+    lines.append("        'PMSessionName': session_name,")
+    lines.append("        'PMFolderName': folder_name,")
+    lines.append("        'PMWorkflowName': os.environ.get('INFA_VAR_PMWorkflowName', ''),")
+    lines.append("        'PMWorkflowRunId': os.environ.get('INFA_VAR_PMWorkflowRunId', '0'),")
+    lines.append("        'PMSessionRunId': os.environ.get('INFA_VAR_PMSessionRunId', '0'),")
+    lines.append("        'PMIntegrationServiceName': os.environ.get('INFA_VAR_PMIntegrationServiceName', ''),")
+    lines.append("        'PMRepositoryServiceName': os.environ.get('INFA_VAR_PMRepositoryServiceName', ''),")
+    lines.append("        'PMSourceDBConnection': os.environ.get('INFA_VAR_PMSourceDBConnection', ''),")
+    lines.append("        'PMTargetDBConnection': os.environ.get('INFA_VAR_PMTargetDBConnection', ''),")
+    lines.append("    }")
+    lines.append("    clean = var_name.lstrip('$').lstrip('PM')")
+    lines.append("    for key, val in builtins.items():")
+    lines.append("        if key.lower() == ('PM' + clean).lower() or key.lower() == var_name.lstrip('$').lower():")
+    lines.append("            return val")
+    lines.append("    return os.environ.get(f'INFA_VAR_{var_name.lstrip(\"$\")}', '')")
+    lines.append("")
+    lines.append("")
+
+
 def _add_db_functions(lines, data_lib):
     lines.append("# ============================================================")
     lines.append("# Database Operations")
     lines.append("# ============================================================")
     lines.append("")
     lines.append("")
+    lines.append("_engine_cache = {}")
+    lines.append("")
+    lines.append("")
     lines.append("def get_db_connection(config, connection_name='default'):")
-    lines.append('    """Create database connection from config."""')
+    lines.append('    """')
+    lines.append("    Create database connection from config.")
+    lines.append("    Prefers SQLAlchemy engine (with connection pooling) over raw drivers.")
+    lines.append("    Config values support ${VAR} env-var placeholders via resolve_env().")
+    lines.append('    """')
     lines.append("    conn_config = config.get('connections', {}).get(connection_name, {})")
-    lines.append("    db_type = conn_config.get('type', 'mssql')")
-    lines.append("    host = conn_config.get('host', 'localhost')")
-    lines.append("    port = conn_config.get('port', 1433)")
-    lines.append("    database = conn_config.get('database', '')")
-    lines.append("    username = conn_config.get('username', '')")
-    lines.append("    password = conn_config.get('password', '')")
-    lines.append("    schema = conn_config.get('schema', 'dbo')")
+    lines.append("    db_type = resolve_env(conn_config.get('type', 'mssql'), config)")
+    lines.append("    host = resolve_env(conn_config.get('host', 'localhost'), config)")
+    lines.append("    port = resolve_env(conn_config.get('port', 1433), config)")
+    lines.append("    database = resolve_env(conn_config.get('database', ''), config)")
+    lines.append("    username = resolve_env(conn_config.get('username', ''), config)")
+    lines.append("    password = resolve_env(conn_config.get('password', ''), config)")
+    lines.append("    schema = resolve_env(conn_config.get('schema', 'dbo'), config)")
+    lines.append("")
+    lines.append("    cache_key = f'{db_type}://{username}@{host}:{port}/{database}'")
+    lines.append("    if cache_key in _engine_cache:")
+    lines.append("        return _engine_cache[cache_key].connect()")
+    lines.append("")
+    lines.append("    try:")
+    lines.append("        from sqlalchemy import create_engine")
+    lines.append("        url = _build_sqlalchemy_url(db_type, host, port, database, username, password)")
+    lines.append("        if url:")
+    lines.append("            engine = create_engine(url, pool_pre_ping=True, pool_size=5)")
+    lines.append("            _engine_cache[cache_key] = engine")
+    lines.append("            return engine.connect()")
+    lines.append("    except ImportError:")
+    lines.append("        logger.info('SQLAlchemy not available, falling back to raw drivers')")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'SQLAlchemy connection failed: {e}, falling back to raw drivers')")
+    lines.append("")
+    lines.append("    return _get_raw_connection(db_type, host, port, database, username, password, conn_config)")
     lines.append("")
+    lines.append("")
+    lines.append("def _build_sqlalchemy_url(db_type, host, port, database, username, password):")
+    lines.append("    from urllib.parse import quote_plus")
+    lines.append("    pw = quote_plus(str(password)) if password else ''")
+    lines.append("    if db_type == 'mssql':")
+    lines.append("        try:")
+    lines.append("            import pyodbc")
+    lines.append("            conn_str = quote_plus(")
+    lines.append("                f'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={host},{port};DATABASE={database};UID={username};PWD={password}'")
+    lines.append("            )")
+    lines.append("            return f'mssql+pyodbc:///?odbc_connect={conn_str}'")
+    lines.append("        except ImportError:")
+    lines.append("            return f'mssql+pymssql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        return f'postgresql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        return f'oracle+cx_oracle://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'mysql':")
+    lines.append("        return f'mysql+pymysql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    return None")
+    lines.append("")
+    lines.append("")
+    lines.append("def _get_raw_connection(db_type, host, port, database, username, password, conn_config):")
     lines.append("    if db_type == 'mssql':")
     lines.append("        try:")
     lines.append("            import pyodbc")
@@ -156,39 +278,19 @@ def _add_db_functions(lines, data_lib):
     lines.append("            return pymssql.connect(server=host, port=int(port), database=database, user=username, password=password)")
     lines.append("        except ImportError:")
     lines.append("            pass")
-    lines.append("        try:")
-    lines.append("            from sqlalchemy import create_engine")
-    lines.append("            engine = create_engine(f'mssql+pymssql://{username}:{password}@{host}:{port}/{database}')")
-    lines.append("            return engine.connect()")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'postgresql':")
-    lines.append("        try:")
-    lines.append("            import psycopg2")
-    lines.append("            return psycopg2.connect(")
-    lines.append("                host=host, port=port, dbname=database,")
-    lines.append("                user=username, password=password")
-    lines.append("            )")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'oracle':")
-    lines.append("        try:")
-    lines.append("            import cx_Oracle")
-    lines.append("            dsn = cx_Oracle.makedsn(host, port, service_name=database)")
-    lines.append("            return cx_Oracle.connect(username, password, dsn)")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        import psycopg2")
+    lines.append("        return psycopg2.connect(host=host, port=port, dbname=database, user=username, password=password)")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        import cx_Oracle")
+    lines.append("        dsn = cx_Oracle.makedsn(host, port, service_name=database)")
+    lines.append("        return cx_Oracle.connect(username, password, dsn)")
     lines.append("")
     lines.append("    jdbc_url = conn_config.get('jdbc_url', '')")
     lines.append("    if jdbc_url:")
-    lines.append("        try:")
-    lines.append("            import jaydebeapi")
-    lines.append("            driver = conn_config.get('jdbc_driver', '')")
-    lines.append("            return jaydebeapi.connect(driver, jdbc_url, [username, password])")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("        import jaydebeapi")
+    lines.append("        driver = conn_config.get('jdbc_driver', '')")
+    lines.append("        return jaydebeapi.connect(driver, jdbc_url, [username, password])")
     lines.append("")
     lines.append("    raise ConnectionError(f'Cannot create connection for type: {db_type}')")
     lines.append("")
@@ -203,6 +305,15 @@ def _add_db_functions(lines, data_lib):
     else:
         read_func = "pd.read_sql"
 
+    lines.append("def _safe_close(conn):")
+    lines.append('    """Close connection safely — handles both SQLAlchemy and raw connections."""')
+    lines.append("    try:")
+    lines.append("        if hasattr(conn, 'close'):")
+    lines.append("            conn.close()")
+    lines.append("    except Exception:")
+    lines.append("        pass")
+    lines.append("")
+    lines.append("")
     lines.append("def read_from_db(config, query, connection_name='default'):")
     lines.append('    """Read data from database using SQL query."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
@@ -220,7 +331,7 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB read error on {{connection_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def write_to_db(config, df, table_name, connection_name='default', if_exists='append', schema=None):")
@@ -242,23 +353,31 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB write error to {{schema}}.{{table_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def execute_sql(config, sql, connection_name='default'):")
     lines.append('    """Execute a SQL statement (INSERT, UPDATE, DELETE, DDL)."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
     lines.append("    try:")
-    lines.append("        cursor = conn.cursor()")
-    lines.append("        cursor.execute(sql)")
-    lines.append("        conn.commit()")
+    lines.append("        if hasattr(conn, 'dialect'):")
+    lines.append("            from sqlalchemy import text")
+    lines.append("            conn.execute(text(sql))")
+    lines.append("            conn.commit()")
+    lines.append("        else:")
+    lines.append("            cursor = conn.cursor()")
+    lines.append("            cursor.execute(sql)")
+    lines.append("            conn.commit()")
     lines.append("        logger.info(f'Executed SQL on {{connection_name}}')")
     lines.append("    except Exception as e:")
     lines.append("        logger.error(f'SQL execution error: {{e}}')")
-    lines.append("        conn.rollback()")
+    lines.append("        try:")
+    lines.append("            conn.rollback()")
+    lines.append("        except Exception:")
+    lines.append("            pass")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
 
@@ -1150,10 +1269,60 @@ def _add_expression_helpers(lines):
     lines.append("    raise SystemExit(message)")
     lines.append("")
     lines.append("")
-    lines.append("def lookup_func(table, condition, *fields):")
-    lines.append('    """Placeholder for Informatica LOOKUP function."""')
-    lines.append("    logger.warning(f'LOOKUP called for table {table} - implement in mapping-specific code')")
-    lines.append("    return None")
+    lines.append("_lookup_cache = {}")
+    lines.append("")
+    lines.append("")
+    lines.append("def lookup_func(table, condition, *fields, config=None, connection_name='default'):")
+    lines.append('    """')
+    lines.append("    Informatica unconnected LOOKUP function.")
+    lines.append("    Loads and caches the lookup table, then filters by condition.")
+    lines.append("    Returns the first matching value of the first return field, or None.")
+    lines.append('    """')
+    lines.append("    global _lookup_cache")
+    lines.append("    if table not in _lookup_cache:")
+    lines.append("        if config is not None:")
+    lines.append("            try:")
+    lines.append("                lkp_conn = connection_name")
+    lines.append("                conns = config.get('connections', {})")
+    lines.append("                for cname, cval in conns.items():")
+    lines.append("                    if isinstance(cval, dict) and cval.get('connection_name', '') == table:")
+    lines.append("                        lkp_conn = cname")
+    lines.append("                        break")
+    lines.append("                df_lkp = read_from_db(config, f'SELECT * FROM {table}', lkp_conn)")
+    lines.append("                _lookup_cache[table] = df_lkp")
+    lines.append("                logger.info(f'Cached lookup table {table}: {len(df_lkp)} rows')")
+    lines.append("            except Exception as e:")
+    lines.append("                logger.warning(f'Could not load lookup table {table}: {e}')")
+    lines.append("                _lookup_cache[table] = None")
+    lines.append("        else:")
+    lines.append("            logger.warning(f'LOOKUP called for {table} without config - returning None')")
+    lines.append("            return None")
+    lines.append("    df_lkp = _lookup_cache.get(table)")
+    lines.append("    if df_lkp is None or df_lkp.empty:")
+    lines.append("        return None")
+    lines.append("    try:")
+    lines.append("        if callable(condition):")
+    lines.append("            matches = df_lkp[condition(df_lkp)]")
+    lines.append("        elif isinstance(condition, str) and '=' in condition:")
+    lines.append("            col, _, val = condition.partition('=')")
+    lines.append("            col = col.strip()")
+    lines.append("            val = val.strip().strip(\"'\")")
+    lines.append("            if col in df_lkp.columns:")
+    lines.append("                matches = df_lkp[df_lkp[col].astype(str) == str(val)]")
+    lines.append("            else:")
+    lines.append("                return None")
+    lines.append("        else:")
+    lines.append("            return None")
+    lines.append("        if matches.empty:")
+    lines.append("            return None")
+    lines.append("        if fields:")
+    lines.append("            field = str(fields[0]).strip()")
+    lines.append("            if field in matches.columns:")
+    lines.append("                return matches.iloc[0][field]")
+    lines.append("        return matches.iloc[0].to_dict()")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'LOOKUP error on {table}: {e}')")
+    lines.append("        return None")
     lines.append("")
     lines.append("")
     lines.append("_param_store = {}")

informatica_python/generators/mapping_gen.py

@@ -316,7 +316,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
                      if t.type in ("Source Qualifier", "Application Source Qualifier")]
     if sq_transforms:
         for sq in sq_transforms:
-            _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides)
+            _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides, mapping_name=mapping.name, folder_name=folder.name)
     else:
         for src_name, src_def in source_map.items():
             safe = _safe_name(src_name)
@@ -347,7 +347,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     for tx in processing_order:
         if tx.type in ("Source Qualifier", "Application Source Qualifier"):
             continue
-        _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib)
+        _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib, mapping_name=mapping.name, folder_name=folder.name)
 
     for tgt_name, tgt_def in target_map.items():
         _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides, validate_casts=validate_casts)
@@ -419,9 +419,10 @@ def _safe_name(name):
     return safe.lower()
 
 
-def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
+def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    ", mapping_name="", session_name="", folder_name=""):
     import re
     params = re.findall(r'\$\$(\w+)', sql_text)
+    pm_vars = re.findall(r'\$(PM\w+)', sql_text)
     lines.append(f"{indent}{sql_var_name} = '''")
     for sql_line in sql_text.strip().split("\n"):
         lines.append(f"{indent}{sql_line}")
@@ -433,6 +434,13 @@ def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
                 continue
             seen.add(p)
             lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('$${p}', str(get_param(config, '{p}')))")
+    if pm_vars:
+        seen_pm = set()
+        for pm in pm_vars:
+            if pm in seen_pm:
+                continue
+            seen_pm.add(pm)
+            lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('${pm}', str(resolve_builtin_variable('{pm}', mapping_name='{mapping_name}', session_name='{session_name}', folder_name='{folder_name}')))")
 
 
 def _flatfile_config_dict(ff):
@@ -618,7 +626,7 @@ def _get_processing_order(transformations, connector_graph, sq_transforms):
     return ordered
 
 
-def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides=None):
+def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides=None, mapping_name="", folder_name=""):
     sq_safe = _safe_name(sq.name)
     sql_override = ""
     pre_sql = ""
@@ -657,7 +665,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
     if not connected_sources:
         sq_src_name = sq.name[3:] if sq.name.upper().startswith("SQ_") else sq.name
         if sql_override:
-            _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override)
+            _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override, mapping_name=mapping_name, folder_name=folder_name)
             lines.append(f"    df_{sq_safe} = read_from_db(config, sql_{sq_safe}, 'default')")
         else:
             lines.append(f"    df_{sq_safe} = read_file(config.get('sources', {{}}).get('{sq_src_name}', {{}}).get('file_path', '{sq_src_name}'),")
@@ -668,7 +676,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
         sq_override = (session_overrides or {}).get(sq.name, {}) or (session_overrides or {}).get(src_name, {})
         conn_name = sq_override.get("connection_name") or (_safe_name(src_def.db_name) if src_def.db_name else "default")
 
-        _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override)
+        _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override, mapping_name=mapping_name, folder_name=folder_name)
         lines.append(f"    df_{sq_safe} = read_from_db(config, sql_{sq_safe}, '{conn_name}')")
     elif len(connected_sources) == 1:
         src_name = next(iter(connected_sources))
@@ -710,7 +718,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
     lines.append("")
 
 
-def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib="pandas"):
+def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib="pandas", mapping_name="", folder_name=""):
     tx_safe = _safe_name(tx.name)
     tx_type = tx.type.lower().strip()
 
@@ -757,7 +765,7 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type in ("joiner",):
         _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph, data_lib)
     elif tx_type in ("lookup procedure", "lookup"):
-        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph, data_lib)
+        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph, data_lib, mapping_name=mapping_name, folder_name=folder_name)
     elif tx_type == "router":
         _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("union",):
@@ -777,7 +785,7 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type in ("java",):
         _gen_java_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("sql",):
-        _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs, mapping_name=mapping_name, folder_name=folder_name)
     else:
         lines.append(f"    # TODO: Unsupported transformation type '{tx.type}' - passing through")
         copy_expr = lib_copy(data_lib, input_df)
@@ -982,7 +990,7 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph=None, data_lib="pandas"):
+def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph=None, data_lib="pandas", mapping_name="", folder_name=""):
     lookup_table = ""
     lookup_sql = ""
     lookup_condition = ""
@@ -1019,7 +1027,7 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_gr
 
     lines.append(f"    # Lookup: {lookup_table or tx.name}")
     if lookup_sql:
-        _emit_sql_with_params(lines, f"lkp_sql_{tx_safe}", lookup_sql)
+        _emit_sql_with_params(lines, f"lkp_sql_{tx_safe}", lookup_sql, mapping_name=mapping_name, folder_name=folder_name)
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, lkp_sql_{tx_safe}, 'default')")
     elif lookup_table:
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, 'SELECT * FROM {lookup_table}', 'default')")
@@ -1415,14 +1423,14 @@ def _gen_java_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs):
+def _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs, mapping_name="", folder_name=""):
     sql_query = ""
     for attr in tx.attributes:
         if attr.name == "Sql Query" and attr.value:
             sql_query = convert_sql_expression(attr.value)
     lines.append(f"    # SQL Transformation: {tx.name}")
     if sql_query:
-        _emit_sql_with_params(lines, f"sql_{tx_safe}", sql_query)
+        _emit_sql_with_params(lines, f"sql_{tx_safe}", sql_query, mapping_name=mapping_name, folder_name=folder_name)
         lines.append(f"    df_{tx_safe} = read_from_db(config, sql_{tx_safe}, 'default')")
     else:
         lines.append(f"    df_{tx_safe} = {input_df}.copy()")
@@ -1461,7 +1469,7 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     if col_mapping:
         lines.append(f"    # Column mapping: source -> target")
         lines.append(f"    target_columns_{tgt_safe} = {col_mapping}")
-        lines.append(f"    df_target_{tgt_safe} = {input_df}.rename(columns={{v: k for k, v in target_columns_{tgt_safe}.items()}})")
+        lines.append(f"    df_target_{tgt_safe} = rename_with_duplicates({input_df}, target_columns_{tgt_safe})")
         target_cols = [f.name for f in tgt_def.fields] if tgt_def.fields else None
         if target_cols:
             lines.append(f"    # Select only target columns")

informatica_python/utils/expression_converter.py

@@ -867,7 +867,7 @@ def _vec_recursive(expr, df_var):
             if v.startswith("'") and v.endswith("'"):
                 vec_parts.append(v)
             else:
-                vec_parts.append(f'{v}.astype(str)')
+                vec_parts.append(f'{v}.fillna(\'\').astype(str)')
         return " + ".join(vec_parts)
 
     for func_name in sorted(INFA_FUNC_MAP.keys(), key=lambda x: -len(x)):

{informatica_python-1.9.4.dist-info → informatica_python-1.9.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.9.4
+Version: 1.9.6
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.9.4.dist-info → informatica_python-1.9.6.dist-info}/RECORD

@@ -1,4 +1,4 @@
-informatica_python/__init__.py,sha256=UiVcrgRjgo439mxIWb1Oz1caKLcmgl0DTlz5-GrgKjs,337
+informatica_python/__init__.py,sha256=JFO8fVMClSWe0SR-CBseX4RaPyyC3rZBdxxjy47ZT5E,337
 informatica_python/cli.py,sha256=gFwg0O99vKM-OLO0HoHA4emd-6qrgjMNqa9T59e4e_s,2905
 informatica_python/converter.py,sha256=xCuWrYzDji0yN72D3QqOgZCVVM2j3k2_CvlGplCWxLU,22779
 informatica_python/models.py,sha256=G_C2WfQL-ykKjNj23m8vKFtLZYrQozp99HJzrLTKG1Y,17293
@@ -6,18 +6,18 @@ informatica_python/parser.py,sha256=v0qoTlAi3RZ3IHN_5g5t6f66XzRpJIjpAfpyMzZ5cuA,
 informatica_python/generators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 informatica_python/generators/config_gen.py,sha256=4tqcNKTB06kyGZIiM4yl0q97q_i3zeCHXTjuE1dNFKY,5726
 informatica_python/generators/error_log_gen.py,sha256=2cc0rEcblydHkb9VAMXlrH7WdSQ-CNqAXcwVk3FYZeM,21319
-informatica_python/generators/helper_gen.py,sha256=D6-UqNh09Qy2V7RimNgP-SzK_uB9YqAlsa0-cgLhf5o,72209
-informatica_python/generators/mapping_gen.py,sha256=TPcd9tIAhOAIZMbzPslUgPvF-FY_XkI7EfvCVf-yU50,71610
+informatica_python/generators/helper_gen.py,sha256=lC30hyZn6RIkbo4e_6sbqdrCfmZHWaXdr-p0tmtfILc,82376
+informatica_python/generators/mapping_gen.py,sha256=a5UZCIoU5E5ff9Q8Nxp_m-6k8wZv2NTZL96o2hDddZo,72603
 informatica_python/generators/sql_gen.py,sha256=O8Y-aJz9EyFJ0DXeuISRt5yKwC3wlp2K3B0BHrmxrXw,4872
 informatica_python/generators/workflow_gen.py,sha256=_uSlBg31ZRMhMlCYk4hWDRBPaBROrepD8_v3QGEWJxE,18089
 informatica_python/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 informatica_python/utils/datatype_map.py,sha256=iLOYg-iBKT4rMecGbrFkTpJj4yqs5S9HeBOTLUIWhX0,2809
-informatica_python/utils/expression_converter.py,sha256=f8sNAhE0Yo8sdcVIPDjrw_uGfd5UTD8k1SN8NYApmhI,46846
+informatica_python/utils/expression_converter.py,sha256=SkkT2CyhIZzUms9TT4cEimZlxjOoVq96AQgGTrO_Lmc,46859
 informatica_python/utils/lib_adapters.py,sha256=1ZtuMbgDg9Ukf-OF_EG1L_BeeR-6JQk8Kx3WwMfvNRU,6516
 informatica_python/utils/sql_dialect.py,sha256=_IHJbfu8a3mT_OvHpybgSfZKqz6mwVy5ItTKDRChqnU,5461
-informatica_python-1.9.4.dist-info/licenses/LICENSE,sha256=77RaRDdXgey1D90YZAjXqEQdBxWfvUQqLQX3pC1qjUE,1061
-informatica_python-1.9.4.dist-info/METADATA,sha256=0KXrQZPNvGQB-47LSGQDZv1hm_nN6YWBlhB8WtBCV4I,26097
-informatica_python-1.9.4.dist-info/WHEEL,sha256=PovZm1ExVWmrRefZoXCfejlbKLnQI5SVIf1SWRV4QQI,97
-informatica_python-1.9.4.dist-info/entry_points.txt,sha256=030jjTrx-1oRRQ16HZz52rdcKS8R8_llnymsTUtn_Xc,67
-informatica_python-1.9.4.dist-info/top_level.txt,sha256=Dngg-WNteYi22XAJU2XKAQS8aZ52yM2LYC0tzxrlbVQ,19
-informatica_python-1.9.4.dist-info/RECORD,,
+informatica_python-1.9.6.dist-info/licenses/LICENSE,sha256=77RaRDdXgey1D90YZAjXqEQdBxWfvUQqLQX3pC1qjUE,1061
+informatica_python-1.9.6.dist-info/METADATA,sha256=4XrzBAs63VuLy6Wf_WsgmbGLWK9iaJeHjIZiyOb2PBw,26097
+informatica_python-1.9.6.dist-info/WHEEL,sha256=PovZm1ExVWmrRefZoXCfejlbKLnQI5SVIf1SWRV4QQI,97
+informatica_python-1.9.6.dist-info/entry_points.txt,sha256=030jjTrx-1oRRQ16HZz52rdcKS8R8_llnymsTUtn_Xc,67
+informatica_python-1.9.6.dist-info/top_level.txt,sha256=Dngg-WNteYi22XAJU2XKAQS8aZ52yM2LYC0tzxrlbVQ,19
+informatica_python-1.9.6.dist-info/RECORD,,