informatica-python 1.9.3__tar.gz → 1.9.5__tar.gz

{informatica_python-1.9.3 → informatica_python-1.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.9.3
+Version: 1.9.5
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT
@@ -430,7 +430,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 - **Generated code formatting**: Consistent `# ---` section headers for Source Qualifiers, Transforms, and Target Writes; metadata comments (database type, field lists); column mapping and write operation comments; clean blank line handling
 - **Source/target detection**: Case-insensitive instance type matching
 - **Session→mapping inference**: Longest-suffix-match strategy for ambiguous mapping names
-- **646 tests** across unit, integration, expression, and formatting test suites
+- **663 tests** across unit, integration, expression, and formatting test suites
 
 ### v1.9.2 (Phase 8)
 - Mapping output files now use real mapping names (e.g., `mapping_m_customer_load.py`) instead of generic numeric indices (`mapping_1.py`)
@@ -495,7 +495,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 cd informatica_python
 pip install -e ".[dev]"
 
-# Run tests (646 tests)
+# Run tests (663 tests)
 pytest tests/ -v
 ```
 

{informatica_python-1.9.3 → informatica_python-1.9.5}/README.md

@@ -403,7 +403,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 - **Generated code formatting**: Consistent `# ---` section headers for Source Qualifiers, Transforms, and Target Writes; metadata comments (database type, field lists); column mapping and write operation comments; clean blank line handling
 - **Source/target detection**: Case-insensitive instance type matching
 - **Session→mapping inference**: Longest-suffix-match strategy for ambiguous mapping names
-- **646 tests** across unit, integration, expression, and formatting test suites
+- **663 tests** across unit, integration, expression, and formatting test suites
 
 ### v1.9.2 (Phase 8)
 - Mapping output files now use real mapping names (e.g., `mapping_m_customer_load.py`) instead of generic numeric indices (`mapping_1.py`)
@@ -468,7 +468,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 cd informatica_python
 pip install -e ".[dev]"
 
-# Run tests (646 tests)
+# Run tests (663 tests)
 pytest tests/ -v
 ```
 

{informatica_python-1.9.3 → informatica_python-1.9.5}/informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.9.3"
+__version__ = "1.9.5"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

{informatica_python-1.9.3 → informatica_python-1.9.5}/informatica_python/generators/helper_gen.py

@@ -11,6 +11,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     lines.append("import os")
+    lines.append("import re")
     lines.append("import sys")
     lines.append("import logging")
     lines.append("import yaml")
@@ -58,6 +59,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     _add_param_file_functions(lines)
+    _add_env_resolution(lines)
     _add_db_functions(lines, data_lib)
     _add_file_functions(lines, data_lib)
     _add_expression_helpers(lines)
@@ -121,23 +123,143 @@ def _add_param_file_functions(lines):
     lines.append("")
 
 
+def _add_env_resolution(lines):
+    lines.append("# ============================================================")
+    lines.append("# Environment Variable Resolution")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("def resolve_env(value, config=None):")
+    lines.append('    """')
+    lines.append("    Resolve ${VAR} placeholders in a string.")
+    lines.append("    Lookup order: OS environment variable -> config connections/variables -> literal.")
+    lines.append('    """')
+    lines.append("    if not isinstance(value, str):")
+    lines.append("        return value")
+    lines.append("    def _replace(m):")
+    lines.append("        var = m.group(1)")
+    lines.append("        env_val = os.environ.get(var)")
+    lines.append("        if env_val is not None:")
+    lines.append("            return env_val")
+    lines.append("        if config:")
+    lines.append("            for section in ('variables', 'connections', 'params'):")
+    lines.append("                sect = config.get(section, {})")
+    lines.append("                if isinstance(sect, dict) and var in sect:")
+    lines.append("                    v = sect[var]")
+    lines.append("                    return str(v) if not isinstance(v, dict) else str(v.get('default_value', ''))")
+    lines.append("        return m.group(0)")
+    lines.append(r"    return re.sub(r'\$\{(\w+)\}', _replace, value)")
+    lines.append("")
+    lines.append("")
+    lines.append("def rename_with_duplicates(df, col_mapping):")
+    lines.append('    """')
+    lines.append("    Rename DataFrame columns supporting one-source-to-many-target mapping.")
+    lines.append("    col_mapping is {target_col: source_col}.")
+    lines.append("    When multiple target cols map to the same source col, we duplicate the column.")
+    lines.append('    """')
+    lines.append("    result = df.copy()")
+    lines.append("    from collections import Counter")
+    lines.append("    src_counts = Counter(col_mapping.values())")
+    lines.append("    simple_rename = {}")
+    lines.append("    for tgt, src in col_mapping.items():")
+    lines.append("        if src_counts[src] == 1 and src in result.columns:")
+    lines.append("            simple_rename[src] = tgt")
+    lines.append("        elif src in result.columns:")
+    lines.append("            result[tgt] = result[src].copy()")
+    lines.append("    if simple_rename:")
+    lines.append("        result = result.rename(columns=simple_rename)")
+    lines.append("    return result")
+    lines.append("")
+    lines.append("")
+
+    lines.append("def resolve_builtin_variable(var_name, mapping_name='', session_name='', folder_name=''):")
+    lines.append('    """Resolve Informatica built-in variables like $PMMappingName, $PMSessionName."""')
+    lines.append("    builtins = {")
+    lines.append("        'PMMappingName': mapping_name,")
+    lines.append("        'PMSessionName': session_name,")
+    lines.append("        'PMFolderName': folder_name,")
+    lines.append("        'PMWorkflowName': os.environ.get('INFA_VAR_PMWorkflowName', ''),")
+    lines.append("        'PMWorkflowRunId': os.environ.get('INFA_VAR_PMWorkflowRunId', '0'),")
+    lines.append("        'PMSessionRunId': os.environ.get('INFA_VAR_PMSessionRunId', '0'),")
+    lines.append("        'PMIntegrationServiceName': os.environ.get('INFA_VAR_PMIntegrationServiceName', ''),")
+    lines.append("        'PMRepositoryServiceName': os.environ.get('INFA_VAR_PMRepositoryServiceName', ''),")
+    lines.append("        'PMSourceDBConnection': os.environ.get('INFA_VAR_PMSourceDBConnection', ''),")
+    lines.append("        'PMTargetDBConnection': os.environ.get('INFA_VAR_PMTargetDBConnection', ''),")
+    lines.append("    }")
+    lines.append("    clean = var_name.lstrip('$').lstrip('PM')")
+    lines.append("    for key, val in builtins.items():")
+    lines.append("        if key.lower() == ('PM' + clean).lower() or key.lower() == var_name.lstrip('$').lower():")
+    lines.append("            return val")
+    lines.append("    return os.environ.get(f'INFA_VAR_{var_name.lstrip(\"$\")}', '')")
+    lines.append("")
+    lines.append("")
+
+
 def _add_db_functions(lines, data_lib):
     lines.append("# ============================================================")
     lines.append("# Database Operations")
     lines.append("# ============================================================")
     lines.append("")
     lines.append("")
+    lines.append("_engine_cache = {}")
+    lines.append("")
+    lines.append("")
     lines.append("def get_db_connection(config, connection_name='default'):")
-    lines.append('    """Create database connection from config."""')
+    lines.append('    """')
+    lines.append("    Create database connection from config.")
+    lines.append("    Prefers SQLAlchemy engine (with connection pooling) over raw drivers.")
+    lines.append("    Config values support ${VAR} env-var placeholders via resolve_env().")
+    lines.append('    """')
     lines.append("    conn_config = config.get('connections', {}).get(connection_name, {})")
-    lines.append("    db_type = conn_config.get('type', 'mssql')")
-    lines.append("    host = conn_config.get('host', 'localhost')")
-    lines.append("    port = conn_config.get('port', 1433)")
-    lines.append("    database = conn_config.get('database', '')")
-    lines.append("    username = conn_config.get('username', '')")
-    lines.append("    password = conn_config.get('password', '')")
-    lines.append("    schema = conn_config.get('schema', 'dbo')")
+    lines.append("    db_type = resolve_env(conn_config.get('type', 'mssql'), config)")
+    lines.append("    host = resolve_env(conn_config.get('host', 'localhost'), config)")
+    lines.append("    port = resolve_env(conn_config.get('port', 1433), config)")
+    lines.append("    database = resolve_env(conn_config.get('database', ''), config)")
+    lines.append("    username = resolve_env(conn_config.get('username', ''), config)")
+    lines.append("    password = resolve_env(conn_config.get('password', ''), config)")
+    lines.append("    schema = resolve_env(conn_config.get('schema', 'dbo'), config)")
+    lines.append("")
+    lines.append("    cache_key = f'{db_type}://{username}@{host}:{port}/{database}'")
+    lines.append("    if cache_key in _engine_cache:")
+    lines.append("        return _engine_cache[cache_key].connect()")
+    lines.append("")
+    lines.append("    try:")
+    lines.append("        from sqlalchemy import create_engine")
+    lines.append("        url = _build_sqlalchemy_url(db_type, host, port, database, username, password)")
+    lines.append("        if url:")
+    lines.append("            engine = create_engine(url, pool_pre_ping=True, pool_size=5)")
+    lines.append("            _engine_cache[cache_key] = engine")
+    lines.append("            return engine.connect()")
+    lines.append("    except ImportError:")
+    lines.append("        logger.info('SQLAlchemy not available, falling back to raw drivers')")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'SQLAlchemy connection failed: {e}, falling back to raw drivers')")
+    lines.append("")
+    lines.append("    return _get_raw_connection(db_type, host, port, database, username, password, conn_config)")
     lines.append("")
+    lines.append("")
+    lines.append("def _build_sqlalchemy_url(db_type, host, port, database, username, password):")
+    lines.append("    from urllib.parse import quote_plus")
+    lines.append("    pw = quote_plus(str(password)) if password else ''")
+    lines.append("    if db_type == 'mssql':")
+    lines.append("        try:")
+    lines.append("            import pyodbc")
+    lines.append("            conn_str = quote_plus(")
+    lines.append("                f'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={host},{port};DATABASE={database};UID={username};PWD={password}'")
+    lines.append("            )")
+    lines.append("            return f'mssql+pyodbc:///?odbc_connect={conn_str}'")
+    lines.append("        except ImportError:")
+    lines.append("            return f'mssql+pymssql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        return f'postgresql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        return f'oracle+cx_oracle://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'mysql':")
+    lines.append("        return f'mysql+pymysql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    return None")
+    lines.append("")
+    lines.append("")
+    lines.append("def _get_raw_connection(db_type, host, port, database, username, password, conn_config):")
     lines.append("    if db_type == 'mssql':")
     lines.append("        try:")
     lines.append("            import pyodbc")
@@ -156,39 +278,19 @@ def _add_db_functions(lines, data_lib):
     lines.append("            return pymssql.connect(server=host, port=int(port), database=database, user=username, password=password)")
     lines.append("        except ImportError:")
     lines.append("            pass")
-    lines.append("        try:")
-    lines.append("            from sqlalchemy import create_engine")
-    lines.append("            engine = create_engine(f'mssql+pymssql://{username}:{password}@{host}:{port}/{database}')")
-    lines.append("            return engine.connect()")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'postgresql':")
-    lines.append("        try:")
-    lines.append("            import psycopg2")
-    lines.append("            return psycopg2.connect(")
-    lines.append("                host=host, port=port, dbname=database,")
-    lines.append("                user=username, password=password")
-    lines.append("            )")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'oracle':")
-    lines.append("        try:")
-    lines.append("            import cx_Oracle")
-    lines.append("            dsn = cx_Oracle.makedsn(host, port, service_name=database)")
-    lines.append("            return cx_Oracle.connect(username, password, dsn)")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        import psycopg2")
+    lines.append("        return psycopg2.connect(host=host, port=port, dbname=database, user=username, password=password)")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        import cx_Oracle")
+    lines.append("        dsn = cx_Oracle.makedsn(host, port, service_name=database)")
+    lines.append("        return cx_Oracle.connect(username, password, dsn)")
     lines.append("")
     lines.append("    jdbc_url = conn_config.get('jdbc_url', '')")
     lines.append("    if jdbc_url:")
-    lines.append("        try:")
-    lines.append("            import jaydebeapi")
-    lines.append("            driver = conn_config.get('jdbc_driver', '')")
-    lines.append("            return jaydebeapi.connect(driver, jdbc_url, [username, password])")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("        import jaydebeapi")
+    lines.append("        driver = conn_config.get('jdbc_driver', '')")
+    lines.append("        return jaydebeapi.connect(driver, jdbc_url, [username, password])")
     lines.append("")
     lines.append("    raise ConnectionError(f'Cannot create connection for type: {db_type}')")
     lines.append("")
@@ -203,6 +305,15 @@ def _add_db_functions(lines, data_lib):
     else:
         read_func = "pd.read_sql"
 
+    lines.append("def _safe_close(conn):")
+    lines.append('    """Close connection safely — handles both SQLAlchemy and raw connections."""')
+    lines.append("    try:")
+    lines.append("        if hasattr(conn, 'close'):")
+    lines.append("            conn.close()")
+    lines.append("    except Exception:")
+    lines.append("        pass")
+    lines.append("")
+    lines.append("")
     lines.append("def read_from_db(config, query, connection_name='default'):")
     lines.append('    """Read data from database using SQL query."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
@@ -220,7 +331,7 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB read error on {{connection_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def write_to_db(config, df, table_name, connection_name='default', if_exists='append', schema=None):")
@@ -242,23 +353,31 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB write error to {{schema}}.{{table_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def execute_sql(config, sql, connection_name='default'):")
     lines.append('    """Execute a SQL statement (INSERT, UPDATE, DELETE, DDL)."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
     lines.append("    try:")
-    lines.append("        cursor = conn.cursor()")
-    lines.append("        cursor.execute(sql)")
-    lines.append("        conn.commit()")
+    lines.append("        if hasattr(conn, 'execute'):")
+    lines.append("            from sqlalchemy import text")
+    lines.append("            conn.execute(text(sql))")
+    lines.append("            conn.commit()")
+    lines.append("        else:")
+    lines.append("            cursor = conn.cursor()")
+    lines.append("            cursor.execute(sql)")
+    lines.append("            conn.commit()")
     lines.append("        logger.info(f'Executed SQL on {{connection_name}}')")
     lines.append("    except Exception as e:")
     lines.append("        logger.error(f'SQL execution error: {{e}}')")
-    lines.append("        conn.rollback()")
+    lines.append("        try:")
+    lines.append("            conn.rollback()")
+    lines.append("        except Exception:")
+    lines.append("            pass")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
 
@@ -1150,10 +1269,60 @@ def _add_expression_helpers(lines):
     lines.append("    raise SystemExit(message)")
     lines.append("")
     lines.append("")
-    lines.append("def lookup_func(table, condition, *fields):")
-    lines.append('    """Placeholder for Informatica LOOKUP function."""')
-    lines.append("    logger.warning(f'LOOKUP called for table {table} - implement in mapping-specific code')")
-    lines.append("    return None")
+    lines.append("_lookup_cache = {}")
+    lines.append("")
+    lines.append("")
+    lines.append("def lookup_func(table, condition, *fields, config=None, connection_name='default'):")
+    lines.append('    """')
+    lines.append("    Informatica unconnected LOOKUP function.")
+    lines.append("    Loads and caches the lookup table, then filters by condition.")
+    lines.append("    Returns the first matching value of the first return field, or None.")
+    lines.append('    """')
+    lines.append("    global _lookup_cache")
+    lines.append("    if table not in _lookup_cache:")
+    lines.append("        if config is not None:")
+    lines.append("            try:")
+    lines.append("                lkp_conn = connection_name")
+    lines.append("                conns = config.get('connections', {})")
+    lines.append("                for cname, cval in conns.items():")
+    lines.append("                    if isinstance(cval, dict) and cval.get('connection_name', '') == table:")
+    lines.append("                        lkp_conn = cname")
+    lines.append("                        break")
+    lines.append("                df_lkp = read_from_db(config, f'SELECT * FROM {table}', lkp_conn)")
+    lines.append("                _lookup_cache[table] = df_lkp")
+    lines.append("                logger.info(f'Cached lookup table {table}: {len(df_lkp)} rows')")
+    lines.append("            except Exception as e:")
+    lines.append("                logger.warning(f'Could not load lookup table {table}: {e}')")
+    lines.append("                _lookup_cache[table] = None")
+    lines.append("        else:")
+    lines.append("            logger.warning(f'LOOKUP called for {table} without config - returning None')")
+    lines.append("            return None")
+    lines.append("    df_lkp = _lookup_cache.get(table)")
+    lines.append("    if df_lkp is None or df_lkp.empty:")
+    lines.append("        return None")
+    lines.append("    try:")
+    lines.append("        if callable(condition):")
+    lines.append("            matches = df_lkp[condition(df_lkp)]")
+    lines.append("        elif isinstance(condition, str) and '=' in condition:")
+    lines.append("            col, _, val = condition.partition('=')")
+    lines.append("            col = col.strip()")
+    lines.append("            val = val.strip().strip(\"'\")")
+    lines.append("            if col in df_lkp.columns:")
+    lines.append("                matches = df_lkp[df_lkp[col].astype(str) == str(val)]")
+    lines.append("            else:")
+    lines.append("                return None")
+    lines.append("        else:")
+    lines.append("            return None")
+    lines.append("        if matches.empty:")
+    lines.append("            return None")
+    lines.append("        if fields:")
+    lines.append("            field = str(fields[0]).strip()")
+    lines.append("            if field in matches.columns:")
+    lines.append("                return matches.iloc[0][field]")
+    lines.append("        return matches.iloc[0].to_dict()")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'LOOKUP error on {table}: {e}')")
+    lines.append("        return None")
     lines.append("")
     lines.append("")
     lines.append("_param_store = {}")

{informatica_python-1.9.3 → informatica_python-1.9.5}/informatica_python/generators/mapping_gen.py

@@ -419,9 +419,10 @@ def _safe_name(name):
     return safe.lower()
 
 
-def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
+def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    ", mapping_name="", session_name="", folder_name=""):
     import re
     params = re.findall(r'\$\$(\w+)', sql_text)
+    pm_vars = re.findall(r'\$(PM\w+)', sql_text)
     lines.append(f"{indent}{sql_var_name} = '''")
     for sql_line in sql_text.strip().split("\n"):
         lines.append(f"{indent}{sql_line}")
@@ -433,6 +434,13 @@ def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
                 continue
             seen.add(p)
             lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('$${p}', str(get_param(config, '{p}')))")
+    if pm_vars:
+        seen_pm = set()
+        for pm in pm_vars:
+            if pm in seen_pm:
+                continue
+            seen_pm.add(pm)
+            lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('${pm}', str(resolve_builtin_variable('{pm}', mapping_name='{mapping_name}', session_name='{session_name}', folder_name='{folder_name}')))")
 
 
 def _flatfile_config_dict(ff):
@@ -757,7 +765,7 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type in ("joiner",):
         _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph, data_lib)
     elif tx_type in ("lookup procedure", "lookup"):
-        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
+        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph, data_lib)
     elif tx_type == "router":
         _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("union",):
@@ -982,7 +990,7 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
+def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph=None, data_lib="pandas"):
     lookup_table = ""
     lookup_sql = ""
     lookup_condition = ""
@@ -1012,6 +1020,11 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pa
 
     all_output_fields = return_fields + lookup_output_fields
 
+    port_to_col = {}
+    if connector_graph and tx.name in connector_graph.get("to", {}):
+        for conn in connector_graph["to"][tx.name]:
+            port_to_col[conn.to_field.lower()] = conn.from_field
+
     lines.append(f"    # Lookup: {lookup_table or tx.name}")
     if lookup_sql:
         _emit_sql_with_params(lines, f"lkp_sql_{tx_safe}", lookup_sql)
@@ -1020,10 +1033,13 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pa
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, 'SELECT * FROM {lookup_table}', 'default')")
     else:
         empty_expr = lib_empty_df(data_lib)
-        lines.append(f"    df_lkp_{tx_safe} = {empty_expr}")
+        lines.append(f"    df_lkp_{tx_safe} = {empty_expr}  # WARNING: no lookup table/SQL override found")
 
     input_keys, lookup_keys = parse_lookup_condition(lookup_condition)
 
+    if input_keys and port_to_col:
+        input_keys = [port_to_col.get(k.lower(), k) for k in input_keys]
+
     if input_keys and lookup_keys:
         lines.append(f"    # Lookup condition: {lookup_condition}")
 
@@ -1078,12 +1094,23 @@ def _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs):
         if "Group Filter Condition" in attr.name:
             group_conditions[attr.name] = attr.value
 
+    remaining_mask_parts = []
     if group_conditions:
         for i, (gname, cond) in enumerate(group_conditions.items()):
-            expr_py = convert_expression(cond) if cond else "True"
-            lines.append(f"    df_{tx_safe}_group{i} = {input_df}[{expr_py}].copy()  # {gname}")
+            if cond and cond.strip():
+                expr_py = convert_filter_vectorized(cond, input_df)
+            else:
+                expr_py = f"pd.Series(True, index={input_df}.index)"
+            mask_var = f"_router_mask_{tx_safe}_{i}"
+            lines.append(f"    {mask_var} = {expr_py}  # {gname}")
+            lines.append(f"    df_{tx_safe}_group{i} = {input_df}[{mask_var}].copy()")
             source_dfs[f"{tx.name}_group{i}"] = f"df_{tx_safe}_group{i}"
-    lines.append(f"    df_{tx_safe} = {input_df}.copy()  # Default group")
+            remaining_mask_parts.append(f"~{mask_var}")
+    if remaining_mask_parts:
+        lines.append(f"    _router_default_mask = {' & '.join(remaining_mask_parts)}")
+        lines.append(f"    df_{tx_safe} = {input_df}[_router_default_mask].copy()  # Default group")
+    else:
+        lines.append(f"    df_{tx_safe} = {input_df}.copy()  # Default group")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -1442,7 +1469,7 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     if col_mapping:
         lines.append(f"    # Column mapping: source -> target")
         lines.append(f"    target_columns_{tgt_safe} = {col_mapping}")
-        lines.append(f"    df_target_{tgt_safe} = {input_df}.rename(columns={{v: k for k, v in target_columns_{tgt_safe}.items()}})")
+        lines.append(f"    df_target_{tgt_safe} = rename_with_duplicates({input_df}, target_columns_{tgt_safe})")
         target_cols = [f.name for f in tgt_def.fields] if tgt_def.fields else None
         if target_cols:
             lines.append(f"    # Select only target columns")

{informatica_python-1.9.3 → informatica_python-1.9.5}/informatica_python/utils/expression_converter.py

@@ -248,6 +248,7 @@ def _convert_infa_date_format(fmt_str):
     fmt = fmt.replace("Mon", "%b").replace("MON", "%b")
     fmt = fmt.replace("HH24", "%H").replace("HH12", "%I").replace("HH", "%H")
     fmt = fmt.replace("MI", "%M").replace("SS", "%S")
+    fmt = fmt.replace("US", "%f").replace("NS", "%f").replace("MS", "%f")
     return fmt
 
 
@@ -548,7 +549,7 @@ def _vec_recursive(expr, df_var):
                         'RTRIM': f'.str.rstrip("{char_arg}")',
                         'TRIM': f'.str.strip("{char_arg}")',
                     }
-                return f'{inner_val}{method_map[func_name.upper()]}'
+                return f'{inner_val}.astype(str){method_map[func_name.upper()]}'
 
     upper_result = _find_func_call(cleaned, 'UPPER')
     if upper_result and upper_result[0] == 0 and upper_result[1] == len(cleaned):
@@ -584,7 +585,7 @@ def _vec_recursive(expr, df_var):
         if len(args) >= 2:
             field_val = _vec_recursive(args[0], df_var)
             try:
-                start = int(args[1].strip()) - 1
+                start = max(int(args[1].strip()) - 1, 0)
             except ValueError:
                 start_val = _vec_recursive(args[1], df_var)
                 if len(args) >= 3:
@@ -722,7 +723,11 @@ def _vec_recursive(expr, df_var):
                 field_val = _vec_recursive(args[0], df_var)
                 pattern_val = args[1].strip().strip("'\"")
                 if func_name == 'REG_EXTRACT':
-                    return f'{field_val}.str.extract(r"({pattern_val})", expand=False)'
+                    if re.search(r'(?<!\\)\((?!\?)', pattern_val):
+                        extract_pat = pattern_val
+                    else:
+                        extract_pat = f'({pattern_val})'
+                    return f'{field_val}.str.extract(r"{extract_pat}", expand=False)'
                 elif func_name == 'REG_REPLACE':
                     replace_val = args[2].strip().strip("'\"") if len(args) >= 3 else ''
                     return f'{field_val}.str.replace(r"{pattern_val}", "{replace_val}", regex=True)'
@@ -862,7 +867,7 @@ def _vec_recursive(expr, df_var):
             if v.startswith("'") and v.endswith("'"):
                 vec_parts.append(v)
             else:
-                vec_parts.append(f'{v}.astype(str)')
+                vec_parts.append(f'{v}.fillna(\'\').astype(str)')
         return " + ".join(vec_parts)
 
     for func_name in sorted(INFA_FUNC_MAP.keys(), key=lambda x: -len(x)):
@@ -894,7 +899,8 @@ def _vec_recursive(expr, df_var):
         'True', 'False', 'None', 'and', 'or', 'not', 'np', 'pd', 'get_variable',
         'str', 'int', 'float', 'bool', 'len', 'abs', 'round',
         'fillna', 'astype', 'isna', 'notna', 'where', 'errors', 'coerce',
-        'lookup_func',
+        'lookup_func', 'expand', 'extract', 'regex', 'contains', 'replace',
+        'upper', 'lower', 'strip', 'lstrip', 'rstrip', 'dt', 'copy',
     }
     converted = _substitute_fields(converted, df_var, skip_words)
 
@@ -904,6 +910,8 @@ def _vec_recursive(expr, df_var):
     converted = re.sub(r'<>', '!=', converted)
     converted = re.sub(r'(?<![<>!=])=(?!=)', '==', converted)
     converted = re.sub(r'\berrors\s*==\s*(["\'])', r'errors=\1', converted)
+    converted = re.sub(r'\bexpand\s*==\s*', 'expand=', converted)
+    converted = re.sub(r'\bregex\s*==\s*', 'regex=', converted)
 
     converted = re.sub(r'\s+', ' ', converted).strip()
 
@@ -1044,8 +1052,14 @@ def _vectorize_simple(part, df_var):
         'True', 'False', 'None', 'and', 'or', 'not', 'np', 'pd',
         'str', 'int', 'float', 'isna', 'notna', 'fillna',
         'get_variable', 'lookup_func', 'isin', 'eq',
+        'expand', 'extract', 'astype', 'errors', 'coerce', 'regex',
+        'contains', 'replace', 'upper', 'lower', 'strip', 'lstrip', 'rstrip',
+        'dt', 'len', 'copy', 'abs', 'round', 'where', 'bool',
     }
     c = _substitute_fields(c, df_var, skip_words)
+    c = re.sub(r'\bexpand\s*==\s*', 'expand=', c)
+    c = re.sub(r'\berrors\s*==\s*', 'errors=', c)
+    c = re.sub(r'\bregex\s*==\s*', 'regex=', c)
 
     return c
 

{informatica_python-1.9.3 → informatica_python-1.9.5}/informatica_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.9.3
+Version: 1.9.5
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT
@@ -430,7 +430,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 - **Generated code formatting**: Consistent `# ---` section headers for Source Qualifiers, Transforms, and Target Writes; metadata comments (database type, field lists); column mapping and write operation comments; clean blank line handling
 - **Source/target detection**: Case-insensitive instance type matching
 - **Session→mapping inference**: Longest-suffix-match strategy for ambiguous mapping names
-- **646 tests** across unit, integration, expression, and formatting test suites
+- **663 tests** across unit, integration, expression, and formatting test suites
 
 ### v1.9.2 (Phase 8)
 - Mapping output files now use real mapping names (e.g., `mapping_m_customer_load.py`) instead of generic numeric indices (`mapping_1.py`)
@@ -495,7 +495,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 cd informatica_python
 pip install -e ".[dev]"
 
-# Run tests (646 tests)
+# Run tests (663 tests)
 pytest tests/ -v
 ```
 

{informatica_python-1.9.3 → informatica_python-1.9.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "informatica-python"
-version = "1.9.3"
+version = "1.9.5"
 description = "Convert Informatica PowerCenter workflow XML to Python/PySpark code"
 readme = "README.md"
 license = {text = "MIT"}

{informatica_python-1.9.3 → informatica_python-1.9.5}/tests/test_integration.py

@@ -2246,3 +2246,465 @@ class TestJoinerFieldRemapping(unittest.TestCase):
             if "left_on" in line and "right_on" in line:
                 assert "Table_Name" in line, \
                     "Merge should use source column name Table_Name"
+
+
+class TestRegExtractConversion(unittest.TestCase):
+    """Tests for REG_EXTRACT capture group and expand parameter handling."""
+
+    def test_no_double_capture_group(self):
+        r = convert_expression_vectorized(r"REG_EXTRACT(col,'(\s+)')", "df")
+        assert r.count("(") - r.count("str.extract") <= 2
+        assert '((\\s+))' not in r
+
+    def test_adds_capture_group_when_missing(self):
+        r = convert_expression_vectorized(r"REG_EXTRACT(col,'\\d+')", "df")
+        assert 'expand=False' in r
+        assert '.str.extract' in r
+
+    def test_expand_is_boolean_not_series(self):
+        r = convert_expression_vectorized(r"REG_EXTRACT(col,'(\s+)')", "df")
+        assert 'expand=False' in r
+        assert 'expand==False' not in r
+        assert 'df["expand"]' not in r
+
+    def test_isnull_reg_extract_nested(self):
+        r = convert_expression_vectorized(
+            "IIF(ISNULL(REG_EXTRACT(PART_BIRTH_DTE,'(\\s+)')),PART_BIRTH_DTE,NULL)", "df_exp"
+        )
+        assert "np.where" in r
+        assert ".isna()" in r
+        assert "expand=False" in r
+        assert 'expand==False' not in r
+        assert 'df_exp["expand"]' not in r
+
+
+class TestDatetimeFormatMask(unittest.TestCase):
+    """Tests for datetime format mask conversion (US/microseconds)."""
+
+    def test_us_to_percent_f(self):
+        from informatica_python.utils.expression_converter import _convert_infa_date_format
+        fmt = _convert_infa_date_format("YYYY-MM-DD HH24.MI.SS.US")
+        assert "%f" in fmt
+        assert "US" not in fmt
+
+    def test_full_format_mask(self):
+        from informatica_python.utils.expression_converter import _convert_infa_date_format
+        fmt = _convert_infa_date_format("YYYY-MM-DD HH24:MI:SS")
+        assert fmt == "%Y-%m-%d %H:%M:%S"
+
+    def test_to_date_with_us_format(self):
+        r = convert_expression_vectorized(
+            "TO_DATE(x, 'YYYY-MM-DD HH24.MI.SS.US')", "df"
+        )
+        assert "%f" in r
+        assert "US" not in r
+
+
+class TestSubstrZeroIndex(unittest.TestCase):
+    """Tests for SUBSTR with 0-based start position."""
+
+    def test_substr_start_0(self):
+        r = convert_expression_vectorized("SUBSTR(x, 0, 11)", "df")
+        assert "str[0:" in r
+        assert "str[-1:" not in r
+
+    def test_substr_start_1(self):
+        r = convert_expression_vectorized("SUBSTR(x, 1, 5)", "df")
+        assert "str[0:" in r
+
+    def test_substr_start_5(self):
+        r = convert_expression_vectorized("SUBSTR(x, 5, 3)", "df")
+        assert "str[4:7]" in r
+
+
+class TestStringOpSafety(unittest.TestCase):
+    """Tests for string operations adding .astype(str) for safety."""
+
+    def test_ltrim_has_astype_str(self):
+        r = convert_expression_vectorized("LTRIM(name)", "df")
+        assert ".astype(str)" in r
+        assert ".str.lstrip()" in r
+
+    def test_rtrim_has_astype_str(self):
+        r = convert_expression_vectorized("RTRIM(name)", "df")
+        assert ".astype(str)" in r
+        assert ".str.rstrip()" in r
+
+    def test_trim_has_astype_str(self):
+        r = convert_expression_vectorized("TRIM(name)", "df")
+        assert ".astype(str)" in r
+        assert ".str.strip()" in r
+
+    def test_ltrim_with_char(self):
+        r = convert_expression_vectorized("LTRIM(name, '0')", "df")
+        assert ".astype(str)" in r
+        assert '.str.lstrip("0")' in r
+
+
+class TestRouterVectorized(unittest.TestCase):
+    """Tests for Router transformation generating vectorized conditions."""
+
+    ROUTER_XML = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE POWERMART SYSTEM "powrmart.dtd">
+<POWERMART CREATION_DATE="01/01/2025" REPOSITORY_VERSION="1">
+<REPOSITORY NAME="repo" VERSION="1" CODEPAGE="UTF-8" DATABASETYPE="Oracle">
+<FOLDER NAME="TEST" OWNER="admin">
+  <SOURCE NAME="SRC" DATABASETYPE="Flat File" DBDNAME="SRC">
+    <FLATFILE DELIMITEDBY="COMMA" HEADERROWPRESENT="YES" PADBYTES="NO" ROWDELIMITER="\\n"/>
+    <SOURCEFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NOTNULL" KEYTYPE="PRIMARY KEY" FIELDNUMBER="1"/>
+    <SOURCEFIELD NAME="STATUS" DATATYPE="string" PRECISION="20" SCALE="0" NULLABLE="NULL" KEYTYPE="NOT A KEY" FIELDNUMBER="2"/>
+  </SOURCE>
+  <TARGET NAME="TGT" DATABASETYPE="Flat File">
+    <TARGETFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NULL" KEYTYPE="NOT A KEY" FIELDNUMBER="1"/>
+  </TARGET>
+  <MAPPING NAME="m_router_test" ISVALID="YES">
+    <TRANSFORMATION NAME="SQ_SRC" TYPE="Source Qualifier" REUSABLE="NO">
+      <TRANSFORMFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" PORTTYPE="OUTPUT"/>
+      <TRANSFORMFIELD NAME="STATUS" DATATYPE="string" PRECISION="20" SCALE="0" PORTTYPE="OUTPUT"/>
+    </TRANSFORMATION>
+    <TRANSFORMATION NAME="RTR_STATUS" TYPE="Router" REUSABLE="NO">
+      <TRANSFORMFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" PORTTYPE="INPUT/OUTPUT"/>
+      <TRANSFORMFIELD NAME="STATUS" DATATYPE="string" PRECISION="20" SCALE="0" PORTTYPE="INPUT/OUTPUT"/>
+      <TABLEATTRIBUTE NAME="Group Filter Condition_ACTIVE" VALUE="STATUS = 'ACTIVE'"/>
+      <TABLEATTRIBUTE NAME="Group Filter Condition_INACTIVE" VALUE="STATUS = 'INACTIVE'"/>
+    </TRANSFORMATION>
+    <INSTANCE NAME="SRC" TYPE="Source Definition" TRANSFORMATION_NAME="SRC"/>
+    <INSTANCE NAME="SQ_SRC" TYPE="Source Qualifier" TRANSFORMATION_NAME="SQ_SRC"/>
+    <INSTANCE NAME="RTR_STATUS" TYPE="Router" TRANSFORMATION_NAME="RTR_STATUS"/>
+    <INSTANCE NAME="TGT" TYPE="Target Definition" TRANSFORMATION_NAME="TGT"/>
+    <CONNECTOR FROMINSTANCE="SRC" FROMFIELD="ID" TOINSTANCE="SQ_SRC" TOFIELD="ID"/>
+    <CONNECTOR FROMINSTANCE="SRC" FROMFIELD="STATUS" TOINSTANCE="SQ_SRC" TOFIELD="STATUS"/>
+    <CONNECTOR FROMINSTANCE="SQ_SRC" FROMFIELD="ID" TOINSTANCE="RTR_STATUS" TOFIELD="ID"/>
+    <CONNECTOR FROMINSTANCE="SQ_SRC" FROMFIELD="STATUS" TOINSTANCE="RTR_STATUS" TOFIELD="STATUS"/>
+    <CONNECTOR FROMINSTANCE="RTR_STATUS" FROMFIELD="ID" TOINSTANCE="TGT" TOFIELD="ID"/>
+  </MAPPING>
+  <CONFIG NAME="default_session_config"/>
+  <WORKFLOW NAME="wf_router_test" ISVALID="YES">
+    <TASK NAME="Start" REUSABLE="NO" TYPE="Start"/>
+    <SESSION NAME="s_m_router_test" ISVALID="YES" REUSABLE="NO" MAPPINGNAME="m_router_test">
+      <CONFIGREFERENCE REFOBJECTNAME="default_session_config" TYPE="Session config"/>
+    </SESSION>
+    <TASKINSTANCE NAME="Start" TASKNAME="Start" TASKTYPE="Start"/>
+    <TASKINSTANCE NAME="s_m_router_test" TASKNAME="s_m_router_test" TASKTYPE="Session"/>
+    <WORKFLOWLINK FROMTASK="Start" TOTASK="s_m_router_test"/>
+  </WORKFLOW>
+</FOLDER>
+</REPOSITORY>
+</POWERMART>'''
+
+    def test_router_generates_group_filters(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(self.ROUTER_XML, output_dir=tmpdir)
+            for fn in os.listdir(tmpdir):
+                if fn.startswith("mapping_") and fn.endswith(".py"):
+                    with open(os.path.join(tmpdir, fn)) as f:
+                        code = f.read()
+                    assert "_router_mask_" in code or "group0" in code, \
+                        "Router should generate group filter masks"
+                    assert "Default group" in code
+                    break
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_router_default_excludes_matched_rows(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(self.ROUTER_XML, output_dir=tmpdir)
+            for fn in os.listdir(tmpdir):
+                if fn.startswith("mapping_") and fn.endswith(".py"):
+                    with open(os.path.join(tmpdir, fn)) as f:
+                        code = f.read()
+                    assert "_router_default_mask" in code or "~" in code, \
+                        "Default group should exclude rows matching other groups"
+                    break
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestLookupWarning(unittest.TestCase):
+    """Tests for lookup empty DataFrame warning."""
+
+    LOOKUP_XML = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE POWERMART SYSTEM "powrmart.dtd">
+<POWERMART CREATION_DATE="01/01/2025" REPOSITORY_VERSION="1">
+<REPOSITORY NAME="repo" VERSION="1" CODEPAGE="UTF-8" DATABASETYPE="Oracle">
+<FOLDER NAME="TEST" OWNER="admin">
+  <SOURCE NAME="SRC" DATABASETYPE="Flat File" DBDNAME="SRC">
+    <FLATFILE DELIMITEDBY="COMMA" HEADERROWPRESENT="YES" PADBYTES="NO" ROWDELIMITER="\\n"/>
+    <SOURCEFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NOTNULL" KEYTYPE="PRIMARY KEY" FIELDNUMBER="1"/>
+  </SOURCE>
+  <TARGET NAME="TGT" DATABASETYPE="Flat File">
+    <TARGETFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NULL" KEYTYPE="NOT A KEY" FIELDNUMBER="1"/>
+  </TARGET>
+  <MAPPING NAME="m_lkp_test" ISVALID="YES">
+    <TRANSFORMATION NAME="SQ_SRC" TYPE="Source Qualifier" REUSABLE="NO">
+      <TRANSFORMFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" PORTTYPE="OUTPUT"/>
+    </TRANSFORMATION>
+    <TRANSFORMATION NAME="LKP_TEST" TYPE="Lookup Procedure" REUSABLE="NO">
+      <TRANSFORMFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" PORTTYPE="INPUT"/>
+      <TRANSFORMFIELD NAME="RESULT" DATATYPE="string" PRECISION="100" SCALE="0" PORTTYPE="OUTPUT/RETURN"/>
+      <TABLEATTRIBUTE NAME="Lookup table name" VALUE="DIM_TABLE"/>
+      <TABLEATTRIBUTE NAME="Lookup condition" VALUE="ID = ID"/>
+    </TRANSFORMATION>
+    <INSTANCE NAME="SRC" TYPE="Source Definition" TRANSFORMATION_NAME="SRC"/>
+    <INSTANCE NAME="SQ_SRC" TYPE="Source Qualifier" TRANSFORMATION_NAME="SQ_SRC"/>
+    <INSTANCE NAME="LKP_TEST" TYPE="Lookup Procedure" TRANSFORMATION_NAME="LKP_TEST"/>
+    <INSTANCE NAME="TGT" TYPE="Target Definition" TRANSFORMATION_NAME="TGT"/>
+    <CONNECTOR FROMINSTANCE="SRC" FROMFIELD="ID" TOINSTANCE="SQ_SRC" TOFIELD="ID"/>
+    <CONNECTOR FROMINSTANCE="SQ_SRC" FROMFIELD="ID" TOINSTANCE="LKP_TEST" TOFIELD="ID"/>
+    <CONNECTOR FROMINSTANCE="LKP_TEST" FROMFIELD="RESULT" TOINSTANCE="TGT" TOFIELD="ID"/>
+  </MAPPING>
+  <CONFIG NAME="default_session_config"/>
+  <WORKFLOW NAME="wf_lkp_test" ISVALID="YES">
+    <TASK NAME="Start" REUSABLE="NO" TYPE="Start"/>
+    <SESSION NAME="s_m_lkp_test" ISVALID="YES" REUSABLE="NO" MAPPINGNAME="m_lkp_test">
+      <CONFIGREFERENCE REFOBJECTNAME="default_session_config" TYPE="Session config"/>
+    </SESSION>
+    <TASKINSTANCE NAME="Start" TASKNAME="Start" TASKTYPE="Start"/>
+    <TASKINSTANCE NAME="s_m_lkp_test" TASKNAME="s_m_lkp_test" TASKTYPE="Session"/>
+    <WORKFLOWLINK FROMTASK="Start" TOTASK="s_m_lkp_test"/>
+  </WORKFLOW>
+</FOLDER>
+</REPOSITORY>
+</POWERMART>'''
+
+    def test_lookup_with_table_reads_from_db(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(self.LOOKUP_XML, output_dir=tmpdir)
+            for fn in os.listdir(tmpdir):
+                if fn.startswith("mapping_") and fn.endswith(".py"):
+                    with open(os.path.join(tmpdir, fn)) as f:
+                        code = f.read()
+                    assert "read_from_db" in code, "Lookup with table should use read_from_db"
+                    assert "DIM_TABLE" in code
+                    break
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestRenameWithDuplicates(unittest.TestCase):
+
+    def test_helper_contains_rename_with_duplicates(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def rename_with_duplicates(" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_target_uses_rename_with_duplicates(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            for fn in os.listdir(tmpdir):
+                if fn.startswith("mapping_") and fn.endswith(".py"):
+                    with open(os.path.join(tmpdir, fn)) as f:
+                        code = f.read()
+                    if "target_columns_" in code:
+                        assert "rename_with_duplicates(" in code, \
+                            "Target rename should use rename_with_duplicates"
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestResolveEnv(unittest.TestCase):
+
+    def test_helper_contains_resolve_env(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def resolve_env(" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_contains_resolve_builtin_variable(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def resolve_builtin_variable(" in code
+            assert "PMMappingName" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestGetDbConnectionSQLAlchemy(unittest.TestCase):
+
+    def test_helper_sqlalchemy_primary(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            sa_pos = code.index("create_engine")
+            pyodbc_pos = code.index("pyodbc")
+            assert sa_pos < pyodbc_pos, "SQLAlchemy should be tried before raw pyodbc"
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_engine_cache(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "_engine_cache" in code
+            assert "pool_pre_ping=True" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_safe_close(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "def _safe_close(" in code
+            assert "_safe_close(conn)" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+    def test_helper_resolve_env_in_db(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "resolve_env(" in code
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestLookupFuncImpl(unittest.TestCase):
+
+    def test_helper_lookup_func_full_impl(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "_lookup_cache" in code
+            assert "def lookup_func(" in code
+            assert "config=None" in code
+            assert "read_from_db" in code.split("def lookup_func")[1]
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestNullSafeConcat(unittest.TestCase):
+
+    def test_concat_fillna(self):
+        result = convert_expression_vectorized("A || B", "df")
+        assert ".fillna('')" in result, f"Concat should use fillna, got: {result}"
+        assert ".astype(str)" in result
+
+    def test_concat_literal_no_fillna(self):
+        result = convert_expression_vectorized("A || '-' || B", "df")
+        assert "'-'" in result
+        parts = result.split(" + ")
+        for part in parts:
+            if part.strip().startswith("'") and part.strip().endswith("'"):
+                assert ".fillna" not in part
+            else:
+                assert ".fillna('')" in part
+
+    def test_concat_three_fields_all_fillna(self):
+        result = convert_expression_vectorized("X || Y || Z", "df")
+        assert result.count(".fillna('')") == 3
+
+
+class TestPMVariableHandling(unittest.TestCase):
+
+    PM_VAR_XML = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE POWERMART SYSTEM "powrmart.dtd">
+<POWERMART CREATION_DATE="01/01/2025" REPOSITORY_VERSION="1">
+<REPOSITORY NAME="repo" VERSION="1" CODEPAGE="UTF-8" DATABASETYPE="Oracle">
+<FOLDER NAME="TEST_FOLDER" OWNER="admin">
+  <SOURCE NAME="SRC_PM" DATABASETYPE="Microsoft SQL Server" DBDNAME="TestDB" OWNERNAME="dbo">
+    <SOURCEFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NOTNULL" KEYTYPE="PRIMARY KEY" FIELDNUMBER="1"/>
+  </SOURCE>
+  <TARGET NAME="TGT_PM" DATABASETYPE="Microsoft SQL Server">
+    <TARGETFIELD NAME="ID" DATATYPE="integer" PRECISION="10" SCALE="0" NULLABLE="NOTNULL" KEYTYPE="PRIMARY KEY" FIELDNUMBER="1"/>
+  </TARGET>
+  <MAPPING NAME="m_pm_vars" ISVALID="YES">
+    <TRANSFORMATION NAME="SQ_SRC_PM" TYPE="Source Qualifier" REUSABLE="NO">
+      <TRANSFORMFIELD NAME="ID" DATATYPE="integer" PORTTYPE="INPUT/OUTPUT" PRECISION="10" SCALE="0"/>
+      <TABLEATTRIBUTE NAME="Sql Query" VALUE="SELECT ID FROM dbo.SRC_PM WHERE mapping_name = &apos;$PMMappingName&apos;"/>
+    </TRANSFORMATION>
+    <INSTANCE NAME="SQ_SRC_PM" TRANSFORMATION_NAME="SQ_SRC_PM" TYPE="Source Qualifier"/>
+    <INSTANCE NAME="SRC_PM" TRANSFORMATION_NAME="SRC_PM" TYPE="Source Definition"/>
+    <INSTANCE NAME="TGT_PM" TRANSFORMATION_NAME="TGT_PM" TYPE="Target Definition"/>
+    <CONNECTOR FROMINSTANCE="SRC_PM" FROMFIELD="ID" TOINSTANCE="SQ_SRC_PM" TOFIELD="ID" FROMINSTANCETYPE="Source Definition" TOINSTANCETYPE="Source Qualifier"/>
+    <CONNECTOR FROMINSTANCE="SQ_SRC_PM" FROMFIELD="ID" TOINSTANCE="TGT_PM" TOFIELD="ID" FROMINSTANCETYPE="Source Qualifier" TOINSTANCETYPE="Target Definition"/>
+  </MAPPING>
+  <CONFIG NAME="default_session_config"/>
+  <WORKFLOW NAME="wf_pm_vars" ISVALID="YES">
+    <SESSION NAME="s_pm_vars" ISVALID="YES" MAPPINGNAME="m_pm_vars"/>
+  </WORKFLOW>
+</FOLDER>
+</REPOSITORY>
+</POWERMART>'''
+
+    def test_pm_variable_resolved_in_sql(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(self.PM_VAR_XML, output_dir=tmpdir)
+            for fn in os.listdir(tmpdir):
+                if fn.startswith("mapping_") and fn.endswith(".py"):
+                    with open(os.path.join(tmpdir, fn)) as f:
+                        code = f.read()
+                    if "$PMMappingName" in code:
+                        assert "resolve_builtin_variable" in code, \
+                            "SQL with $PMMappingName should call resolve_builtin_variable"
+                        break
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestExecuteSqlAlchemy(unittest.TestCase):
+
+    def test_execute_sql_handles_sqlalchemy(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            exec_block = code.split("def execute_sql(")[1]
+            assert "sqlalchemy" in exec_block or "text(sql)" in exec_block
+        finally:
+            shutil.rmtree(tmpdir)
+
+
+class TestImportRe(unittest.TestCase):
+
+    def test_helper_imports_re(self):
+        converter = InformaticaConverter()
+        tmpdir = tempfile.mkdtemp()
+        try:
+            converter.convert_string(MINIMAL_XML, output_dir=tmpdir)
+            with open(os.path.join(tmpdir, "helper_functions.py")) as f:
+                code = f.read()
+            assert "import re" in code
+        finally:
+            shutil.rmtree(tmpdir)