informatica-python 1.9.3__py3-none-any.whl → 1.9.5__py3-none-any.whl

informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.9.3"
+__version__ = "1.9.5"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

informatica_python/generators/helper_gen.py

@@ -11,6 +11,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     lines.append("import os")
+    lines.append("import re")
     lines.append("import sys")
     lines.append("import logging")
     lines.append("import yaml")
@@ -58,6 +59,7 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append("")
 
     _add_param_file_functions(lines)
+    _add_env_resolution(lines)
     _add_db_functions(lines, data_lib)
     _add_file_functions(lines, data_lib)
     _add_expression_helpers(lines)
@@ -121,23 +123,143 @@ def _add_param_file_functions(lines):
     lines.append("")
 
 
+def _add_env_resolution(lines):
+    lines.append("# ============================================================")
+    lines.append("# Environment Variable Resolution")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("def resolve_env(value, config=None):")
+    lines.append('    """')
+    lines.append("    Resolve ${VAR} placeholders in a string.")
+    lines.append("    Lookup order: OS environment variable -> config connections/variables -> literal.")
+    lines.append('    """')
+    lines.append("    if not isinstance(value, str):")
+    lines.append("        return value")
+    lines.append("    def _replace(m):")
+    lines.append("        var = m.group(1)")
+    lines.append("        env_val = os.environ.get(var)")
+    lines.append("        if env_val is not None:")
+    lines.append("            return env_val")
+    lines.append("        if config:")
+    lines.append("            for section in ('variables', 'connections', 'params'):")
+    lines.append("                sect = config.get(section, {})")
+    lines.append("                if isinstance(sect, dict) and var in sect:")
+    lines.append("                    v = sect[var]")
+    lines.append("                    return str(v) if not isinstance(v, dict) else str(v.get('default_value', ''))")
+    lines.append("        return m.group(0)")
+    lines.append(r"    return re.sub(r'\$\{(\w+)\}', _replace, value)")
+    lines.append("")
+    lines.append("")
+    lines.append("def rename_with_duplicates(df, col_mapping):")
+    lines.append('    """')
+    lines.append("    Rename DataFrame columns supporting one-source-to-many-target mapping.")
+    lines.append("    col_mapping is {target_col: source_col}.")
+    lines.append("    When multiple target cols map to the same source col, we duplicate the column.")
+    lines.append('    """')
+    lines.append("    result = df.copy()")
+    lines.append("    from collections import Counter")
+    lines.append("    src_counts = Counter(col_mapping.values())")
+    lines.append("    simple_rename = {}")
+    lines.append("    for tgt, src in col_mapping.items():")
+    lines.append("        if src_counts[src] == 1 and src in result.columns:")
+    lines.append("            simple_rename[src] = tgt")
+    lines.append("        elif src in result.columns:")
+    lines.append("            result[tgt] = result[src].copy()")
+    lines.append("    if simple_rename:")
+    lines.append("        result = result.rename(columns=simple_rename)")
+    lines.append("    return result")
+    lines.append("")
+    lines.append("")
+
+    lines.append("def resolve_builtin_variable(var_name, mapping_name='', session_name='', folder_name=''):")
+    lines.append('    """Resolve Informatica built-in variables like $PMMappingName, $PMSessionName."""')
+    lines.append("    builtins = {")
+    lines.append("        'PMMappingName': mapping_name,")
+    lines.append("        'PMSessionName': session_name,")
+    lines.append("        'PMFolderName': folder_name,")
+    lines.append("        'PMWorkflowName': os.environ.get('INFA_VAR_PMWorkflowName', ''),")
+    lines.append("        'PMWorkflowRunId': os.environ.get('INFA_VAR_PMWorkflowRunId', '0'),")
+    lines.append("        'PMSessionRunId': os.environ.get('INFA_VAR_PMSessionRunId', '0'),")
+    lines.append("        'PMIntegrationServiceName': os.environ.get('INFA_VAR_PMIntegrationServiceName', ''),")
+    lines.append("        'PMRepositoryServiceName': os.environ.get('INFA_VAR_PMRepositoryServiceName', ''),")
+    lines.append("        'PMSourceDBConnection': os.environ.get('INFA_VAR_PMSourceDBConnection', ''),")
+    lines.append("        'PMTargetDBConnection': os.environ.get('INFA_VAR_PMTargetDBConnection', ''),")
+    lines.append("    }")
+    lines.append("    clean = var_name.lstrip('$').lstrip('PM')")
+    lines.append("    for key, val in builtins.items():")
+    lines.append("        if key.lower() == ('PM' + clean).lower() or key.lower() == var_name.lstrip('$').lower():")
+    lines.append("            return val")
+    lines.append("    return os.environ.get(f'INFA_VAR_{var_name.lstrip(\"$\")}', '')")
+    lines.append("")
+    lines.append("")
+
+
 def _add_db_functions(lines, data_lib):
     lines.append("# ============================================================")
     lines.append("# Database Operations")
     lines.append("# ============================================================")
     lines.append("")
     lines.append("")
+    lines.append("_engine_cache = {}")
+    lines.append("")
+    lines.append("")
     lines.append("def get_db_connection(config, connection_name='default'):")
-    lines.append('    """Create database connection from config."""')
+    lines.append('    """')
+    lines.append("    Create database connection from config.")
+    lines.append("    Prefers SQLAlchemy engine (with connection pooling) over raw drivers.")
+    lines.append("    Config values support ${VAR} env-var placeholders via resolve_env().")
+    lines.append('    """')
     lines.append("    conn_config = config.get('connections', {}).get(connection_name, {})")
-    lines.append("    db_type = conn_config.get('type', 'mssql')")
-    lines.append("    host = conn_config.get('host', 'localhost')")
-    lines.append("    port = conn_config.get('port', 1433)")
-    lines.append("    database = conn_config.get('database', '')")
-    lines.append("    username = conn_config.get('username', '')")
-    lines.append("    password = conn_config.get('password', '')")
-    lines.append("    schema = conn_config.get('schema', 'dbo')")
+    lines.append("    db_type = resolve_env(conn_config.get('type', 'mssql'), config)")
+    lines.append("    host = resolve_env(conn_config.get('host', 'localhost'), config)")
+    lines.append("    port = resolve_env(conn_config.get('port', 1433), config)")
+    lines.append("    database = resolve_env(conn_config.get('database', ''), config)")
+    lines.append("    username = resolve_env(conn_config.get('username', ''), config)")
+    lines.append("    password = resolve_env(conn_config.get('password', ''), config)")
+    lines.append("    schema = resolve_env(conn_config.get('schema', 'dbo'), config)")
+    lines.append("")
+    lines.append("    cache_key = f'{db_type}://{username}@{host}:{port}/{database}'")
+    lines.append("    if cache_key in _engine_cache:")
+    lines.append("        return _engine_cache[cache_key].connect()")
+    lines.append("")
+    lines.append("    try:")
+    lines.append("        from sqlalchemy import create_engine")
+    lines.append("        url = _build_sqlalchemy_url(db_type, host, port, database, username, password)")
+    lines.append("        if url:")
+    lines.append("            engine = create_engine(url, pool_pre_ping=True, pool_size=5)")
+    lines.append("            _engine_cache[cache_key] = engine")
+    lines.append("            return engine.connect()")
+    lines.append("    except ImportError:")
+    lines.append("        logger.info('SQLAlchemy not available, falling back to raw drivers')")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'SQLAlchemy connection failed: {e}, falling back to raw drivers')")
+    lines.append("")
+    lines.append("    return _get_raw_connection(db_type, host, port, database, username, password, conn_config)")
     lines.append("")
+    lines.append("")
+    lines.append("def _build_sqlalchemy_url(db_type, host, port, database, username, password):")
+    lines.append("    from urllib.parse import quote_plus")
+    lines.append("    pw = quote_plus(str(password)) if password else ''")
+    lines.append("    if db_type == 'mssql':")
+    lines.append("        try:")
+    lines.append("            import pyodbc")
+    lines.append("            conn_str = quote_plus(")
+    lines.append("                f'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={host},{port};DATABASE={database};UID={username};PWD={password}'")
+    lines.append("            )")
+    lines.append("            return f'mssql+pyodbc:///?odbc_connect={conn_str}'")
+    lines.append("        except ImportError:")
+    lines.append("            return f'mssql+pymssql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        return f'postgresql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        return f'oracle+cx_oracle://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    elif db_type == 'mysql':")
+    lines.append("        return f'mysql+pymysql://{username}:{pw}@{host}:{port}/{database}'")
+    lines.append("    return None")
+    lines.append("")
+    lines.append("")
+    lines.append("def _get_raw_connection(db_type, host, port, database, username, password, conn_config):")
     lines.append("    if db_type == 'mssql':")
     lines.append("        try:")
     lines.append("            import pyodbc")
@@ -156,39 +278,19 @@ def _add_db_functions(lines, data_lib):
     lines.append("            return pymssql.connect(server=host, port=int(port), database=database, user=username, password=password)")
     lines.append("        except ImportError:")
     lines.append("            pass")
-    lines.append("        try:")
-    lines.append("            from sqlalchemy import create_engine")
-    lines.append("            engine = create_engine(f'mssql+pymssql://{username}:{password}@{host}:{port}/{database}')")
-    lines.append("            return engine.connect()")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'postgresql':")
-    lines.append("        try:")
-    lines.append("            import psycopg2")
-    lines.append("            return psycopg2.connect(")
-    lines.append("                host=host, port=port, dbname=database,")
-    lines.append("                user=username, password=password")
-    lines.append("            )")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
-    lines.append("")
-    lines.append("    if db_type == 'oracle':")
-    lines.append("        try:")
-    lines.append("            import cx_Oracle")
-    lines.append("            dsn = cx_Oracle.makedsn(host, port, service_name=database)")
-    lines.append("            return cx_Oracle.connect(username, password, dsn)")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("    elif db_type == 'postgresql':")
+    lines.append("        import psycopg2")
+    lines.append("        return psycopg2.connect(host=host, port=port, dbname=database, user=username, password=password)")
+    lines.append("    elif db_type == 'oracle':")
+    lines.append("        import cx_Oracle")
+    lines.append("        dsn = cx_Oracle.makedsn(host, port, service_name=database)")
+    lines.append("        return cx_Oracle.connect(username, password, dsn)")
     lines.append("")
     lines.append("    jdbc_url = conn_config.get('jdbc_url', '')")
     lines.append("    if jdbc_url:")
-    lines.append("        try:")
-    lines.append("            import jaydebeapi")
-    lines.append("            driver = conn_config.get('jdbc_driver', '')")
-    lines.append("            return jaydebeapi.connect(driver, jdbc_url, [username, password])")
-    lines.append("        except ImportError:")
-    lines.append("            pass")
+    lines.append("        import jaydebeapi")
+    lines.append("        driver = conn_config.get('jdbc_driver', '')")
+    lines.append("        return jaydebeapi.connect(driver, jdbc_url, [username, password])")
     lines.append("")
     lines.append("    raise ConnectionError(f'Cannot create connection for type: {db_type}')")
     lines.append("")
@@ -203,6 +305,15 @@ def _add_db_functions(lines, data_lib):
     else:
         read_func = "pd.read_sql"
 
+    lines.append("def _safe_close(conn):")
+    lines.append('    """Close connection safely — handles both SQLAlchemy and raw connections."""')
+    lines.append("    try:")
+    lines.append("        if hasattr(conn, 'close'):")
+    lines.append("            conn.close()")
+    lines.append("    except Exception:")
+    lines.append("        pass")
+    lines.append("")
+    lines.append("")
     lines.append("def read_from_db(config, query, connection_name='default'):")
     lines.append('    """Read data from database using SQL query."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
@@ -220,7 +331,7 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB read error on {{connection_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def write_to_db(config, df, table_name, connection_name='default', if_exists='append', schema=None):")
@@ -242,23 +353,31 @@ def _add_db_functions(lines, data_lib):
     lines.append("        logger.error(f'DB write error to {{schema}}.{{table_name}}: {{e}}')")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
     lines.append("def execute_sql(config, sql, connection_name='default'):")
     lines.append('    """Execute a SQL statement (INSERT, UPDATE, DELETE, DDL)."""')
     lines.append("    conn = get_db_connection(config, connection_name)")
     lines.append("    try:")
-    lines.append("        cursor = conn.cursor()")
-    lines.append("        cursor.execute(sql)")
-    lines.append("        conn.commit()")
+    lines.append("        if hasattr(conn, 'execute'):")
+    lines.append("            from sqlalchemy import text")
+    lines.append("            conn.execute(text(sql))")
+    lines.append("            conn.commit()")
+    lines.append("        else:")
+    lines.append("            cursor = conn.cursor()")
+    lines.append("            cursor.execute(sql)")
+    lines.append("            conn.commit()")
     lines.append("        logger.info(f'Executed SQL on {{connection_name}}')")
     lines.append("    except Exception as e:")
     lines.append("        logger.error(f'SQL execution error: {{e}}')")
-    lines.append("        conn.rollback()")
+    lines.append("        try:")
+    lines.append("            conn.rollback()")
+    lines.append("        except Exception:")
+    lines.append("            pass")
     lines.append("        raise")
     lines.append("    finally:")
-    lines.append("        conn.close()")
+    lines.append("        _safe_close(conn)")
     lines.append("")
     lines.append("")
 
@@ -1150,10 +1269,60 @@ def _add_expression_helpers(lines):
     lines.append("    raise SystemExit(message)")
     lines.append("")
     lines.append("")
-    lines.append("def lookup_func(table, condition, *fields):")
-    lines.append('    """Placeholder for Informatica LOOKUP function."""')
-    lines.append("    logger.warning(f'LOOKUP called for table {table} - implement in mapping-specific code')")
-    lines.append("    return None")
+    lines.append("_lookup_cache = {}")
+    lines.append("")
+    lines.append("")
+    lines.append("def lookup_func(table, condition, *fields, config=None, connection_name='default'):")
+    lines.append('    """')
+    lines.append("    Informatica unconnected LOOKUP function.")
+    lines.append("    Loads and caches the lookup table, then filters by condition.")
+    lines.append("    Returns the first matching value of the first return field, or None.")
+    lines.append('    """')
+    lines.append("    global _lookup_cache")
+    lines.append("    if table not in _lookup_cache:")
+    lines.append("        if config is not None:")
+    lines.append("            try:")
+    lines.append("                lkp_conn = connection_name")
+    lines.append("                conns = config.get('connections', {})")
+    lines.append("                for cname, cval in conns.items():")
+    lines.append("                    if isinstance(cval, dict) and cval.get('connection_name', '') == table:")
+    lines.append("                        lkp_conn = cname")
+    lines.append("                        break")
+    lines.append("                df_lkp = read_from_db(config, f'SELECT * FROM {table}', lkp_conn)")
+    lines.append("                _lookup_cache[table] = df_lkp")
+    lines.append("                logger.info(f'Cached lookup table {table}: {len(df_lkp)} rows')")
+    lines.append("            except Exception as e:")
+    lines.append("                logger.warning(f'Could not load lookup table {table}: {e}')")
+    lines.append("                _lookup_cache[table] = None")
+    lines.append("        else:")
+    lines.append("            logger.warning(f'LOOKUP called for {table} without config - returning None')")
+    lines.append("            return None")
+    lines.append("    df_lkp = _lookup_cache.get(table)")
+    lines.append("    if df_lkp is None or df_lkp.empty:")
+    lines.append("        return None")
+    lines.append("    try:")
+    lines.append("        if callable(condition):")
+    lines.append("            matches = df_lkp[condition(df_lkp)]")
+    lines.append("        elif isinstance(condition, str) and '=' in condition:")
+    lines.append("            col, _, val = condition.partition('=')")
+    lines.append("            col = col.strip()")
+    lines.append("            val = val.strip().strip(\"'\")")
+    lines.append("            if col in df_lkp.columns:")
+    lines.append("                matches = df_lkp[df_lkp[col].astype(str) == str(val)]")
+    lines.append("            else:")
+    lines.append("                return None")
+    lines.append("        else:")
+    lines.append("            return None")
+    lines.append("        if matches.empty:")
+    lines.append("            return None")
+    lines.append("        if fields:")
+    lines.append("            field = str(fields[0]).strip()")
+    lines.append("            if field in matches.columns:")
+    lines.append("                return matches.iloc[0][field]")
+    lines.append("        return matches.iloc[0].to_dict()")
+    lines.append("    except Exception as e:")
+    lines.append("        logger.warning(f'LOOKUP error on {table}: {e}')")
+    lines.append("        return None")
     lines.append("")
     lines.append("")
     lines.append("_param_store = {}")

informatica_python/generators/mapping_gen.py

@@ -419,9 +419,10 @@ def _safe_name(name):
     return safe.lower()
 
 
-def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
+def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    ", mapping_name="", session_name="", folder_name=""):
     import re
     params = re.findall(r'\$\$(\w+)', sql_text)
+    pm_vars = re.findall(r'\$(PM\w+)', sql_text)
     lines.append(f"{indent}{sql_var_name} = '''")
     for sql_line in sql_text.strip().split("\n"):
         lines.append(f"{indent}{sql_line}")
@@ -433,6 +434,13 @@ def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
                 continue
             seen.add(p)
             lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('$${p}', str(get_param(config, '{p}')))")
+    if pm_vars:
+        seen_pm = set()
+        for pm in pm_vars:
+            if pm in seen_pm:
+                continue
+            seen_pm.add(pm)
+            lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('${pm}', str(resolve_builtin_variable('{pm}', mapping_name='{mapping_name}', session_name='{session_name}', folder_name='{folder_name}')))")
 
 
 def _flatfile_config_dict(ff):
@@ -757,7 +765,7 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type in ("joiner",):
         _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph, data_lib)
     elif tx_type in ("lookup procedure", "lookup"):
-        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
+        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph, data_lib)
     elif tx_type == "router":
         _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("union",):
@@ -982,7 +990,7 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
+def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, connector_graph=None, data_lib="pandas"):
     lookup_table = ""
     lookup_sql = ""
     lookup_condition = ""
@@ -1012,6 +1020,11 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pa
 
     all_output_fields = return_fields + lookup_output_fields
 
+    port_to_col = {}
+    if connector_graph and tx.name in connector_graph.get("to", {}):
+        for conn in connector_graph["to"][tx.name]:
+            port_to_col[conn.to_field.lower()] = conn.from_field
+
     lines.append(f"    # Lookup: {lookup_table or tx.name}")
     if lookup_sql:
         _emit_sql_with_params(lines, f"lkp_sql_{tx_safe}", lookup_sql)
@@ -1020,10 +1033,13 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pa
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, 'SELECT * FROM {lookup_table}', 'default')")
     else:
         empty_expr = lib_empty_df(data_lib)
-        lines.append(f"    df_lkp_{tx_safe} = {empty_expr}")
+        lines.append(f"    df_lkp_{tx_safe} = {empty_expr}  # WARNING: no lookup table/SQL override found")
 
     input_keys, lookup_keys = parse_lookup_condition(lookup_condition)
 
+    if input_keys and port_to_col:
+        input_keys = [port_to_col.get(k.lower(), k) for k in input_keys]
+
     if input_keys and lookup_keys:
         lines.append(f"    # Lookup condition: {lookup_condition}")
 
@@ -1078,12 +1094,23 @@ def _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs):
         if "Group Filter Condition" in attr.name:
             group_conditions[attr.name] = attr.value
 
+    remaining_mask_parts = []
     if group_conditions:
         for i, (gname, cond) in enumerate(group_conditions.items()):
-            expr_py = convert_expression(cond) if cond else "True"
-            lines.append(f"    df_{tx_safe}_group{i} = {input_df}[{expr_py}].copy()  # {gname}")
+            if cond and cond.strip():
+                expr_py = convert_filter_vectorized(cond, input_df)
+            else:
+                expr_py = f"pd.Series(True, index={input_df}.index)"
+            mask_var = f"_router_mask_{tx_safe}_{i}"
+            lines.append(f"    {mask_var} = {expr_py}  # {gname}")
+            lines.append(f"    df_{tx_safe}_group{i} = {input_df}[{mask_var}].copy()")
             source_dfs[f"{tx.name}_group{i}"] = f"df_{tx_safe}_group{i}"
-    lines.append(f"    df_{tx_safe} = {input_df}.copy()  # Default group")
+            remaining_mask_parts.append(f"~{mask_var}")
+    if remaining_mask_parts:
+        lines.append(f"    _router_default_mask = {' & '.join(remaining_mask_parts)}")
+        lines.append(f"    df_{tx_safe} = {input_df}[_router_default_mask].copy()  # Default group")
+    else:
+        lines.append(f"    df_{tx_safe} = {input_df}.copy()  # Default group")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -1442,7 +1469,7 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     if col_mapping:
         lines.append(f"    # Column mapping: source -> target")
         lines.append(f"    target_columns_{tgt_safe} = {col_mapping}")
-        lines.append(f"    df_target_{tgt_safe} = {input_df}.rename(columns={{v: k for k, v in target_columns_{tgt_safe}.items()}})")
+        lines.append(f"    df_target_{tgt_safe} = rename_with_duplicates({input_df}, target_columns_{tgt_safe})")
         target_cols = [f.name for f in tgt_def.fields] if tgt_def.fields else None
         if target_cols:
             lines.append(f"    # Select only target columns")

informatica_python/utils/expression_converter.py

@@ -248,6 +248,7 @@ def _convert_infa_date_format(fmt_str):
     fmt = fmt.replace("Mon", "%b").replace("MON", "%b")
     fmt = fmt.replace("HH24", "%H").replace("HH12", "%I").replace("HH", "%H")
     fmt = fmt.replace("MI", "%M").replace("SS", "%S")
+    fmt = fmt.replace("US", "%f").replace("NS", "%f").replace("MS", "%f")
     return fmt
 
 
@@ -548,7 +549,7 @@ def _vec_recursive(expr, df_var):
                         'RTRIM': f'.str.rstrip("{char_arg}")',
                         'TRIM': f'.str.strip("{char_arg}")',
                     }
-                return f'{inner_val}{method_map[func_name.upper()]}'
+                return f'{inner_val}.astype(str){method_map[func_name.upper()]}'
 
     upper_result = _find_func_call(cleaned, 'UPPER')
     if upper_result and upper_result[0] == 0 and upper_result[1] == len(cleaned):
@@ -584,7 +585,7 @@ def _vec_recursive(expr, df_var):
         if len(args) >= 2:
             field_val = _vec_recursive(args[0], df_var)
             try:
-                start = int(args[1].strip()) - 1
+                start = max(int(args[1].strip()) - 1, 0)
             except ValueError:
                 start_val = _vec_recursive(args[1], df_var)
                 if len(args) >= 3:
@@ -722,7 +723,11 @@ def _vec_recursive(expr, df_var):
                 field_val = _vec_recursive(args[0], df_var)
                 pattern_val = args[1].strip().strip("'\"")
                 if func_name == 'REG_EXTRACT':
-                    return f'{field_val}.str.extract(r"({pattern_val})", expand=False)'
+                    if re.search(r'(?<!\\)\((?!\?)', pattern_val):
+                        extract_pat = pattern_val
+                    else:
+                        extract_pat = f'({pattern_val})'
+                    return f'{field_val}.str.extract(r"{extract_pat}", expand=False)'
                 elif func_name == 'REG_REPLACE':
                     replace_val = args[2].strip().strip("'\"") if len(args) >= 3 else ''
                     return f'{field_val}.str.replace(r"{pattern_val}", "{replace_val}", regex=True)'
@@ -862,7 +867,7 @@ def _vec_recursive(expr, df_var):
             if v.startswith("'") and v.endswith("'"):
                 vec_parts.append(v)
             else:
-                vec_parts.append(f'{v}.astype(str)')
+                vec_parts.append(f'{v}.fillna(\'\').astype(str)')
         return " + ".join(vec_parts)
 
     for func_name in sorted(INFA_FUNC_MAP.keys(), key=lambda x: -len(x)):
@@ -894,7 +899,8 @@ def _vec_recursive(expr, df_var):
         'True', 'False', 'None', 'and', 'or', 'not', 'np', 'pd', 'get_variable',
         'str', 'int', 'float', 'bool', 'len', 'abs', 'round',
         'fillna', 'astype', 'isna', 'notna', 'where', 'errors', 'coerce',
-        'lookup_func',
+        'lookup_func', 'expand', 'extract', 'regex', 'contains', 'replace',
+        'upper', 'lower', 'strip', 'lstrip', 'rstrip', 'dt', 'copy',
     }
     converted = _substitute_fields(converted, df_var, skip_words)
 
@@ -904,6 +910,8 @@ def _vec_recursive(expr, df_var):
     converted = re.sub(r'<>', '!=', converted)
     converted = re.sub(r'(?<![<>!=])=(?!=)', '==', converted)
     converted = re.sub(r'\berrors\s*==\s*(["\'])', r'errors=\1', converted)
+    converted = re.sub(r'\bexpand\s*==\s*', 'expand=', converted)
+    converted = re.sub(r'\bregex\s*==\s*', 'regex=', converted)
 
     converted = re.sub(r'\s+', ' ', converted).strip()
 
@@ -1044,8 +1052,14 @@ def _vectorize_simple(part, df_var):
         'True', 'False', 'None', 'and', 'or', 'not', 'np', 'pd',
         'str', 'int', 'float', 'isna', 'notna', 'fillna',
         'get_variable', 'lookup_func', 'isin', 'eq',
+        'expand', 'extract', 'astype', 'errors', 'coerce', 'regex',
+        'contains', 'replace', 'upper', 'lower', 'strip', 'lstrip', 'rstrip',
+        'dt', 'len', 'copy', 'abs', 'round', 'where', 'bool',
     }
     c = _substitute_fields(c, df_var, skip_words)
+    c = re.sub(r'\bexpand\s*==\s*', 'expand=', c)
+    c = re.sub(r'\berrors\s*==\s*', 'errors=', c)
+    c = re.sub(r'\bregex\s*==\s*', 'regex=', c)
 
     return c
 

{informatica_python-1.9.3.dist-info → informatica_python-1.9.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.9.3
+Version: 1.9.5
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT
@@ -430,7 +430,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 - **Generated code formatting**: Consistent `# ---` section headers for Source Qualifiers, Transforms, and Target Writes; metadata comments (database type, field lists); column mapping and write operation comments; clean blank line handling
 - **Source/target detection**: Case-insensitive instance type matching
 - **Session→mapping inference**: Longest-suffix-match strategy for ambiguous mapping names
-- **646 tests** across unit, integration, expression, and formatting test suites
+- **663 tests** across unit, integration, expression, and formatting test suites
 
 ### v1.9.2 (Phase 8)
 - Mapping output files now use real mapping names (e.g., `mapping_m_customer_load.py`) instead of generic numeric indices (`mapping_1.py`)
@@ -495,7 +495,7 @@ The generated `helper_functions.py` provides a complete runtime library:
 cd informatica_python
 pip install -e ".[dev]"
 
-# Run tests (646 tests)
+# Run tests (663 tests)
 pytest tests/ -v
 ```
 

{informatica_python-1.9.3.dist-info → informatica_python-1.9.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-informatica_python/__init__.py,sha256=o9kEVkHnEwXAD7hhY8YbN6G8RP4Mqby_q8CpjfbiknQ,337
+informatica_python/__init__.py,sha256=sbOT0MiP4hfEvssyO1fr_bI8Mxs6ZN6SO7rBglvkJFU,337
 informatica_python/cli.py,sha256=gFwg0O99vKM-OLO0HoHA4emd-6qrgjMNqa9T59e4e_s,2905
 informatica_python/converter.py,sha256=xCuWrYzDji0yN72D3QqOgZCVVM2j3k2_CvlGplCWxLU,22779
 informatica_python/models.py,sha256=G_C2WfQL-ykKjNj23m8vKFtLZYrQozp99HJzrLTKG1Y,17293
@@ -6,18 +6,18 @@ informatica_python/parser.py,sha256=v0qoTlAi3RZ3IHN_5g5t6f66XzRpJIjpAfpyMzZ5cuA,
 informatica_python/generators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 informatica_python/generators/config_gen.py,sha256=4tqcNKTB06kyGZIiM4yl0q97q_i3zeCHXTjuE1dNFKY,5726
 informatica_python/generators/error_log_gen.py,sha256=2cc0rEcblydHkb9VAMXlrH7WdSQ-CNqAXcwVk3FYZeM,21319
-informatica_python/generators/helper_gen.py,sha256=D6-UqNh09Qy2V7RimNgP-SzK_uB9YqAlsa0-cgLhf5o,72209
-informatica_python/generators/mapping_gen.py,sha256=gBVArcb8uODbgY3epdsldCbUywS-qo8CiKr7hcNjMnc,70654
+informatica_python/generators/helper_gen.py,sha256=ylhZnZb5yQ23kQswa9jrf45DvCcHq-3K7KCT2eCwCVM,82376
+informatica_python/generators/mapping_gen.py,sha256=Aic4srfI6SfbtB1ggx7Rn27STVOV2olo3YKiRysmFps,72055
 informatica_python/generators/sql_gen.py,sha256=O8Y-aJz9EyFJ0DXeuISRt5yKwC3wlp2K3B0BHrmxrXw,4872
 informatica_python/generators/workflow_gen.py,sha256=_uSlBg31ZRMhMlCYk4hWDRBPaBROrepD8_v3QGEWJxE,18089
 informatica_python/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 informatica_python/utils/datatype_map.py,sha256=iLOYg-iBKT4rMecGbrFkTpJj4yqs5S9HeBOTLUIWhX0,2809
-informatica_python/utils/expression_converter.py,sha256=CqkkTESMKxcYmVsDpNfn7VcZZe771uCIMy_0YQYq6pc,45946
+informatica_python/utils/expression_converter.py,sha256=SkkT2CyhIZzUms9TT4cEimZlxjOoVq96AQgGTrO_Lmc,46859
 informatica_python/utils/lib_adapters.py,sha256=1ZtuMbgDg9Ukf-OF_EG1L_BeeR-6JQk8Kx3WwMfvNRU,6516
 informatica_python/utils/sql_dialect.py,sha256=_IHJbfu8a3mT_OvHpybgSfZKqz6mwVy5ItTKDRChqnU,5461
-informatica_python-1.9.3.dist-info/licenses/LICENSE,sha256=77RaRDdXgey1D90YZAjXqEQdBxWfvUQqLQX3pC1qjUE,1061
-informatica_python-1.9.3.dist-info/METADATA,sha256=VbfZWdzKE382RnkR7F2rs7PNL397g3PfglvugN4XVTw,26097
-informatica_python-1.9.3.dist-info/WHEEL,sha256=PovZm1ExVWmrRefZoXCfejlbKLnQI5SVIf1SWRV4QQI,97
-informatica_python-1.9.3.dist-info/entry_points.txt,sha256=030jjTrx-1oRRQ16HZz52rdcKS8R8_llnymsTUtn_Xc,67
-informatica_python-1.9.3.dist-info/top_level.txt,sha256=Dngg-WNteYi22XAJU2XKAQS8aZ52yM2LYC0tzxrlbVQ,19
-informatica_python-1.9.3.dist-info/RECORD,,
+informatica_python-1.9.5.dist-info/licenses/LICENSE,sha256=77RaRDdXgey1D90YZAjXqEQdBxWfvUQqLQX3pC1qjUE,1061
+informatica_python-1.9.5.dist-info/METADATA,sha256=I5YvXFM4ctZFRE_RNNi6aeZb7vLMlZM3Az-QTuXypYU,26097
+informatica_python-1.9.5.dist-info/WHEEL,sha256=PovZm1ExVWmrRefZoXCfejlbKLnQI5SVIf1SWRV4QQI,97
+informatica_python-1.9.5.dist-info/entry_points.txt,sha256=030jjTrx-1oRRQ16HZz52rdcKS8R8_llnymsTUtn_Xc,67
+informatica_python-1.9.5.dist-info/top_level.txt,sha256=Dngg-WNteYi22XAJU2XKAQS8aZ52yM2LYC0tzxrlbVQ,19
+informatica_python-1.9.5.dist-info/RECORD,,