informatica-python 1.9.1__py3-none-any.whl → 1.9.3__py3-none-any.whl

informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.9.1"
+__version__ = "1.9.3"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

informatica_python/generators/helper_gen.py

@@ -151,6 +151,17 @@ def _add_db_functions(lines, data_lib):
     lines.append("            return pyodbc.connect(conn_str)")
     lines.append("        except ImportError:")
     lines.append("            pass")
+    lines.append("        try:")
+    lines.append("            import pymssql")
+    lines.append("            return pymssql.connect(server=host, port=int(port), database=database, user=username, password=password)")
+    lines.append("        except ImportError:")
+    lines.append("            pass")
+    lines.append("        try:")
+    lines.append("            from sqlalchemy import create_engine")
+    lines.append("            engine = create_engine(f'mssql+pymssql://{username}:{password}@{host}:{port}/{database}')")
+    lines.append("            return engine.connect()")
+    lines.append("        except ImportError:")
+    lines.append("            pass")
     lines.append("")
     lines.append("    if db_type == 'postgresql':")
     lines.append("        try:")

informatica_python/generators/mapping_gen.py

@@ -1,3 +1,4 @@
+import os
 from typing import List, Dict
 from informatica_python.models import (
     MappingDef, FolderDef, SourceDef, TargetDef,
@@ -227,6 +228,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append("")
     lines.append("import logging")
     lines.append("import numpy as np")
+    lines.append("import pandas as pd")
     lines.append("from helper_functions import *")
     lines.append("")
     lines.append("logger = logging.getLogger(__name__)")
@@ -373,7 +375,40 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append(f"    run_{_safe_name(mapping.name)}(config)")
     lines.append("")
 
-    return "\n".join(lines)
+    code = "\n".join(lines)
+    func_sig = f"def run_{_safe_name(mapping.name)}(config):"
+    sig_idx = code.index(func_sig) + len(func_sig)
+    docstring_end = code.index('"""', code.index('"""', sig_idx) + 3) + 3
+    before_body = code[:docstring_end]
+    after_docstring = code[docstring_end:]
+    main_sentinel = "\n\nif __name__"
+    body_end_idx = after_docstring.index(main_sentinel)
+    body = after_docstring[:body_end_idx]
+    rest = after_docstring[body_end_idx:]
+    body_lines = body.split("\n")
+    while body_lines and body_lines[0].strip() == "":
+        body_lines.pop(0)
+    while body_lines and body_lines[-1].strip() == "":
+        body_lines.pop()
+    wrapped = []
+    wrapped.append("")
+    wrapped.append("    try:")
+    prev_blank = False
+    for bl in body_lines:
+        if bl.strip() == "":
+            if not prev_blank:
+                wrapped.append("")
+            prev_blank = True
+        else:
+            wrapped.append("    " + bl)
+            prev_blank = False
+    wrapped.append("")
+    wrapped.append("    except Exception as _exc:")
+    wrapped.append(f"        logger.error(f'Mapping {mapping.name} failed: {{_exc}}')")
+    wrapped.append("        raise")
+    wrapped.append("")
+
+    return before_body + "\n".join(wrapped) + rest
 
 
 def _safe_name(name):
@@ -384,6 +419,22 @@ def _safe_name(name):
     return safe.lower()
 
 
+def _emit_sql_with_params(lines, sql_var_name, sql_text, indent="    "):
+    import re
+    params = re.findall(r'\$\$(\w+)', sql_text)
+    lines.append(f"{indent}{sql_var_name} = '''")
+    for sql_line in sql_text.strip().split("\n"):
+        lines.append(f"{indent}{sql_line}")
+    lines.append(f"{indent}'''")
+    if params:
+        seen = set()
+        for p in params:
+            if p in seen:
+                continue
+            seen.add(p)
+            lines.append(f"{indent}{sql_var_name} = {sql_var_name}.replace('$${p}', str(get_param(config, '{p}')))")
+
+
 def _flatfile_config_dict(ff):
     cfg = {}
     if not ff:
@@ -502,7 +553,7 @@ def _emit_flatfile_write(lines, var_name, tgt_def, indent="    ", file_path_over
 def _build_source_map(mapping, folder):
     source_map = {}
     for inst in mapping.instances:
-        if inst.type == "Source Definition":
+        if inst.type.upper() in ("SOURCE DEFINITION", "SOURCE"):
             tx_name = inst.transformation_name or inst.name
             for src in folder.sources:
                 if src.name == tx_name:
@@ -516,7 +567,7 @@ def _build_source_map(mapping, folder):
 def _build_target_map(mapping, folder):
     target_map = {}
     for inst in mapping.instances:
-        if inst.type == "Target Definition":
+        if inst.type.upper() in ("TARGET DEFINITION", "TARGET"):
             tx_name = inst.transformation_name or inst.name
             for tgt in folder.targets:
                 if tgt.name == tx_name:
@@ -592,7 +643,9 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
         if not connected_sources and source_map:
             connected_sources.add(next(iter(source_map)))
 
+    lines.append(f"    # -------------------------------------------------------------------")
     lines.append(f"    # Source Qualifier: {sq.name}")
+    lines.append(f"    # -------------------------------------------------------------------")
 
     if pre_sql:
         lines.append(f"    # Pre-SQL")
@@ -604,10 +657,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
     if not connected_sources:
         sq_src_name = sq.name[3:] if sq.name.upper().startswith("SQ_") else sq.name
         if sql_override:
-            lines.append(f"    sql_{sq_safe} = '''")
-            for sql_line in sql_override.strip().split("\n"):
-                lines.append(f"    {sql_line}")
-            lines.append(f"    '''")
+            _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override)
             lines.append(f"    df_{sq_safe} = read_from_db(config, sql_{sq_safe}, 'default')")
         else:
             lines.append(f"    df_{sq_safe} = read_file(config.get('sources', {{}}).get('{sq_src_name}', {{}}).get('file_path', '{sq_src_name}'),")
@@ -618,10 +668,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
         sq_override = (session_overrides or {}).get(sq.name, {}) or (session_overrides or {}).get(src_name, {})
         conn_name = sq_override.get("connection_name") or (_safe_name(src_def.db_name) if src_def.db_name else "default")
 
-        lines.append(f"    sql_{sq_safe} = '''")
-        for sql_line in sql_override.strip().split("\n"):
-            lines.append(f"    {sql_line}")
-        lines.append(f"    '''")
+        _emit_sql_with_params(lines, f"sql_{sq_safe}", sql_override)
         lines.append(f"    df_{sq_safe} = read_from_db(config, sql_{sq_safe}, '{conn_name}')")
     elif len(connected_sources) == 1:
         src_name = next(iter(connected_sources))
@@ -654,10 +701,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
         lines.append(f"    df_{sq_safe} = df_{_safe_name(next(iter(connected_sources)))}")
 
     source_dfs[sq.name] = f"df_{sq_safe}"
-    lines.append(f"    try:")
-    lines.append(f"        logger.info(f'Source {sq.name}: {{len(df_{sq_safe})}} rows read')")
-    lines.append(f"    except Exception:")
-    lines.append(f"        logger.info('Source {sq.name}: rows read (count unavailable)')")
+    lines.append(f"    logger.info(f'Source {sq.name}: {{len(df_{sq_safe})}} rows read')")
 
     if post_sql:
         lines.append(f"    # Post-SQL")
@@ -697,10 +741,10 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     lines.append(f"    # Input fields: {', '.join(in_fields[:10])}{' ...' if len(in_fields) > 10 else ''}")
     lines.append(f"    # Output fields: {', '.join(out_fields[:10])}{' ...' if len(out_fields) > 10 else ''}")
     lines.append(f"    # -------------------------------------------------------------------")
-    lines.append(f"    try:")
-    lines.append(f"        _input_rows_{tx_safe} = len({input_df})")
-    lines.append(f"    except Exception:")
-    lines.append(f"        _input_rows_{tx_safe} = -1")
+    if input_df == "df_input":
+        lines.append(f"    _input_rows_{tx_safe} = -1")
+    else:
+        lines.append(f"    _input_rows_{tx_safe} = len({input_df})")
 
     if tx_type == "expression":
         _gen_expression_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
@@ -740,28 +784,26 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
         lines.append(f"    df_{tx_safe} = {copy_expr}")
         source_dfs[tx.name] = f"df_{tx_safe}"
 
-    lines.append(f"    try:")
-    lines.append(f"        _output_rows_{tx_safe} = len(df_{tx_safe})")
-    lines.append(f"    except Exception:")
-    lines.append(f"        _output_rows_{tx_safe} = -1")
+    lines.append(f"    _output_rows_{tx_safe} = len(df_{tx_safe})")
     lines.append(f"    logger.info(f'{tx.name} ({tx.type}): {{_input_rows_{tx_safe}}} input rows -> {{_output_rows_{tx_safe}}} output rows')")
     lines.append("")
 
 
 def _gen_expression_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
-    copy_expr = lib_copy(data_lib, input_df)
-    lines.append(f"    df_{tx_safe} = {copy_expr}")
-    has_expressions = False
+    active_fields = []
     for fld in tx.fields:
-        if fld.expression and fld.expression.strip() and fld.expression.strip() != fld.name:
-            has_expressions = True
+        if fld.expression and fld.expression.strip() and fld.expression.strip().lower() != fld.name.lower():
+            active_fields.append(fld)
+
+    if active_fields:
+        copy_expr = lib_copy(data_lib, input_df)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
+        for fld in active_fields:
             expr_vec = convert_expression_vectorized(fld.expression, f"df_{tx_safe}")
             lines.append(f"    # {fld.name} = {fld.expression}")
-            if fld.porttype and "OUTPUT" in fld.porttype.upper() and "INPUT" not in fld.porttype.upper():
-                lines.append(f"    df_{tx_safe}['{fld.name}'] = {expr_vec}")
-            else:
-                lines.append(f"    df_{tx_safe}['{fld.name}'] = {expr_vec}")
-    if not has_expressions:
+            lines.append(f"    df_{tx_safe}['{fld.name}'] = {expr_vec}")
+    else:
+        lines.append(f"    df_{tx_safe} = {input_df}")
         lines.append(f"    # Pass-through expression (no transformations)")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
@@ -840,7 +882,11 @@ def _gen_sorter_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pa
     sort_dirs = []
     for fld in tx.fields:
         sort_keys.append(fld.name)
-        sort_dirs.append(True)
+        direction = 'ASCENDING'
+        for fa in getattr(fld, 'field_attributes', []):
+            if isinstance(fa, dict) and fa.get('name', '').upper() == 'SORTDIRECTION':
+                direction = fa.get('value', 'ASCENDING') or 'ASCENDING'
+        sort_dirs.append(direction.upper() != 'DESCENDING')
     if sort_keys:
         sort_expr = lib_sort(data_lib, input_df, sort_keys, sort_dirs)
         lines.append(f"    df_{tx_safe} = {sort_expr}")
@@ -879,13 +925,23 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
     master_src = None
     detail_src = None
     input_conns = connector_graph.get("to", {}).get(tx.name, []) if connector_graph else []
+
+    port_to_col = {}
+    master_fields_lower = {f.lower() for f in master_fields}
+    detail_fields_lower = {f.lower() for f in detail_fields}
     for conn in input_conns:
         to_field = conn.to_field
-        if to_field in master_fields:
+        port_to_col[to_field] = conn.from_field
+        port_to_col[to_field.lower()] = conn.from_field
+        if to_field in master_fields or to_field.lower() in master_fields_lower:
             master_src = conn.from_instance
-        elif to_field in detail_fields:
+        elif to_field in detail_fields or to_field.lower() in detail_fields_lower:
             detail_src = conn.from_instance
 
+    if left_keys and right_keys and port_to_col:
+        left_keys = [port_to_col.get(k, port_to_col.get(k.lower(), k)) for k in left_keys]
+        right_keys = [port_to_col.get(k, port_to_col.get(k.lower(), k)) for k in right_keys]
+
     src_list = list(input_sources)
     if not master_src and not detail_src and len(src_list) >= 2:
         master_src = src_list[0]
@@ -958,10 +1014,7 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pa
 
     lines.append(f"    # Lookup: {lookup_table or tx.name}")
     if lookup_sql:
-        lines.append(f"    lkp_sql_{tx_safe} = '''")
-        for sql_line in lookup_sql.strip().split("\n"):
-            lines.append(f"    {sql_line}")
-        lines.append(f"    '''")
+        _emit_sql_with_params(lines, f"lkp_sql_{tx_safe}", lookup_sql)
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, lkp_sql_{tx_safe}, 'default')")
     elif lookup_table:
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, 'SELECT * FROM {lookup_table}', 'default')")
@@ -996,7 +1049,11 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pa
             lines.append(f"    if _lkp_drop:")
             lines.append(f"        df_{tx_safe} = df_{tx_safe}.drop(columns=_lkp_drop)")
 
+        seen_output_cols = set()
         for rf in all_output_fields:
+            if rf.name in seen_output_cols:
+                continue
+            seen_output_cols.add(rf.name)
             lines.append(f"    if '{rf.name}' not in df_{tx_safe}.columns:")
             lines.append(f"        df_{tx_safe}['{rf.name}'] = None")
             if rf.default_value:
@@ -1071,14 +1128,19 @@ def _gen_update_strategy(lines, tx, tx_safe, input_df, source_dfs):
         for dd_const, label in dd_map.items():
             expr = expr.replace(dd_const, f"'{label}'")
         try:
-            converted = convert_expression(expr)
+            expr_vec = convert_expression_vectorized(expr, f"df_{tx_safe}")
             lines.append(f"    # Original expression: {strategy_expr}")
-            lines.append(f"    def _resolve_strategy(row):")
-            lines.append(f"        return {converted}")
-            lines.append(f"    df_{tx_safe}['_update_strategy'] = df_{tx_safe}.apply(_resolve_strategy, axis=1)")
+            lines.append(f"    df_{tx_safe}['_update_strategy'] = {expr_vec}")
         except Exception:
-            lines.append(f"    # Could not parse strategy expression: {strategy_expr}")
-            lines.append(f"    df_{tx_safe}['_update_strategy'] = 'INSERT'")
+            try:
+                converted = convert_expression(expr)
+                lines.append(f"    # Original expression: {strategy_expr}")
+                lines.append(f"    def _resolve_strategy(row):")
+                lines.append(f"        return {converted}")
+                lines.append(f"    df_{tx_safe}['_update_strategy'] = df_{tx_safe}.apply(_resolve_strategy, axis=1)")
+            except Exception:
+                lines.append(f"    # Could not parse strategy expression: {strategy_expr}")
+                lines.append(f"    df_{tx_safe}['_update_strategy'] = 'INSERT'")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -1341,7 +1403,7 @@ def _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs):
             sql_query = convert_sql_expression(attr.value)
     lines.append(f"    # SQL Transformation: {tx.name}")
     if sql_query:
-        lines.append(f"    sql_{tx_safe} = '''{sql_query}'''")
+        _emit_sql_with_params(lines, f"sql_{tx_safe}", sql_query)
         lines.append(f"    df_{tx_safe} = read_from_db(config, sql_{tx_safe}, 'default')")
     else:
         lines.append(f"    df_{tx_safe} = {input_df}.copy()")
@@ -1369,12 +1431,21 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     for c in to_conns:
         col_mapping[c.to_field] = c.from_field
 
+    lines.append(f"    # -------------------------------------------------------------------")
     lines.append(f"    # Write to target: {tgt_def.name}")
+    if tgt_def.database_type:
+        lines.append(f"    # Database type: {tgt_def.database_type}")
+    target_field_names = [f.name for f in tgt_def.fields] if tgt_def.fields else []
+    if target_field_names:
+        lines.append(f"    # Target fields: {', '.join(target_field_names[:10])}{' ...' if len(target_field_names) > 10 else ''}")
+    lines.append(f"    # -------------------------------------------------------------------")
     if col_mapping:
+        lines.append(f"    # Column mapping: source -> target")
         lines.append(f"    target_columns_{tgt_safe} = {col_mapping}")
         lines.append(f"    df_target_{tgt_safe} = {input_df}.rename(columns={{v: k for k, v in target_columns_{tgt_safe}.items()}})")
         target_cols = [f.name for f in tgt_def.fields] if tgt_def.fields else None
         if target_cols:
+            lines.append(f"    # Select only target columns")
             lines.append(f"    available_cols = [c for c in {target_cols} if c in df_target_{tgt_safe}.columns]")
             lines.append(f"    if '_update_strategy' in df_target_{tgt_safe}.columns and '_update_strategy' not in available_cols:")
             lines.append(f"        available_cols.append('_update_strategy')")
@@ -1387,17 +1458,37 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     tgt_override = (session_overrides or {}).get(tgt_name, {})
     tgt_conn = tgt_override.get("connection_name")
 
+    _FILE_EXTENSIONS = {".csv", ".dat", ".txt", ".xml", ".json", ".parquet", ".xlsx", ".xls", ".tsv", ".avro"}
+    _is_file_target = bool(
+        tgt_override.get("output_file_directory") or tgt_override.get("output_filename")
+        or tgt_def.flatfile
+        or (tgt_def.database_type and tgt_def.database_type == "Flat File")
+        or os.path.splitext(tgt_def.name)[1].lower() in _FILE_EXTENSIONS
+    )
+    _is_db_target = bool(
+        tgt_def.database_type and tgt_def.database_type != "Flat File"
+    )
+
     if tgt_override.get("output_file_directory") or tgt_override.get("output_filename"):
         out_dir = tgt_override.get("output_file_directory", ".")
         out_file = tgt_override.get("output_filename", tgt_def.name)
+        lines.append(f"    # Write to file (session override path)")
         lines.append(f"    _tgt_path_{tgt_safe} = config.get('targets', {{}}).get('{tgt_def.name}', {{}}).get('file_path',")
         lines.append(f"        os.path.join('{out_dir}', '{out_file}'))")
         if tgt_def.flatfile:
             _emit_flatfile_write(lines, tgt_safe, tgt_def, file_path_override=True)
         else:
             lines.append(f"    write_file(df_target_{tgt_safe}, _tgt_path_{tgt_safe}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
-    elif tgt_def.database_type and tgt_def.database_type != "Flat File":
+    elif tgt_def.flatfile:
+        lines.append(f"    # Write to flat file")
+        _emit_flatfile_write(lines, tgt_safe, tgt_def)
+    elif _is_file_target and not _is_db_target:
+        lines.append(f"    # Write to file")
+        lines.append(f"    write_file(df_target_{tgt_safe}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}).get('file_path', '{tgt_def.name}'),")
+        lines.append(f"              config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
+    else:
         conn_label = tgt_conn or "target"
+        lines.append(f"    # Write to database table")
         lines.append(f"    if '_update_strategy' in df_target_{tgt_safe}.columns:")
         key_cols = [f.name for f in tgt_def.fields if getattr(f, 'keytype', 'NOT A KEY') == 'PRIMARY KEY'] or None
         if key_cols:
@@ -1406,15 +1497,8 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
             lines.append(f"        write_with_update_strategy(config, df_target_{tgt_safe}, '{tgt_def.name}', '{conn_label}')")
         lines.append(f"    else:")
         lines.append(f"        write_to_db(config, df_target_{tgt_safe}, '{tgt_def.name}', '{conn_label}')")
-    elif tgt_def.flatfile:
-        _emit_flatfile_write(lines, tgt_safe, tgt_def)
-    else:
-        lines.append(f"    write_file(df_target_{tgt_safe}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}).get('file_path', '{tgt_def.name}'),")
-        lines.append(f"              config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
-    lines.append(f"    try:")
-    lines.append(f"        logger.info(f'Target {tgt_def.name}: {{len(df_target_{tgt_safe})}} rows written')")
-    lines.append(f"    except Exception:")
-    lines.append(f"        logger.info('Target {tgt_def.name}: rows written (count unavailable)')")
+    lines.append(f"    logger.info(f'Target {tgt_def.name}: {{len(df_target_{tgt_safe})}} rows written')")
+    lines.append("")
 
 
 CAST_MAP = {

informatica_python/generators/workflow_gen.py

@@ -179,24 +179,41 @@ def _generate_workflow_function(lines, wf: WorkflowDef, folder: FolderDef, workl
 def _emit_task_code(lines, task, mapping_name_map, session_to_mapping, wf, worklets):
     task_safe = _safe_name(task.name)
 
-    if task.task_type == "Start Task":
+    if task.task_type in ("Start Task", "Start"):
         lines.append(f"    # Start Task: {task.name}")
         lines.append(f"    logger.info('Workflow started')")
         lines.append("")
         return
 
     if task.task_type == "Session":
-        mapping_name = session_to_mapping.get(task.task_name or task.name, "")
+        session_key = task.task_name or task.name
+        mapping_name = session_to_mapping.get(session_key, "")
         run_func = mapping_name_map.get(mapping_name, None)
 
+        if not run_func:
+            best_match = None
+            best_len = 0
+            session_lower = session_key.lower()
+            for mname, rfunc in mapping_name_map.items():
+                safe_mname = _safe_name(mname)
+                if session_lower.endswith(safe_mname) and len(safe_mname) > best_len:
+                    best_match = rfunc
+                    best_len = len(safe_mname)
+            if not best_match:
+                for mname, rfunc in mapping_name_map.items():
+                    safe_mname = _safe_name(mname)
+                    if safe_mname in session_lower and len(safe_mname) > best_len:
+                        best_match = rfunc
+                        best_len = len(safe_mname)
+            run_func = best_match
+
         lines.append(f"    # Session: {task.name}")
         lines.append(f"    try:")
         lines.append(f"        logger.info('Executing session: {task.name}')")
         if run_func:
             lines.append(f"        {run_func}(config)")
         else:
-            lines.append(f"        # TODO: Map session '{task.name}' to corresponding mapping function")
-            lines.append(f"        logger.warning('Session {task.name} has no mapped function')")
+            lines.append(f"        logger.warning('Session {task.name}: no mapped function found — verify mapping linkage')")
         lines.append(f"    except Exception as e:")
         lines.append(f"        logger.error(f'Session {task.name} failed: {{e}}')")