informatica-python 1.2.0__tar.gz → 1.3.0__tar.gz

{informatica_python-1.2.0 → informatica_python-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.2.0
+Version: 1.3.0
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 License-Expression: MIT
 Requires-Python: >=3.8

{informatica_python-1.2.0 → informatica_python-1.3.0}/informatica_python/generators/mapping_gen.py

@@ -1,7 +1,7 @@
 from typing import List, Dict
 from informatica_python.models import (
     MappingDef, FolderDef, SourceDef, TargetDef,
-    TransformationDef, ConnectorDef, InstanceDef,
+    TransformationDef, ConnectorDef, InstanceDef, MappletDef,
 )
 from informatica_python.utils.expression_converter import (
     convert_expression, convert_sql_expression,
@@ -11,6 +11,130 @@ from informatica_python.utils.expression_converter import (
 from informatica_python.utils.datatype_map import get_python_type
 
 
+def _inline_mapplets(mapping, folder):
+    mapplet_map = {m.name: m for m in folder.mapplets}
+    extra_transforms = []
+    extra_connectors = []
+    mapplet_instances = set()
+
+    for inst in mapping.instances:
+        if inst.type == "Mapplet" or (inst.transformation_type or "").lower() == "mapplet":
+            mapplet_name = inst.transformation_name or inst.name
+            mapplet = mapplet_map.get(mapplet_name)
+            if not mapplet:
+                continue
+            mapplet_instances.add(inst.name)
+            prefix = inst.name
+
+            for tx in mapplet.transformations:
+                inlined = TransformationDef(
+                    name=f"{prefix}__{tx.name}",
+                    type=tx.type,
+                    description=tx.description,
+                    reusable=tx.reusable,
+                    fields=list(tx.fields),
+                    attributes=list(tx.attributes),
+                    groups=list(tx.groups),
+                    metadata_extensions=list(tx.metadata_extensions),
+                )
+                extra_transforms.append(inlined)
+
+            for conn in mapplet.connectors:
+                from informatica_python.models import ConnectorDef
+                new_from = f"{prefix}__{conn.from_instance}" if conn.from_instance in {t.name for t in mapplet.transformations} else conn.from_instance
+                new_to = f"{prefix}__{conn.to_instance}" if conn.to_instance in {t.name for t in mapplet.transformations} else conn.to_instance
+                inlined_conn = ConnectorDef(
+                    from_instance=new_from,
+                    from_field=conn.from_field,
+                    from_instance_type=conn.from_instance_type,
+                    to_instance=new_to,
+                    to_field=conn.to_field,
+                    to_instance_type=conn.to_instance_type,
+                )
+                extra_connectors.append(inlined_conn)
+
+    rewired_connectors = []
+    mapplet_internal_names = set()
+    for inst_name in mapplet_instances:
+        mapplet_name = None
+        for inst in mapping.instances:
+            if inst.name == inst_name:
+                mapplet_name = inst.transformation_name or inst.name
+                break
+        mapplet = mapplet_map.get(mapplet_name) if mapplet_name else None
+        if mapplet:
+            for tx in mapplet.transformations:
+                mapplet_internal_names.add(f"{inst_name}__{tx.name}")
+
+    for conn in mapping.connectors:
+        if conn.to_instance in mapplet_instances:
+            first_tx = None
+            for ec in extra_connectors:
+                if ec.from_instance == conn.to_instance or ec.to_instance.startswith(f"{conn.to_instance}__"):
+                    for et in extra_transforms:
+                        if et.name.startswith(f"{conn.to_instance}__"):
+                            has_input = any(
+                                "INPUT" in (f.porttype or "").upper()
+                                for f in et.fields
+                                if f.name == conn.to_field
+                            )
+                            if has_input:
+                                first_tx = et.name
+                                break
+                    if first_tx:
+                        break
+            if not first_tx:
+                for et in extra_transforms:
+                    if et.name.startswith(f"{conn.to_instance}__"):
+                        first_tx = et.name
+                        break
+            if first_tx:
+                from informatica_python.models import ConnectorDef
+                rewired_connectors.append(ConnectorDef(
+                    from_instance=conn.from_instance,
+                    from_field=conn.from_field,
+                    from_instance_type=conn.from_instance_type,
+                    to_instance=first_tx,
+                    to_field=conn.to_field,
+                    to_instance_type=conn.to_instance_type,
+                ))
+            else:
+                rewired_connectors.append(conn)
+        elif conn.from_instance in mapplet_instances:
+            last_tx = None
+            for et in reversed(extra_transforms):
+                if et.name.startswith(f"{conn.from_instance}__"):
+                    has_output = any(
+                        "OUTPUT" in (f.porttype or "").upper()
+                        for f in et.fields
+                        if f.name == conn.from_field
+                    )
+                    if has_output:
+                        last_tx = et.name
+                        break
+            if not last_tx:
+                for et in reversed(extra_transforms):
+                    if et.name.startswith(f"{conn.from_instance}__"):
+                        last_tx = et.name
+                        break
+            if last_tx:
+                from informatica_python.models import ConnectorDef
+                rewired_connectors.append(ConnectorDef(
+                    from_instance=last_tx,
+                    from_field=conn.from_field,
+                    from_instance_type=conn.from_instance_type,
+                    to_instance=conn.to_instance,
+                    to_field=conn.to_field,
+                    to_instance_type=conn.to_instance_type,
+                ))
+            else:
+                rewired_connectors.append(conn)
+        else:
+            rewired_connectors.append(conn)
+
+    return extra_transforms, extra_connectors + rewired_connectors, mapplet_instances
+
+
 def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
                           data_lib: str = "pandas", mapping_index: int = 1) -> str:
     lines = []
@@ -24,10 +148,17 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append("")
     lines.append("")
 
+    inlined_transforms, inlined_connectors, mapplet_instance_names = _inline_mapplets(mapping, folder)
+
+    all_transforms = list(mapping.transformations) + inlined_transforms
+    all_connectors = [c for c in mapping.connectors
+                      if c.from_instance not in mapplet_instance_names
+                      and c.to_instance not in mapplet_instance_names] + inlined_connectors
+
     source_map = _build_source_map(mapping, folder)
     target_map = _build_target_map(mapping, folder)
-    transform_map = {t.name: t for t in mapping.transformations}
-    connector_graph = _build_connector_graph(mapping.connectors)
+    transform_map = {t.name: t for t in all_transforms}
+    connector_graph = _build_connector_graph(all_connectors)
     instance_map = {i.name: i for i in mapping.instances}
 
     lines.append(f"def run_{_safe_name(mapping.name)}(config):")
@@ -50,7 +181,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
         safe = _safe_name(src_name)
         source_dfs[src_name] = f"df_{safe}"
 
-    sq_transforms = [t for t in mapping.transformations
+    sq_transforms = [t for t in all_transforms
                      if t.type in ("Source Qualifier", "Application Source Qualifier")]
     if sq_transforms:
         for sq in sq_transforms:
@@ -63,12 +194,14 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
                 conn_name = _safe_name(src_def.db_name) if src_def.db_name else "default"
                 schema = src_def.owner_name or "dbo"
                 lines.append(f"    df_{safe} = read_from_db(config, 'SELECT * FROM {schema}.{src_name}', '{conn_name}')")
+            elif src_def.flatfile:
+                _emit_flatfile_read(lines, safe, src_def)
             else:
                 lines.append(f"    df_{safe} = read_file(config.get('sources', {{}}).get('{src_name}', {{}}).get('file_path', '{src_name}'),")
                 lines.append(f"                          config.get('sources', {{}}).get('{src_name}', {{}}))")
             lines.append("")
 
-    processing_order = _get_processing_order(mapping.transformations, connector_graph, sq_transforms)
+    processing_order = _get_processing_order(all_transforms, connector_graph, sq_transforms)
 
     for tx in processing_order:
         if tx.type in ("Source Qualifier", "Application Source Qualifier"):
@@ -99,6 +232,107 @@ def _safe_name(name):
     return safe.lower()
 
 
+def _flatfile_config_dict(ff):
+    cfg = {}
+    if not ff:
+        return cfg
+    if ff.delimiter and ff.delimiter != ",":
+        d = ff.delimiter
+        DELIMITER_MAP = {
+            "COMMA": ",", "TAB": "\\t", "PIPE": "|", "SEMICOLON": ";",
+            "SPACE": " ", "TILDE": "~", "CARET": "^",
+        }
+        d = DELIMITER_MAP.get(d.upper(), d)
+        cfg["delimiter"] = d
+    if ff.is_fixed_width == "YES":
+        cfg["fixed_width"] = True
+    if ff.header_lines:
+        cfg["header_lines"] = ff.header_lines
+    if ff.skip_rows:
+        cfg["skip_rows"] = ff.skip_rows
+    if ff.text_qualifier:
+        cfg["quotechar"] = ff.text_qualifier
+    if ff.escape_character:
+        cfg["escapechar"] = ff.escape_character
+    if ff.strip_trailing_blanks == "YES":
+        cfg["strip_trailing_blanks"] = True
+    if ff.code_page:
+        cfg["encoding"] = ff.code_page
+    if ff.row_delimiter:
+        cfg["lineterminator"] = ff.row_delimiter
+    return cfg
+
+
+def _emit_flatfile_read(lines, var_name, src_def, indent="    "):
+    ff = src_def.flatfile
+    fc = _flatfile_config_dict(ff)
+    if fc.get("fixed_width"):
+        widths = []
+        for fld in src_def.fields:
+            widths.append(fld.precision if fld.precision else 10)
+        lines.append(f"{indent}df_{var_name} = pd.read_fwf(")
+        lines.append(f"{indent}    config.get('sources', {{}}).get('{src_def.name}', {{}}).get('file_path', '{src_def.name}'),")
+        lines.append(f"{indent}    widths={widths},")
+        hdr = fc.get("header_lines", 0)
+        if hdr:
+            lines.append(f"{indent}    header={hdr - 1},")
+        else:
+            lines.append(f"{indent}    header=None,")
+        skip = fc.get("skip_rows", 0)
+        if skip:
+            lines.append(f"{indent}    skiprows={skip},")
+        lines.append(f"{indent})")
+        return
+
+    file_cfg = {}
+    if "delimiter" in fc:
+        file_cfg["delimiter"] = fc["delimiter"]
+    if "quotechar" in fc:
+        file_cfg["quotechar"] = fc["quotechar"]
+    if "escapechar" in fc:
+        file_cfg["escapechar"] = fc["escapechar"]
+    if "encoding" in fc:
+        file_cfg["encoding"] = fc["encoding"]
+    if "lineterminator" in fc:
+        file_cfg["lineterminator"] = fc["lineterminator"]
+    hdr = fc.get("header_lines", 0)
+    if hdr:
+        file_cfg["header"] = True
+        file_cfg["header_row"] = hdr - 1
+    if fc.get("skip_rows"):
+        file_cfg["skip_rows"] = fc["skip_rows"]
+    if fc.get("strip_trailing_blanks"):
+        file_cfg["strip_trailing_blanks"] = True
+
+    if file_cfg:
+        lines.append(f"{indent}ff_cfg_{var_name} = {repr(file_cfg)}")
+        lines.append(f"{indent}ff_cfg_{var_name}.update(config.get('sources', {{}}).get('{src_def.name}', {{}}))")
+        lines.append(f"{indent}df_{var_name} = read_file(ff_cfg_{var_name}.get('file_path', '{src_def.name}'), ff_cfg_{var_name})")
+    else:
+        lines.append(f"{indent}df_{var_name} = read_file(config.get('sources', {{}}).get('{src_def.name}', {{}}).get('file_path', '{src_def.name}'),")
+        lines.append(f"{indent}                          config.get('sources', {{}}).get('{src_def.name}', {{}}))")
+
+
+def _emit_flatfile_write(lines, var_name, tgt_def, indent="    "):
+    ff = tgt_def.flatfile
+    fc = _flatfile_config_dict(ff)
+    file_cfg = {}
+    if "delimiter" in fc:
+        file_cfg["delimiter"] = fc["delimiter"]
+    if "quotechar" in fc:
+        file_cfg["quotechar"] = fc["quotechar"]
+    if "encoding" in fc:
+        file_cfg["encoding"] = fc["encoding"]
+
+    if file_cfg:
+        lines.append(f"{indent}ff_cfg_{var_name} = {repr(file_cfg)}")
+        lines.append(f"{indent}ff_cfg_{var_name}.update(config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
+        lines.append(f"{indent}write_file(df_target_{var_name}, ff_cfg_{var_name}.get('file_path', '{tgt_def.name}'), ff_cfg_{var_name})")
+    else:
+        lines.append(f"{indent}write_file(df_target_{var_name}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}).get('file_path', '{tgt_def.name}'),")
+        lines.append(f"{indent}          config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
+
+
 def _build_source_map(mapping, folder):
     source_map = {}
     for inst in mapping.instances:
@@ -221,6 +455,8 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
                 schema = src_def.owner_name or "dbo"
                 cols = ", ".join(f.name for f in src_def.fields) if src_def.fields else "*"
                 lines.append(f"    df_{sq_safe} = read_from_db(config, 'SELECT {cols} FROM {schema}.{src_def.name}', '{conn_name}')")
+            elif src_def.flatfile:
+                _emit_flatfile_read(lines, sq_safe, src_def)
             else:
                 lines.append(f"    df_{sq_safe} = read_file(config.get('sources', {{}}).get('{src_def.name}', {{}}).get('file_path', '{src_def.name}'),")
                 lines.append(f"                              config.get('sources', {{}}).get('{src_def.name}', {{}}))")
@@ -232,6 +468,8 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
                     conn_name = _safe_name(src_def.db_name) if src_def.db_name else "default"
                     schema = src_def.owner_name or "dbo"
                     lines.append(f"    df_{safe_src} = read_from_db(config, 'SELECT * FROM {schema}.{src_def.name}', '{conn_name}')")
+                elif src_def.flatfile:
+                    _emit_flatfile_read(lines, safe_src, src_def)
                 else:
                     lines.append(f"    df_{safe_src} = read_file(config.get('sources', {{}}).get('{src_def.name}', {{}}).get('file_path', '{src_def.name}'),")
                     lines.append(f"                              config.get('sources', {{}}).get('{src_def.name}', {{}}))")
@@ -278,7 +516,7 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type == "sorter":
         _gen_sorter_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("joiner",):
-        _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs)
+        _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph)
     elif tx_type in ("lookup procedure", "lookup"):
         _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type == "router":
@@ -410,7 +648,7 @@ def _gen_sorter_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs):
+def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph=None):
     join_type = "inner"
     join_condition = ""
     for attr in tx.attributes:
@@ -436,10 +674,31 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
 
     left_keys, right_keys = parse_join_condition(join_condition)
 
+    master_src = None
+    detail_src = None
+    input_conns = connector_graph.get("to", {}).get(tx.name, []) if connector_graph else []
+    for conn in input_conns:
+        to_field = conn.to_field
+        if to_field in master_fields:
+            master_src = conn.from_instance
+        elif to_field in detail_fields:
+            detail_src = conn.from_instance
+
     src_list = list(input_sources)
-    if len(src_list) >= 2:
-        df_master = source_dfs.get(src_list[0], f"df_{_safe_name(src_list[0])}")
-        df_detail = source_dfs.get(src_list[1], f"df_{_safe_name(src_list[1])}")
+    if not master_src and not detail_src and len(src_list) >= 2:
+        master_src = src_list[0]
+        detail_src = src_list[1]
+    elif not master_src and len(src_list) >= 1:
+        master_src = src_list[0]
+    if not detail_src:
+        for s in src_list:
+            if s != master_src:
+                detail_src = s
+                break
+
+    if master_src and detail_src:
+        df_master = source_dfs.get(master_src, f"df_{_safe_name(master_src)}")
+        df_detail = source_dfs.get(detail_src, f"df_{_safe_name(detail_src)}")
 
         lines.append(f"    # Join ({join_type}): {join_condition or 'auto'}")
         if left_keys and right_keys:
@@ -451,9 +710,7 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
             lines.append(f"        suffixes=('', '_master')")
             lines.append(f"    )")
         else:
-            common_cols = []
-            if master_fields and detail_fields:
-                common_cols = [f for f in detail_fields if f in master_fields]
+            common_cols = [f for f in detail_fields if f in master_fields]
             if common_cols:
                 lines.append(f"    df_{tx_safe} = {df_detail}.merge(")
                 lines.append(f"        {df_master},")
@@ -539,9 +796,13 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs):
 
         drop_cols = [k for k in lookup_keys if k not in input_keys]
         if drop_cols:
-            lines.append(f"    lkp_drop = [c for c in {drop_cols} if c in df_{tx_safe}.columns and c + '_lkp' not in df_{tx_safe}.columns]")
+            lines.append(f"    _lkp_drop = [c for c in {drop_cols} if c in df_{tx_safe}.columns]")
+            lines.append(f"    if _lkp_drop:")
+            lines.append(f"        df_{tx_safe} = df_{tx_safe}.drop(columns=_lkp_drop)")
 
         for rf in all_output_fields:
+            lines.append(f"    if '{rf.name}' not in df_{tx_safe}.columns:")
+            lines.append(f"        df_{tx_safe}['{rf.name}'] = None")
             if rf.default_value:
                 lines.append(f"    df_{tx_safe}['{rf.name}'] = df_{tx_safe}['{rf.name}'].fillna({repr(rf.default_value)})")
     else:
@@ -629,26 +890,127 @@ def _gen_sequence_generator(lines, tx, tx_safe, input_df, source_dfs):
 
 
 def _gen_normalizer_transform(lines, tx, tx_safe, input_df, source_dfs):
-    lines.append(f"    # TODO: Normalizer transformation - implement based on specific normalization logic")
-    lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+    input_ports = []
+    output_ports = []
+    occurs_cols = []
+    id_cols = []
+
+    for fld in tx.fields:
+        pt = (fld.porttype or "").upper()
+        if "INPUT" in pt:
+            input_ports.append(fld)
+        if "OUTPUT" in pt:
+            output_ports.append(fld)
+
+    for fld in tx.fields:
+        if fld.field_number > 0:
+            occurs_cols.append(fld.name)
+
+    if not occurs_cols:
+        import re
+        base_groups = {}
+        for fld in input_ports:
+            m = re.match(r'^(.+?)(\d+)$', fld.name)
+            if m:
+                base = m.group(1)
+                idx = int(m.group(2))
+                if base not in base_groups:
+                    base_groups[base] = []
+                base_groups[base].append(fld.name)
+            else:
+                id_cols.append(fld.name)
+
+        if base_groups:
+            longest_group = max(base_groups.values(), key=len)
+            occurs_cols = longest_group
+            id_cols = [f.name for f in input_ports if f.name not in occurs_cols]
+        else:
+            for fld in input_ports:
+                pt = (fld.porttype or "").upper()
+                if "INPUT" in pt and "OUTPUT" in pt:
+                    id_cols.append(fld.name)
+                elif "INPUT" in pt and "OUTPUT" not in pt:
+                    occurs_cols.append(fld.name)
+
+    if not id_cols:
+        id_cols = [f.name for f in input_ports if f.name not in occurs_cols]
+
+    gk_field = None
+    for fld in output_ports:
+        if "GK" in fld.name.upper() or "GENERATED" in fld.name.upper() or "KEY" in fld.name.upper():
+            gk_field = fld.name
+            break
+
+    lines.append(f"    # Normalizer: unpivot repeated columns into rows")
+    if occurs_cols and id_cols:
+        lines.append(f"    df_{tx_safe} = {input_df}.melt(")
+        lines.append(f"        id_vars={id_cols},")
+        lines.append(f"        value_vars={occurs_cols},")
+        lines.append(f"        var_name='_norm_variable',")
+        lines.append(f"        value_name='_norm_value'")
+        lines.append(f"    )")
+        lines.append(f"    df_{tx_safe} = df_{tx_safe}.dropna(subset=['_norm_value']).reset_index(drop=True)")
+    elif occurs_cols:
+        lines.append(f"    df_{tx_safe} = {input_df}[{occurs_cols}].stack().reset_index(drop=True).to_frame('_norm_value')")
+    else:
+        lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+
+    if gk_field:
+        lines.append(f"    df_{tx_safe}['{gk_field}'] = range(1, len(df_{tx_safe}) + 1)")
+
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
 def _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs):
     rank_port = None
-    group_by = []
+    group_by_ports = []
+    top_bottom = "TOP"
+    top_n = 0
+
     for fld in tx.fields:
-        if "RANK" in fld.name.upper():
+        pt = (fld.porttype or "").upper()
+        if "INPUT" in pt and "OUTPUT" in pt:
+            group_by_ports.append(fld.name)
+
+    for fld in tx.fields:
+        if fld.expression and fld.expression.strip() and fld.name.upper() not in ("RANKINDEX",):
             rank_port = fld.name
-    top_bottom = "TOP"
+            break
+    if not rank_port:
+        for fld in tx.fields:
+            if fld.name.upper() == "RANKINDEX":
+                continue
+            pt = (fld.porttype or "").upper()
+            if "INPUT" in pt and "OUTPUT" not in pt:
+                rank_port = fld.name
+                break
+
     for attr in tx.attributes:
         if attr.name == "Top/Bottom":
             top_bottom = attr.value
+        elif attr.name == "Number Of Ranks":
+            try:
+                top_n = int(attr.value)
+            except (ValueError, TypeError):
+                top_n = 0
+
+    ascending = top_bottom.upper() != "TOP"
 
     lines.append(f"    df_{tx_safe} = {input_df}.copy()")
-    if rank_port:
-        ascending = top_bottom.upper() != "TOP"
-        lines.append(f"    df_{tx_safe}['RANK_INDEX'] = df_{tx_safe}['{rank_port}'].rank(ascending={ascending})")
+    if rank_port and group_by_ports:
+        lines.append(f"    # Rank by '{rank_port}' within groups {group_by_ports}")
+        lines.append(f"    df_{tx_safe}['RANKINDEX'] = df_{tx_safe}.groupby({group_by_ports})['{rank_port}'].rank(")
+        lines.append(f"        method='min', ascending={ascending}")
+        lines.append(f"    ).astype(int)")
+        if top_n:
+            lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['RANKINDEX'] <= {top_n}].reset_index(drop=True)")
+    elif rank_port:
+        lines.append(f"    # Rank by '{rank_port}' (no group-by)")
+        lines.append(f"    df_{tx_safe}['RANKINDEX'] = df_{tx_safe}['{rank_port}'].rank(method='min', ascending={ascending}).astype(int)")
+        if top_n:
+            lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['RANKINDEX'] <= {top_n}].reset_index(drop=True)")
+    else:
+        lines.append(f"    df_{tx_safe}['RANKINDEX'] = range(1, len(df_{tx_safe}) + 1)")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -759,6 +1121,8 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
 
     if tgt_def.database_type and tgt_def.database_type != "Flat File":
         lines.append(f"    write_to_db(config, df_target_{tgt_safe}, '{tgt_def.name}', 'target')")
+    elif tgt_def.flatfile:
+        _emit_flatfile_write(lines, tgt_safe, tgt_def)
     else:
         lines.append(f"    write_file(df_target_{tgt_safe}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}).get('file_path', '{tgt_def.name}'),")
         lines.append(f"              config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")

{informatica_python-1.2.0 → informatica_python-1.3.0}/informatica_python/generators/workflow_gen.py

@@ -1,4 +1,5 @@
 from informatica_python.models import FolderDef, WorkflowDef, TaskInstanceDef
+from informatica_python.utils.expression_converter import convert_expression
 
 
 def generate_workflow_code(folder: FolderDef) -> str:
@@ -127,14 +128,51 @@ def _generate_workflow_function(lines, wf: WorkflowDef, folder: FolderDef):
             lines.append("")
 
         elif task.task_type == "Decision":
-            lines.append(f"    # Decision Task: {task.name}")
             decision_cond = ""
+            decision_name = ""
             for attr in task.attributes:
                 if attr.name == "Decision Condition":
                     decision_cond = attr.value
+                elif attr.name == "Decision Name":
+                    decision_name = attr.value
+
+            lines.append(f"    # Decision Task: {task.name}")
             if decision_cond:
-                lines.append(f"    # Condition: {decision_cond}")
-            lines.append(f"    logger.info('Decision task: {task.name}')")
+                py_cond = _convert_decision_condition(decision_cond)
+                lines.append(f"    # Original condition: {decision_cond}")
+                lines.append(f"    decision_{task_safe} = {py_cond}")
+                lines.append(f"    logger.info(f'Decision {task.name}: {{decision_{task_safe}}}')")
+
+                succ_targets = []
+                fail_targets = []
+                for link in wf.links:
+                    if link.from_instance == task.name:
+                        cond_text = (link.condition or "").strip()
+                        if cond_text and ("$" in cond_text or "SUCCEEDED" in cond_text.upper()
+                                          or "TRUE" in cond_text.upper()):
+                            succ_targets.append(link.to_instance)
+                        elif cond_text and ("FAILED" in cond_text.upper()
+                                            or "FALSE" in cond_text.upper()):
+                            fail_targets.append(link.to_instance)
+                        else:
+                            succ_targets.append(link.to_instance)
+
+                if succ_targets or fail_targets:
+                    lines.append(f"    if decision_{task_safe}:")
+                    if succ_targets:
+                        for t in succ_targets:
+                            lines.append(f"        logger.info('Decision {task.name} => proceeding to {t}')")
+                    else:
+                        lines.append(f"        pass")
+                    if fail_targets:
+                        lines.append(f"    else:")
+                        for t in fail_targets:
+                            lines.append(f"        logger.info('Decision {task.name} => proceeding to {t}')")
+                else:
+                    lines.append(f"    if not decision_{task_safe}:")
+                    lines.append(f"        logger.warning('Decision {task.name} evaluated to False')")
+            else:
+                lines.append(f"    logger.info('Decision task: {task.name} (no condition specified)')")
             lines.append("")
 
         elif task.task_type == "Timer":
@@ -226,6 +264,24 @@ def _get_task_execution_order(wf: WorkflowDef):
     return ordered
 
 
+def _convert_decision_condition(condition):
+    import re
+    cond = condition.strip()
+    cond = re.sub(r'\$\$(\w+)', r'\1', cond)
+    cond = re.sub(r'\$(\w+)\.(\w+)\.(Status|PrevTaskStatus)', r"'\2_status'", cond)
+    cond = re.sub(r'\bSUCCEEDED\b', "'SUCCEEDED'", cond, flags=re.IGNORECASE)
+    cond = re.sub(r'\bFAILED\b', "'FAILED'", cond, flags=re.IGNORECASE)
+    cond = re.sub(r'\bABORTED\b', "'ABORTED'", cond, flags=re.IGNORECASE)
+    cond = re.sub(r'\bAND\b', 'and', cond, flags=re.IGNORECASE)
+    cond = re.sub(r'\bOR\b', 'or', cond, flags=re.IGNORECASE)
+    cond = re.sub(r'\bNOT\b', 'not', cond, flags=re.IGNORECASE)
+    cond = re.sub(r'\bTRUE\b', 'True', cond, flags=re.IGNORECASE)
+    cond = re.sub(r'\bFALSE\b', 'False', cond, flags=re.IGNORECASE)
+    cond = re.sub(r'(?<!=)=(?!=)', '==', cond)
+    cond = cond.replace('<>', '!=')
+    return cond
+
+
 def _safe_name(name):
     import re
     safe = re.sub(r'[^a-zA-Z0-9_]', '_', name)

{informatica_python-1.2.0 → informatica_python-1.3.0}/informatica_python/utils/expression_converter.py

@@ -95,11 +95,11 @@ INFA_FUNC_MAP = {
 }
 
 
-AGG_FUNC_NAMES = {
-    "SUM", "COUNT", "AVG", "MAX", "MIN", "MEDIAN",
-    "STDDEV", "VARIANCE", "PERCENTILE", "FIRST", "LAST",
-    "MOVINGAVG", "MOVINGSUM", "CUME",
-}
+AGG_FUNC_NAMES = [
+    "MOVINGAVG", "MOVINGSUM", "PERCENTILE", "VARIANCE",
+    "STDDEV", "MEDIAN", "COUNT", "FIRST", "LAST",
+    "CUME", "SUM", "AVG", "MAX", "MIN",
+]
 
 
 def convert_expression(expr):
@@ -131,6 +131,8 @@ def convert_expression(expr):
 
     converted = re.sub(r'<>', '!=', converted)
 
+    converted = re.sub(r'(?<![<>!])=(?!=)', '==', converted)
+
     converted = re.sub(r':LKP\.(\w+)\(', r'lookup_func("\1", ', converted)
 
     converted = re.sub(r'\$\$(\w+)', r'get_variable("\1")', converted)
@@ -202,8 +204,11 @@ def parse_aggregate_expression(expr):
     cleaned = expr.strip()
 
     for func_name in AGG_FUNC_NAMES:
-        pattern = re.compile(r'\b' + func_name + r'\s*\(\s*([^)]*)\s*\)', re.IGNORECASE)
-        match = pattern.search(cleaned)
+        pattern = re.compile(
+            r'^\s*' + func_name + r'\s*\(\s*([A-Za-z_][A-Za-z0-9_]*|\*)\s*\)\s*$',
+            re.IGNORECASE
+        )
+        match = pattern.match(cleaned)
         if match:
             col = match.group(1).strip()
             return func_name.lower(), col

{informatica_python-1.2.0 → informatica_python-1.3.0}/informatica_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.2.0
+Version: 1.3.0
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 License-Expression: MIT
 Requires-Python: >=3.8

{informatica_python-1.2.0 → informatica_python-1.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "informatica-python"
-version = "1.2.0"
+version = "1.3.0"
 description = "Convert Informatica PowerCenter workflow XML to Python/PySpark code"
 readme = "README.md"
 license = "MIT"

{informatica_python-1.2.0 → informatica_python-1.3.0}/tests/test_converter.py

@@ -239,6 +239,8 @@ def test_expression_converter_expanded():
 
     result = convert_expression("IIF(STATUS = 'A', 'Active', 'Inactive')")
     assert "iif_expr" in result
+    assert "==" in result, f"Expected == in result, got: {result}"
+    assert "= =" not in result
 
     result = convert_expression("DECODE(TYPE, 1, 'One', 2, 'Two', 'Other')")
     assert "decode_expr" in result
@@ -301,6 +303,17 @@ def test_expression_converter_expanded():
     result = convert_expression("STATUS <> 'X'")
     assert "!=" in result
 
+    result = convert_expression("AMOUNT >= 100")
+    assert ">=" in result
+    assert ">==" not in result
+
+    result = convert_expression("AMOUNT <= 100")
+    assert "<=" in result
+    assert "<==" not in result
+
+    result = convert_expression("SUM(A)/COUNT(*)")
+    assert "sum_val" in result or "count_val" in result
+
     result = convert_expression("$$MY_VARIABLE")
     assert 'get_variable("MY_VARIABLE")' in result
 
@@ -405,6 +418,13 @@ def test_parse_aggregate_expression():
     assert func is None
     assert col is None
 
+    func, col = parse_aggregate_expression("SUM(A)/COUNT(*)")
+    assert func is None, f"Compound expression should not match, got func={func}"
+    assert col is None
+
+    func, col = parse_aggregate_expression("AVG(A+B)")
+    assert func is None, f"Expression with operators should not match, got func={func}"
+
     print("PASS: test_parse_aggregate_expression")
 
 
@@ -459,7 +479,7 @@ def test_generated_joiner_code():
     lines = []
     source_dfs = {"SRC_CUST": "df_src_cust", "SRC_ORDER": "df_src_order"}
     input_sources = {"SRC_CUST", "SRC_ORDER"}
-    _gen_joiner_transform(lines, tx, "jnr_cust_order", "df_src_cust", input_sources, source_dfs)
+    _gen_joiner_transform(lines, tx, "jnr_cust_order", "df_src_cust", input_sources, source_dfs, connector_graph=None)
     code = "\n".join(lines)
 
     assert "merge" in code
@@ -508,6 +528,254 @@ def test_generated_lookup_code():
     print(f"PASS: test_generated_lookup_code")
 
 
+def test_flatfile_metadata_read():
+    from informatica_python.models import (
+        MappingDef, FolderDef, SourceDef, TargetDef, FlatFileDef,
+        FieldDef, TransformationDef, ConnectorDef, InstanceDef, TableAttribute,
+    )
+    from informatica_python.generators.mapping_gen import generate_mapping_code
+
+    ff = FlatFileDef(
+        name="test_file",
+        delimiter="|",
+        header_lines=1,
+        text_qualifier='"',
+        skip_rows=2,
+        code_page="UTF-8",
+    )
+    src = SourceDef(
+        name="PIPE_SOURCE",
+        database_type="Flat File",
+        flatfile=ff,
+        fields=[FieldDef(name="COL_A", datatype="string"), FieldDef(name="COL_B", datatype="integer")],
+    )
+    tgt_ff = FlatFileDef(name="tgt_file", delimiter="~")
+    tgt = TargetDef(
+        name="TILDE_TARGET",
+        database_type="Flat File",
+        flatfile=tgt_ff,
+        fields=[FieldDef(name="COL_A", datatype="string")],
+    )
+    mapping = MappingDef(
+        name="m_flatfile_test",
+        transformations=[],
+        connectors=[ConnectorDef(from_instance="PIPE_SOURCE", from_field="COL_A",
+                                  from_instance_type="Source Definition",
+                                  to_instance="TILDE_TARGET", to_field="COL_A",
+                                  to_instance_type="Target Definition")],
+        instances=[
+            InstanceDef(name="PIPE_SOURCE", type="Source Definition", transformation_name="PIPE_SOURCE"),
+            InstanceDef(name="TILDE_TARGET", type="Target Definition", transformation_name="TILDE_TARGET"),
+        ],
+    )
+    folder = FolderDef(name="test", sources=[src], targets=[tgt], mappings=[mapping])
+    code = generate_mapping_code(mapping, folder)
+
+    assert "ff_cfg_" in code, "Should emit flatfile config dict"
+    assert "'delimiter': '|'" in code, "Pipe delimiter should appear"
+    assert "'skip_rows': 2" in code, "Skip rows should appear"
+    assert "'~'" in code, "Tilde delimiter should appear for target"
+    print("PASS: test_flatfile_metadata_read")
+
+
+def test_flatfile_fixed_width():
+    from informatica_python.models import (
+        MappingDef, FolderDef, SourceDef, FlatFileDef,
+        FieldDef, InstanceDef, TargetDef, ConnectorDef,
+    )
+    from informatica_python.generators.mapping_gen import generate_mapping_code
+
+    ff = FlatFileDef(name="fw_file", is_fixed_width="YES", header_lines=0)
+    src = SourceDef(
+        name="FW_SOURCE",
+        database_type="Flat File",
+        flatfile=ff,
+        fields=[FieldDef(name="F1", datatype="string", precision=10),
+                FieldDef(name="F2", datatype="string", precision=20)],
+    )
+    mapping = MappingDef(
+        name="m_fw_test",
+        transformations=[],
+        connectors=[],
+        instances=[InstanceDef(name="FW_SOURCE", type="Source Definition", transformation_name="FW_SOURCE")],
+    )
+    folder = FolderDef(name="test", sources=[src], targets=[], mappings=[mapping])
+    code = generate_mapping_code(mapping, folder)
+
+    assert "read_fwf" in code, "Fixed-width should use pd.read_fwf"
+    assert "[10, 20]" in code, "Widths should be derived from field precision"
+    print("PASS: test_flatfile_fixed_width")
+
+
+def test_normalizer_transform():
+    from informatica_python.models import (
+        MappingDef, FolderDef, SourceDef, FieldDef,
+        TransformationDef, ConnectorDef, InstanceDef, TableAttribute, TargetDef,
+    )
+    from informatica_python.generators.mapping_gen import _gen_normalizer_transform
+
+    tx = TransformationDef(
+        name="NRM_PHONES",
+        type="Normalizer",
+        fields=[
+            FieldDef(name="CUST_ID", datatype="integer", porttype="INPUT/OUTPUT"),
+            FieldDef(name="PHONE1", datatype="string", porttype="INPUT"),
+            FieldDef(name="PHONE2", datatype="string", porttype="INPUT"),
+            FieldDef(name="PHONE3", datatype="string", porttype="INPUT"),
+            FieldDef(name="GK", datatype="integer", porttype="OUTPUT"),
+        ],
+    )
+    lines = []
+    source_dfs = {}
+    _gen_normalizer_transform(lines, tx, "nrm_phones", "df_input", source_dfs)
+    code = "\n".join(lines)
+
+    assert "melt(" in code, "Normalizer should use pd.melt()"
+    assert "PHONE1" in code, "Should reference PHONE columns"
+    assert "CUST_ID" in code, "Should reference ID column"
+    assert "GK" in code, "Should generate GK sequence"
+    assert source_dfs["NRM_PHONES"] == "df_nrm_phones"
+    print("PASS: test_normalizer_transform")
+
+
+def test_rank_with_groupby():
+    from informatica_python.models import (
+        FieldDef, TransformationDef, TableAttribute,
+    )
+    from informatica_python.generators.mapping_gen import _gen_rank_transform
+
+    tx = TransformationDef(
+        name="RNK_SALES",
+        type="Rank",
+        fields=[
+            FieldDef(name="REGION", datatype="string", porttype="INPUT/OUTPUT"),
+            FieldDef(name="AMOUNT", datatype="decimal", porttype="INPUT", expression="AMOUNT"),
+            FieldDef(name="RANKINDEX", datatype="integer", porttype="OUTPUT"),
+        ],
+        attributes=[
+            TableAttribute(name="Top/Bottom", value="TOP"),
+            TableAttribute(name="Number Of Ranks", value="5"),
+        ],
+    )
+    lines = []
+    source_dfs = {}
+    _gen_rank_transform(lines, tx, "rnk_sales", "df_input", source_dfs)
+    code = "\n".join(lines)
+
+    assert "groupby" in code, "Should use groupby for group-by rank"
+    assert "REGION" in code, "Should group by REGION"
+    assert "AMOUNT" in code, "Should rank by AMOUNT"
+    assert "RANKINDEX" in code, "Should produce RANKINDEX column"
+    assert "<= 5" in code, "Should filter top 5"
+    assert source_dfs["RNK_SALES"] == "df_rnk_sales"
+    print("PASS: test_rank_with_groupby")
+
+
+def test_decision_task_if_else():
+    from informatica_python.models import (
+        FolderDef, WorkflowDef, TaskInstanceDef, WorkflowLink,
+        TableAttribute, MappingDef,
+    )
+    from informatica_python.generators.workflow_gen import generate_workflow_code
+
+    wf = WorkflowDef(
+        name="wf_test_decision",
+        task_instances=[
+            TaskInstanceDef(name="Start", task_name="Start", task_type="Start Task"),
+            TaskInstanceDef(
+                name="dec_check_status",
+                task_name="dec_check_status",
+                task_type="Decision",
+                attributes=[TableAttribute(name="Decision Condition", value="$$LOAD_FLAG = TRUE")],
+            ),
+            TaskInstanceDef(name="s_load_data", task_name="s_load_data", task_type="Session"),
+            TaskInstanceDef(name="s_skip_load", task_name="s_skip_load", task_type="Session"),
+        ],
+        links=[
+            WorkflowLink(from_instance="Start", to_instance="dec_check_status"),
+            WorkflowLink(from_instance="dec_check_status", to_instance="s_load_data", condition="$dec_check_status.SUCCEEDED"),
+            WorkflowLink(from_instance="dec_check_status", to_instance="s_skip_load", condition="$dec_check_status.FAILED"),
+        ],
+    )
+    folder = FolderDef(name="test", workflows=[wf], mappings=[])
+    code = generate_workflow_code(folder)
+
+    assert "decision_dec_check_status" in code, "Should create decision variable"
+    assert "if decision_dec_check_status" in code, "Should generate if branch"
+    assert "LOAD_FLAG" in code, "Should convert $$LOAD_FLAG"
+    assert "True" in code, "Should convert TRUE to Python True"
+    print("PASS: test_decision_task_if_else")
+
+
+def test_inline_mapplet():
+    from informatica_python.models import (
+        MappingDef, FolderDef, SourceDef, TargetDef, MappletDef,
+        TransformationDef, ConnectorDef, InstanceDef, FieldDef,
+        TableAttribute,
+    )
+    from informatica_python.generators.mapping_gen import generate_mapping_code
+
+    mplt = MappletDef(
+        name="mplt_clean_name",
+        transformations=[
+            TransformationDef(
+                name="EXP_UPPER",
+                type="Expression",
+                fields=[
+                    FieldDef(name="FULL_NAME", datatype="string", porttype="INPUT/OUTPUT",
+                             expression="UPPER(FULL_NAME)"),
+                ],
+            ),
+        ],
+        connectors=[],
+    )
+
+    mapping = MappingDef(
+        name="m_with_mapplet",
+        transformations=[
+            TransformationDef(name="SQ_INPUT", type="Source Qualifier",
+                              fields=[FieldDef(name="FULL_NAME", datatype="string", porttype="INPUT/OUTPUT")]),
+        ],
+        connectors=[
+            ConnectorDef(from_instance="SRC", from_field="FULL_NAME",
+                         from_instance_type="Source Definition",
+                         to_instance="SQ_INPUT", to_field="FULL_NAME",
+                         to_instance_type="Source Qualifier"),
+            ConnectorDef(from_instance="SQ_INPUT", from_field="FULL_NAME",
+                         from_instance_type="Source Qualifier",
+                         to_instance="MPLT_INST", to_field="FULL_NAME",
+                         to_instance_type="Mapplet"),
+            ConnectorDef(from_instance="MPLT_INST", from_field="FULL_NAME",
+                         from_instance_type="Mapplet",
+                         to_instance="TGT", to_field="FULL_NAME",
+                         to_instance_type="Target Definition"),
+        ],
+        instances=[
+            InstanceDef(name="SRC", type="Source Definition", transformation_name="SRC"),
+            InstanceDef(name="SQ_INPUT", type="Source Qualifier"),
+            InstanceDef(name="MPLT_INST", type="Mapplet", transformation_name="mplt_clean_name",
+                        transformation_type="Mapplet"),
+            InstanceDef(name="TGT", type="Target Definition", transformation_name="TGT"),
+        ],
+    )
+
+    src = SourceDef(name="SRC", fields=[FieldDef(name="FULL_NAME", datatype="string")])
+    tgt = TargetDef(name="TGT", fields=[FieldDef(name="FULL_NAME", datatype="string")])
+    folder = FolderDef(
+        name="test",
+        sources=[src],
+        targets=[tgt],
+        mappings=[mapping],
+        mapplets=[mplt],
+    )
+    code = generate_mapping_code(mapping, folder)
+
+    assert "MPLT_INST__EXP_UPPER" in code or "mplt_inst__exp_upper" in code, \
+        "Inlined mapplet transform should appear with prefix"
+    assert "UPPER" in code, "UPPER expression from mapplet should be present"
+    print("PASS: test_inline_mapplet")
+
+
 if __name__ == "__main__":
     print("=" * 60)
     print("Running informatica-python tests")
@@ -531,6 +799,12 @@ if __name__ == "__main__":
         test_generated_aggregator_code,
         test_generated_joiner_code,
         test_generated_lookup_code,
+        test_flatfile_metadata_read,
+        test_flatfile_fixed_width,
+        test_normalizer_transform,
+        test_rank_with_groupby,
+        test_decision_task_if_else,
+        test_inline_mapplet,
     ]
 
     passed = 0