informatica-python 1.4.2__tar.gz → 1.5.1__tar.gz

{informatica_python-1.4.2 → informatica_python-1.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.4.2
+Version: 1.5.1
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.4.2 → informatica_python-1.5.1}/informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.4.2"
+__version__ = "1.5.1"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

{informatica_python-1.4.2 → informatica_python-1.5.1}/informatica_python/cli.py

@@ -41,6 +41,11 @@ def main():
         default=None,
         help="Save parsed JSON to a file",
     )
+    parser.add_argument(
+        "--param-file",
+        default=None,
+        help="Path to Informatica .param file for variable substitution",
+    )
 
     args = parser.parse_args()
 
@@ -61,6 +66,7 @@ def main():
                 args.input_file,
                 output_dir=args.output,
                 output_zip=args.zip,
+                param_file=args.param_file,
             )
             print(f"Conversion complete! Output: {output_path}")
             print(f"Files generated:")

{informatica_python-1.4.2 → informatica_python-1.5.1}/informatica_python/converter.py

@@ -33,7 +33,8 @@ class InformaticaConverter:
         return self._powermart_to_dict(self.powermart)
 
     def convert(self, file_path: str, output_dir: str = "output",
-                output_zip: Optional[str] = None) -> str:
+                output_zip: Optional[str] = None,
+                param_file: Optional[str] = None) -> str:
         self.powermart = self.parser.parse_file(file_path)
 
         if not self.powermart.repositories:
@@ -47,7 +48,7 @@ class InformaticaConverter:
             raise ValueError("No folder found in XML file")
 
         if len(all_folders) == 1:
-            return self._convert_folder(all_folders[0], output_dir, output_zip)
+            return self._convert_folder(all_folders[0], output_dir, output_zip, param_file)
 
         result_path = output_dir if not output_zip else os.path.dirname(output_zip) or "."
         for folder in all_folders:
@@ -56,7 +57,7 @@ class InformaticaConverter:
             if output_zip:
                 base, ext = os.path.splitext(output_zip)
                 folder_zip = f"{base}_{folder.name}{ext}"
-            self._convert_folder(folder, folder_dir, folder_zip)
+            self._convert_folder(folder, folder_dir, folder_zip, param_file)
         return result_path
 
     def convert_string(self, xml_string: str, output_dir: str = "output",
@@ -87,7 +88,8 @@ class InformaticaConverter:
         return result_path
 
     def _convert_folder(self, folder: FolderDef, output_dir: str,
-                        output_zip: Optional[str] = None) -> str:
+                        output_zip: Optional[str] = None,
+                        param_file: Optional[str] = None) -> str:
         files = {}
 
         files["helper_functions.py"] = generate_helper_functions(folder, self.data_lib)

{informatica_python-1.4.2 → informatica_python-1.5.1}/informatica_python/generators/helper_gen.py

@@ -44,13 +44,20 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     lines.append('logger = logging.getLogger("informatica_converter")')
     lines.append("")
     lines.append("")
-    lines.append("def load_config(config_path='config.yml'):")
-    lines.append('    """Load configuration from YAML file."""')
+    lines.append("def load_config(config_path='config.yml', param_file=None):")
+    lines.append('    """Load configuration from YAML file, optionally merging Informatica .param file."""')
     lines.append("    with open(config_path, 'r') as f:")
-    lines.append("        return yaml.safe_load(f)")
+    lines.append("        config = yaml.safe_load(f) or {}")
+    lines.append("    if param_file:")
+    lines.append("        params = parse_param_file(param_file)")
+    lines.append("        config['params'] = params")
+    lines.append("        for key, val in params.items():")
+    lines.append("            os.environ[f'INFA_VAR_{key}'] = str(val)")
+    lines.append("    return config")
     lines.append("")
     lines.append("")
 
+    _add_param_file_functions(lines)
     _add_db_functions(lines, data_lib)
     _add_file_functions(lines, data_lib)
     _add_expression_helpers(lines)
@@ -59,6 +66,61 @@ def generate_helper_functions(folder: FolderDef, data_lib: str = "pandas") -> st
     return "\n".join(lines)
 
 
+def _add_param_file_functions(lines):
+    lines.append("# ============================================================")
+    lines.append("# Informatica Parameter File Support")
+    lines.append("# ============================================================")
+    lines.append("")
+    lines.append("")
+    lines.append("def parse_param_file(param_path):")
+    lines.append('    """')
+    lines.append("    Parse an Informatica .param file into a flat dict of variable names to values.")
+    lines.append("    Supports standard Informatica parameter file format:")
+    lines.append("      [Global]")
+    lines.append("      $$VAR_NAME=value")
+    lines.append("      [folder_name.WF:workflow_name.ST:session_name]")
+    lines.append("      $$CONN_NAME=value")
+    lines.append('    """')
+    lines.append("    params = {}")
+    lines.append("    if not os.path.exists(param_path):")
+    lines.append("        logger.warning(f'Parameter file not found: {param_path}')")
+    lines.append("        return params")
+    lines.append("")
+    lines.append("    current_section = 'Global'")
+    lines.append("    with open(param_path, 'r') as f:")
+    lines.append("        for line_num, line in enumerate(f, 1):")
+    lines.append("            line = line.strip()")
+    lines.append("            if not line or line.startswith('#'):")
+    lines.append("                continue")
+    lines.append("            if line.startswith('[') and line.endswith(']'):")
+    lines.append("                current_section = line[1:-1].strip()")
+    lines.append("                continue")
+    lines.append("            if '=' in line:")
+    lines.append("                key, _, value = line.partition('=')")
+    lines.append("                key = key.strip()")
+    lines.append("                value = value.strip()")
+    lines.append("                clean_key = key.lstrip('$')")
+    lines.append("                params[clean_key] = value")
+    lines.append("                if current_section != 'Global':")
+    lines.append("                    params[f'{current_section}.{clean_key}'] = value")
+    lines.append("    logger.info(f'Loaded {len(params)} parameters from {param_path}')")
+    lines.append("    return params")
+    lines.append("")
+    lines.append("")
+    lines.append("def get_param(config, var_name, default=''):")
+    lines.append('    """Get a parameter value from config params, then env vars, then default."""')
+    lines.append("    clean = var_name.lstrip('$')")
+    lines.append("    params = config.get('params', {})")
+    lines.append("    if clean in params:")
+    lines.append("        return params[clean]")
+    lines.append("    env_val = os.environ.get(f'INFA_VAR_{clean}')")
+    lines.append("    if env_val is not None:")
+    lines.append("        return env_val")
+    lines.append("    return default")
+    lines.append("")
+    lines.append("")
+
+
 def _add_db_functions(lines, data_lib):
     lines.append("# ============================================================")
     lines.append("# Database Operations")
@@ -1060,14 +1122,26 @@ def _add_expression_helpers(lines):
     lines.append("    return None")
     lines.append("")
     lines.append("")
-    lines.append("def get_variable(var_name):")
-    lines.append('    """Get workflow/mapping variable value."""')
-    lines.append("    return os.environ.get(f'INFA_VAR_{var_name}', '')")
+    lines.append("_param_store = {}")
+    lines.append("")
+    lines.append("")
+    lines.append("def get_variable(var_name, config=None):")
+    lines.append('    """Get workflow/mapping variable value from params, env vars, or param store."""')
+    lines.append("    clean = var_name.lstrip('$')")
+    lines.append("    if config and 'params' in config:")
+    lines.append("        val = config['params'].get(clean)")
+    lines.append("        if val is not None:")
+    lines.append("            return val")
+    lines.append("    if clean in _param_store:")
+    lines.append("        return _param_store[clean]")
+    lines.append("    return os.environ.get(f'INFA_VAR_{clean}', '')")
     lines.append("")
     lines.append("")
     lines.append("def set_variable(var_name, value):")
     lines.append('    """Set workflow/mapping variable value."""')
-    lines.append("    os.environ[f'INFA_VAR_{var_name}'] = str(value)")
+    lines.append("    clean = var_name.lstrip('$')")
+    lines.append("    _param_store[clean] = value")
+    lines.append("    os.environ[f'INFA_VAR_{clean}'] = str(value)")
     lines.append("    return value")
     lines.append("")
     lines.append("")

{informatica_python-1.4.2 → informatica_python-1.5.1}/informatica_python/generators/mapping_gen.py

@@ -4,11 +4,16 @@ from informatica_python.models import (
     TransformationDef, ConnectorDef, InstanceDef, MappletDef,
 )
 from informatica_python.utils.expression_converter import (
-    convert_expression, convert_sql_expression,
+    convert_expression, convert_expression_vectorized,
+    convert_sql_expression, convert_filter_vectorized,
     parse_join_condition, parse_lookup_condition,
     parse_aggregate_expression, PANDAS_AGG_MAP,
 )
 from informatica_python.utils.datatype_map import get_python_type
+from informatica_python.utils.lib_adapters import (
+    lib_merge, lib_sort, lib_groupby_agg, lib_groupby_first,
+    lib_concat, lib_empty_df, lib_copy, lib_rank,
+)
 
 
 def _inline_mapplets(mapping, folder):
@@ -184,6 +189,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append(f"Auto-generated by informatica-python")
     lines.append('"""')
     lines.append("")
+    lines.append("import numpy as np")
     lines.append("from helper_functions import *")
     lines.append("")
     lines.append("")
@@ -266,7 +272,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     for tx in processing_order:
         if tx.type in ("Source Qualifier", "Application Source Qualifier"):
             continue
-        _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map)
+        _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib)
 
     for tgt_name, tgt_def in target_map.items():
         _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides)
@@ -277,7 +283,12 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append("")
     lines.append("")
     lines.append("if __name__ == '__main__':")
-    lines.append("    config = load_config()")
+    lines.append("    import argparse as _ap")
+    lines.append("    _parser = _ap.ArgumentParser()")
+    lines.append("    _parser.add_argument('--param-file', default=None)")
+    lines.append("    _parser.add_argument('--config', default='config.yml')")
+    lines.append("    _args = _parser.parse_args()")
+    lines.append("    config = load_config(_args.config, param_file=_args.param_file)")
     lines.append(f"    run_{_safe_name(mapping.name)}(config)")
     lines.append("")
 
@@ -560,7 +571,7 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
     lines.append("")
 
 
-def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map):
+def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib="pandas"):
     tx_safe = _safe_name(tx.name)
     tx_type = tx.type.lower().strip()
 
@@ -584,21 +595,21 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     lines.append(f"    # Transformation: {tx.name} (Type: {tx.type})")
 
     if tx_type == "expression":
-        _gen_expression_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_expression_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
     elif tx_type == "filter":
-        _gen_filter_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_filter_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
     elif tx_type in ("aggregator",):
-        _gen_aggregator_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_aggregator_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
     elif tx_type == "sorter":
-        _gen_sorter_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_sorter_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
     elif tx_type in ("joiner",):
-        _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph)
+        _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph, data_lib)
     elif tx_type in ("lookup procedure", "lookup"):
-        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
     elif tx_type == "router":
         _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("union",):
-        _gen_union_transform(lines, tx, tx_safe, input_sources, source_dfs)
+        _gen_union_transform(lines, tx, tx_safe, input_sources, source_dfs, data_lib)
     elif tx_type in ("update strategy",):
         _gen_update_strategy(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type == "sequence generator":
@@ -606,9 +617,9 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
     elif tx_type in ("normalizer",):
         _gen_normalizer_transform(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("rank",):
-        _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs)
+        _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib)
     elif tx_type in ("custom transformation",):
-        _gen_custom_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs)
+        _gen_custom_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, data_lib)
     elif tx_type in ("stored procedure",):
         _gen_stored_proc(lines, tx, tx_safe, input_df, source_dfs)
     elif tx_type in ("java",):
@@ -617,44 +628,48 @@ def _generate_transformation(lines, tx, connector_graph, source_dfs, transform_m
         _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs)
     else:
         lines.append(f"    # TODO: Unsupported transformation type '{tx.type}' - passing through")
-        lines.append(f"    df_{tx_safe} = {input_df}.copy() if hasattr({input_df}, 'copy') else {input_df}")
+        copy_expr = lib_copy(data_lib, input_df)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
         source_dfs[tx.name] = f"df_{tx_safe}"
 
     lines.append("")
 
 
-def _gen_expression_transform(lines, tx, tx_safe, input_df, source_dfs):
-    lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+def _gen_expression_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
+    copy_expr = lib_copy(data_lib, input_df)
+    lines.append(f"    df_{tx_safe} = {copy_expr}")
     has_expressions = False
     for fld in tx.fields:
         if fld.expression and fld.expression.strip() and fld.expression.strip() != fld.name:
             has_expressions = True
-            expr_py = convert_expression(fld.expression)
+            expr_vec = convert_expression_vectorized(fld.expression, f"df_{tx_safe}")
             lines.append(f"    # {fld.name} = {fld.expression}")
             if fld.porttype and "OUTPUT" in fld.porttype.upper() and "INPUT" not in fld.porttype.upper():
-                lines.append(f"    df_{tx_safe}['{fld.name}'] = {expr_py}  # output-only port")
+                lines.append(f"    df_{tx_safe}['{fld.name}'] = {expr_vec}")
             else:
-                lines.append(f"    df_{tx_safe}['{fld.name}'] = {expr_py}")
+                lines.append(f"    df_{tx_safe}['{fld.name}'] = {expr_vec}")
     if not has_expressions:
         lines.append(f"    # Pass-through expression (no transformations)")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_filter_transform(lines, tx, tx_safe, input_df, source_dfs):
+def _gen_filter_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
     filter_condition = ""
     for attr in tx.attributes:
         if attr.name == "Filter Condition":
             filter_condition = attr.value
     if filter_condition:
-        expr_py = convert_expression(filter_condition)
+        expr_vec = convert_filter_vectorized(filter_condition, input_df)
         lines.append(f"    # Filter: {filter_condition}")
-        lines.append(f"    df_{tx_safe} = {input_df}[{expr_py}].copy()")
+        copy_expr = lib_copy(data_lib, f"{input_df}[{expr_vec}]")
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
     else:
-        lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+        copy_expr = lib_copy(data_lib, input_df)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_aggregator_transform(lines, tx, tx_safe, input_df, source_dfs):
+def _gen_aggregator_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
     group_by_ports = []
     agg_ports = []
     for fld in tx.fields:
@@ -686,22 +701,18 @@ def _gen_aggregator_transform(lines, tx, tx_safe, input_df, source_dfs):
 
     if group_by_ports and agg_dict:
         lines.append(f"    # Aggregator: group by {group_by_ports}")
-        agg_spec = {}
-        for out_name, (col, func) in agg_dict.items():
-            agg_spec[out_name] = f"pd.NamedAgg(column='{col}', aggfunc='{func}')"
-
-        lines.append(f"    df_{tx_safe} = {input_df}.groupby({group_by_ports}, as_index=False).agg(")
-        for out_name, spec in agg_spec.items():
-            lines.append(f"        {out_name}={spec},")
-        lines.append(f"    )")
+        agg_expr = lib_groupby_agg(data_lib, input_df, group_by_ports, agg_dict)
+        lines.append(f"    df_{tx_safe} = {agg_expr}")
 
         if rename_map:
             lines.append(f"    df_{tx_safe} = df_{tx_safe}.rename(columns={rename_map})")
     elif group_by_ports:
         lines.append(f"    # Aggregator: group by {group_by_ports}")
-        lines.append(f"    df_{tx_safe} = {input_df}.groupby({group_by_ports}, as_index=False).agg('first')")
+        first_expr = lib_groupby_first(data_lib, input_df, group_by_ports)
+        lines.append(f"    df_{tx_safe} = {first_expr}")
     else:
-        lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+        copy_expr = lib_copy(data_lib, input_df)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
 
     for col_name, expr_text in computed_aggs:
         expr_py = convert_expression(expr_text)
@@ -711,20 +722,22 @@ def _gen_aggregator_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_sorter_transform(lines, tx, tx_safe, input_df, source_dfs):
+def _gen_sorter_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
     sort_keys = []
     sort_dirs = []
     for fld in tx.fields:
         sort_keys.append(fld.name)
         sort_dirs.append(True)
     if sort_keys:
-        lines.append(f"    df_{tx_safe} = {input_df}.sort_values(by={sort_keys}, ascending={sort_dirs}).reset_index(drop=True)")
+        sort_expr = lib_sort(data_lib, input_df, sort_keys, sort_dirs)
+        lines.append(f"    df_{tx_safe} = {sort_expr}")
     else:
-        lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+        copy_expr = lib_copy(data_lib, input_df)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph=None):
+def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, connector_graph=None, data_lib="pandas"):
     join_type = "inner"
     join_condition = ""
     for attr in tx.attributes:
@@ -778,33 +791,29 @@ def _gen_joiner_transform(lines, tx, tx_safe, input_df, input_sources, source_df
 
         lines.append(f"    # Join ({join_type}): {join_condition or 'auto'}")
         if left_keys and right_keys:
-            lines.append(f"    df_{tx_safe} = {df_detail}.merge(")
-            lines.append(f"        {df_master},")
-            lines.append(f"        left_on={left_keys},")
-            lines.append(f"        right_on={right_keys},")
-            lines.append(f"        how='{join_type}',")
-            lines.append(f"        suffixes=('', '_master')")
-            lines.append(f"    )")
+            merge_expr = lib_merge(data_lib, df_detail, df_master,
+                                   left_on=left_keys, right_on=right_keys, how=join_type)
+            lines.append(f"    df_{tx_safe} = {merge_expr}")
         else:
             common_cols = [f for f in detail_fields if f in master_fields]
             if common_cols:
-                lines.append(f"    df_{tx_safe} = {df_detail}.merge(")
-                lines.append(f"        {df_master},")
-                lines.append(f"        on={common_cols},")
-                lines.append(f"        how='{join_type}',")
-                lines.append(f"        suffixes=('', '_master')")
-                lines.append(f"    )")
+                merge_expr = lib_merge(data_lib, df_detail, df_master,
+                                       on=common_cols, how=join_type)
+                lines.append(f"    df_{tx_safe} = {merge_expr}")
             else:
-                lines.append(f"    df_{tx_safe} = {df_detail}.merge({df_master}, how='{join_type}', suffixes=('', '_master'))")
+                merge_expr = lib_merge(data_lib, df_detail, df_master, how=join_type)
+                lines.append(f"    df_{tx_safe} = {merge_expr}")
     elif len(src_list) == 1:
         df1 = source_dfs.get(src_list[0], f"df_{_safe_name(src_list[0])}")
-        lines.append(f"    df_{tx_safe} = {df1}.copy()")
+        copy_expr = lib_copy(data_lib, df1)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
     else:
-        lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+        copy_expr = lib_copy(data_lib, input_df)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs):
+def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
     lookup_table = ""
     lookup_sql = ""
     lookup_condition = ""
@@ -844,7 +853,8 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs):
     elif lookup_table:
         lines.append(f"    df_lkp_{tx_safe} = read_from_db(config, 'SELECT * FROM {lookup_table}', 'default')")
     else:
-        lines.append(f"    df_lkp_{tx_safe} = pd.DataFrame()")
+        empty_expr = lib_empty_df(data_lib)
+        lines.append(f"    df_lkp_{tx_safe} = {empty_expr}")
 
     input_keys, lookup_keys = parse_lookup_condition(lookup_condition)
 
@@ -862,13 +872,10 @@ def _gen_lookup_transform(lines, tx, tx_safe, input_df, source_dfs):
         else:
             lines.append(f"    df_lkp_{tx_safe} = df_lkp_{tx_safe}[lkp_select_cols_{tx_safe}].drop_duplicates(subset={lookup_keys}, keep='first')")
 
-        lines.append(f"    df_{tx_safe} = {input_df}.merge(")
-        lines.append(f"        df_lkp_{tx_safe},")
-        lines.append(f"        left_on={input_keys},")
-        lines.append(f"        right_on={lookup_keys},")
-        lines.append(f"        how='left',")
-        lines.append(f"        suffixes=('', '_lkp')")
-        lines.append(f"    )")
+        merge_expr = lib_merge(data_lib, input_df, f"df_lkp_{tx_safe}",
+                               left_on=input_keys, right_on=lookup_keys,
+                               how="left", suffixes=("", "_lkp"))
+        lines.append(f"    df_{tx_safe} = {merge_expr}")
 
         drop_cols = [k for k in lookup_keys if k not in input_keys]
         if drop_cols:
@@ -910,7 +917,7 @@ def _gen_router_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_union_transform(lines, tx, tx_safe, input_sources, source_dfs):
+def _gen_union_transform(lines, tx, tx_safe, input_sources, source_dfs, data_lib="pandas"):
     dfs_to_union = []
     for src in input_sources:
         df_name = source_dfs.get(src, f"df_{_safe_name(src)}")
@@ -918,11 +925,14 @@ def _gen_union_transform(lines, tx, tx_safe, input_sources, source_dfs):
 
     if len(dfs_to_union) > 1:
         df_list = ", ".join(dfs_to_union)
-        lines.append(f"    df_{tx_safe} = pd.concat([{df_list}], ignore_index=True)")
+        concat_expr = lib_concat(data_lib, df_list)
+        lines.append(f"    df_{tx_safe} = {concat_expr}")
     elif dfs_to_union:
-        lines.append(f"    df_{tx_safe} = {dfs_to_union[0]}.copy()")
+        copy_expr = lib_copy(data_lib, dfs_to_union[0])
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
     else:
-        lines.append(f"    df_{tx_safe} = pd.DataFrame()")
+        empty_expr = lib_empty_df(data_lib)
+        lines.append(f"    df_{tx_safe} = {empty_expr}")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -1037,7 +1047,7 @@ def _gen_normalizer_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs):
+def _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs, data_lib="pandas"):
     rank_port = None
     group_by_ports = []
     top_bottom = "TOP"
@@ -1080,19 +1090,15 @@ def _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs):
                 rank_out_field = fld.name
                 break
 
-    lines.append(f"    df_{tx_safe} = {input_df}.copy()")
-    if rank_port and group_by_ports:
-        lines.append(f"    # Rank by '{rank_port}' within groups {group_by_ports}")
-        lines.append(f"    _rank_vals = df_{tx_safe}.groupby({group_by_ports})['{rank_port}'].rank(")
-        lines.append(f"        method='min', ascending={ascending}")
-        lines.append(f"    )")
-        lines.append(f"    df_{tx_safe}['{rank_out_field}'] = _rank_vals.fillna(0).astype(int)")
-        if top_n:
-            lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['{rank_out_field}'] <= {top_n}].reset_index(drop=True)")
-    elif rank_port:
-        lines.append(f"    # Rank by '{rank_port}' (no group-by)")
-        lines.append(f"    _rank_vals = df_{tx_safe}['{rank_port}'].rank(method='min', ascending={ascending})")
-        lines.append(f"    df_{tx_safe}['{rank_out_field}'] = _rank_vals.fillna(0).astype(int)")
+    copy_expr = lib_copy(data_lib, input_df)
+    lines.append(f"    df_{tx_safe} = {copy_expr}")
+    if rank_port:
+        rank_code = lib_rank(data_lib, f"df_{tx_safe}", group_by_ports, rank_port, ascending, rank_out_field)
+        if group_by_ports:
+            lines.append(f"    # Rank by '{rank_port}' within groups {group_by_ports}")
+        else:
+            lines.append(f"    # Rank by '{rank_port}' (no group-by)")
+        lines.append(f"    {rank_code}")
         if top_n:
             lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['{rank_out_field}'] <= {top_n}].reset_index(drop=True)")
     else:
@@ -1100,7 +1106,7 @@ def _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _gen_custom_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs):
+def _gen_custom_transform(lines, tx, tx_safe, input_df, input_sources, source_dfs, data_lib="pandas"):
     is_union = False
     output_fields = []
     input_groups = {}
@@ -1108,11 +1114,9 @@ def _gen_custom_transform(lines, tx, tx_safe, input_df, input_sources, source_df
     for fld in tx.fields:
         if "OUTPUT" in (fld.porttype or "").upper():
             output_fields.append(fld)
-        group_suffix_match = None
         import re
         m = re.match(r'^(.+?)(\d+)$', fld.name)
         if m and "INPUT" in (fld.porttype or "").upper():
-            base_name = m.group(1)
             group_idx = m.group(2)
             if group_idx not in input_groups:
                 input_groups[group_idx] = []
@@ -1128,14 +1132,18 @@ def _gen_custom_transform(lines, tx, tx_safe, input_df, input_sources, source_df
             dfs_to_union.append(df_name)
         if len(dfs_to_union) > 1:
             df_list = ", ".join(dfs_to_union)
-            lines.append(f"    df_{tx_safe} = pd.concat([{df_list}], ignore_index=True)")
+            concat_expr = lib_concat(data_lib, df_list)
+            lines.append(f"    df_{tx_safe} = {concat_expr}")
         elif dfs_to_union:
-            lines.append(f"    df_{tx_safe} = {dfs_to_union[0]}.copy()")
+            copy_expr = lib_copy(data_lib, dfs_to_union[0])
+            lines.append(f"    df_{tx_safe} = {copy_expr}")
         else:
-            lines.append(f"    df_{tx_safe} = pd.DataFrame()")
+            empty_expr = lib_empty_df(data_lib)
+            lines.append(f"    df_{tx_safe} = {empty_expr}")
     else:
         lines.append(f"    # Custom transformation: {tx.name}")
-        lines.append(f"    df_{tx_safe} = {input_df}.copy()")
+        copy_expr = lib_copy(data_lib, input_df)
+        lines.append(f"    df_{tx_safe} = {copy_expr}")
 
     source_dfs[tx.name] = f"df_{tx_safe}"
 

{informatica_python-1.4.2 → informatica_python-1.5.1}/informatica_python/generators/workflow_gen.py

@@ -51,7 +51,12 @@ def generate_workflow_code(folder: FolderDef) -> str:
 
     lines.append("")
     lines.append("if __name__ == '__main__':")
-    lines.append("    config = load_config()")
+    lines.append("    import argparse as _ap")
+    lines.append("    _parser = _ap.ArgumentParser()")
+    lines.append("    _parser.add_argument('--param-file', default=None)")
+    lines.append("    _parser.add_argument('--config', default='config.yml')")
+    lines.append("    _args = _parser.parse_args()")
+    lines.append("    config = load_config(_args.config, param_file=_args.param_file)")
     lines.append("    success = run_workflow(config)")
     lines.append("    sys.exit(0 if success else 1)")
     lines.append("")