informatica-python 1.3.0__tar.gz → 1.4.0__tar.gz

{informatica_python-1.3.0 → informatica_python-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.3.0
+Version: 1.4.0
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 License-Expression: MIT
 Requires-Python: >=3.8

{informatica_python-1.3.0 → informatica_python-1.4.0}/informatica_python/generators/helper_gen.py

@@ -208,7 +208,11 @@ def _add_file_functions(lines, data_lib):
     lines.append("    delimiter = file_config.get('delimiter', ',')")
     lines.append("    header = file_config.get('header', True)")
     lines.append("    encoding = file_config.get('encoding', 'utf-8')")
-    lines.append("    header_row = 0 if header else None")
+    lines.append("    header_row = file_config.get('header_row', 0 if header else None)")
+    lines.append("    skip_rows = file_config.get('skip_rows', 0)")
+    lines.append("    quotechar = file_config.get('quotechar', '\"')")
+    lines.append("    escapechar = file_config.get('escapechar', None)")
+    lines.append("    lineterminator = file_config.get('lineterminator', None)")
     lines.append("")
     lines.append("    logger.info(f'Reading file: {file_path} (ext={ext})')")
     lines.append("")
@@ -245,7 +249,17 @@ def _add_file_functions(lines, data_lib):
         lines.append("            return dd.read_csv(file_path, sep=delimiter, header=header_row)")
     else:
         lines.append("        if ext in ('.csv', '.dat', '.txt', ''):")
-        lines.append("            return pd.read_csv(file_path, sep=delimiter, header=header_row, encoding=encoding)")
+        lines.append("            csv_kwargs = dict(sep=delimiter, header=header_row, encoding=encoding,")
+        lines.append("                              quotechar=quotechar, escapechar=escapechar)")
+        lines.append("            if skip_rows:")
+        lines.append("                csv_kwargs['skiprows'] = skip_rows")
+        lines.append("            if lineterminator:")
+        lines.append("                csv_kwargs['lineterminator'] = lineterminator")
+        lines.append("            df = pd.read_csv(file_path, **csv_kwargs)")
+        lines.append("            if file_config.get('strip_trailing_blanks'):")
+        lines.append("                str_cols = df.select_dtypes(include=['object']).columns")
+        lines.append("                df[str_cols] = df[str_cols].apply(lambda c: c.str.rstrip())")
+        lines.append("            return df")
         lines.append("        elif ext in ('.xlsx', '.xls'):")
         lines.append("            return pd.read_excel(file_path, header=header_row)")
         lines.append("        elif ext == '.xml':")
@@ -271,6 +285,7 @@ def _add_file_functions(lines, data_lib):
     lines.append("    delimiter = file_config.get('delimiter', ',')")
     lines.append("    header = file_config.get('header', True)")
     lines.append("    encoding = file_config.get('encoding', 'utf-8')")
+    lines.append("    quotechar = file_config.get('quotechar', '\"')")
     lines.append("")
     lines.append("    os.makedirs(os.path.dirname(file_path) or '.', exist_ok=True)")
     lines.append("    logger.info(f'Writing file: {file_path}')")
@@ -290,7 +305,7 @@ def _add_file_functions(lines, data_lib):
         lines.append("            df.write_csv(file_path, separator=delimiter, has_header=header)")
     elif data_lib == "dask":
         lines.append("        if ext in ('.csv', '.dat', '.txt', ''):")
-        lines.append("            df.compute().to_csv(file_path, sep=delimiter, header=header, index=False, encoding=encoding)")
+        lines.append("            df.compute().to_csv(file_path, sep=delimiter, header=header, index=False, encoding=encoding, quotechar=quotechar)")
         lines.append("        elif ext in ('.xlsx', '.xls'):")
         lines.append("            df.compute().to_excel(file_path, header=header, index=False)")
         lines.append("        elif ext == '.json':")
@@ -301,7 +316,7 @@ def _add_file_functions(lines, data_lib):
         lines.append("            df.compute().to_csv(file_path, sep=delimiter, header=header, index=False)")
     else:
         lines.append("        if ext in ('.csv', '.dat', '.txt', ''):")
-        lines.append("            df.to_csv(file_path, sep=delimiter, header=header, index=False, encoding=encoding)")
+        lines.append("            df.to_csv(file_path, sep=delimiter, header=header, index=False, encoding=encoding, quotechar=quotechar)")
         lines.append("        elif ext in ('.xlsx', '.xls'):")
         lines.append("            df.to_excel(file_path, header=header, index=False)")
         lines.append("        elif ext == '.json':")

{informatica_python-1.3.0 → informatica_python-1.4.0}/informatica_python/generators/mapping_gen.py

@@ -135,6 +135,46 @@ def _inline_mapplets(mapping, folder):
     return extra_transforms, extra_connectors + rewired_connectors, mapplet_instances
 
 
+def _build_session_conn_overrides(mapping, folder):
+    overrides = {}
+    for session in folder.sessions:
+        if session.mapping_name != mapping.name:
+            continue
+        for sti in session.transform_instances:
+            inst_name = sti.instance_name or sti.transformation_name
+            for conn_ref in sti.connections:
+                conn_key = conn_ref.connection_name or conn_ref.variable
+                if conn_key:
+                    overrides[inst_name] = {
+                        "connection_name": conn_ref.connection_name,
+                        "connection_type": conn_ref.connection_type,
+                        "connection_subtype": conn_ref.connection_subtype,
+                        "variable": conn_ref.variable,
+                    }
+            for attr in sti.attributes:
+                if attr.name == "Connection Information" and attr.value:
+                    if inst_name not in overrides:
+                        overrides[inst_name] = {}
+                    overrides[inst_name]["connection_info"] = attr.value
+                elif attr.name == "Source File Directory" and attr.value:
+                    if inst_name not in overrides:
+                        overrides[inst_name] = {}
+                    overrides[inst_name]["source_file_directory"] = attr.value
+                elif attr.name == "Source filename" and attr.value:
+                    if inst_name not in overrides:
+                        overrides[inst_name] = {}
+                    overrides[inst_name]["source_filename"] = attr.value
+                elif attr.name == "Output File Directory" and attr.value:
+                    if inst_name not in overrides:
+                        overrides[inst_name] = {}
+                    overrides[inst_name]["output_file_directory"] = attr.value
+                elif attr.name == "Output filename" and attr.value:
+                    if inst_name not in overrides:
+                        overrides[inst_name] = {}
+                    overrides[inst_name]["output_filename"] = attr.value
+    return overrides
+
+
 def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
                           data_lib: str = "pandas", mapping_index: int = 1) -> str:
     lines = []
@@ -151,15 +191,20 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     inlined_transforms, inlined_connectors, mapplet_instance_names = _inline_mapplets(mapping, folder)
 
     all_transforms = list(mapping.transformations) + inlined_transforms
-    all_connectors = [c for c in mapping.connectors
-                      if c.from_instance not in mapplet_instance_names
-                      and c.to_instance not in mapplet_instance_names] + inlined_connectors
+    if mapplet_instance_names:
+        kept_originals = [c for c in mapping.connectors
+                          if c.from_instance not in mapplet_instance_names
+                          and c.to_instance not in mapplet_instance_names]
+        all_connectors = kept_originals + inlined_connectors
+    else:
+        all_connectors = list(mapping.connectors)
 
     source_map = _build_source_map(mapping, folder)
     target_map = _build_target_map(mapping, folder)
     transform_map = {t.name: t for t in all_transforms}
     connector_graph = _build_connector_graph(all_connectors)
     instance_map = {i.name: i for i in mapping.instances}
+    session_overrides = _build_session_conn_overrides(mapping, folder)
 
     lines.append(f"def run_{_safe_name(mapping.name)}(config):")
     lines.append(f'    """Execute mapping: {mapping.name}"""')
@@ -176,6 +221,11 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
             lines.append(f"    {safe_var} = {default}")
         lines.append("")
 
+    if session_overrides:
+        lines.append("    # Session connection overrides")
+        lines.append(f"    _sess_overrides = {repr(session_overrides)}")
+        lines.append("")
+
     source_dfs = {}
     for src_name, src_def in source_map.items():
         safe = _safe_name(src_name)
@@ -185,13 +235,23 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
                      if t.type in ("Source Qualifier", "Application Source Qualifier")]
     if sq_transforms:
         for sq in sq_transforms:
-            _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map)
+            _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides)
     else:
         for src_name, src_def in source_map.items():
             safe = _safe_name(src_name)
+            override = session_overrides.get(src_name, {})
             lines.append(f"    # Read source: {src_name}")
-            if src_def.database_type and src_def.database_type != "Flat File":
-                conn_name = _safe_name(src_def.db_name) if src_def.db_name else "default"
+            if override.get("source_file_directory") or override.get("source_filename"):
+                src_dir = override.get("source_file_directory", ".")
+                src_file = override.get("source_filename", src_def.name)
+                lines.append(f"    _src_path_{safe} = config.get('sources', {{}}).get('{src_def.name}', {{}}).get('file_path',")
+                lines.append(f"        os.path.join('{src_dir}', '{src_file}'))")
+                if src_def.flatfile:
+                    _emit_flatfile_read(lines, safe, src_def)
+                else:
+                    lines.append(f"    df_{safe} = read_file(_src_path_{safe}, config.get('sources', {{}}).get('{src_def.name}', {{}}))")
+            elif src_def.database_type and src_def.database_type != "Flat File":
+                conn_name = override.get("connection_name") or (_safe_name(src_def.db_name) if src_def.db_name else "default")
                 schema = src_def.owner_name or "dbo"
                 lines.append(f"    df_{safe} = read_from_db(config, 'SELECT * FROM {schema}.{src_name}', '{conn_name}')")
             elif src_def.flatfile:
@@ -209,7 +269,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
         _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map)
 
     for tgt_name, tgt_def in target_map.items():
-        _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map)
+        _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides)
 
     lines.append("")
     lines.append(f"    log_mapping_end('{mapping.name}', start_time)")
@@ -401,7 +461,7 @@ def _get_processing_order(transformations, connector_graph, sq_transforms):
     return ordered
 
 
-def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map):
+def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_graph, instance_map, session_overrides=None):
     sq_safe = _safe_name(sq.name)
     sql_override = ""
     pre_sql = ""
@@ -438,7 +498,8 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
     if sql_override:
         src_name = next(iter(connected_sources)) if connected_sources else "source"
         src_def = source_map.get(src_name, SourceDef(name=src_name))
-        conn_name = _safe_name(src_def.db_name) if src_def.db_name else "default"
+        sq_override = (session_overrides or {}).get(sq.name, {}) or (session_overrides or {}).get(src_name, {})
+        conn_name = sq_override.get("connection_name") or (_safe_name(src_def.db_name) if src_def.db_name else "default")
 
         lines.append(f"    sql_{sq_safe} = '''")
         for sql_line in sql_override.strip().split("\n"):
@@ -450,8 +511,9 @@ def _generate_source_qualifier(lines, sq, source_map, source_dfs, connector_grap
             src_name = next(iter(connected_sources))
             src_def = source_map.get(src_name, SourceDef(name=src_name))
             safe_src = _safe_name(src_name)
+            src_override = (session_overrides or {}).get(sq.name, {}) or (session_overrides or {}).get(src_name, {})
             if src_def.database_type and src_def.database_type != "Flat File":
-                conn_name = _safe_name(src_def.db_name) if src_def.db_name else "default"
+                conn_name = src_override.get("connection_name") or (_safe_name(src_def.db_name) if src_def.db_name else "default")
                 schema = src_def.owner_name or "dbo"
                 cols = ", ".join(f.name for f in src_def.fields) if src_def.fields else "*"
                 lines.append(f"    df_{sq_safe} = read_from_db(config, 'SELECT {cols} FROM {schema}.{src_def.name}', '{conn_name}')")
@@ -996,21 +1058,31 @@ def _gen_rank_transform(lines, tx, tx_safe, input_df, source_dfs):
 
     ascending = top_bottom.upper() != "TOP"
 
+    rank_out_field = "RANKINDEX"
+    for fld in tx.fields:
+        if fld.name.upper() == "RANKINDEX" or "RANK" in fld.name.upper():
+            pt = (fld.porttype or "").upper()
+            if "OUTPUT" in pt and "INPUT" not in pt:
+                rank_out_field = fld.name
+                break
+
     lines.append(f"    df_{tx_safe} = {input_df}.copy()")
     if rank_port and group_by_ports:
         lines.append(f"    # Rank by '{rank_port}' within groups {group_by_ports}")
-        lines.append(f"    df_{tx_safe}['RANKINDEX'] = df_{tx_safe}.groupby({group_by_ports})['{rank_port}'].rank(")
+        lines.append(f"    _rank_vals = df_{tx_safe}.groupby({group_by_ports})['{rank_port}'].rank(")
         lines.append(f"        method='min', ascending={ascending}")
-        lines.append(f"    ).astype(int)")
+        lines.append(f"    )")
+        lines.append(f"    df_{tx_safe}['{rank_out_field}'] = _rank_vals.fillna(0).astype(int)")
         if top_n:
-            lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['RANKINDEX'] <= {top_n}].reset_index(drop=True)")
+            lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['{rank_out_field}'] <= {top_n}].reset_index(drop=True)")
     elif rank_port:
         lines.append(f"    # Rank by '{rank_port}' (no group-by)")
-        lines.append(f"    df_{tx_safe}['RANKINDEX'] = df_{tx_safe}['{rank_port}'].rank(method='min', ascending={ascending}).astype(int)")
+        lines.append(f"    _rank_vals = df_{tx_safe}['{rank_port}'].rank(method='min', ascending={ascending})")
+        lines.append(f"    df_{tx_safe}['{rank_out_field}'] = _rank_vals.fillna(0).astype(int)")
         if top_n:
-            lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['RANKINDEX'] <= {top_n}].reset_index(drop=True)")
+            lines.append(f"    df_{tx_safe} = df_{tx_safe}[df_{tx_safe}['{rank_out_field}'] <= {top_n}].reset_index(drop=True)")
     else:
-        lines.append(f"    df_{tx_safe}['RANKINDEX'] = range(1, len(df_{tx_safe}) + 1)")
+        lines.append(f"    df_{tx_safe}['{rank_out_field}'] = range(1, len(df_{tx_safe}) + 1)")
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
@@ -1087,7 +1159,7 @@ def _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map):
+def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides=None):
     tgt_safe = _safe_name(tgt_name)
 
     to_conns = connector_graph.get("to", {}).get(tgt_name, [])
@@ -1119,10 +1191,92 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     else:
         lines.append(f"    df_target_{tgt_safe} = {input_df}")
 
-    if tgt_def.database_type and tgt_def.database_type != "Flat File":
-        lines.append(f"    write_to_db(config, df_target_{tgt_safe}, '{tgt_def.name}', 'target')")
+    _emit_type_casting(lines, tgt_safe, tgt_def)
+
+    tgt_override = (session_overrides or {}).get(tgt_name, {})
+    tgt_conn = tgt_override.get("connection_name")
+
+    if tgt_override.get("output_file_directory") or tgt_override.get("output_filename"):
+        out_dir = tgt_override.get("output_file_directory", ".")
+        out_file = tgt_override.get("output_filename", tgt_def.name)
+        lines.append(f"    _tgt_path_{tgt_safe} = config.get('targets', {{}}).get('{tgt_def.name}', {{}}).get('file_path',")
+        lines.append(f"        os.path.join('{out_dir}', '{out_file}'))")
+        if tgt_def.flatfile:
+            _emit_flatfile_write(lines, tgt_safe, tgt_def)
+        else:
+            lines.append(f"    write_file(df_target_{tgt_safe}, _tgt_path_{tgt_safe}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
+    elif tgt_def.database_type and tgt_def.database_type != "Flat File":
+        conn_label = tgt_conn or "target"
+        lines.append(f"    write_to_db(config, df_target_{tgt_safe}, '{tgt_def.name}', '{conn_label}')")
     elif tgt_def.flatfile:
         _emit_flatfile_write(lines, tgt_safe, tgt_def)
     else:
         lines.append(f"    write_file(df_target_{tgt_safe}, config.get('targets', {{}}).get('{tgt_def.name}', {{}}).get('file_path', '{tgt_def.name}'),")
         lines.append(f"              config.get('targets', {{}}).get('{tgt_def.name}', {{}}))")
+
+
+CAST_MAP = {
+    "bigint": ("int", "Int64"),
+    "integer": ("int", "Int32"),
+    "int": ("int", "Int32"),
+    "small integer": ("int", "Int16"),
+    "smallint": ("int", "Int16"),
+    "tinyint": ("int", "Int8"),
+    "numeric": ("float", "float64"),
+    "decimal": ("float", "float64"),
+    "float": ("float", "float64"),
+    "double": ("float", "float64"),
+    "real": ("float", "float32"),
+    "money": ("float", "float64"),
+    "smallmoney": ("float", "float64"),
+    "string": ("str", "object"),
+    "nstring": ("str", "object"),
+    "text": ("str", "object"),
+    "ntext": ("str", "object"),
+    "varchar": ("str", "object"),
+    "nvarchar": ("str", "object"),
+    "char": ("str", "object"),
+    "nchar": ("str", "object"),
+    "date/time": ("str", "datetime64[ns]"),
+    "datetime": ("str", "datetime64[ns]"),
+    "datetime2": ("str", "datetime64[ns]"),
+    "date": ("str", "datetime64[ns]"),
+    "timestamp": ("str", "datetime64[ns]"),
+    "bit": ("bool", "boolean"),
+    "boolean": ("bool", "boolean"),
+}
+
+
+def _emit_type_casting(lines, tgt_safe, tgt_def):
+    cast_ops = []
+    for fld in tgt_def.fields:
+        dt_key = fld.datatype.lower().strip()
+        if dt_key not in CAST_MAP:
+            continue
+        py_type, pd_dtype = CAST_MAP[dt_key]
+        if pd_dtype in ("datetime64[ns]",):
+            cast_ops.append((fld.name, "datetime", pd_dtype, fld.nullable == "NULL"))
+        elif pd_dtype in ("Int64", "Int32", "Int16", "Int8"):
+            cast_ops.append((fld.name, "int", pd_dtype, fld.nullable == "NULL"))
+        elif pd_dtype in ("float64", "float32"):
+            cast_ops.append((fld.name, "float", pd_dtype, fld.nullable == "NULL"))
+        elif pd_dtype == "boolean":
+            cast_ops.append((fld.name, "bool", pd_dtype, fld.nullable == "NULL"))
+
+    if not cast_ops:
+        return
+
+    lines.append(f"    # Type casting for target fields")
+    for col_name, cast_type, pd_dtype, nullable in cast_ops:
+        lines.append(f"    if '{col_name}' in df_target_{tgt_safe}.columns:")
+        if cast_type == "datetime":
+            lines.append(f"        df_target_{tgt_safe}['{col_name}'] = pd.to_datetime(df_target_{tgt_safe}['{col_name}'], errors='coerce')")
+        elif cast_type == "int":
+            if nullable:
+                lines.append(f"        df_target_{tgt_safe}['{col_name}'] = pd.to_numeric(df_target_{tgt_safe}['{col_name}'], errors='coerce').astype('{pd_dtype}')")
+            else:
+                lines.append(f"        df_target_{tgt_safe}['{col_name}'] = pd.to_numeric(df_target_{tgt_safe}['{col_name}'], errors='coerce').fillna(0).astype(int)")
+        elif cast_type == "float":
+            lines.append(f"        df_target_{tgt_safe}['{col_name}'] = pd.to_numeric(df_target_{tgt_safe}['{col_name}'], errors='coerce')")
+        elif cast_type == "bool":
+            lines.append(f"        df_target_{tgt_safe}['{col_name}'] = df_target_{tgt_safe}['{col_name}'].astype('{pd_dtype}')")