informatica-python 1.5.1__tar.gz → 1.6.0__tar.gz

{informatica_python-1.5.1 → informatica_python-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.5.1
+Version: 1.6.0
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python/__init__.py

@@ -7,7 +7,7 @@ Licensed under the MIT License.
 
 from informatica_python.converter import InformaticaConverter
 
-__version__ = "1.5.1"
+__version__ = "1.6.0"
 __author__ = "Nick"
 __license__ = "MIT"
 __all__ = ["InformaticaConverter"]

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python/cli.py

@@ -46,6 +46,11 @@ def main():
         default=None,
         help="Path to Informatica .param file for variable substitution",
     )
+    parser.add_argument(
+        "--validate-casts",
+        action="store_true",
+        help="Generate data quality validation code that logs warnings on type coercion",
+    )
 
     args = parser.parse_args()
 
@@ -67,6 +72,7 @@ def main():
                 output_dir=args.output,
                 output_zip=args.zip,
                 param_file=args.param_file,
+                validate_casts=args.validate_casts,
             )
             print(f"Conversion complete! Output: {output_path}")
             print(f"Files generated:")

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python/converter.py

@@ -34,7 +34,8 @@ class InformaticaConverter:
 
     def convert(self, file_path: str, output_dir: str = "output",
                 output_zip: Optional[str] = None,
-                param_file: Optional[str] = None) -> str:
+                param_file: Optional[str] = None,
+                validate_casts: bool = False) -> str:
         self.powermart = self.parser.parse_file(file_path)
 
         if not self.powermart.repositories:
@@ -48,7 +49,7 @@ class InformaticaConverter:
             raise ValueError("No folder found in XML file")
 
         if len(all_folders) == 1:
-            return self._convert_folder(all_folders[0], output_dir, output_zip, param_file)
+            return self._convert_folder(all_folders[0], output_dir, output_zip, param_file, validate_casts)
 
         result_path = output_dir if not output_zip else os.path.dirname(output_zip) or "."
         for folder in all_folders:
@@ -57,7 +58,7 @@ class InformaticaConverter:
             if output_zip:
                 base, ext = os.path.splitext(output_zip)
                 folder_zip = f"{base}_{folder.name}{ext}"
-            self._convert_folder(folder, folder_dir, folder_zip, param_file)
+            self._convert_folder(folder, folder_dir, folder_zip, param_file, validate_casts)
         return result_path
 
     def convert_string(self, xml_string: str, output_dir: str = "output",
@@ -89,13 +90,18 @@ class InformaticaConverter:
 
     def _convert_folder(self, folder: FolderDef, output_dir: str,
                         output_zip: Optional[str] = None,
-                        param_file: Optional[str] = None) -> str:
+                        param_file: Optional[str] = None,
+                        validate_casts: bool = False) -> str:
+        if param_file:
+            from informatica_python.utils.expression_converter import parse_param_file
+            parse_param_file(param_file)
+
         files = {}
 
         files["helper_functions.py"] = generate_helper_functions(folder, self.data_lib)
 
         for i, mapping in enumerate(folder.mappings, 1):
-            code = generate_mapping_code(mapping, folder, self.data_lib, i)
+            code = generate_mapping_code(mapping, folder, self.data_lib, i, validate_casts=validate_casts)
             files[f"mapping_{i}.py"] = code
 
         files["workflow.py"] = generate_workflow_code(folder)

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python/generators/error_log_gen.py

@@ -222,6 +222,123 @@ def generate_error_log(folder: FolderDef, parser_errors=None, parser_warnings=No
             for part in sti.partitions:
                 lines.append(f"[INFO] Session '{session.name}': Partition '{part.name}' (type={part.partition_type}) on '{sti.instance_name}'")
 
+    lines.append("")
+    lines.append("-" * 70)
+    lines.append("UNSUPPORTED TRANSFORMS (Require Manual Review)")
+    lines.append("-" * 70)
+    lines.append("")
+
+    unsupported_types = {
+        "Custom Transformation", "Java", "Stored Procedure",
+        "External Procedure", "HTTP Transformation",
+        "Web Service Consumer", "SQL",
+    }
+    skipped_items = []
+    for mapping in folder.mappings:
+        for tx in mapping.transformations:
+            if tx.type in unsupported_types:
+                skipped_attrs = []
+                for attr in tx.attributes:
+                    if attr.value and attr.value.strip():
+                        skipped_attrs.append(attr.name)
+                skipped_items.append({
+                    "mapping": mapping.name,
+                    "transform": tx.name,
+                    "type": tx.type,
+                    "field_count": len(tx.fields),
+                    "skipped_attrs": skipped_attrs,
+                })
+
+    if skipped_items:
+        for item in skipped_items:
+            lines.append(f"  Mapping: {item['mapping']}")
+            lines.append(f"    Transform: {item['transform']} (type={item['type']}, {item['field_count']} fields)")
+            if item['skipped_attrs']:
+                lines.append(f"    Skipped attributes: {', '.join(item['skipped_attrs'])}")
+            lines.append("")
+    else:
+        lines.append("  None - all transformations are supported")
+        lines.append("")
+
+    lines.append("-" * 70)
+    lines.append("UNMAPPED PORTS (Fields with no connectors)")
+    lines.append("-" * 70)
+    lines.append("")
+
+    for mapping in folder.mappings:
+        connected_fields = set()
+        for conn in mapping.connectors:
+            connected_fields.add((conn.from_instance, conn.from_field))
+            connected_fields.add((conn.to_instance, conn.to_field))
+
+        unmapped = []
+        for tx in mapping.transformations:
+            for fld in tx.fields:
+                pt = (fld.porttype or "").upper()
+                if "OUTPUT" in pt or "INPUT/OUTPUT" in pt:
+                    if (tx.name, fld.name) not in connected_fields:
+                        unmapped.append((tx.name, tx.type, fld.name, fld.porttype or ""))
+
+        if unmapped:
+            lines.append(f"  Mapping: {mapping.name}")
+            for tx_name, tx_type, fld_name, port_type in unmapped:
+                lines.append(f"    {tx_name} ({tx_type}): {fld_name} [{port_type}]")
+            lines.append("")
+
+    lines.append("-" * 70)
+    lines.append("UNSUPPORTED EXPRESSION FUNCTIONS")
+    lines.append("-" * 70)
+    lines.append("")
+
+    import re
+    known_functions = {
+        "IIF", "DECODE", "CHOOSE", "IN", "LTRIM", "RTRIM", "TRIM",
+        "UPPER", "LOWER", "INITCAP", "SUBSTR", "LPAD", "RPAD",
+        "REVERSE", "CHR", "ASCII", "LEFT", "RIGHT", "INDEXOF",
+        "TO_CHAR", "TO_DATE", "TO_TIMESTAMP", "TO_INTEGER", "TO_BIGINT",
+        "TO_FLOAT", "TO_DECIMAL", "CAST", "SYSDATE", "SYSTIMESTAMP",
+        "GET_DATE_PART", "SET_DATE_PART", "ADD_TO_DATE", "DATE_DIFF",
+        "DATE_COMPARE", "LAST_DAY", "MAKE_DATE_TIME", "TRUNC", "ROUND",
+        "ABS", "CEIL", "CEILING", "FLOOR", "MOD", "POWER", "SQRT",
+        "LOG", "EXP", "SIGN", "LENGTH", "CONCAT", "INSTR", "REPLACE",
+        "REPLACESTR", "REPLACECHR", "REG_EXTRACT", "REG_REPLACE",
+        "REG_MATCH", "IS_SPACES", "IS_NUMBER", "IS_DATE", "NVL",
+        "NVL2", "ISNULL", "MAX", "MIN", "SUM", "AVG", "COUNT",
+        "FIRST", "LAST", "MEDIAN", "PERCENTILE", "VARIANCE", "STDDEV",
+        "LOOKUP", "ERROR", "ABORT", "SESSSTARTTIME",
+        "METAPHONE", "SOUNDEX", "COMPRESS", "DECOMPRESS",
+        "RANK", "MOVINGAVG", "MOVINGSUM", "CUME",
+    }
+    func_pattern = re.compile(r'\b([A-Z_][A-Z0-9_]*)\s*\(', re.IGNORECASE)
+    unsupported_funcs = {}
+    for mapping in folder.mappings:
+        for tx in mapping.transformations:
+            for fld in tx.fields:
+                if not fld.expression:
+                    continue
+                for m in func_pattern.finditer(fld.expression):
+                    func_name = m.group(1).upper()
+                    if func_name not in known_functions:
+                        key = func_name
+                        if key not in unsupported_funcs:
+                            unsupported_funcs[key] = []
+                        unsupported_funcs[key].append(
+                            f"{mapping.name} > {tx.name} > {fld.name}"
+                        )
+
+    if unsupported_funcs:
+        for func_name in sorted(unsupported_funcs.keys()):
+            locations = unsupported_funcs[func_name]
+            lines.append(f"  {func_name}() — found in {len(locations)} field(s):")
+            for loc in locations[:5]:
+                lines.append(f"    - {loc}")
+            if len(locations) > 5:
+                lines.append(f"    ... and {len(locations) - 5} more")
+            lines.append("")
+    else:
+        lines.append("  None - all expression functions are recognized")
+        lines.append("")
+
     lines.append("")
     lines.append("-" * 70)
     lines.append("PARSED XML TAG COVERAGE")

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python/generators/mapping_gen.py

@@ -16,6 +16,62 @@ from informatica_python.utils.lib_adapters import (
 )
 
 
+def _expand_mapplet_recursive(mapplet, mapplet_map, prefix, depth=0, max_depth=10, visited=None):
+    if visited is None:
+        visited = set()
+    if depth > max_depth:
+        return [], []
+    if mapplet.name in visited:
+        return [], []
+    visited.add(mapplet.name)
+
+    transforms = []
+    connectors = []
+    tx_names = {t.name for t in mapplet.transformations}
+
+    for tx in mapplet.transformations:
+        inlined = TransformationDef(
+            name=f"{prefix}__{tx.name}",
+            type=tx.type,
+            description=tx.description,
+            reusable=tx.reusable,
+            fields=list(tx.fields),
+            attributes=list(tx.attributes),
+            groups=list(tx.groups),
+            metadata_extensions=list(tx.metadata_extensions),
+        )
+        transforms.append(inlined)
+
+    for conn in mapplet.connectors:
+        from informatica_python.models import ConnectorDef
+        new_from = f"{prefix}__{conn.from_instance}" if conn.from_instance in tx_names else conn.from_instance
+        new_to = f"{prefix}__{conn.to_instance}" if conn.to_instance in tx_names else conn.to_instance
+        connectors.append(ConnectorDef(
+            from_instance=new_from,
+            from_field=conn.from_field,
+            from_instance_type=conn.from_instance_type,
+            to_instance=new_to,
+            to_field=conn.to_field,
+            to_instance_type=conn.to_instance_type,
+        ))
+
+    for inst in getattr(mapplet, 'instances', []):
+        if inst.type == "Mapplet" or (inst.transformation_type or "").lower() == "mapplet":
+            nested_name = inst.transformation_name or inst.name
+            nested_mapplet = mapplet_map.get(nested_name)
+            if not nested_mapplet:
+                continue
+            nested_prefix = f"{prefix}__{inst.name}"
+            nested_tx, nested_conn = _expand_mapplet_recursive(
+                nested_mapplet, mapplet_map, nested_prefix,
+                depth + 1, max_depth, visited.copy()
+            )
+            transforms.extend(nested_tx)
+            connectors.extend(nested_conn)
+
+    return transforms, connectors
+
+
 def _inline_mapplets(mapping, folder):
     mapplet_map = {m.name: m for m in folder.mapplets}
     extra_transforms = []
@@ -31,32 +87,11 @@ def _inline_mapplets(mapping, folder):
             mapplet_instances.add(inst.name)
             prefix = inst.name
 
-            for tx in mapplet.transformations:
-                inlined = TransformationDef(
-                    name=f"{prefix}__{tx.name}",
-                    type=tx.type,
-                    description=tx.description,
-                    reusable=tx.reusable,
-                    fields=list(tx.fields),
-                    attributes=list(tx.attributes),
-                    groups=list(tx.groups),
-                    metadata_extensions=list(tx.metadata_extensions),
-                )
-                extra_transforms.append(inlined)
-
-            for conn in mapplet.connectors:
-                from informatica_python.models import ConnectorDef
-                new_from = f"{prefix}__{conn.from_instance}" if conn.from_instance in {t.name for t in mapplet.transformations} else conn.from_instance
-                new_to = f"{prefix}__{conn.to_instance}" if conn.to_instance in {t.name for t in mapplet.transformations} else conn.to_instance
-                inlined_conn = ConnectorDef(
-                    from_instance=new_from,
-                    from_field=conn.from_field,
-                    from_instance_type=conn.from_instance_type,
-                    to_instance=new_to,
-                    to_field=conn.to_field,
-                    to_instance_type=conn.to_instance_type,
-                )
-                extra_connectors.append(inlined_conn)
+            nested_tx, nested_conn = _expand_mapplet_recursive(
+                mapplet, mapplet_map, prefix
+            )
+            extra_transforms.extend(nested_tx)
+            extra_connectors.extend(nested_conn)
 
     rewired_connectors = []
     mapplet_internal_names = set()
@@ -181,7 +216,8 @@ def _build_session_conn_overrides(mapping, folder):
 
 
 def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
-                          data_lib: str = "pandas", mapping_index: int = 1) -> str:
+                          data_lib: str = "pandas", mapping_index: int = 1,
+                          validate_casts: bool = False) -> str:
     lines = []
     lines.append('"""')
     lines.append(f"Mapping: {mapping.name}")
@@ -275,7 +311,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
         _generate_transformation(lines, tx, connector_graph, source_dfs, transform_map, instance_map, data_lib)
 
     for tgt_name, tgt_def in target_map.items():
-        _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides)
+        _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides, validate_casts=validate_casts)
 
     lines.append("")
     lines.append(f"    log_mapping_end('{mapping.name}', start_time)")
@@ -287,6 +323,7 @@ def generate_mapping_code(mapping: MappingDef, folder: FolderDef,
     lines.append("    _parser = _ap.ArgumentParser()")
     lines.append("    _parser.add_argument('--param-file', default=None)")
     lines.append("    _parser.add_argument('--config', default='config.yml')")
+    lines.append("    _parser.add_argument('--validate-casts', action='store_true', help='Log data quality warnings on type coercion')")
     lines.append("    _args = _parser.parse_args()")
     lines.append("    config = load_config(_args.config, param_file=_args.param_file)")
     lines.append(f"    run_{_safe_name(mapping.name)}(config)")
@@ -1181,7 +1218,7 @@ def _gen_sql_transform(lines, tx, tx_safe, input_df, source_dfs):
     source_dfs[tx.name] = f"df_{tx_safe}"
 
 
-def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides=None):
+def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs, transform_map, instance_map, session_overrides=None, validate_casts=False):
     tgt_safe = _safe_name(tgt_name)
 
     to_conns = connector_graph.get("to", {}).get(tgt_name, [])
@@ -1213,7 +1250,7 @@ def _generate_target_write(lines, tgt_name, tgt_def, connector_graph, source_dfs
     else:
         lines.append(f"    df_target_{tgt_safe} = {input_df}")
 
-    _emit_type_casting(lines, tgt_safe, tgt_def)
+    _emit_type_casting(lines, tgt_safe, tgt_def, validate_casts=validate_casts)
 
     tgt_override = (session_overrides or {}).get(tgt_name, {})
     tgt_conn = tgt_override.get("connection_name")
@@ -1269,7 +1306,7 @@ CAST_MAP = {
 }
 
 
-def _emit_type_casting(lines, tgt_safe, tgt_def):
+def _emit_type_casting(lines, tgt_safe, tgt_def, validate_casts=False):
     cast_ops = []
     for fld in tgt_def.fields:
         dt_key = fld.datatype.lower().strip()
@@ -1289,8 +1326,12 @@ def _emit_type_casting(lines, tgt_safe, tgt_def):
         return
 
     lines.append(f"    # Type casting for target fields")
+    if validate_casts:
+        lines.append(f"    _cast_warnings = []")
     for col_name, cast_type, pd_dtype, nullable in cast_ops:
         lines.append(f"    if '{col_name}' in df_target_{tgt_safe}.columns:")
+        if validate_casts:
+            lines.append(f"        _pre_null_{_safe_name(col_name)} = df_target_{tgt_safe}['{col_name}'].isna().sum()")
         if cast_type == "datetime":
             lines.append(f"        df_target_{tgt_safe}['{col_name}'] = pd.to_datetime(df_target_{tgt_safe}['{col_name}'], errors='coerce')")
         elif cast_type == "int":
@@ -1302,3 +1343,12 @@ def _emit_type_casting(lines, tgt_safe, tgt_def):
             lines.append(f"        df_target_{tgt_safe}['{col_name}'] = pd.to_numeric(df_target_{tgt_safe}['{col_name}'], errors='coerce')")
         elif cast_type == "bool":
             lines.append(f"        df_target_{tgt_safe}['{col_name}'] = df_target_{tgt_safe}['{col_name}'].astype('{pd_dtype}')")
+        if validate_casts:
+            lines.append(f"        _post_null_{_safe_name(col_name)} = df_target_{tgt_safe}['{col_name}'].isna().sum()")
+            lines.append(f"        _coerced_{_safe_name(col_name)} = int(_post_null_{_safe_name(col_name)} - _pre_null_{_safe_name(col_name)})")
+            lines.append(f"        if _coerced_{_safe_name(col_name)} > 0:")
+            lines.append(f"            _cast_warnings.append('{col_name}: {{}} values coerced to null during {cast_type} cast'.format(_coerced_{_safe_name(col_name)}))")
+            lines.append(f"            logger.warning('Column {col_name}: %d values coerced to null during {cast_type} cast', _coerced_{_safe_name(col_name)})")
+    if validate_casts:
+        lines.append(f"    if _cast_warnings:")
+        lines.append(f"        logger.warning('Data quality warnings for target {tgt_safe}: %s', '; '.join(_cast_warnings))")

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python/generators/sql_gen.py

@@ -1,5 +1,16 @@
 from informatica_python.models import FolderDef
 from informatica_python.utils.expression_converter import convert_sql_expression, detect_sql_dialect
+from informatica_python.utils.sql_dialect import translate_sql
+
+
+SQL_ATTR_NAMES = {
+    "Sql Query": "Sql Query",
+    "Lookup Sql Override": "Lookup SQL Override",
+    "Pre SQL": "Pre-SQL",
+    "Post SQL": "Post-SQL",
+    "User Defined Join": "User Defined Join",
+    "Source Filter": "Source Filter",
+}
 
 
 def generate_sql_file(folder: FolderDef) -> str:
@@ -7,82 +18,34 @@ def generate_sql_file(folder: FolderDef) -> str:
     lines.append("-- ============================================================")
     lines.append(f"-- All SQL Queries extracted from folder: {folder.name}")
     lines.append("-- Auto-generated by informatica-python")
+    lines.append("-- Includes ANSI SQL translations where dialect was detected")
     lines.append("-- ============================================================")
     lines.append("")
 
     sql_count = 0
+    translated_count = 0
 
     for mapping in folder.mappings:
         mapping_sqls = []
 
         for tx in mapping.transformations:
             for attr in tx.attributes:
-                if attr.name == "Sql Query" and attr.value and attr.value.strip():
-                    sql = convert_sql_expression(attr.value)
-                    dialect = detect_sql_dialect(sql)
-                    mapping_sqls.append({
-                        "transformation": tx.name,
-                        "type": tx.type,
-                        "attribute": "Sql Query",
-                        "sql": sql,
-                        "dialect": dialect,
-                    })
-                    sql_count += 1
-
-                elif attr.name == "Lookup Sql Override" and attr.value and attr.value.strip():
+                display_name = SQL_ATTR_NAMES.get(attr.name)
+                if display_name and attr.value and attr.value.strip():
                     sql = convert_sql_expression(attr.value)
                     dialect = detect_sql_dialect(sql)
-                    mapping_sqls.append({
+                    ansi = translate_sql(sql, source_dialect=dialect.lower())
+                    entry = {
                         "transformation": tx.name,
                         "type": tx.type,
-                        "attribute": "Lookup SQL Override",
+                        "attribute": display_name,
                         "sql": sql,
                         "dialect": dialect,
-                    })
-                    sql_count += 1
-
-                elif attr.name == "Pre SQL" and attr.value and attr.value.strip():
-                    sql = convert_sql_expression(attr.value)
-                    mapping_sqls.append({
-                        "transformation": tx.name,
-                        "type": tx.type,
-                        "attribute": "Pre-SQL",
-                        "sql": sql,
-                        "dialect": detect_sql_dialect(sql),
-                    })
-                    sql_count += 1
-
-                elif attr.name == "Post SQL" and attr.value and attr.value.strip():
-                    sql = convert_sql_expression(attr.value)
-                    mapping_sqls.append({
-                        "transformation": tx.name,
-                        "type": tx.type,
-                        "attribute": "Post-SQL",
-                        "sql": sql,
-                        "dialect": detect_sql_dialect(sql),
-                    })
-                    sql_count += 1
-
-                elif attr.name == "User Defined Join" and attr.value and attr.value.strip():
-                    sql = convert_sql_expression(attr.value)
-                    mapping_sqls.append({
-                        "transformation": tx.name,
-                        "type": tx.type,
-                        "attribute": "User Defined Join",
-                        "sql": sql,
-                        "dialect": detect_sql_dialect(sql),
-                    })
-                    sql_count += 1
-
-                elif attr.name == "Source Filter" and attr.value and attr.value.strip():
-                    sql = convert_sql_expression(attr.value)
-                    mapping_sqls.append({
-                        "transformation": tx.name,
-                        "type": tx.type,
-                        "attribute": "Source Filter",
-                        "sql": sql,
-                        "dialect": detect_sql_dialect(sql),
-                    })
+                    }
+                    if ansi.strip() != sql.strip():
+                        entry["translated"] = ansi
+                        translated_count += 1
+                    mapping_sqls.append(entry)
                     sql_count += 1
 
         if mapping_sqls:
@@ -98,6 +61,10 @@ def generate_sql_file(folder: FolderDef) -> str:
                 lines.append(f"-- ----")
                 lines.append(sq["sql"].rstrip())
                 lines.append("")
+                if "translated" in sq:
+                    lines.append(f"-- >> ANSI SQL Translation:")
+                    lines.append(sq["translated"].rstrip())
+                    lines.append("")
                 lines.append("")
 
     for session in folder.sessions:
@@ -106,11 +73,18 @@ def generate_sql_file(folder: FolderDef) -> str:
             for attr in sti.attributes:
                 if "sql" in attr.name.lower() and attr.value and attr.value.strip():
                     sql = convert_sql_expression(attr.value)
-                    session_sqls.append({
+                    dialect = detect_sql_dialect(sql)
+                    ansi = translate_sql(sql, source_dialect=dialect.lower())
+                    entry = {
                         "instance": sti.instance_name,
                         "attribute": attr.name,
                         "sql": sql,
-                    })
+                        "dialect": dialect,
+                    }
+                    if ansi.strip() != sql.strip():
+                        entry["translated"] = ansi
+                        translated_count += 1
+                    session_sqls.append(entry)
                     sql_count += 1
 
         if session_sqls:
@@ -121,12 +95,19 @@ def generate_sql_file(folder: FolderDef) -> str:
             for sq in session_sqls:
                 lines.append(f"-- Instance: {sq['instance']}")
                 lines.append(f"-- Attribute: {sq['attribute']}")
+                lines.append(f"-- Detected dialect: {sq['dialect']}")
                 lines.append(f"-- ----")
                 lines.append(sq["sql"].rstrip())
                 lines.append("")
+                if "translated" in sq:
+                    lines.append(f"-- >> ANSI SQL Translation:")
+                    lines.append(sq["translated"].rstrip())
+                    lines.append("")
                 lines.append("")
 
     lines.append(f"-- Total SQL queries extracted: {sql_count}")
+    if translated_count:
+        lines.append(f"-- SQL queries with dialect translation: {translated_count}")
     lines.append("")
 
     return "\n".join(lines)

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python/utils/expression_converter.py

@@ -295,8 +295,8 @@ def _vectorize_value(val, df_var="df"):
     return val
 
 
-def _vectorize_condition(cond, df_var="df"):
-    c = cond.strip()
+def _vectorize_simple(part, df_var):
+    c = part.strip()
 
     c = re.sub(r'\bISNULL\s*\(\s*([A-Za-z_]\w*)\s*\)',
                lambda m: f'{df_var}["{m.group(1)}"].isna()', c, flags=re.IGNORECASE)
@@ -305,9 +305,6 @@ def _vectorize_condition(cond, df_var="df"):
     c = re.sub(r'\b([A-Za-z_]\w*)\s*IS\s+NULL\b',
                lambda m: f'{df_var}["{m.group(1)}"].isna()', c, flags=re.IGNORECASE)
 
-    c = re.sub(r'\bAND\b', ' & ', c, flags=re.IGNORECASE)
-    c = re.sub(r'\bOR\b', ' | ', c, flags=re.IGNORECASE)
-    c = re.sub(r'\bNOT\s+', ' ~', c, flags=re.IGNORECASE)
     c = re.sub(r'<>', '!=', c)
     c = re.sub(r'(?<![<>!=])=(?!=)', '==', c)
 
@@ -322,6 +319,45 @@ def _vectorize_condition(cond, df_var="df"):
     return c
 
 
+def _vectorize_condition(cond, df_var="df"):
+    c = cond.strip()
+
+    tokens = re.split(r'\b(AND|OR)\b', c, flags=re.IGNORECASE)
+
+    parts = []
+    ops = []
+    for tok in tokens:
+        stripped = tok.strip()
+        if stripped.upper() in ('AND', 'OR'):
+            ops.append('&' if stripped.upper() == 'AND' else '|')
+        elif stripped:
+            parts.append(stripped)
+
+    if not parts:
+        return "True"
+
+    vectorized = []
+    for part in parts:
+        negate = False
+        inner = part.strip()
+        if re.match(r'^NOT\s+', inner, flags=re.IGNORECASE):
+            negate = True
+            inner = re.sub(r'^NOT\s+', '', inner, flags=re.IGNORECASE).strip()
+        v = _vectorize_simple(inner, df_var)
+        if negate:
+            v = f"~({v})"
+        vectorized.append(v)
+
+    if len(vectorized) == 1:
+        return vectorized[0]
+
+    result_parts = [f"({vectorized[0]})"]
+    for i, op in enumerate(ops):
+        result_parts.append(f" {op} ")
+        result_parts.append(f"({vectorized[i + 1]})")
+    return "".join(result_parts)
+
+
 def convert_filter_expression(expr):
     if not expr or not expr.strip():
         return "True"

informatica_python-1.6.0/informatica_python/utils/sql_dialect.py

@@ -0,0 +1,174 @@
+import re
+
+
+ORACLE_TO_ANSI = [
+    (re.compile(r'\bNVL2\s*\(\s*([^,]+?)\s*,\s*([^,]+?)\s*,\s*([^)]+?)\s*\)', re.IGNORECASE),
+     r'CASE WHEN \1 IS NOT NULL THEN \2 ELSE \3 END'),
+    (re.compile(r'\bNVL\s*\(\s*([^,]+?)\s*,\s*([^)]+?)\s*\)', re.IGNORECASE),
+     r'COALESCE(\1, \2)'),
+    (re.compile(r'\bSYSDATE\b', re.IGNORECASE), 'CURRENT_TIMESTAMP'),
+    (re.compile(r'\bSYSTIMESTAMP\b', re.IGNORECASE), 'CURRENT_TIMESTAMP'),
+    (re.compile(r'\|\|', re.IGNORECASE), ' || '),
+]
+
+MSSQL_TO_ANSI = [
+    (re.compile(r'\bISNULL\s*\(\s*([^,]+?)\s*,\s*([^)]+?)\s*\)', re.IGNORECASE),
+     r'COALESCE(\1, \2)'),
+    (re.compile(r'\bGETDATE\s*\(\s*\)', re.IGNORECASE), 'CURRENT_TIMESTAMP'),
+    (re.compile(r'\bCONVERT\s*\(\s*VARCHAR\s*,\s*([^,)]+?)\s*,\s*\d+\s*\)', re.IGNORECASE),
+     r'CAST(\1 AS VARCHAR)'),
+    (re.compile(r'\bLEN\s*\(', re.IGNORECASE), 'LENGTH('),
+    (re.compile(r'\bCHARINDEX\s*\(\s*([^,]+?)\s*,\s*([^)]+?)\s*\)', re.IGNORECASE),
+     r'POSITION(\1 IN \2)'),
+]
+
+_DECODE_RE = re.compile(
+    r'\bDECODE\s*\(', re.IGNORECASE
+)
+
+_ORACLE_JOIN_RE = re.compile(
+    r'(\w+\.\w+)\s*=\s*(\w+\.\w+)\s*\(\+\)'
+)
+
+_ORACLE_JOIN_RE2 = re.compile(
+    r'(\w+\.\w+)\s*\(\+\)\s*=\s*(\w+\.\w+)'
+)
+
+_ROWNUM_RE = re.compile(
+    r'\bAND\s+ROWNUM\s*<=?\s*(\d+)\b|\bWHERE\s+ROWNUM\s*<=?\s*(\d+)\b',
+    re.IGNORECASE
+)
+
+_TOP_RE = re.compile(
+    r'\bSELECT\s+TOP\s+(\d+)\b', re.IGNORECASE
+)
+
+
+def _convert_decode(sql):
+    result = sql
+    idx = 0
+    while True:
+        m = _DECODE_RE.search(result, idx)
+        if not m:
+            break
+        start = m.start()
+        paren_start = m.end() - 1
+        depth = 1
+        pos = paren_start + 1
+        while pos < len(result) and depth > 0:
+            if result[pos] == '(':
+                depth += 1
+            elif result[pos] == ')':
+                depth -= 1
+            pos += 1
+        if depth != 0:
+            idx = pos
+            continue
+        inner = result[paren_start + 1:pos - 1]
+        args = _split_args(inner)
+        if len(args) < 3:
+            idx = pos
+            continue
+        expr = args[0].strip()
+        pairs = args[1:]
+        case_parts = [f"CASE {expr}"]
+        i = 0
+        while i < len(pairs) - 1:
+            case_parts.append(f" WHEN {pairs[i].strip()} THEN {pairs[i+1].strip()}")
+            i += 2
+        if i < len(pairs):
+            case_parts.append(f" ELSE {pairs[i].strip()}")
+        case_parts.append(" END")
+        replacement = "".join(case_parts)
+        result = result[:start] + replacement + result[pos:]
+        idx = start + len(replacement)
+    return result
+
+
+def _split_args(s):
+    args = []
+    depth = 0
+    current = []
+    for ch in s:
+        if ch == '(':
+            depth += 1
+            current.append(ch)
+        elif ch == ')':
+            depth -= 1
+            current.append(ch)
+        elif ch == ',' and depth == 0:
+            args.append(''.join(current))
+            current = []
+        else:
+            current.append(ch)
+    if current:
+        args.append(''.join(current))
+    return args
+
+
+def _convert_oracle_outer_join(sql):
+    result = _ORACLE_JOIN_RE.sub(
+        lambda m: f'{m.group(1)} = {m.group(2)} -- (+) converted: use LEFT JOIN',
+        sql
+    )
+    result = _ORACLE_JOIN_RE2.sub(
+        lambda m: f'{m.group(1)} = {m.group(2)} -- (+) converted: use RIGHT JOIN',
+        result
+    )
+    return result
+
+
+def _convert_rownum(sql):
+    m = _ROWNUM_RE.search(sql)
+    if m:
+        limit_val = m.group(1) or m.group(2)
+        cleaned = _ROWNUM_RE.sub('', sql).strip()
+        if cleaned.endswith('AND'):
+            cleaned = cleaned[:-3].strip()
+        if cleaned.endswith('WHERE'):
+            cleaned = cleaned[:-5].strip()
+        cleaned = cleaned.rstrip(';')
+        return f"{cleaned}\nLIMIT {limit_val}"
+    return sql
+
+
+def _convert_top(sql):
+    m = _TOP_RE.search(sql)
+    if m:
+        limit_val = m.group(1)
+        cleaned = _TOP_RE.sub('SELECT', sql)
+        cleaned = cleaned.rstrip(';')
+        return f"{cleaned}\nLIMIT {limit_val}"
+    return sql
+
+
+def translate_sql(sql, source_dialect="auto", target_dialect="ansi"):
+    if not sql or not sql.strip():
+        return sql
+
+    from informatica_python.utils.expression_converter import detect_sql_dialect
+
+    if source_dialect == "auto":
+        source_dialect = detect_sql_dialect(sql).lower()
+
+    translated = sql
+
+    if source_dialect == "oracle":
+        translated = _convert_oracle_outer_join(translated)
+        translated = _convert_decode(translated)
+        translated = _convert_rownum(translated)
+        for pattern, replacement in ORACLE_TO_ANSI:
+            translated = pattern.sub(replacement, translated)
+
+    elif source_dialect in ("mssql", "sql server"):
+        translated = _convert_top(translated)
+        for pattern, replacement in MSSQL_TO_ANSI:
+            translated = pattern.sub(replacement, translated)
+
+    elif source_dialect in ("generic", "postgresql"):
+        for pattern, replacement in ORACLE_TO_ANSI:
+            translated = pattern.sub(replacement, translated)
+        for pattern, replacement in MSSQL_TO_ANSI:
+            translated = pattern.sub(replacement, translated)
+
+    return translated

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.5.1
+Version: 1.6.0
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
 Author: Nick
 License: MIT

{informatica_python-1.5.1 → informatica_python-1.6.0}/informatica_python.egg-info/SOURCES.txt

@@ -23,5 +23,6 @@ informatica_python/utils/__init__.py
 informatica_python/utils/datatype_map.py
 informatica_python/utils/expression_converter.py
 informatica_python/utils/lib_adapters.py
+informatica_python/utils/sql_dialect.py
 tests/test_converter.py
 tests/test_integration.py

{informatica_python-1.5.1 → informatica_python-1.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "informatica-python"
-version = "1.5.1"
+version = "1.6.0"
 description = "Convert Informatica PowerCenter workflow XML to Python/PySpark code"
 readme = "README.md"
 license = {text = "MIT"}

{informatica_python-1.5.1 → informatica_python-1.6.0}/tests/test_integration.py

@@ -210,10 +210,18 @@ class TestFilterVectorized:
         assert 'df["A"]' in result
         assert 'df["B"]' in result
         assert "AND" not in result
+        assert "(df[" in result
 
     def test_or_condition(self):
         result = convert_filter_vectorized("STATUS = 'A' OR STATUS = 'B'", "df")
         assert "|" in result
+        assert "(df[" in result
+
+    def test_not_condition(self):
+        result = convert_filter_vectorized("NOT A = 1", "df")
+        assert "~(" in result
+        assert 'df["A"]' in result
+        assert "==" in result
 
     def test_is_null_filter(self):
         result = convert_filter_vectorized("NAME IS NULL", "df_src")
@@ -232,6 +240,7 @@ class TestFilterVectorized:
         result = convert_expression_vectorized("IIF(A > 1 AND B < 2, 1, 0)", "df")
         assert "np.where" in result
         assert "&" in result
+        assert "(" in result
 
 
 class TestLibAdapters:
@@ -538,3 +547,269 @@ class TestCLIParamFile:
         help_text = f.getvalue()
         from informatica_python.cli import main as cli_main
         assert callable(cli_main)
+
+
+class TestSQLDialectTranslation:
+
+    def test_nvl_to_coalesce(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT NVL(COL1, 0) FROM T", source_dialect="oracle")
+        assert "COALESCE" in result
+        assert "NVL" not in result
+
+    def test_sysdate_to_current_timestamp(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT SYSDATE FROM DUAL", source_dialect="oracle")
+        assert "CURRENT_TIMESTAMP" in result
+        assert "SYSDATE" not in result
+
+    def test_decode_to_case(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT DECODE(STATUS, 'A', 'Active', 'I', 'Inactive', 'Unknown') FROM T", source_dialect="oracle")
+        assert "CASE" in result
+        assert "WHEN" in result
+        assert "ELSE" in result
+
+    def test_nvl2_to_case(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT NVL2(COL1, 'has value', 'null') FROM T", source_dialect="oracle")
+        assert "CASE WHEN" in result
+        assert "IS NOT NULL" in result
+
+    def test_getdate_to_current_timestamp(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT GETDATE() FROM T", source_dialect="mssql")
+        assert "CURRENT_TIMESTAMP" in result
+
+    def test_isnull_mssql_to_coalesce(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT ISNULL(COL1, 0) FROM T", source_dialect="mssql")
+        assert "COALESCE" in result
+
+    def test_top_to_limit(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT TOP 10 * FROM T", source_dialect="mssql")
+        assert "LIMIT 10" in result
+        assert "TOP" not in result
+
+    def test_rownum_to_limit(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT * FROM T WHERE ROWNUM <= 5", source_dialect="oracle")
+        assert "LIMIT 5" in result
+
+    def test_auto_dialect_detection(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        result = translate_sql("SELECT NVL(A, 0), SYSDATE FROM T")
+        assert "COALESCE" in result
+        assert "CURRENT_TIMESTAMP" in result
+
+    def test_no_change_for_clean_sql(self):
+        from informatica_python.utils.sql_dialect import translate_sql
+        sql = "SELECT * FROM employees WHERE id = 1"
+        result = translate_sql(sql, source_dialect="generic")
+        assert result.strip() == sql.strip()
+
+    def test_sql_gen_includes_translation(self):
+        from informatica_python.generators.sql_gen import generate_sql_file
+        from informatica_python.models import (
+            FolderDef, MappingDef, TransformationDef, TableAttribute, FieldDef
+        )
+        tx = TransformationDef(
+            name="SQ_TEST", type="Source Qualifier",
+            attributes=[TableAttribute(name="Sql Query", value="SELECT NVL(A, 0), SYSDATE FROM T")],
+        )
+        mapping = MappingDef(name="m_test", transformations=[tx])
+        folder = FolderDef(name="F", mappings=[mapping])
+        result = generate_sql_file(folder)
+        assert "ANSI SQL Translation" in result
+        assert "COALESCE" in result
+
+
+class TestEnhancedErrorReporting:
+
+    def test_unsupported_transforms_section(self):
+        from informatica_python.generators.error_log_gen import generate_error_log
+        from informatica_python.models import (
+            FolderDef, MappingDef, TransformationDef, FieldDef, TableAttribute
+        )
+        tx = TransformationDef(
+            name="JAVA_TX", type="Java",
+            attributes=[TableAttribute(name="Class Name", value="com.example.Transform")],
+            fields=[FieldDef(name="OUT1", datatype="string", porttype="OUTPUT")],
+        )
+        mapping = MappingDef(name="m_test", transformations=[tx])
+        folder = FolderDef(name="F", mappings=[mapping])
+        result = generate_error_log(folder)
+        assert "UNSUPPORTED TRANSFORMS" in result
+        assert "JAVA_TX" in result
+        assert "Java" in result
+        assert "Class Name" in result
+
+    def test_unmapped_ports_section(self):
+        from informatica_python.generators.error_log_gen import generate_error_log
+        from informatica_python.models import (
+            FolderDef, MappingDef, TransformationDef, FieldDef, ConnectorDef
+        )
+        tx = TransformationDef(
+            name="EXP1", type="Expression",
+            fields=[
+                FieldDef(name="IN1", datatype="string", porttype="INPUT"),
+                FieldDef(name="OUT1", datatype="string", porttype="OUTPUT"),
+                FieldDef(name="OUT2", datatype="string", porttype="OUTPUT"),
+            ],
+        )
+        conn = ConnectorDef(
+            from_instance="EXP1", from_field="OUT1",
+            from_instance_type="Expression",
+            to_instance="TGT", to_field="COL1",
+            to_instance_type="Target Definition",
+        )
+        mapping = MappingDef(name="m_test", transformations=[tx], connectors=[conn])
+        folder = FolderDef(name="F", mappings=[mapping])
+        result = generate_error_log(folder)
+        assert "UNMAPPED PORTS" in result
+        assert "OUT2" in result
+
+    def test_unsupported_functions_section(self):
+        from informatica_python.generators.error_log_gen import generate_error_log
+        from informatica_python.models import (
+            FolderDef, MappingDef, TransformationDef, FieldDef
+        )
+        tx = TransformationDef(
+            name="EXP1", type="Expression",
+            fields=[
+                FieldDef(name="OUT1", datatype="string", porttype="OUTPUT",
+                         expression="CUSTOM_FUNC(IN1, 'abc')"),
+            ],
+        )
+        mapping = MappingDef(name="m_test", transformations=[tx])
+        folder = FolderDef(name="F", mappings=[mapping])
+        result = generate_error_log(folder)
+        assert "UNSUPPORTED EXPRESSION FUNCTIONS" in result
+        assert "CUSTOM_FUNC" in result
+
+
+class TestNestedMapplets:
+
+    def test_recursive_expansion(self):
+        from informatica_python.generators.mapping_gen import _expand_mapplet_recursive
+        from informatica_python.models import (
+            MappletDef, TransformationDef, FieldDef, ConnectorDef, InstanceDef
+        )
+        inner_mapplet = MappletDef(
+            name="INNER_MPL",
+            transformations=[
+                TransformationDef(name="INNER_EXP", type="Expression",
+                                  fields=[FieldDef(name="F1", datatype="string", porttype="INPUT/OUTPUT")]),
+            ],
+            connectors=[],
+        )
+        outer_mapplet = MappletDef(
+            name="OUTER_MPL",
+            transformations=[
+                TransformationDef(name="OUTER_EXP", type="Expression",
+                                  fields=[FieldDef(name="F1", datatype="string", porttype="INPUT/OUTPUT")]),
+            ],
+            connectors=[],
+            instances=[
+                InstanceDef(name="INNER_INST", type="Mapplet",
+                            transformation_name="INNER_MPL", transformation_type="Mapplet"),
+            ],
+        )
+        mapplet_map = {"INNER_MPL": inner_mapplet, "OUTER_MPL": outer_mapplet}
+        transforms, connectors = _expand_mapplet_recursive(outer_mapplet, mapplet_map, "MPL1")
+        names = [t.name for t in transforms]
+        assert "MPL1__OUTER_EXP" in names
+        assert "MPL1__INNER_INST__INNER_EXP" in names
+
+    def test_circular_reference_protection(self):
+        from informatica_python.generators.mapping_gen import _expand_mapplet_recursive
+        from informatica_python.models import (
+            MappletDef, TransformationDef, FieldDef, InstanceDef
+        )
+        circular = MappletDef(
+            name="SELF_REF",
+            transformations=[
+                TransformationDef(name="EXP1", type="Expression",
+                                  fields=[FieldDef(name="F1", datatype="string")]),
+            ],
+            connectors=[],
+            instances=[
+                InstanceDef(name="SELF", type="Mapplet",
+                            transformation_name="SELF_REF", transformation_type="Mapplet"),
+            ],
+        )
+        mapplet_map = {"SELF_REF": circular}
+        transforms, _ = _expand_mapplet_recursive(circular, mapplet_map, "M")
+        assert len(transforms) == 1
+
+    def test_depth_limit(self):
+        from informatica_python.generators.mapping_gen import _expand_mapplet_recursive
+        from informatica_python.models import (
+            MappletDef, TransformationDef, FieldDef, InstanceDef
+        )
+        mapplets = {}
+        for i in range(15):
+            name = f"MPL_{i}"
+            instances = []
+            if i < 14:
+                instances = [InstanceDef(name=f"NEST_{i+1}", type="Mapplet",
+                                         transformation_name=f"MPL_{i+1}",
+                                         transformation_type="Mapplet")]
+            mapplets[name] = MappletDef(
+                name=name,
+                transformations=[
+                    TransformationDef(name=f"TX_{i}", type="Expression",
+                                      fields=[FieldDef(name="F", datatype="string")]),
+                ],
+                connectors=[],
+                instances=instances,
+            )
+        transforms, _ = _expand_mapplet_recursive(mapplets["MPL_0"], mapplets, "ROOT")
+        assert len(transforms) <= 11
+
+
+class TestDataQualityValidation:
+
+    def test_validate_casts_generates_warnings(self):
+        from informatica_python.generators.mapping_gen import _emit_type_casting, _safe_name
+        from informatica_python.models import FieldDef
+        class FakeTgt:
+            fields = [
+                FieldDef(name="AGE", datatype="integer", nullable="NULL"),
+                FieldDef(name="CREATED", datatype="date/time", nullable="NULL"),
+            ]
+        lines = []
+        _emit_type_casting(lines, "TGT1", FakeTgt(), validate_casts=True)
+        code = "\n".join(lines)
+        assert "_cast_warnings" in code
+        assert "_pre_null_" in code
+        assert "_post_null_" in code
+        assert "coerced to null" in code
+        assert "logger.warning" in code
+
+    def test_no_validation_by_default(self):
+        from informatica_python.generators.mapping_gen import _emit_type_casting
+        from informatica_python.models import FieldDef
+        class FakeTgt:
+            fields = [
+                FieldDef(name="AGE", datatype="integer", nullable="NULL"),
+            ]
+        lines = []
+        _emit_type_casting(lines, "TGT1", FakeTgt())
+        code = "\n".join(lines)
+        assert "_cast_warnings" not in code
+        assert "_pre_null_" not in code
+
+    def test_validate_casts_cli_flag(self):
+        import io, contextlib
+        from informatica_python.cli import main
+        f = io.StringIO()
+        with contextlib.redirect_stdout(f):
+            try:
+                sys.argv = ["informatica-python", "--help"]
+                main()
+            except SystemExit:
+                pass
+        help_text = f.getvalue()
+        assert "--validate-casts" in help_text or "validate_casts" in help_text