informatica-python 1.0.0__py3-none-any.whl

informatica_python/__init__.py

@@ -0,0 +1,4 @@
+from informatica_python.converter import InformaticaConverter
+
+__version__ = "1.0.0"
+__all__ = ["InformaticaConverter"]

informatica_python/cli.py

@@ -0,0 +1,83 @@
+import argparse
+import sys
+import json
+from informatica_python.converter import InformaticaConverter
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="informatica-python",
+        description="Convert Informatica PowerCenter workflow XML to Python/PySpark code",
+    )
+
+    parser.add_argument(
+        "input_file",
+        help="Path to Informatica workflow XML file",
+    )
+    parser.add_argument(
+        "-o", "--output",
+        default="output",
+        help="Output directory for generated files (default: output)",
+    )
+    parser.add_argument(
+        "-z", "--zip",
+        default=None,
+        help="Output as zip file (provide zip file path)",
+    )
+    parser.add_argument(
+        "--data-lib",
+        choices=["pandas", "dask", "polars", "vaex", "modin"],
+        default="pandas",
+        help="Data manipulation library to use (default: pandas)",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        dest="output_json",
+        help="Output parsed XML as JSON (no code generation)",
+    )
+    parser.add_argument(
+        "--json-file",
+        default=None,
+        help="Save parsed JSON to a file",
+    )
+
+    args = parser.parse_args()
+
+    converter = InformaticaConverter(data_lib=args.data_lib)
+
+    try:
+        if args.output_json or args.json_file:
+            result = converter.parse_file(args.input_file)
+            json_str = json.dumps(result, indent=2, ensure_ascii=False)
+            if args.json_file:
+                with open(args.json_file, "w", encoding="utf-8") as f:
+                    f.write(json_str)
+                print(f"JSON saved to: {args.json_file}")
+            else:
+                print(json_str)
+        else:
+            output_path = converter.convert(
+                args.input_file,
+                output_dir=args.output,
+                output_zip=args.zip,
+            )
+            print(f"Conversion complete! Output: {output_path}")
+            print(f"Files generated:")
+            if args.zip:
+                import zipfile
+                with zipfile.ZipFile(output_path, "r") as zf:
+                    for name in zf.namelist():
+                        print(f"  - {name}")
+            else:
+                import os
+                for f in sorted(os.listdir(output_path)):
+                    print(f"  - {f}")
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

informatica_python/converter.py

@@ -0,0 +1,285 @@
+import os
+import json
+import zipfile
+import tempfile
+from typing import Optional
+from informatica_python.parser import InformaticaParser
+from informatica_python.models import PowermartDef, FolderDef
+from informatica_python.generators.helper_gen import generate_helper_functions
+from informatica_python.generators.mapping_gen import generate_mapping_code
+from informatica_python.generators.workflow_gen import generate_workflow_code
+from informatica_python.generators.config_gen import generate_config
+from informatica_python.generators.sql_gen import generate_sql_file
+from informatica_python.generators.error_log_gen import generate_error_log
+
+
+class InformaticaConverter:
+    def __init__(self, data_lib: str = "pandas"):
+        self.data_lib = data_lib
+        self.parser = InformaticaParser()
+        self.powermart = None
+
+    def parse_file(self, file_path: str) -> dict:
+        self.powermart = self.parser.parse_file(file_path)
+        return self.to_json()
+
+    def parse_string(self, xml_string: str) -> dict:
+        self.powermart = self.parser.parse_string(xml_string)
+        return self.to_json()
+
+    def to_json(self) -> dict:
+        if not self.powermart:
+            return {}
+        return self._powermart_to_dict(self.powermart)
+
+    def convert(self, file_path: str, output_dir: str = "output",
+                output_zip: Optional[str] = None) -> str:
+        self.powermart = self.parser.parse_file(file_path)
+
+        if not self.powermart.repositories:
+            raise ValueError("No repository found in XML file")
+
+        all_folders = []
+        for repo in self.powermart.repositories:
+            all_folders.extend(repo.folders)
+
+        if not all_folders:
+            raise ValueError("No folder found in XML file")
+
+        if len(all_folders) == 1:
+            return self._convert_folder(all_folders[0], output_dir, output_zip)
+
+        result_path = output_dir if not output_zip else os.path.dirname(output_zip) or "."
+        for folder in all_folders:
+            folder_dir = os.path.join(output_dir, folder.name)
+            folder_zip = None
+            if output_zip:
+                base, ext = os.path.splitext(output_zip)
+                folder_zip = f"{base}_{folder.name}{ext}"
+            self._convert_folder(folder, folder_dir, folder_zip)
+        return result_path
+
+    def convert_string(self, xml_string: str, output_dir: str = "output",
+                       output_zip: Optional[str] = None) -> str:
+        self.powermart = self.parser.parse_string(xml_string)
+
+        if not self.powermart.repositories:
+            raise ValueError("No repository found in XML")
+
+        all_folders = []
+        for repo in self.powermart.repositories:
+            all_folders.extend(repo.folders)
+
+        if not all_folders:
+            raise ValueError("No folder found in XML")
+
+        if len(all_folders) == 1:
+            return self._convert_folder(all_folders[0], output_dir, output_zip)
+
+        result_path = output_dir if not output_zip else os.path.dirname(output_zip) or "."
+        for folder in all_folders:
+            folder_dir = os.path.join(output_dir, folder.name)
+            folder_zip = None
+            if output_zip:
+                base, ext = os.path.splitext(output_zip)
+                folder_zip = f"{base}_{folder.name}{ext}"
+            self._convert_folder(folder, folder_dir, folder_zip)
+        return result_path
+
+    def _convert_folder(self, folder: FolderDef, output_dir: str,
+                        output_zip: Optional[str] = None) -> str:
+        files = {}
+
+        files["helper_functions.py"] = generate_helper_functions(folder, self.data_lib)
+
+        for i, mapping in enumerate(folder.mappings, 1):
+            code = generate_mapping_code(mapping, folder, self.data_lib, i)
+            files[f"mapping_{i}.py"] = code
+
+        files["workflow.py"] = generate_workflow_code(folder)
+
+        files["config.yml"] = generate_config(folder, self.data_lib)
+
+        files["all_sql_queries.sql"] = generate_sql_file(folder)
+
+        files["error_log.txt"] = generate_error_log(
+            folder,
+            parser_errors=self.parser.errors,
+            parser_warnings=self.parser.warnings,
+        )
+
+        if output_zip:
+            return self._write_zip(files, output_zip)
+        else:
+            return self._write_files(files, output_dir)
+
+    def _write_files(self, files: dict, output_dir: str) -> str:
+        os.makedirs(output_dir, exist_ok=True)
+        for filename, content in files.items():
+            filepath = os.path.join(output_dir, filename)
+            with open(filepath, "w", encoding="utf-8") as f:
+                f.write(content)
+        return output_dir
+
+    def _write_zip(self, files: dict, zip_path: str) -> str:
+        os.makedirs(os.path.dirname(zip_path) or ".", exist_ok=True)
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for filename, content in files.items():
+                zf.writestr(filename, content)
+        return zip_path
+
+    def _powermart_to_dict(self, pm: PowermartDef) -> dict:
+        result = {
+            "creation_date": pm.creation_date,
+            "repository_version": pm.repository_version,
+            "repositories": [],
+        }
+        for repo in pm.repositories:
+            repo_dict = {
+                "name": repo.name,
+                "version": repo.version,
+                "codepage": repo.codepage,
+                "database_type": repo.database_type,
+                "folders": [],
+            }
+            for folder in repo.folders:
+                folder_dict = self._folder_to_dict(folder)
+                repo_dict["folders"].append(folder_dict)
+            result["repositories"].append(repo_dict)
+        return result
+
+    def _folder_to_dict(self, folder: FolderDef) -> dict:
+        return {
+            "name": folder.name,
+            "owner": folder.owner,
+            "description": folder.description,
+            "sources": [self._source_to_dict(s) for s in folder.sources],
+            "targets": [self._target_to_dict(t) for t in folder.targets],
+            "mappings": [self._mapping_to_dict(m) for m in folder.mappings],
+            "sessions": [{"name": s.name, "mapping_name": s.mapping_name} for s in folder.sessions],
+            "workflows": [self._workflow_to_dict(w) for w in folder.workflows],
+            "tasks": [{"name": t.name, "type": t.type} for t in folder.tasks],
+            "configs": [{"name": c.name} for c in folder.configs],
+            "schedulers": [{"name": s.name} for s in folder.schedulers],
+            "shortcuts": [{"name": s.name, "reference": s.reference_name} for s in folder.shortcuts],
+            "mapplets": [{"name": m.name} for m in folder.mapplets],
+        }
+
+    def _source_to_dict(self, src):
+        return {
+            "name": src.name,
+            "database_type": src.database_type,
+            "db_name": src.db_name,
+            "owner_name": src.owner_name,
+            "fields": [
+                {
+                    "name": f.name,
+                    "datatype": f.datatype,
+                    "precision": f.precision,
+                    "scale": f.scale,
+                    "nullable": f.nullable,
+                    "keytype": f.keytype,
+                }
+                for f in src.fields
+            ],
+        }
+
+    def _target_to_dict(self, tgt):
+        return {
+            "name": tgt.name,
+            "database_type": tgt.database_type,
+            "fields": [
+                {
+                    "name": f.name,
+                    "datatype": f.datatype,
+                    "precision": f.precision,
+                    "scale": f.scale,
+                    "nullable": f.nullable,
+                    "keytype": f.keytype,
+                }
+                for f in tgt.fields
+            ],
+        }
+
+    def _mapping_to_dict(self, mapping):
+        return {
+            "name": mapping.name,
+            "description": mapping.description,
+            "is_valid": mapping.is_valid,
+            "transformations": [
+                {
+                    "name": tx.name,
+                    "type": tx.type,
+                    "fields": [
+                        {
+                            "name": f.name,
+                            "datatype": f.datatype,
+                            "expression": f.expression,
+                            "porttype": f.porttype,
+                        }
+                        for f in tx.fields
+                    ],
+                    "attributes": [
+                        {"name": a.name, "value": a.value}
+                        for a in tx.attributes
+                    ],
+                }
+                for tx in mapping.transformations
+            ],
+            "connectors": [
+                {
+                    "from_field": c.from_field,
+                    "from_instance": c.from_instance,
+                    "to_field": c.to_field,
+                    "to_instance": c.to_instance,
+                }
+                for c in mapping.connectors
+            ],
+            "instances": [
+                {
+                    "name": i.name,
+                    "type": i.type,
+                    "transformation_name": i.transformation_name,
+                }
+                for i in mapping.instances
+            ],
+            "variables": [
+                {
+                    "name": v.name,
+                    "datatype": v.datatype,
+                    "default_value": v.default_value,
+                }
+                for v in mapping.variables
+            ],
+        }
+
+    def _workflow_to_dict(self, wf):
+        return {
+            "name": wf.name,
+            "description": wf.description,
+            "is_valid": wf.is_valid,
+            "task_instances": [
+                {
+                    "name": t.name,
+                    "task_name": t.task_name,
+                    "task_type": t.task_type,
+                }
+                for t in wf.task_instances
+            ],
+            "links": [
+                {
+                    "from": l.from_instance,
+                    "to": l.to_instance,
+                    "condition": l.condition,
+                }
+                for l in wf.links
+            ],
+            "variables": [
+                {
+                    "name": v.name,
+                    "datatype": v.datatype,
+                    "default_value": v.default_value,
+                }
+                for v in wf.variables
+            ],
+        }

informatica_python/generators/config_gen.py

@@ -0,0 +1,159 @@
+import yaml
+from typing import List, Dict
+from informatica_python.models import FolderDef, SessionDef
+from informatica_python.utils.datatype_map import get_db_type
+
+
+def generate_config(folder: FolderDef, data_lib: str = "pandas") -> str:
+    config = {
+        "workflow": {
+            "name": "",
+            "folder": folder.name,
+        },
+        "data_library": data_lib,
+        "connections": {},
+        "sources": {},
+        "targets": {},
+        "variables": {},
+    }
+
+    if folder.workflows:
+        config["workflow"]["name"] = folder.workflows[0].name
+
+    connection_names = set()
+
+    for src in folder.sources:
+        src_config = {
+            "database_type": src.database_type or "unknown",
+            "db_name": src.db_name or "",
+            "owner": src.owner_name or "dbo",
+            "fields": [],
+        }
+
+        for fld in src.fields:
+            src_config["fields"].append({
+                "name": fld.name,
+                "datatype": fld.datatype,
+                "precision": fld.precision,
+                "scale": fld.scale,
+                "nullable": fld.nullable,
+            })
+
+        config["sources"][src.name] = src_config
+
+        if src.database_type and src.database_type != "Flat File":
+            conn_key = src.db_name or "default"
+            if conn_key not in connection_names:
+                connection_names.add(conn_key)
+                config["connections"][conn_key] = {
+                    "type": get_db_type(src.database_type),
+                    "host": "${DB_HOST}",
+                    "port": _get_default_port(src.database_type),
+                    "database": src.db_name or "${DB_NAME}",
+                    "username": "${DB_USER}",
+                    "password": "${DB_PASSWORD}",
+                    "schema": src.owner_name or "dbo",
+                }
+        else:
+            config["sources"][src.name]["file_path"] = f"${{INPUT_DIR}}/{src.name}"
+            config["sources"][src.name]["delimiter"] = ","
+            config["sources"][src.name]["header"] = True
+            config["sources"][src.name]["encoding"] = "utf-8"
+
+    for tgt in folder.targets:
+        tgt_config = {
+            "database_type": tgt.database_type or "unknown",
+            "fields": [],
+        }
+
+        for fld in tgt.fields:
+            tgt_config["fields"].append({
+                "name": fld.name,
+                "datatype": fld.datatype,
+                "precision": fld.precision,
+                "scale": fld.scale,
+                "nullable": fld.nullable,
+                "keytype": fld.keytype,
+            })
+
+        config["targets"][tgt.name] = tgt_config
+
+        if tgt.database_type and tgt.database_type != "Flat File":
+            conn_key = "target"
+            if conn_key not in connection_names:
+                connection_names.add(conn_key)
+                config["connections"][conn_key] = {
+                    "type": get_db_type(tgt.database_type),
+                    "host": "${DB_HOST}",
+                    "port": _get_default_port(tgt.database_type),
+                    "database": "${TARGET_DB_NAME}",
+                    "username": "${DB_USER}",
+                    "password": "${DB_PASSWORD}",
+                    "schema": "dbo",
+                }
+        else:
+            config["targets"][tgt.name]["file_path"] = f"${{OUTPUT_DIR}}/{tgt.name}"
+
+    _extract_session_connections(folder, config, connection_names)
+
+    for mapping in folder.mappings:
+        for var in mapping.variables:
+            var_name = var.name.replace("$$", "")
+            config["variables"][var_name] = {
+                "datatype": var.datatype,
+                "default_value": var.default_value or "",
+                "is_persistent": var.is_persistent,
+            }
+
+    for wf in folder.workflows:
+        for var in wf.variables:
+            var_name = var.name.replace("$$", "")
+            config["variables"][var_name] = {
+                "datatype": var.datatype,
+                "default_value": var.default_value or "",
+                "is_persistent": var.is_persistent,
+            }
+
+    if not config["connections"]:
+        config["connections"]["default"] = {
+            "type": "mssql",
+            "host": "${DB_HOST}",
+            "port": 1433,
+            "database": "${DB_NAME}",
+            "username": "${DB_USER}",
+            "password": "${DB_PASSWORD}",
+            "schema": "dbo",
+        }
+
+    return yaml.dump(config, default_flow_style=False, sort_keys=False, allow_unicode=True)
+
+
+def _get_default_port(database_type):
+    port_map = {
+        "Microsoft SQL Server": 1433,
+        "Oracle": 1521,
+        "Sybase": 5000,
+        "DB2": 50000,
+        "Teradata": 1025,
+        "Informix": 9088,
+    }
+    return port_map.get(database_type, 1433)
+
+
+def _extract_session_connections(folder, config, connection_names):
+    for session in folder.sessions:
+        for sti in session.transform_instances:
+            for conn_ref in sti.connections:
+                conn_name = conn_ref.connection_name or conn_ref.variable or "default"
+                if conn_name and conn_name not in connection_names:
+                    connection_names.add(conn_name)
+                    config["connections"][conn_name] = {
+                        "type": conn_ref.connection_type or "relational",
+                        "connection_name": conn_ref.connection_name,
+                        "connection_subtype": conn_ref.connection_subtype,
+                        "host": "${DB_HOST}",
+                        "port": 1433,
+                        "database": "${DB_NAME}",
+                        "username": "${DB_USER}",
+                        "password": "${DB_PASSWORD}",
+                    }

informatica_python/generators/error_log_gen.py

@@ -0,0 +1,140 @@
+from datetime import datetime
+from informatica_python.models import FolderDef
+
+
+def generate_error_log(folder: FolderDef, parser_errors=None, parser_warnings=None) -> str:
+    lines = []
+    lines.append("=" * 70)
+    lines.append(f"Informatica-Python Conversion Log")
+    lines.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    lines.append(f"Folder: {folder.name}")
+    lines.append("=" * 70)
+    lines.append("")
+
+    lines.append(f"Sources found: {len(folder.sources)}")
+    for src in folder.sources:
+        lines.append(f"  - {src.name} ({src.database_type}, {len(src.fields)} fields)")
+    lines.append("")
+
+    lines.append(f"Targets found: {len(folder.targets)}")
+    for tgt in folder.targets:
+        lines.append(f"  - {tgt.name} ({tgt.database_type}, {len(tgt.fields)} fields)")
+    lines.append("")
+
+    lines.append(f"Mappings found: {len(folder.mappings)}")
+    for mapping in folder.mappings:
+        lines.append(f"  - {mapping.name} (valid={mapping.is_valid})")
+        lines.append(f"    Transformations: {len(mapping.transformations)}")
+        tx_types = {}
+        for tx in mapping.transformations:
+            tx_types[tx.type] = tx_types.get(tx.type, 0) + 1
+        for tx_type, count in sorted(tx_types.items()):
+            lines.append(f"      {tx_type}: {count}")
+        lines.append(f"    Connectors: {len(mapping.connectors)}")
+        lines.append(f"    Instances: {len(mapping.instances)}")
+        lines.append(f"    Variables: {len(mapping.variables)}")
+    lines.append("")
+
+    lines.append(f"Sessions found: {len(folder.sessions)}")
+    for session in folder.sessions:
+        lines.append(f"  - {session.name} (mapping: {session.mapping_name})")
+    lines.append("")
+
+    lines.append(f"Workflows found: {len(folder.workflows)}")
+    for wf in folder.workflows:
+        lines.append(f"  - {wf.name} (valid={wf.is_valid})")
+        lines.append(f"    Task Instances: {len(wf.task_instances)}")
+        lines.append(f"    Links: {len(wf.links)}")
+        lines.append(f"    Variables: {len(wf.variables)}")
+    lines.append("")
+
+    if folder.mapplets:
+        lines.append(f"Mapplets found: {len(folder.mapplets)}")
+        for m in folder.mapplets:
+            lines.append(f"  - {m.name}")
+        lines.append("")
+
+    if folder.shortcuts:
+        lines.append(f"Shortcuts found: {len(folder.shortcuts)}")
+        for s in folder.shortcuts:
+            lines.append(f"  - {s.name} -> {s.reference_name}")
+        lines.append("")
+
+    if folder.configs:
+        lines.append(f"Configs found: {len(folder.configs)}")
+        for c in folder.configs:
+            lines.append(f"  - {c.name}")
+        lines.append("")
+
+    if folder.schedulers:
+        lines.append(f"Schedulers found: {len(folder.schedulers)}")
+        for s in folder.schedulers:
+            lines.append(f"  - {s.name}")
+        lines.append("")
+
+    lines.append("-" * 70)
+    lines.append("WARNINGS AND DETECTIONS")
+    lines.append("-" * 70)
+    lines.append("")
+
+    if parser_errors:
+        for err in parser_errors:
+            lines.append(f"[ERROR] {err}")
+        lines.append("")
+
+    if parser_warnings:
+        for warn in parser_warnings:
+            lines.append(f"[WARNING] {warn}")
+        lines.append("")
+
+    for mapping in folder.mappings:
+        for tx in mapping.transformations:
+            if tx.type in ("Custom Transformation", "Java", "Stored Procedure"):
+                lines.append(f"[WARNING] Mapping '{mapping.name}': Transformation '{tx.name}' is type '{tx.type}' - manual review needed")
+
+            if tx.type == "Lookup Procedure":
+                has_sql_override = False
+                for attr in tx.attributes:
+                    if attr.name == "Lookup Sql Override" and attr.value:
+                        has_sql_override = True
+                if has_sql_override:
+                    lines.append(f"[INFO] Mapping '{mapping.name}': Lookup '{tx.name}' has SQL override")
+
+            if tx.type == "Source Qualifier":
+                for attr in tx.attributes:
+                    if attr.name == "Sql Query" and attr.value:
+                        lines.append(f"[INFO] Mapping '{mapping.name}': Source Qualifier '{tx.name}' has SQL override")
+
+            for fld in tx.fields:
+                if fld.expression:
+                    expr_upper = fld.expression.upper()
+                    if "ERROR(" in expr_upper or "ABORT(" in expr_upper:
+                        lines.append(f"[WARNING] Mapping '{mapping.name}': Field '{fld.name}' in '{tx.name}' contains ERROR/ABORT function")
+                    if ":LKP." in fld.expression:
+                        lines.append(f"[INFO] Mapping '{mapping.name}': Field '{fld.name}' uses inline lookup")
+
+    lines.append("")
+    lines.append("-" * 70)
+    lines.append("CONVERSION SUMMARY")
+    lines.append("-" * 70)
+    lines.append("")
+
+    total_transforms = sum(len(m.transformations) for m in folder.mappings)
+    unsupported = 0
+    for mapping in folder.mappings:
+        for tx in mapping.transformations:
+            if tx.type in ("Custom Transformation", "Java", "Stored Procedure",
+                           "External Procedure", "HTTP Transformation",
+                           "Web Service Consumer"):
+                unsupported += 1
+
+    supported = total_transforms - unsupported
+    pct = (supported / total_transforms * 100) if total_transforms > 0 else 100
+
+    lines.append(f"Total transformations: {total_transforms}")
+    lines.append(f"Supported/converted: {supported}")
+    lines.append(f"Needs manual review: {unsupported}")
+    lines.append(f"Conversion coverage: {pct:.1f}%")
+    lines.append("")
+
+    return "\n".join(lines)