informatica-python 1.0.0__tar.gz → 1.1.1__tar.gz

{informatica_python-1.0.0 → informatica_python-1.1.1}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: informatica-python
-Version: 1.0.0
+Version: 1.1.1
 Summary: Convert Informatica PowerCenter workflow XML to Python/PySpark code
-License: MIT
+License-Expression: MIT
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: lxml>=4.9.0

informatica_python-1.1.1/informatica_python/converter.py

@@ -0,0 +1,450 @@
+import os
+import json
+import zipfile
+import tempfile
+from typing import Optional
+from informatica_python.parser import InformaticaParser
+from informatica_python.models import PowermartDef, FolderDef
+from informatica_python.generators.helper_gen import generate_helper_functions
+from informatica_python.generators.mapping_gen import generate_mapping_code
+from informatica_python.generators.workflow_gen import generate_workflow_code
+from informatica_python.generators.config_gen import generate_config
+from informatica_python.generators.sql_gen import generate_sql_file
+from informatica_python.generators.error_log_gen import generate_error_log
+
+
+class InformaticaConverter:
+    def __init__(self, data_lib: str = "pandas"):
+        self.data_lib = data_lib
+        self.parser = InformaticaParser()
+        self.powermart = None
+
+    def parse_file(self, file_path: str) -> dict:
+        self.powermart = self.parser.parse_file(file_path)
+        return self.to_json()
+
+    def parse_string(self, xml_string: str) -> dict:
+        self.powermart = self.parser.parse_string(xml_string)
+        return self.to_json()
+
+    def to_json(self) -> dict:
+        if not self.powermart:
+            return {}
+        return self._powermart_to_dict(self.powermart)
+
+    def convert(self, file_path: str, output_dir: str = "output",
+                output_zip: Optional[str] = None) -> str:
+        self.powermart = self.parser.parse_file(file_path)
+
+        if not self.powermart.repositories:
+            raise ValueError("No repository found in XML file")
+
+        all_folders = []
+        for repo in self.powermart.repositories:
+            all_folders.extend(repo.folders)
+
+        if not all_folders:
+            raise ValueError("No folder found in XML file")
+
+        if len(all_folders) == 1:
+            return self._convert_folder(all_folders[0], output_dir, output_zip)
+
+        result_path = output_dir if not output_zip else os.path.dirname(output_zip) or "."
+        for folder in all_folders:
+            folder_dir = os.path.join(output_dir, folder.name)
+            folder_zip = None
+            if output_zip:
+                base, ext = os.path.splitext(output_zip)
+                folder_zip = f"{base}_{folder.name}{ext}"
+            self._convert_folder(folder, folder_dir, folder_zip)
+        return result_path
+
+    def convert_string(self, xml_string: str, output_dir: str = "output",
+                       output_zip: Optional[str] = None) -> str:
+        self.powermart = self.parser.parse_string(xml_string)
+
+        if not self.powermart.repositories:
+            raise ValueError("No repository found in XML")
+
+        all_folders = []
+        for repo in self.powermart.repositories:
+            all_folders.extend(repo.folders)
+
+        if not all_folders:
+            raise ValueError("No folder found in XML")
+
+        if len(all_folders) == 1:
+            return self._convert_folder(all_folders[0], output_dir, output_zip)
+
+        result_path = output_dir if not output_zip else os.path.dirname(output_zip) or "."
+        for folder in all_folders:
+            folder_dir = os.path.join(output_dir, folder.name)
+            folder_zip = None
+            if output_zip:
+                base, ext = os.path.splitext(output_zip)
+                folder_zip = f"{base}_{folder.name}{ext}"
+            self._convert_folder(folder, folder_dir, folder_zip)
+        return result_path
+
+    def _convert_folder(self, folder: FolderDef, output_dir: str,
+                        output_zip: Optional[str] = None) -> str:
+        files = {}
+
+        files["helper_functions.py"] = generate_helper_functions(folder, self.data_lib)
+
+        for i, mapping in enumerate(folder.mappings, 1):
+            code = generate_mapping_code(mapping, folder, self.data_lib, i)
+            files[f"mapping_{i}.py"] = code
+
+        files["workflow.py"] = generate_workflow_code(folder)
+
+        files["config.yml"] = generate_config(folder, self.data_lib)
+
+        files["all_sql_queries.sql"] = generate_sql_file(folder)
+
+        files["error_log.txt"] = generate_error_log(
+            folder,
+            parser_errors=self.parser.errors,
+            parser_warnings=self.parser.warnings,
+        )
+
+        if output_zip:
+            return self._write_zip(files, output_zip)
+        else:
+            return self._write_files(files, output_dir)
+
+    def _write_files(self, files: dict, output_dir: str) -> str:
+        os.makedirs(output_dir, exist_ok=True)
+        for filename, content in files.items():
+            filepath = os.path.join(output_dir, filename)
+            with open(filepath, "w", encoding="utf-8") as f:
+                f.write(content)
+        return output_dir
+
+    def _write_zip(self, files: dict, zip_path: str) -> str:
+        os.makedirs(os.path.dirname(zip_path) or ".", exist_ok=True)
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for filename, content in files.items():
+                zf.writestr(filename, content)
+        return zip_path
+
+    def _powermart_to_dict(self, pm: PowermartDef) -> dict:
+        result = {
+            "creation_date": pm.creation_date,
+            "repository_version": pm.repository_version,
+            "repositories": [],
+        }
+        for repo in pm.repositories:
+            repo_dict = {
+                "name": repo.name,
+                "version": repo.version,
+                "codepage": repo.codepage,
+                "database_type": repo.database_type,
+                "folders": [],
+            }
+            for folder in repo.folders:
+                folder_dict = self._folder_to_dict(folder)
+                repo_dict["folders"].append(folder_dict)
+            result["repositories"].append(repo_dict)
+        return result
+
+    def _folder_to_dict(self, folder) -> dict:
+        return {
+            "name": folder.name,
+            "owner": folder.owner,
+            "description": folder.description,
+            "folder_versions": [
+                {"folder_name": fv.folder_name, "version_number": fv.version_number, **fv.attributes}
+                for fv in folder.folder_versions
+            ],
+            "sources": [self._source_to_dict(s) for s in folder.sources],
+            "targets": [self._target_to_dict(t) for t in folder.targets],
+            "mappings": [self._mapping_to_dict(m) for m in folder.mappings],
+            "mapplets": [self._mapplet_to_dict(m) for m in folder.mapplets],
+            "sessions": [self._session_to_dict(s) for s in folder.sessions],
+            "workflows": [self._workflow_to_dict(w) for w in folder.workflows],
+            "tasks": [self._task_to_dict(t) for t in folder.tasks],
+            "configs": [self._config_to_dict(c) for c in folder.configs],
+            "schedulers": [self._scheduler_to_dict(s) for s in folder.schedulers],
+            "shortcuts": [self._shortcut_to_dict(s) for s in folder.shortcuts],
+            "transformations": [self._transformation_to_dict(tx) for tx in folder.transformations],
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in folder.metadata_extensions],
+        }
+
+    def _field_to_dict(self, f):
+        d = {
+            "name": f.name,
+            "datatype": f.datatype,
+            "precision": f.precision,
+            "scale": f.scale,
+            "nullable": f.nullable,
+            "keytype": f.keytype,
+        }
+        if f.expression:
+            d["expression"] = f.expression
+        if f.porttype:
+            d["porttype"] = f.porttype
+        if f.default_value:
+            d["default_value"] = f.default_value
+        if f.field_attributes:
+            d["field_attributes"] = f.field_attributes
+        return d
+
+    def _meta_ext_to_dict(self, me):
+        return {"name": me.name, "value": me.value, "datatype": me.datatype}
+
+    def _source_to_dict(self, src):
+        d = {
+            "name": src.name,
+            "database_type": src.database_type,
+            "db_name": src.db_name,
+            "owner_name": src.owner_name,
+            "fields": [self._field_to_dict(f) for f in src.fields],
+            "attributes": [{"name": a.name, "value": a.value} for a in src.attributes],
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in src.metadata_extensions],
+        }
+        if src.flatfile:
+            d["flatfile"] = {"delimiter": src.flatfile.delimiter, "header_lines": src.flatfile.header_lines,
+                             "is_fixed_width": src.flatfile.is_fixed_width, "code_page": src.flatfile.code_page}
+        if src.xmlinfo:
+            d["xmlinfo"] = {"xml_type": src.xmlinfo.xml_type, "root_element": src.xmlinfo.root_element,
+                            "xml_texts": src.xmlinfo.xml_texts}
+        if src.groups:
+            d["groups"] = [{"name": g.name, "type": g.type, "fields": [self._field_to_dict(f) for f in g.fields]} for g in src.groups]
+        if src.keywords:
+            d["keywords"] = [{"name": k.name, "value": k.value} for k in src.keywords]
+        if src.erp_src_info:
+            d["erp_src_info"] = {"name": src.erp_src_info.name, "source_type": src.erp_src_info.source_type}
+        return d
+
+    def _target_to_dict(self, tgt):
+        d = {
+            "name": tgt.name,
+            "database_type": tgt.database_type,
+            "fields": [self._field_to_dict(f) for f in tgt.fields],
+            "attributes": [{"name": a.name, "value": a.value} for a in tgt.attributes],
+            "indexes": [
+                {"name": idx.name, "index_type": idx.index_type, "unique": idx.unique,
+                 "fields": [{"name": idf.name, "expression": idf.expression, "sort_direction": idf.sort_direction} for idf in idx.fields]}
+                for idx in tgt.indexes
+            ],
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in tgt.metadata_extensions],
+        }
+        if tgt.flatfile:
+            d["flatfile"] = {"delimiter": tgt.flatfile.delimiter, "header_lines": tgt.flatfile.header_lines}
+        if tgt.xmlinfo:
+            d["xmlinfo"] = {"xml_type": tgt.xmlinfo.xml_type, "root_element": tgt.xmlinfo.root_element}
+        if tgt.groups:
+            d["groups"] = [{"name": g.name, "type": g.type, "fields": [self._field_to_dict(f) for f in g.fields]} for g in tgt.groups]
+        if tgt.keywords:
+            d["keywords"] = [{"name": k.name, "value": k.value} for k in tgt.keywords]
+        return d
+
+    def _transformation_to_dict(self, tx):
+        d = {
+            "name": tx.name,
+            "type": tx.type,
+            "description": tx.description,
+            "reusable": tx.reusable,
+            "fields": [self._field_to_dict(f) for f in tx.fields],
+            "attributes": [{"name": a.name, "value": a.value} for a in tx.attributes],
+            "metadata": tx.metadata,
+        }
+        if tx.field_attrs:
+            d["field_attrs"] = [{"name": fa.name, "value": fa.value, "field_name": fa.field_name} for fa in tx.field_attrs]
+        if tx.field_attr_defs:
+            d["field_attr_defs"] = [{"name": fad.name, "datatype": fad.datatype, "default_value": fad.default_value} for fad in tx.field_attr_defs]
+        if tx.init_props:
+            d["init_props"] = [{"name": ip.name, "value": ip.value} for ip in tx.init_props]
+        if tx.erp_info:
+            d["erp_info"] = {"name": tx.erp_info.name, "erp_type": tx.erp_info.erp_type}
+        if tx.groups:
+            d["groups"] = [{"name": g.name, "type": g.type, "fields": [self._field_to_dict(f) for f in g.fields]} for g in tx.groups]
+        if tx.metadata_extensions:
+            d["metadata_extensions"] = [self._meta_ext_to_dict(me) for me in tx.metadata_extensions]
+        if tx.sap_functions:
+            d["sap_functions"] = [self._sap_function_to_dict(sf) for sf in tx.sap_functions]
+        return d
+
+    def _sap_function_to_dict(self, sf):
+        return {
+            "name": sf.name, "function_type": sf.function_type,
+            "structures": [{"name": s.name, "type": s.structure_type} for s in sf.structures],
+            "output_ports": [{"name": p.name, "datatype": p.datatype} for p in sf.output_ports],
+            "variables": [{"name": v.name, "datatype": v.datatype, "default_value": v.default_value} for v in sf.variables],
+            "table_params": [{"name": t.name, "table_name": t.table_name, "direction": t.direction} for t in sf.table_params],
+            "programs": [{"name": p.name, "program_type": p.program_type,
+                          "flow_objects": [{"name": fo.name, "object_type": fo.object_type} for fo in p.flow_objects]} for p in sf.programs],
+        }
+
+    def _mapping_to_dict(self, mapping):
+        return {
+            "name": mapping.name,
+            "description": mapping.description,
+            "is_valid": mapping.is_valid,
+            "transformations": [self._transformation_to_dict(tx) for tx in mapping.transformations],
+            "connectors": [
+                {"from_field": c.from_field, "from_instance": c.from_instance,
+                 "from_instance_type": c.from_instance_type,
+                 "to_field": c.to_field, "to_instance": c.to_instance,
+                 "to_instance_type": c.to_instance_type}
+                for c in mapping.connectors
+            ],
+            "instances": [
+                {"name": i.name, "type": i.type, "transformation_name": i.transformation_name,
+                 "transformation_type": i.transformation_type,
+                 "associated_source_instances": [
+                     {"name": a.name, "source_instance": a.source_instance}
+                     for a in i.associated_source_instances
+                 ]}
+                for i in mapping.instances
+            ],
+            "target_load_orders": [
+                {"order": tlo.order, "target_instance": tlo.target_instance}
+                for tlo in mapping.target_load_orders
+            ],
+            "variables": [
+                {"name": v.name, "datatype": v.datatype, "default_value": v.default_value,
+                 "is_persistent": v.is_persistent, "usage_type": v.usage_type}
+                for v in mapping.variables
+            ],
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in mapping.metadata_extensions],
+            "map_dependencies": [
+                {"name": md.name, "from_mapping": md.from_mapping, "to_mapping": md.to_mapping}
+                for md in mapping.map_dependencies
+            ],
+            "field_dependencies": [
+                {"name": fd.name, "from_field": fd.from_field, "from_instance": fd.from_instance,
+                 "to_field": fd.to_field, "to_instance": fd.to_instance, "expression": fd.expression}
+                for fd in mapping.field_dependencies
+            ],
+        }
+
+    def _mapplet_to_dict(self, mapplet):
+        return {
+            "name": mapplet.name, "description": mapplet.description, "is_valid": mapplet.is_valid,
+            "transformations": [self._transformation_to_dict(tx) for tx in mapplet.transformations],
+            "connectors": [{"from_field": c.from_field, "from_instance": c.from_instance,
+                            "to_field": c.to_field, "to_instance": c.to_instance} for c in mapplet.connectors],
+            "instances": [{"name": i.name, "type": i.type, "transformation_name": i.transformation_name,
+                           "associated_source_instances": [{"name": a.name, "source_instance": a.source_instance} for a in i.associated_source_instances]}
+                          for i in mapplet.instances],
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in mapplet.metadata_extensions],
+            "map_dependencies": [{"name": md.name, "from_mapping": md.from_mapping, "to_mapping": md.to_mapping} for md in mapplet.map_dependencies],
+            "field_dependencies": [{"name": fd.name, "from_field": fd.from_field, "from_instance": fd.from_instance,
+                                    "to_field": fd.to_field, "to_instance": fd.to_instance, "expression": fd.expression} for fd in mapplet.field_dependencies],
+        }
+
+    def _session_to_dict(self, session):
+        return {
+            "name": session.name, "mapping_name": session.mapping_name,
+            "description": session.description, "is_valid": session.is_valid, "reusable": session.reusable,
+            "transform_instances": [
+                {"instance_name": sti.instance_name, "pipeline": sti.pipeline, "stage": sti.stage,
+                 "transformation_name": sti.transformation_name, "transformation_type": sti.transformation_type,
+                 "is_partitionable": sti.is_partitionable,
+                 "attributes": [{"name": a.name, "value": a.value} for a in sti.attributes],
+                 "connections": [{"connection_name": c.connection_name, "connection_type": c.connection_type} for c in sti.connections],
+                 "partitions": [{"name": p.name, "partition_type": p.partition_type,
+                                 "hash_keys": [{"name": hk.name, "expression": hk.expression} for hk in p.hash_keys],
+                                 "key_ranges": [{"name": kr.name, "low": kr.low_value, "high": kr.high_value} for kr in p.key_ranges]}
+                                for p in sti.partitions]}
+                for sti in session.transform_instances
+            ],
+            "transform_groups": [
+                {"name": stg.name, "transform_instances": [
+                    {"instance_name": sti.instance_name, "transformation_name": sti.transformation_name}
+                    for sti in stg.transform_instances
+                ]}
+                for stg in session.transform_groups
+            ],
+            "config_references": session.config_references,
+            "components": session.components,
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in session.metadata_extensions],
+        }
+
+    def _task_to_dict(self, task):
+        d = {
+            "name": task.name, "type": task.type, "description": task.description, "reusable": task.reusable,
+            "attributes": [{"name": a.name, "value": a.value} for a in task.attributes],
+            "value_pairs": [{"name": vp.name, "value": vp.value, "type": vp.type} for vp in task.value_pairs],
+        }
+        if task.timer:
+            d["timer"] = {"name": task.timer.name, "start_type": task.timer.start_type,
+                          "start_date": task.timer.start_date, "start_time": task.timer.start_time,
+                          "end_type": task.timer.end_type, "end_date": task.timer.end_date, "end_time": task.timer.end_time}
+        if task.metadata_extensions:
+            d["metadata_extensions"] = [self._meta_ext_to_dict(me) for me in task.metadata_extensions]
+        return d
+
+    def _config_to_dict(self, cfg):
+        return {
+            "name": cfg.name, "description": cfg.description, "is_valid": cfg.is_valid,
+            "attributes": [{"name": a.name, "value": a.value} for a in cfg.attributes],
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in cfg.metadata_extensions],
+        }
+
+    def _scheduler_to_dict(self, sched):
+        d = {
+            "name": sched.name, "description": sched.description, "reusable": sched.reusable,
+            "attributes": [{"name": a.name, "value": a.value} for a in sched.attributes],
+        }
+        if sched.schedule_info:
+            d["schedule_info"] = {"schedule_type": sched.schedule_info.schedule_type, **sched.schedule_info.attributes}
+        if sched.start_options:
+            d["start_options"] = sched.start_options.attributes
+        if sched.end_options:
+            d["end_options"] = sched.end_options.attributes
+        if sched.schedule_options:
+            d["schedule_options"] = sched.schedule_options.attributes
+        if sched.recurring:
+            d["recurring"] = sched.recurring.attributes
+        if sched.custom:
+            d["custom"] = sched.custom.attributes
+        if sched.daily_frequency:
+            d["daily_frequency"] = sched.daily_frequency.attributes
+        if sched.repeat:
+            d["repeat"] = sched.repeat.attributes
+        if sched.scheduler_filter:
+            d["filter"] = sched.scheduler_filter.attributes
+        return d
+
+    def _shortcut_to_dict(self, sc):
+        return {
+            "name": sc.name, "shortcut_type": sc.shortcut_type, "reference_name": sc.reference_name,
+            "folder_name": sc.folder_name, "repository_name": sc.repository_name,
+            "object_type": sc.object_type, "object_subtype": sc.object_subtype, "dbdname": sc.dbdname,
+        }
+
+    def _workflow_to_dict(self, wf):
+        return {
+            "name": wf.name,
+            "description": wf.description,
+            "is_valid": wf.is_valid,
+            "scheduler_name": wf.scheduler_name,
+            "is_worklet": wf.metadata.get("is_worklet", "NO"),
+            "task_instances": [
+                {"name": t.name, "task_name": t.task_name, "task_type": t.task_type,
+                 "fail_parent_if_instance_fails": t.fail_parent_if_instance_fails,
+                 "treat_input_link_as_and": t.treat_input_link_as_and}
+                for t in wf.task_instances
+            ],
+            "links": [
+                {"from": l.from_instance, "to": l.to_instance,
+                 "condition": l.condition, "link_type": l.link_type}
+                for l in wf.links
+            ],
+            "variables": [
+                {"name": v.name, "datatype": v.datatype, "default_value": v.default_value,
+                 "is_persistent": v.is_persistent, "usage_type": v.usage_type}
+                for v in wf.variables
+            ],
+            "events": [
+                {"name": e.name, "event_type": e.event_type, "description": e.description,
+                 "attributes": [{"name": a.name, "value": a.value} for a in e.attributes]}
+                for e in wf.events
+            ],
+            "attributes": [{"name": a.name, "value": a.value} for a in wf.attributes],
+            "metadata": wf.metadata,
+            "metadata_extensions": [self._meta_ext_to_dict(me) for me in wf.metadata_extensions],
+        }