pyegp-parser 0.1.0__py3-none-any.whl

pyegp_parser/__init__.py

@@ -0,0 +1,356 @@
+"""EGP Parser — Parse SAS Enterprise Guide .egp project files into structured JSON."""
+
+import datetime
+import logging
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Any
+
+from .models.bulk import BulkResult
+from .models.log_code import LogElement
+from .models.project import ParsedProject
+
+__version__ = "0.1.0"
+
+logger = logging.getLogger(__name__)
+
+
+def parse_file(
+    egp_path: str | Path,
+    output_dir: str | Path | None = None,
+) -> ParsedProject:
+    """Parse a single .egp file into a structured project object.
+
+    Opens the archive, parses project.xml, extracts all elements and artifacts,
+    validates completeness, and optionally writes JSON output.
+
+    Args:
+        egp_path: Path to the .egp file to parse.
+        output_dir: Optional directory for JSON output. If None, no file is written.
+
+    Returns:
+        A ParsedProject dataclass instance containing all extracted data.
+
+    Raises:
+        FileNotFoundError: If egp_path does not exist.
+        ValueError: If file is not a valid EGP archive.
+    """
+    from .archive import open_archive
+    from .models.elements import ElementCategory
+    from .models.process_flow import ProcessFlowContainer
+    from .models.project import ParsedProject, SourceInfo
+    from .parsers.data_parser import parse_data_list, parse_external_file_list
+    from .parsers.element_parser import parse_elements
+    from .parsers.external_objects_parser import parse_external_objects
+    from .parsers.layout_parser import parse_open_project_view, parse_visual_layout
+    from .parsers.log_code_parser import (
+        extract_execution_logs,
+        extract_project_log,
+        parse_code_element,
+        parse_log_element,
+    )
+    from .parsers.ods_parser import (
+        extract_ods_results,
+        match_ods_results_to_tasks,
+    )
+    from .parsers.pfd_parser import parse_process_flow
+    from .parsers.project_parser import parse_project_xml
+    from .parsers.query_parser import parse_query
+    from .parsers.shortcut_parser import parse_shortcut
+    from .parsers.task_parser import (
+        parse_append_task,
+        parse_code_task,
+        parse_eg_task,
+        parse_export_task,
+        parse_import_task,
+    )
+    from .serializer import serialize_project
+    from .validator import validate_completeness
+
+    path = Path(egp_path).resolve()
+
+    # Step 1: Open and validate the archive
+    inventory = open_archive(path)
+    try:
+        # Track which paths we process for completeness validation
+        processed_paths: set[str] = set()
+
+        # Step 2: Read project.xml (handling UTF-8, UTF-16, BOM encodings)
+        xml_content = _read_project_xml(inventory)
+        processed_paths.add("project.xml")
+
+        # Parse the XML root for sub-parsers that need it
+        root = ET.fromstring(xml_content)
+
+        # Step 3: Parse project metadata via project_parser
+        partial_project = parse_project_xml(xml_content)
+
+        # Step 4: Parse data list
+        data_list = parse_data_list(root)
+
+        # Step 5: Parse external files (catch ValueError for malformed DNA)
+        external_files: list = []
+        try:
+            external_files = parse_external_file_list(root)
+        except ValueError as e:
+            logger.warning("Malformed external file DNA: %s", e)
+
+        # Step 6: Parse elements
+        parsed_elements = parse_elements(root)
+
+        # Step 6b: Invoke typed parsers for each element category.
+        # ValueError from typed parsers propagates to the caller — no silent
+        # error swallowing. This ensures missing required sections are surfaced
+        # rather than silently producing incomplete output.
+        queries: list = []
+        tasks: list = []
+        shortcuts: list = []
+        log_elements_list: list = []
+        code_elements_list: list = []
+
+        for elem in parsed_elements:
+            if elem.category == ElementCategory.QUERY:
+                submitable, query_model = parse_query(elem.xml_node)
+                queries.append(
+                    {
+                        "metadata": elem.metadata,
+                        "submitable": submitable,
+                        "query_model": query_model,
+                    }
+                )
+            elif elem.category == ElementCategory.IMPORT_TASK:
+                # `result` is rebound to several element types across branches.
+                result: Any = parse_import_task(
+                    elem.xml_node, elem.metadata, archive=inventory
+                )
+                tasks.append(result)
+            elif elem.category == ElementCategory.CODE_TASK:
+                result = parse_code_task(
+                    elem.xml_node, elem.metadata, archive=inventory
+                )
+                tasks.append(result)
+            elif elem.category == ElementCategory.EG_TASK:
+                result = parse_eg_task(elem.xml_node, elem.metadata, archive=inventory)
+                tasks.append(result)
+            elif elem.category == ElementCategory.EXPORT_TASK:
+                result = parse_export_task(elem.xml_node, elem.metadata)
+                tasks.append(result)
+            elif elem.category == ElementCategory.APPEND_TASK:
+                result = parse_append_task(elem.xml_node, elem.metadata)
+                tasks.append(result)
+            elif elem.category == ElementCategory.SHORTCUT_TO_DATA:
+                result = parse_shortcut(elem.xml_node, elem.metadata, is_data=True)
+                shortcuts.append(result)
+            elif elem.category == ElementCategory.SHORTCUT_TO_FILE:
+                result = parse_shortcut(elem.xml_node, elem.metadata, is_data=False)
+                shortcuts.append(result)
+            elif elem.category == ElementCategory.LOG:
+                try:
+                    result = parse_log_element(elem.xml_node, elem.metadata)
+                except ValueError as e:
+                    logger.warning(
+                        "Failed to parse log element '%s': %s", elem.metadata.id, e
+                    )
+                    # Still record the element, without its display settings.
+                    result = LogElement(metadata=elem.metadata)
+                log_elements_list.append(result)
+            elif elem.category == ElementCategory.CODE:
+                result = parse_code_element(elem.xml_node, elem.metadata)
+                code_elements_list.append(result)
+
+        # Step 6c: Parse External_Objects section (ValueError propagates)
+        external_objects = parse_external_objects(root)
+
+        # Step 7: Parse process flow containers and build DAGs
+        containers: list[ProcessFlowContainer] = []
+        for elem in parsed_elements:
+            if elem.category == ElementCategory.PROCESS_FLOW_CONTAINER:
+                try:
+                    dag_model = parse_process_flow(elem.xml_node)
+                    containers.append(
+                        ProcessFlowContainer(
+                            metadata=elem.metadata,
+                            dag=dag_model,
+                        )
+                    )
+                except ValueError as e:
+                    logger.warning(
+                        "Failed to parse process flow for element '%s': %s",
+                        elem.metadata.id,
+                        e,
+                    )
+                    # Still record the container without a DAG
+                    containers.append(
+                        ProcessFlowContainer(metadata=elem.metadata, dag=None)
+                    )
+
+        # Step 8: Extract execution logs. NOTE: extraction is invoked to validate
+        # the log entries (and mark them processed below), but the returned logs
+        # are not currently attached to the ParsedProject.
+        try:
+            extract_execution_logs(inventory)
+            # Mark log paths as processed
+            from .archive import EntryCategory
+
+            for entry in inventory.entries:
+                if entry.category == EntryCategory.EXECUTION_LOG:
+                    processed_paths.add(entry.path)
+        except ValueError as e:
+            logger.warning("Failed to extract execution logs: %s", e)
+
+        # Step 9: Extract project log (catch ValueError for missing ProjectLog)
+        project_log = None
+        try:
+            project_log = extract_project_log(inventory, root)
+            # Mark project log paths as processed
+            from .archive import EntryCategory
+
+            for entry in inventory.entries:
+                if entry.category == EntryCategory.PROJECT_LOG:
+                    processed_paths.add(entry.path)
+        except ValueError as e:
+            logger.warning("Missing or invalid project log: %s", e)
+
+        # Step 10: Extract ODS results
+        ods_entries = extract_ods_results(inventory)
+        ods_entries = match_ods_results_to_tasks(ods_entries, parsed_elements)
+        # Mark ODS paths as processed
+        from .archive import EntryCategory
+
+        for entry in inventory.entries:
+            if entry.category == EntryCategory.ODS_RESULT:
+                processed_paths.add(entry.path)
+
+        # Also mark task configs, code files as processed
+        for entry in inventory.entries:
+            if entry.category in (
+                EntryCategory.TASK_CONFIG,
+                EntryCategory.CODE_FILE,
+            ):
+                processed_paths.add(entry.path)
+
+        # Step 11: Parse visual layout
+        element_ids = {
+            elem.metadata.id for elem in parsed_elements if elem.metadata.id is not None
+        }
+        visual_layout = parse_visual_layout(root, element_ids=element_ids)
+        open_project_view = parse_open_project_view(root)
+
+        # Step 12: Validate completeness
+        completeness_summary, unprocessed_entries, completeness_warning = (
+            validate_completeness(inventory, processed_paths)
+        )
+
+        # Step 13: Build source info (Req 15.16, 15.17)
+        source = SourceInfo(
+            file_path=str(path),
+            file_name=path.name,
+            file_size_bytes=path.stat().st_size,
+            parsed_at=datetime.datetime.now().isoformat(),
+            total_zip_entries=len(inventory.entries),
+        )
+
+        # Assemble the full ParsedProject with all sections populated
+        # Convert ParsedElement objects to their metadata for serialization
+        # (ParsedElement contains xml_node which is not JSON-serializable)
+        serializable_elements = [elem.metadata for elem in parsed_elements]
+
+        project = ParsedProject(
+            source=source,
+            metadata=partial_project.metadata,
+            settings=partial_project.settings,
+            data_list=data_list,
+            external_files=external_files,
+            elements=serializable_elements,
+            containers=containers,
+            parameters=partial_project.parameters,
+            project_log=project_log,
+            visual_layout=visual_layout,
+            completeness_summary=completeness_summary,
+            unprocessed_entries=unprocessed_entries,
+            completeness_warning=completeness_warning,
+            binary_entries=ods_entries,
+            application_overrides=partial_project.application_overrides,
+            metadata_info=partial_project.metadata_info,
+            open_project_view=open_project_view,
+            queries=queries,
+            tasks=tasks,
+            shortcuts=shortcuts,
+            log_elements=log_elements_list,
+            code_elements=code_elements_list,
+            external_objects=external_objects,
+        )
+
+        # Step 14: Write output if output_dir is specified (Req 15.6)
+        if output_dir is not None:
+            out_path = Path(output_dir)
+            serialize_project(project, out_path)
+
+        return project
+    finally:
+        inventory.close()
+
+
+def _read_project_xml(inventory) -> str:
+    """Read project.xml from the archive, handling UTF-8, UTF-16, and BOM encodings.
+
+    Tries UTF-8 first, then falls back to UTF-16 and UTF-8-sig (BOM) if needed.
+
+    Args:
+        inventory: The open ArchiveInventory.
+
+    Returns:
+        The decoded XML content string.
+
+    Raises:
+        ValueError: If project.xml cannot be decoded with any supported encoding.
+    """
+    raw_bytes = inventory.get_bytes("project.xml")
+
+    # Try UTF-8 BOM first (utf-8-sig handles BOM transparently)
+    if raw_bytes.startswith(b"\xef\xbb\xbf"):
+        return raw_bytes.decode("utf-8-sig")
+
+    # Try UTF-16 BOM (both LE and BE)
+    if raw_bytes.startswith(b"\xff\xfe") or raw_bytes.startswith(b"\xfe\xff"):
+        return raw_bytes.decode("utf-16")
+
+    # Default: try UTF-8
+    try:
+        return raw_bytes.decode("utf-8")
+    except UnicodeDecodeError:
+        pass
+
+    # Fallback: try UTF-16 without BOM
+    try:
+        return raw_bytes.decode("utf-16")
+    except UnicodeDecodeError:
+        pass
+
+    raise ValueError(
+        "project.xml cannot be decoded: tried UTF-8, UTF-8-sig, and UTF-16"
+    )
+
+
+def parse_directory(
+    directory: str | Path,
+    output_dir: str | Path | None = None,
+) -> BulkResult:
+    """Parse all .egp files in a directory recursively.
+
+    Discovers .egp files, parses each independently, and returns a structured
+    result with successes, failures, and summary counts.
+
+    Args:
+        directory: Root directory to search for .egp files.
+        output_dir: Optional directory for JSON output. If None, no file is written.
+
+    Returns:
+        A BulkResult dataclass instance with successes, failures, and summary.
+
+    Raises:
+        ValueError: If directory does not exist or is not a directory.
+    """
+    from .bulk import process_directory
+
+    return process_directory(directory, output_dir=output_dir)

pyegp_parser/archive.py

@@ -0,0 +1,114 @@
+"""ZIP extraction and entry classification for EGP archives."""
+
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from zipfile import BadZipFile, ZipFile
+
+
+class EntryCategory(Enum):
+    """Classification categories for ZIP archive entries."""
+
+    PROJECT_XML = "project_xml"
+    TASK_CONFIG = "task_config"
+    CODE_FILE = "code_file"
+    EXECUTION_LOG = "execution_log"
+    PROJECT_LOG = "project_log"
+    ODS_RESULT = "ods_result"
+    EMPTY_DIRECTORY = "empty_directory"
+    UNKNOWN = "unknown"
+
+
+@dataclass(frozen=True)
+class ArchiveEntry:
+    """A single entry in the EGP ZIP archive with its classification."""
+
+    path: str
+    category: EntryCategory
+    compressed_size: int
+    uncompressed_size: int
+
+
+@dataclass
+class ArchiveInventory:
+    """Collection of classified archive entries with access to the open ZIP file."""
+
+    entries: list[ArchiveEntry]
+    zip_file: ZipFile  # kept open for content reads
+
+    def get_content(self, path: str, encoding: str = "utf-8") -> str:
+        """Read a text entry from the archive.
+
+        Args:
+            path: The archive entry path to read.
+            encoding: Text encoding to use (default: utf-8).
+
+        Returns:
+            Decoded text content of the entry.
+        """
+        return self.zip_file.read(path).decode(encoding)
+
+    def get_bytes(self, path: str) -> bytes:
+        """Read raw bytes from the archive.
+
+        Args:
+            path: The archive entry path to read.
+
+        Returns:
+            Raw bytes of the entry.
+        """
+        return self.zip_file.read(path)
+
+    def close(self) -> None:
+        """Close the underlying ZIP file."""
+        self.zip_file.close()
+
+
+def open_archive(egp_path: Path) -> ArchiveInventory:
+    """Open an EGP file and classify all entries.
+
+    Args:
+        egp_path: Path to the .egp file.
+
+    Returns:
+        ArchiveInventory with classified entries and an open ZipFile handle.
+
+    Raises:
+        FileNotFoundError: If path does not exist.
+        ValueError: If not a valid ZIP or missing project.xml.
+    """
+    # Import here to avoid circular dependency (classifier imports EntryCategory from this module)
+    from .classifier import classify_entry
+
+    path = Path(egp_path)
+
+    if not path.exists():
+        raise FileNotFoundError(f"EGP file not found: {path}")
+
+    try:
+        zf = ZipFile(path, "r")
+    except BadZipFile as e:
+        raise ValueError(f"Not a valid EGP archive (invalid ZIP): {path}") from e
+
+    # Enumerate and classify all entries
+    entries: list[ArchiveEntry] = []
+    has_project_xml = False
+
+    for info in zf.infolist():
+        category = classify_entry(info.filename)
+        if category == EntryCategory.PROJECT_XML:
+            has_project_xml = True
+        entries.append(
+            ArchiveEntry(
+                path=info.filename,
+                category=category,
+                compressed_size=info.compress_size,
+                uncompressed_size=info.file_size,
+            )
+        )
+
+    if not has_project_xml:
+        zf.close()
+        raise ValueError(f"Not a valid EGP archive (missing project.xml): {path}")
+
+    return ArchiveInventory(entries=entries, zip_file=zf)

pyegp_parser/bulk.py

@@ -0,0 +1,113 @@
+"""Bulk directory processing for EGP files.
+
+Discovers all .egp files recursively within a directory, parses each
+independently, records successes and failures, and optionally writes
+per-file JSON outputs preserving subdirectory structure.
+
+Requirements: 19.1, 19.2, 19.3, 19.4, 19.5, 19.6, 19.7
+"""
+
+from pathlib import Path
+
+from .models.bulk import BulkFileFailure, BulkFileSuccess, BulkResult, BulkSummary
+
+
+def discover_egp_files(directory: Path) -> list[Path]:
+    """Discover all .egp files recursively, sorted lexicographically by full path.
+
+    Args:
+        directory: Root directory to search.
+
+    Returns:
+        Sorted list of Path objects pointing to .egp files.
+
+    Raises:
+        ValueError: If directory does not exist or is not a directory.
+    """
+    if not directory.exists():
+        raise ValueError(f"Invalid directory path: '{directory}' does not exist")
+    if not directory.is_dir():
+        raise ValueError(f"Invalid directory path: '{directory}' is not a directory")
+    egp_files = list(directory.rglob("*.egp"))
+    # Sort lexicographically by full path string for consistent ordering
+    egp_files.sort(key=lambda p: str(p))
+    return egp_files
+
+
+def _get_parse_file():
+    """Lazy import of parse_file to avoid circular dependencies."""
+    import pyegp_parser
+
+    return pyegp_parser.parse_file
+
+
+def process_directory(
+    directory: str | Path,
+    output_dir: str | Path | None = None,
+) -> BulkResult:
+    """Parse all .egp files in a directory recursively.
+
+    Discovers .egp files, parses each independently, and collects results.
+    If output_dir is provided, writes per-file JSON outputs preserving
+    subdirectory structure relative to the input directory.
+
+    Args:
+        directory: Root directory to scan for .egp files.
+        output_dir: Optional directory for per-file JSON output.
+            Output paths preserve subdirectory structure:
+            {output_dir}/{relative_path_stem}.json
+
+    Returns:
+        BulkResult with successes, failures, and summary counts.
+
+    Raises:
+        ValueError: If directory does not exist or is not a directory.
+    """
+    parse_file = _get_parse_file()
+
+    directory = Path(directory).resolve()
+
+    # Validate directory and discover files (raises ValueError if invalid)
+    egp_files = discover_egp_files(directory)
+
+    output_path = Path(output_dir).resolve() if output_dir else None
+
+    successes: list[BulkFileSuccess] = []
+    failures: list[BulkFileFailure] = []
+
+    for egp_file in egp_files:
+        try:
+            # Determine per-file output directory if output_dir is specified
+            file_output_dir = None
+            if output_path is not None:
+                # Preserve subdirectory structure relative to input directory
+                relative = egp_file.relative_to(directory)
+                # Use stem (filename without extension) as the output subdirectory
+                file_output_dir = output_path / relative.parent / relative.stem
+
+            project = parse_file(egp_file, output_dir=file_output_dir)
+            successes.append(
+                BulkFileSuccess(
+                    file_path=str(egp_file),
+                    project=project,
+                )
+            )
+        except Exception as e:
+            failures.append(
+                BulkFileFailure(
+                    file_path=str(egp_file),
+                    error_message=str(e),
+                )
+            )
+
+    summary = BulkSummary(
+        total_files=len(egp_files),
+        success_count=len(successes),
+        failure_count=len(failures),
+    )
+
+    return BulkResult(
+        successes=successes,
+        failures=failures,
+        summary=summary,
+    )

pyegp_parser/classifier.py

@@ -0,0 +1,60 @@
+"""Path pattern matching for ZIP entry classification."""
+
+import re
+
+from .archive import EntryCategory
+
+# Compiled regex patterns for categories that don't need backreference validation
+_SIMPLE_PATTERNS: dict[EntryCategory, re.Pattern] = {
+    EntryCategory.PROJECT_XML: re.compile(r"^project\.xml$"),
+    EntryCategory.CODE_FILE: re.compile(r"^CodeTask-(?P<id>[A-Za-z0-9]+)/code\.sas$"),
+    EntryCategory.EXECUTION_LOG: re.compile(
+        r"^(?P<task_type>\w+)-(?P<task_id>[A-Za-z0-9]+)/Log-(?P<log_id>[A-Za-z0-9]+)/result\.log$"
+    ),
+    EntryCategory.ODS_RESULT: re.compile(
+        r"^ODSResults/ODSResult-(?P<id>[A-Za-z0-9]+)/.*$"
+    ),
+}
+
+# Patterns that require backreference-like validation (checked via function)
+_TASK_CONFIG_PATTERN = re.compile(
+    r"^(?P<task_type>\w+)-(?P<id>[A-Za-z0-9]+)/(?P<repeat_type>\w+)-(?P<repeat_id>[A-Za-z0-9]+)\.xml$"
+)
+
+_PROJECT_LOG_PATTERN = re.compile(
+    r"^ProjectLog-(?P<id>[A-Za-z0-9]+)/ProjectLog-(?P<repeat_id>[A-Za-z0-9]+)/result\.log$"
+)
+
+
+def classify_entry(path: str) -> EntryCategory:
+    """Classify a ZIP entry path into its category.
+
+    Empty directory entries (ending with '/') are classified as EMPTY_DIRECTORY.
+    Paths matching known patterns get their respective category.
+    All other paths are classified as UNKNOWN.
+    """
+    if path.endswith("/"):
+        return EntryCategory.EMPTY_DIRECTORY
+
+    # Check simple patterns first
+    for category, pattern in _SIMPLE_PATTERNS.items():
+        if pattern.match(path):
+            return category
+
+    # Check TASK_CONFIG with backreference validation:
+    # Pattern: {TaskType}-{ID}/{TaskType}-{ID}.xml (type and ID must repeat)
+    m = _TASK_CONFIG_PATTERN.match(path)
+    if (
+        m
+        and m.group("task_type") == m.group("repeat_type")
+        and m.group("id") == m.group("repeat_id")
+    ):
+        return EntryCategory.TASK_CONFIG
+
+    # Check PROJECT_LOG with backreference validation:
+    # Pattern: ProjectLog-{ID}/ProjectLog-{ID}/result.log (ID must repeat)
+    m = _PROJECT_LOG_PATTERN.match(path)
+    if m and m.group("id") == m.group("repeat_id"):
+        return EntryCategory.PROJECT_LOG
+
+    return EntryCategory.UNKNOWN