prophecy-libs 2.0.0.dev3__tar.gz → 2.0.0.dev5__tar.gz

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: prophecy-libs
-Version: 2.0.0.dev3
+Version: 2.0.0.dev5
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/componentruns/component_runs_dao.py

@@ -26,7 +26,6 @@ from prophecy.executionmetrics.evolutions.models import (
     HiveStorageMetadata,
 )
 from prophecy.executionmetrics.logging_spark_session import (
-    SparkSession,
     refresh_table_if_exists,
     sql_with_logging,
     write_to_delta_with_logging,

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/evolutions/delta_metrics_storage_initializer.py

@@ -6,6 +6,7 @@ This module handles initialization of Delta Lake tables for storing execution me
 
 import logging
 
+from pyspark.sql import SparkSession
 from pyspark.sql.types import *
 from pyspark.sql.utils import AnalysisException
 
@@ -14,7 +15,6 @@ from prophecy.executionmetrics.evolutions.models import (
     MetricsStorageDetails,
 )
 from prophecy.executionmetrics.logging_spark_session import (
-    SparkSession,
     sql_with_logging,
 )
 from prophecy.executionmetrics.evolutions.metrics_storage_initializer import (

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/evolutions/hive_parquet_storage_initializer.py

@@ -6,6 +6,7 @@ This module handles initialization of Hive tables with Parquet format for storin
 
 import logging
 
+from pyspark.sql import SparkSession
 
 from prophecy.executionmetrics.evolutions.models import (
     HiveStorageMetadata,
@@ -18,7 +19,6 @@ from prophecy.executionmetrics.evolutions.package import (
     perform_up_evolutions,
 )
 from prophecy.executionmetrics.logging_spark_session import (
-    SparkSession,
     sql_with_logging,
 )
 from prophecy.executionmetrics.evolutions.metrics_storage_initializer import (

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/evolutions/metrics_storage_initializer.py

@@ -10,13 +10,14 @@ import re
 from abc import ABC, abstractmethod
 from typing import Optional
 
+from pyspark.sql import SparkSession
+
 from prophecy.executionmetrics.evolutions.models import (
     MetricsStorageDetails,
     MetricsStore,
     StorageMetadata,
     TableMetadata,
 )
-from prophecy.executionmetrics.logging_spark_session import SparkSession
 from prophecy.executionmetrics.schemas.external import MetricsTableNames
 from prophecy.executionmetrics.utils.common import is_databricks_environment
 from prophecy.executionmetrics.utils.external import create_db_suffix_from_url

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/evolutions/models.py

@@ -3,7 +3,8 @@ from dataclasses import dataclass
 from enum import Enum, auto
 from typing import Any, Optional
 
-from prophecy.executionmetrics.logging_spark_session import SparkSession
+from pyspark.sql import SparkSession
+
 from prophecy.executionmetrics.utils.common import is_databricks_environment
 
 

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/in_memory_store.py

@@ -143,8 +143,7 @@ def extract_slug_to_process_mapping(
     return slug_to_process
 
 
-def _parse_workflow_graph(data: Dict[str, Any]) -> WorkflowGraph:
-    """Parse workflow graph from dictionary."""
+def _parse_workflow_nodes(data: Dict[str, Any]) -> Dict[str, "WorkflowNode"]:
     processes = {}
     for proc_id, proc_data in data.get("processes", {}).items():
         if proc_data.get("component") == "Subgraph":
@@ -154,7 +153,7 @@ def _parse_workflow_graph(data: Dict[str, Any]) -> WorkflowGraph:
                 metadata=proc_data.get("metadata", {}),
                 ports=proc_data.get("ports", {}),
                 properties=proc_data.get("properties", {}),
-                processes=proc_data.get("processes", {}),
+                processes=_parse_workflow_nodes(proc_data),
                 connections=proc_data.get("connections", []),
             )
         else:
@@ -167,6 +166,13 @@ def _parse_workflow_graph(data: Dict[str, Any]) -> WorkflowGraph:
             )
         processes[proc_id] = node
 
+    return processes
+
+
+def _parse_workflow_graph(data: Dict[str, Any]) -> WorkflowGraph:
+    """Parse workflow graph from dictionary."""
+    processes = _parse_workflow_nodes(data)
+
     connections = []
     for conn_data in data.get("connections", []):
         connections.append(

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/interims/delta_interims.py

@@ -8,10 +8,9 @@ during pipeline execution.
 import logging
 from typing import List, Dict
 
-from pyspark.sql import DataFrame
+from pyspark.sql import DataFrame, SparkSession
 
 from prophecy.executionmetrics.logging_spark_session import (
-    SparkSession,
     sql_with_logging,
     write_to_delta_with_logging,
 )

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/interims/hive_parquet_interims.py

@@ -8,14 +8,13 @@ for interim data during pipeline execution.
 import logging
 from typing import List, Dict, Optional
 
-from pyspark.sql import DataFrame
+from pyspark.sql import DataFrame, SparkSession
 
 from prophecy.executionmetrics.evolutions.package import (
     InterimsSchema,
     perform_up_evolutions,
 )
 from prophecy.executionmetrics.logging_spark_session import (
-    SparkSession,
     sql_with_logging,
     write_to_hive_with_logging,
 )

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/interims/interims_table.py

@@ -9,7 +9,7 @@ import logging
 from abc import ABC, abstractmethod
 from typing import List, Dict, Optional
 
-from pyspark.sql import DataFrame
+from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import StructType, StructField, StringType, TimestampType
 
 from prophecy.executionmetrics.evolutions.models import (
@@ -18,7 +18,6 @@ from prophecy.executionmetrics.evolutions.models import (
     StorageMetadata,
 )
 from prophecy.executionmetrics.logging_spark_session import (
-    SparkSession,
     refresh_table_if_exists,
     sql_with_logging,
 )

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/pipelineruns/pipeline_run_service.py

@@ -14,13 +14,13 @@ from typing import List, Dict, Optional, Tuple, Any
 from datetime import datetime
 import uuid
 
-from pyspark.sql import DataFrame
+from pyspark.sql import DataFrame, SparkSession
 
 from prophecy.executionmetrics.componentruns.component_run_service import (
     ComponentRunService,
 )
 from prophecy.executionmetrics.evolutions.models import StorageMetadata
-from prophecy.executionmetrics.logging_spark_session import SparkSession
+
 from prophecy.executionmetrics.package import (
     ComponentRuns,
     FileContent,

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/executionmetrics/pipelineruns/pipeline_runs_dao.py

@@ -8,7 +8,7 @@ CRUD operations and queries for pipeline execution metrics.
 import logging
 from typing import List, Optional
 
-from pyspark.sql import DataFrame, Row
+from pyspark.sql import DataFrame, Row, SparkSession
 from pyspark.sql.types import (
     StructType,
     StructField,
@@ -22,7 +22,6 @@ from pyspark.sql.types import (
 )
 
 from prophecy.executionmetrics.logging_spark_session import (
-    SparkSession,
     refresh_table_if_exists,
     sql_with_logging,
     write_to_delta_with_logging,

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/utils/initial_python_code.py

@@ -7,16 +7,15 @@ from functools import wraps
 import logging
 import os
 import sys
+import re
 
-# Check if 'spark' is present in globals() or locals(), and is not None.
-# If not, attempt to initialize it using SparkSessionProxy for serverless
-if not globals().get("spark", None):
-    try:
-        from server_rest import SparkSessionProxy
-        spark = SparkSessionProxy.get_instance()
-    except Exception as e:
-        print(f"Failed to initialize 'spark' instance. Error: {e}")
-        raise Exception("Failed to initialize 'spark' instance")
+# Attempt to initialize it using SparkSessionProxy for serverless
+try:
+    from server_rest import SparkSessionProxy
+    spark = SparkSessionProxy.get_instance()
+except Exception as e:
+    print(f"Failed to initialize 'spark' instance. Error: {e}")
+    raise Exception("Failed to initialize 'spark' instance")
 
 # Monkey Patch Caching
 try:
@@ -311,6 +310,7 @@ from pyspark.sql.types import (
     DecimalType,
     DoubleType,
     IntegerType,
+    StructField,
 )
 from datetime import datetime, date, timedelta
 from decimal import Decimal
@@ -747,6 +747,26 @@ class DataSampleLoader:
         Lightweight replacement for Databricks' built-in ``display`` that returns the
         exact JSON payload your code receives through the DBX Commands API.
         """
+
+        
+
+        def to_ddl(dt: DataType) -> str:
+            _valid_ident = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+            def quote_ident(name: str) -> str:
+                return name if _valid_ident.fullmatch(name) else f"`{name.replace('`', '``')}`"
+            
+            if isinstance(dt, ArrayType):
+                return f"array<{to_ddl(dt.elementType)}>"
+            if isinstance(dt, MapType):
+                return f"map<{to_ddl(dt.keyType)},{to_ddl(dt.valueType)}>"
+            if isinstance(dt, StructType):
+                inner = ",".join(
+                    f"{quote_ident(f.name)}:{to_ddl(f.dataType)}"
+                    for f in dt.fields
+                )
+                return f"struct<{inner}>"
+            return dt.simpleString()
+
         if df is None:
             return
 
@@ -755,8 +775,8 @@ class DataSampleLoader:
         schema: list[dict] = [
             {
                 "name": field.name,
-                "type": json.dumps(field.dataType.simpleString()),  # e.g. "\"long\""
-                "metadata": json.dumps(field.metadata or {}),  # e.g. "\"{}\""
+                "type": json.dumps(to_ddl(field.dataType)),
+                "metadata": json.dumps(field.metadata or {}),
             }
             for field in df.schema  # type: StructField
         ]
@@ -770,11 +790,14 @@ class DataSampleLoader:
         OUTPUT_FILE = "/tmp/display_output.json"
         if not Path(OUTPUT_FILE).exists():
             try:
+                serialized = cls._json_serialize(display_res)
+                if isinstance(serialized, bytes):
+                    serialized = serialized.decode("utf-8")
                 Path(OUTPUT_FILE).write_text(
-                    json.dumps(display_res, ensure_ascii=False), encoding="utf-8"
+                    serialized, encoding="utf-8"
                 )
                 logging.info(
-                    f"Successfully wrote display_res: {display_res} to {OUTPUT_FILE}"
+                    f"Successfully wrote display_res: {serialized} to {OUTPUT_FILE}"
                 )
             except Exception as e:
                 logging.error("Failed to write %s: %s", OUTPUT_FILE, e)

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/utils/request_processor.py

@@ -76,6 +76,7 @@ async def dispatch_em_request_async(
         result = await handler(req)  # type: ignore[arg-type]
         return SuccessResponse(id=req_msg.id, result=result)  # type: ignore[return-value]
     except Exception as exc:  # noqa: BLE001
+        logging.info(f"Failed procesing request: {req}, error: {exc}, trace: {traceback.format_exc()}")
         err = JsonRpcError(message=str(exc), trace=traceback.format_exc().splitlines())
         return ErrorResponse(id=req_msg.id, error=err)  # type: ignore[return-value]
 

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy/utils/utils.py

@@ -51,6 +51,17 @@ except ImportError:
 from prophecy.libs.utils import *
 from prophecy.utils.pipeline_monitoring import *
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+if not logger.hasHandlers():
+    handler = logging.StreamHandler()
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    )
+    logger.addHandler(handler)
+
+
 is_serverless = is_serverless_env()
 logger.info(f"is_serverless is {is_serverless}")
 
@@ -444,7 +455,6 @@ class InterimConfig:
         self.session = sessionForInteractive
         # It's `JavaClass` when scala-libs are present, and `JavaPackage` when they are not present.
         try:
-            return
             if (
                 type(spark.sparkContext._jvm.org.apache.spark.sql.InterimOutputHive2)
                 == JavaPackage
@@ -453,11 +463,11 @@ class InterimConfig:
                 raise Exception(
                     "Scala Prophecy Libs jar was not found in the classpath. Please add Scala Prophecy Libs and retry the operation"
                 )
-            # self.interimOutput = (
-            #     spark.sparkContext._jvm.org.apache.spark.sql.InterimOutputHive2.apply(
-            #         sessionForInteractive
-            #     )
-            # )
+            self.interimOutput = (
+                spark.sparkContext._jvm.org.apache.spark.sql.InterimOutputHive2.apply(
+                    sessionForInteractive
+                )
+            )
         except PySparkAttributeError as ae:  # spark >= 3.4.1
             InterimConfig.jvm_accessible = False
 
@@ -877,7 +887,9 @@ class MetricsCollector:
                 spark, session_for_interactive, pipeline_id, pipeline_config_opt
             )
         except Exception as e:
-            logger.info(f"Failed to start for serverless: {e}")
+            logger.info(
+                f"Failed to start for serverless: {e} -- {traceback.format_exc()}"
+            )
 
     @classmethod
     def _start_for_serverless(

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/prophecy_libs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: prophecy-libs
-Version: 2.0.0.dev3
+Version: 2.0.0.dev5
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

{prophecy-libs-2.0.0.dev3 → prophecy-libs-2.0.0.dev5}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="prophecy-libs",
-    version="2.0.0.dev3",
+    version="2.0.0.dev5",
     url="https://github.com/SimpleDataLabsInc/prophecy-python-libs",
     packages=find_packages(exclude=["test.*", "test"]),
     description="Helper library for prophecy generated code",