prophecy-libs 2.1.6.dev0__tar.gz → 2.1.6.dev2__tar.gz

{prophecy-libs-2.1.6.dev0/prophecy_libs.egg-info → prophecy_libs-2.1.6.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
-Name: prophecy-libs
-Version: 2.1.6.dev0
+Name: prophecy_libs
+Version: 2.1.6.dev2
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

{prophecy-libs-2.1.6.dev0 → prophecy_libs-2.1.6.dev2}/prophecy/udfs/scala_udf_wrapper.py

@@ -2,8 +2,7 @@ from typing import Optional
 
 from pyspark.sql.functions import udf
 from pyspark.sql.types import IntegerType
-from pyspark.sql.column import Column
-from pyspark.sql.functions import lit
+from pyspark.sql.column import Column, _to_java_column, _to_seq
 
 
 class UDFBase:
@@ -34,19 +33,6 @@ def rest_api(*cols):
 
 
 def call_udf(udfName: str, *cols):
-    """
-    Cross-version UDF caller that avoids private _to_java_column/_to_seq.
-    Accepts Column or Python literals.
-    """
-    sc = udfConfig.sparkSession.sparkContext
-
-    jcols = []
-    for c in cols:
-        # Accept Column or anything exposing a _jc (Java Column)
-        if not hasattr(c, "_jc"):
-            c = lit(c)
-        jcols.append(c._jc)
-
-    jseq = sc._jvm.PythonUtils.toSeq(jcols)
-    jcol = udfConfig.UDFUtils.call_udf(udfName, jseq)
-    return Column(jcol)
+    _cols = _to_seq(udfConfig.sparkSession.sparkContext, cols, _to_java_column)
+    call_udf_result = udfConfig.UDFUtils.call_udf(udfName, _cols)
+    return Column(call_udf_result)

{prophecy-libs-2.1.6.dev0 → prophecy_libs-2.1.6.dev2}/prophecy/utils/initial_python_code.py

@@ -293,6 +293,17 @@ class ProphecyRequests:
             print(f"Interims Request Failed: {str(e)}")
             raise e
 
+    @staticmethod
+    @prophecy_track_time('send_dataframe_records_count_payload')
+    def send_dataframe_records_count_payload(key: str, job: str):
+        try:
+            payload = DataSampleLoader.get_records_count_payload(key, job)
+            if (payload != None):
+                response = HttpClient.post("/interims", payload)
+        except Exception as e:
+            print(f"Records Count Request Failed: {str(e)}")
+            raise e
+
 
 ProphecyRequests.ping()
 
@@ -814,6 +825,22 @@ class DataSampleLoader:
         df, _, _ = cls._get_entry_from_dataframes_map(key)
         return df
 
+    @classmethod
+    def get_records_count_payload(cls, key: str, job: str) -> Optional[str]:
+        """Get payload with proper JSON handling."""
+        df, _, _ = cls._get_entry_from_dataframes_map(key)
+
+        if df is None:
+            return None
+
+        try:
+            cnt = df.count()
+            result = {'job': job, 'schema':'', 'data': str(cnt), 'op': 'count'}
+            return cls._json_serialize(result)
+        except Exception as e:
+            print(f"Error creating payload for records count: {str(e)}")
+            raise ValueError(f"Error creating payload: {str(e)}")
+
 
 import itertools
 import re

{prophecy-libs-2.1.6.dev0 → prophecy_libs-2.1.6.dev2/prophecy_libs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: prophecy-libs
-Version: 2.1.6.dev0
+Version: 2.1.6.dev2
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

{prophecy-libs-2.1.6.dev0 → prophecy_libs-2.1.6.dev2}/setup.py

@@ -1,8 +1,8 @@
 from setuptools import setup, find_packages
 
 setup(
-    name="prophecy-libs",
-    version="2.1.6.dev0",
+    name="prophecy_libs",
+    version="2.1.6.dev2",
     url="https://github.com/SimpleDataLabsInc/prophecy-python-libs",
     packages=find_packages(exclude=["test.*", "test"]),
     package_data={