prophecy-libs 2.0.4.dev0__tar.gz → 2.0.6.dev69__tar.gz

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: prophecy-libs
-Version: 2.0.4.dev0
+Version: 2.0.6.dev69
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

prophecy-libs-2.0.6.dev69/prophecy/data/__init__.py

@@ -0,0 +1 @@
+# Package data module

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/prophecy/utils/datasampleloader.py

@@ -437,7 +437,12 @@ class DataSampleLoaderLib:
             except Exception as e:
                 from pyspark.sql.functions import struct, col, to_json
 
-                return result.select(to_json(struct(col("*")))).first()
+                row = result.select(to_json(struct(col("*")))).first()
+                if row is not None:
+                    # Extract the string from the Row (first value)
+                    return row[0]
+                else:
+                    return None
         except Exception as e:
             print(f"Error creating payload: {str(e)}")  # Log error before raising
             raise ValueError(f"Error creating payload: {str(e)}")

prophecy-libs-2.0.6.dev69/prophecy/utils/dbx_serverless_helper.py

@@ -0,0 +1,56 @@
+"""
+Databricks Serverless Helper utilities.
+"""
+
+import sys
+from importlib import resources
+
+
+def print_serverless_requirements():
+    """Print the contents of the serverless requirements file to stdout"""
+    try:
+        # Python 3.9+
+        with resources.files("prophecy.data").joinpath("requirements.dbxserverless_sandbox.txt").open() as f:
+            print(f.read(), end="")
+    except AttributeError:
+        # Python 3.8 fallback
+        with resources.open_text("prophecy.data", "requirements.dbxserverless_sandbox.txt") as f:
+            print(f.read(), end="")
+
+
+def get_serverless_requirements_path():
+    """Get the file path to the serverless requirements file"""
+    try:
+        # Python 3.9+
+        files = resources.files("prophecy.data")
+        req_file = files.joinpath("requirements.dbxserverless_sandbox.txt")
+        with resources.as_file(req_file) as path:
+            return str(path)
+    except AttributeError:
+        # Python 3.8 fallback
+        import pkg_resources
+        return pkg_resources.resource_filename("prophecy.data", "requirements.dbxserverless_sandbox.txt")
+
+
+def main():
+    """Main CLI entry point"""
+    if len(sys.argv) == 1:
+        # Default behavior: print requirements
+        print_serverless_requirements()
+    elif len(sys.argv) == 2:
+        command = sys.argv[1]
+        if command == "print":
+            print_serverless_requirements()
+        elif command == "path":
+            print(get_serverless_requirements_path())
+        else:
+            print(f"Unknown command: {command}")
+            print("Usage: python -m prophecy.utils.dbx_serverless_helper [print|path]")
+            sys.exit(1)
+    else:
+        print("Usage: python -m prophecy.utils.dbx_serverless_helper [print|path]")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/prophecy/utils/diff.py

@@ -14,7 +14,8 @@ from pyspark.sql.types import (
     StringType,
     MapType,
 )
-import uuid, enum
+import uuid, enum, json
+from prophecy.config.config_base import is_serverless
 
 
 class DiffKeys(enum.Enum):
@@ -613,7 +614,7 @@ class DataFrameDiff:
                 "duplicatePkCount": duplicate_df.count(),
             }
 
-        return {
+        diff_summary = {
             "label": diff_key,
             "data": {
                 "summaryTiles": [
@@ -669,6 +670,12 @@ class DataFrameDiff:
             },
         }
 
+        if is_serverless:
+            # printing diff summary to support serverless
+            # do not print if not serverless
+            print(json.dumps(diff_summary))
+        return diff_summary
+
     @classmethod
     def clean_joined_df(cls, joined_df, key_columns, value_columns, left_df, right_df):
         """

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/prophecy/utils/httpclient.py

@@ -161,8 +161,8 @@ class ProphecyRequestsLib:
         try:
             payload = DataSampleLoaderLib.get_payload(key, job, df_offset)
             with HttpClientLib(base_url, token) as client:
-                response = client.post_compressed(endpoint, payload)
+                response = client.post_compressed(endpoint, payload or '')
         except HTTPClientError as e:
-            print(f"Interims Request Failed HTTPClientError: {str(e)}")
+            print(f"Interims Request Failed HTTPClientError: {str(e)}. Payload: {payload} Endpoint: {endpoint}")
         except Exception as e:
             print(f"Interims Request Failed: {str(e)}")

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/prophecy/utils/secrets.py

@@ -415,7 +415,9 @@ class SecretManager:
     # ── Health & listings (HashiCorp) ─────────────────────────────────────
     @staticmethod
     def health(conn: HashiCorpConnectionDetails):
-        return VaultUtils.vault_health(conn)
+        resp = VaultUtils.vault_health(conn)
+        VaultUtils.token_health(conn)
+        return resp
 
     @staticmethod
     def listHashiCorpSecrets(conn: HashiCorpConnectionDetails) -> Dict[str, List[str]]:
@@ -492,7 +494,7 @@ async def handle_secrets_crud(req: SecretCrudRequest) -> LSecretsResponse:
             try:
                 resp = await asyncio.to_thread(SecretManager.health, conn)
                 out = HashiCorpHealthResponse.from_rest_response(resp)
-                logger.info(f"Returning health response ")
+                logger.info(f"Returning health response {out}")
                 return LSecretsResponse.from_request(req, out, True)
             except Exception as exc:
                 msg = (
@@ -802,7 +804,12 @@ class VaultUtils:
         cli = VaultUtils._client_from_conn(conn)
         mount, inner = _split_mount_and_path(full_path)
         kv = VaultUtils._kv_version_for_mount(cli, mount)
-        data = VaultUtils._read_kv(cli, mount, inner, kv) or {}
+        data = {}
+        try:
+            data = VaultUtils._read_kv(cli, mount, inner, kv)
+        except hvac.exceptions.InvalidPath:
+            logger.info(f"Secret path '{mount}/{inner}' does not exist – creating it now.")
+
         data[key] = value
         VaultUtils._write_kv(cli, mount, inner, data, kv)
 
@@ -856,22 +863,25 @@ class VaultUtils:
         acc: Dict[str, List[str]],
     ):
         """Depth-first walk; for every leaf secret record  {full_path: [keys…]}."""
+        path = inner or ""
         try:
             if kv_ver == 2:
                 resp = cli.secrets.kv.v2.list_secrets(
-                    path=inner or "", mount_point=mount
+                    path=path, mount_point=mount
                 )
             elif kv_ver == 1:
                 resp = cli.secrets.kv.v1.list_secrets(
-                    path=inner or "", mount_point=mount
+                    path=path, mount_point=mount
                 )
             else:
                 try:
-                    resp = cli.secrets.kv.v2.list_secrets(path=inner or "", mount_point=mount)
+                    resp = cli.secrets.kv.v2.list_secrets(path=path, mount_point=mount)
                     kv_ver = 2
-                except hvac.exceptions.Forbidden:
-                    resp = cli.secrets.kv.v1.list_secrets(path=inner or "", mount_point=mount)
+                    logger.info(f'Setting kv engine version as 2 for mount={mount} and path={path}')
+                except Exception as e:
+                    resp = cli.secrets.kv.v1.list_secrets(path=path, mount_point=mount)
                     kv_ver = 1
+                    logger.info(f'Setting kv engine version as 1 for mount={mount} and path={path}')
 
             keys = resp["data"]["keys"]
         except hvac.exceptions.InvalidPath:
@@ -904,4 +914,4 @@ class VaultUtils:
     @staticmethod
     async def aio(func, *args, **kwargs):
         loop = asyncio.get_running_loop()
-        return await loop.run_in_executor(None, partial(func, *args, **kwargs))
+        return await loop.run_in_executor(None, partial(func, *args, **kwargs))

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/prophecy/utils/transpiler/dataframe_fcns.py

@@ -30,10 +30,6 @@ def writeFixedFile(df, schema, path, mode="overwrite"):
                                                                                      createScalaOption(spark, None),
                                                                                      _getSaveMode(spark, mode))
 
-def index(string, substring, occurence):
-    spark = ScalaUtil.getAbiLib().spark
-    return spark.sparkContext._jvm.io.prophecy.datastage.SparkFunctions.index(string, substring, occurence)
-
 def flattenSchema(df, jsonParsedColumnName, distributeColumnsOnType):
     spark = ScalaUtil.getAbiLib().spark
     result = spark.sparkContext._jvm.io.prophecy.alteryx.SparkFunctions.flattenSchema(df._jdf, jsonParsedColumnName, distributeColumnsOnType, spark._jsparkSession)
@@ -403,23 +399,23 @@ def deduplicate(
 def generate_xml(df, rootElement, namespacePrefixes, namespaceURIs, columnNames, xmlPaths, isAttributes, writeNilElement, schemaLocation=None):
     spark = df.sparkSession
     sc = spark.sparkContext
-
+    
     java_namespacePrefixes = sc._jvm.java.util.ArrayList()
     for item in namespacePrefixes:
         java_namespacePrefixes.add(item)
-
+    
     java_namespaceURIs = sc._jvm.java.util.ArrayList()
     for item in namespaceURIs:
         java_namespaceURIs.add(item)
-
+    
     java_columnNames = sc._jvm.java.util.ArrayList()
     for item in columnNames:
         java_columnNames.add(item)
-
+    
     java_xmlPaths = sc._jvm.java.util.ArrayList()
     for item in xmlPaths:
         java_xmlPaths.add(item)
-
+    
     java_isAttributes = sc._jvm.java.util.ArrayList()
     for item in isAttributes:
         java_isAttributes.add(item)

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/prophecy_libs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: prophecy-libs
-Version: 2.0.4.dev0
+Version: 2.0.6.dev69
 Summary: Helper library for prophecy generated code
 Home-page: https://github.com/SimpleDataLabsInc/prophecy-python-libs
 Author: UNKNOWN

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/prophecy_libs.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ prophecy/config/__init__.py
 prophecy/config/config_base.py
 prophecy/config/config_base_test.py
 prophecy/config/utils.py
+prophecy/data/__init__.py
 prophecy/executionmetrics/__init__.py
 prophecy/executionmetrics/execution_metrics_handler.py
 prophecy/executionmetrics/in_memory_store.py
@@ -59,6 +60,7 @@ prophecy/utils/__init__.py
 prophecy/utils/constants.py
 prophecy/utils/data_profiler.py
 prophecy/utils/datasampleloader.py
+prophecy/utils/dbx_serverless_helper.py
 prophecy/utils/diff.py
 prophecy/utils/functions.py
 prophecy/utils/gems_utils.py

{prophecy-libs-2.0.4.dev0 → prophecy-libs-2.0.6.dev69}/setup.py

@@ -2,9 +2,13 @@ from setuptools import setup, find_packages
 
 setup(
     name="prophecy-libs",
-    version="2.0.4.dev0",
+    version="2.0.6.dev69",
     url="https://github.com/SimpleDataLabsInc/prophecy-python-libs",
     packages=find_packages(exclude=["test.*", "test"]),
+    package_data={
+        "prophecy": ["data/requirements.dbxserverless_sandbox.txt"],
+    },
+    include_package_data=True,
     description="Helper library for prophecy generated code",
     long_description=open("README.md").read(),
     install_requires=[