dataflow-core 2.1.15rc4__py3-none-any.whl → 2.1.16__py3-none-any.whl

dataflow/dataflow.py

@@ -1,6 +1,7 @@
 import os, requests
 from .database_manager import DatabaseManager
 import json
+import base64
 from .configuration import ConfigurationManager
 
 
@@ -177,6 +178,105 @@ class Dataflow:
         except Exception as e:
             print(f"[Dataflow.secret] Exception occurred: {e}")
             return None
+        
+    def secret_file(self, secret_name: str):
+        """
+        Retrieve a Dataflow secret file.
+
+        Args:
+            secret_name (str): Name of the secret to retrieve
+            
+        Returns:
+            str or None: Secret value if found, None otherwise
+        """
+        try:
+            host_name = os.environ.get("HOSTNAME", "")
+            runtime = os.environ.get("RUNTIME")
+            slug = os.environ.get("SLUG")
+            org_id = os.environ.get("ORGANIZATION")
+
+            dataflow_config = ConfigurationManager('/dataflow/app/auth_config/dataflow_auth.cfg')
+            query_params = {
+                "key": secret_name
+            }
+
+            if runtime:
+                secret_api = dataflow_config.get_config_value("auth", "secret_ui_api")
+                query_params["runtime"] = runtime
+                query_params["slug"] = slug
+                query_params["org_id"] = org_id
+            else:
+                secret_api = dataflow_config.get_config_value("auth", "secret_manager_api")
+            if not secret_api:
+                print("[Dataflow.secret] Secret API Unreachable")
+                return None
+
+            response = requests.get(secret_api, params=query_params)
+
+            if response.status_code == 404:
+                return None
+            elif response.status_code >= 500:
+                response.raise_for_status()
+            elif response.status_code >= 400:
+                print(f"[Dataflow.secret] Client error {response.status_code} for secret '{secret_name}'")
+                return None
+            elif response.status_code != 200:
+                print(f"[Dataflow.secret] Unexpected status {response.status_code} for secret '{secret_name}'")
+                return None
+            
+            response_data = response.json()
+            if response.status_code == 200 and response_data.get('filename'):
+                # For runtime mode, create file and return filepath
+                if runtime:
+                    import tempfile
+                    from pathlib import Path
+
+                    # Create /tmp/secrets directory if it doesn't exist
+                    secrets_dir = Path("/tmp/secrets")
+                    secrets_dir.mkdir(parents=True, exist_ok=True)
+                    
+                    # Get filename and content
+                    filename = response_data.get('filename')
+                    file_content = response_data.get('value')
+                    
+                    if not filename or not file_content:
+                        print(f"[Dataflow.secret] Missing filename or content for secret '{secret_name}'")
+                        return None
+                    
+                    file_path = os.path.join(secrets_dir, filename)
+        
+                    # Detect if content is Base64 encoded binary or text
+                    try:
+                        # Try to decode as Base64
+                        decoded_content = base64.b64decode(file_content, validate=True)
+                        # Check if it contains non-printable characters (likely binary)
+                        is_binary = not all(32 <= byte <= 126 or byte in (9, 10, 13) for byte in decoded_content[:100])
+                        
+                        if is_binary:
+                            # Write as binary
+                            with open(file_path, 'wb') as f:
+                                f.write(decoded_content)
+                        else:
+                            # Decode and write as text
+                            with open(file_path, 'w', encoding='utf-8') as f:
+                                f.write(decoded_content.decode('utf-8'))
+                    except Exception:
+                        # Not Base64 or decode failed, treat as text
+                        with open(file_path, 'w', encoding='utf-8') as f:
+                            f.write(file_content)
+                    return str(file_path)
+                else:
+                    # For non-runtime mode, return the value as-is
+                    return response_data.get('value')
+            else:
+                print(f"[Dataflow.secret] No file found for secret '{secret_name}'! If it is a non-file secret, please use the 'secret' method.")
+                return None
+        
+        except requests.exceptions.RequestException as e:
+            raise RuntimeError(f"[Dataflow.secret] Failed to fetch secret '{secret_name}'") from e
+        except Exception as e:
+            print(f"[Dataflow.secret] Exception occurred: {e}")
+            return None
 
     def connection(self, conn_id: str, mode="session"):
         """

dataflow/environment.py

@@ -55,7 +55,8 @@ class EnvironmentManager:
                 status="published",
                 mode="create",
                 yaml_file_path=yaml_path,
-                version=int(env_version)
+                version=int(env_version),
+                py_version=py_version
             )
         elif status == "draft":
             mode = "create" if env_version == '1' else "update"
@@ -65,7 +66,8 @@ class EnvironmentManager:
                 mode=mode,
                 yaml_file_path=yaml_path,
                 log_file_location=log_file_location,
-                version=int(env_version)
+                version=int(env_version),
+                py_version=py_version
             )
             
             # Update job log status if db was provided
@@ -191,6 +193,7 @@ class EnvironmentManager:
         version: int,
         source_path=None, 
         log_file_location=None,
+        py_version=None
     ):
         """
         Executes environment operations (create or clone).
@@ -222,7 +225,7 @@ class EnvironmentManager:
 
             if mode == "create":                
                 create_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/create_environment.sh')
-                command = ["bash", create_env_script_path, yaml_file_path, conda_env_path]
+                command = ["bash", create_env_script_path, yaml_file_path, conda_env_path, py_version]
 
             elif mode == "update":
                 update_env_script_path = pkg_resources.resource_filename('dataflow', 'scripts/update_environment.sh')

dataflow/scripts/clone_environment.sh

@@ -4,6 +4,7 @@ set -e
 source_env_name=$1
 target_env_path=$2
 yaml_file_path=$3
+py_version=$4
 
 # Extract just the env name (basename) from the target path
 env_name=$(basename "$target_env_path")
@@ -12,6 +13,9 @@ env_name=$(basename "$target_env_path")
 export CONDA_PKGS_DIRS="/dataflow/envs/cache/${env_name}"
 mkdir -p "$CONDA_PKGS_DIRS"
 
+export PIP_CONSTRAINT="/dataflow/setup/pip_constraints/py${py_version}-constraints.txt"
+export NO_CONDA_PLUGIN_PIP_CONSTRAINT="true"
+
 # 1. Cloning conda env
 conda create --clone ${source_env_name} --prefix ${target_env_path} --yes
 

{dataflow_core-2.1.15rc4.dist-info → dataflow_core-2.1.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dataflow-core
-Version: 2.1.15rc4
+Version: 2.1.16
 Summary: Dataflow core package
 Author: Dataflow
 Author-email: 

{dataflow_core-2.1.15rc4.dist-info → dataflow_core-2.1.16.dist-info}/RECORD

@@ -5,9 +5,9 @@ authenticator/dataflowsupersetauthenticator.py,sha256=NkAmDaIc-ui-qEolu4xz_UY7P_
 dataflow/__init__.py,sha256=WTRg8HMpMWSgxYJ9ZGVldx4k07fAbta3mBmZ1hG9mWE,30
 dataflow/configuration.py,sha256=7To6XwH1eESiYp39eqPcswXWwrdBUdPF6xN6WnazOF0,663
 dataflow/database_manager.py,sha256=tJHMuOZ9Muskrh9t4uLRlTuFU0VkHAzoHlGP5DORIC4,899
-dataflow/dataflow.py,sha256=EzeOMtMCqG0WJ0vld6ArKbKNFAmA9sbjBqVJ8sd2sQw,13026
+dataflow/dataflow.py,sha256=kUuGZ1zV-kiu4IVP7hTKrAmLt33nXni-ppAEDOUsmyc,17622
 dataflow/db.py,sha256=73ojGqpCTRVTlPszD73Ozhjih_BI2KTHmazqxxL6iWk,3780
-dataflow/environment.py,sha256=7XiSojSIOnMOz5YVk-k6w1yQCKbBTTvUxv5cnn6cqJg,28830
+dataflow/environment.py,sha256=xMHCSjUzpK3dGuPhQPG28lZKtNtXHD0s4wfQVy9yTNs,28944
 dataflow/models/__init__.py,sha256=FHus-b2oTgLSJ9H7dOCNJ14XpjTzHARqK_Vwh8VhCtM,1178
 dataflow/models/app_types.py,sha256=BNGtjwpQU6MSpBGp0pDzUI4uDZNqvE3_x33itIInasM,689
 dataflow/models/blacklist_library.py,sha256=B2oi3Z8GcR_glhLAyinFk0W8c9txXvm3uOER6dY-q7I,991
@@ -37,7 +37,7 @@ dataflow/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 dataflow/schemas/connection.py,sha256=ut2sqz06yOjmFKzHry92FEt7DN09Bj30GYse35__Cuw,2467
 dataflow/schemas/git_ssh.py,sha256=N1O7HM6ZbygIBZn2rKvNR0e7IM3ZJMAH6aJtjaghDr0,1283
 dataflow/schemas/secret.py,sha256=wMSCn6zoBHS-R4NoKwljq2JUad8p9JY542UNJFa86X8,1183
-dataflow/scripts/clone_environment.sh,sha256=Qy0GylsA3kUVUL_L1MirxIWujOFhT1tikKqXNtCTWd4,506
+dataflow/scripts/clone_environment.sh,sha256=KHz29m_lN0SJeCTzd7-vmdmS7-aiM8KogduKp2Ttw8s,654
 dataflow/scripts/create_environment.sh,sha256=3FHgNplJuEZvyTsLqlCJNX9oyfXgsfqn80VZk2xtvso,828
 dataflow/scripts/update_environment.sh,sha256=2dtn2xlNi6frpig-sqlGE1_IKRbbkqYOCpf_qyMKKII,992
 dataflow/secrets_manager/__init__.py,sha256=idGqIDtYl0De2WIK9Obl-N7SDPSYtVM0D-wXfZjCiy4,559
@@ -56,8 +56,8 @@ dfmigration/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dfmigration/env.py,sha256=w_3Kzr3LJCU5l1zBGsNcoNGZyR8fZEtMRsxE-cEqfHQ,1110
 dfmigration/versions/001_initial_baseline_migration.py,sha256=lxjs7LZLs9-c7FQj7t7t49EOfLPPHMLI6iNp0PIBMRA,299
 dfmigration/versions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dataflow_core-2.1.15rc4.dist-info/METADATA,sha256=NM9-0YqN16BHxRN6iHZVfofebsfMK7zVVaL615WttpY,466
-dataflow_core-2.1.15rc4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dataflow_core-2.1.15rc4.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
-dataflow_core-2.1.15rc4.dist-info/top_level.txt,sha256=-gGoIBh-bUMtzOdna2jsDOdkE2CRu3w3aA7yEFCmNAI,35
-dataflow_core-2.1.15rc4.dist-info/RECORD,,
+dataflow_core-2.1.16.dist-info/METADATA,sha256=vu039NfRwGVU7aR0TnR5Yk8Hzfzy-cW6Hul9nAN2xuU,463
+dataflow_core-2.1.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dataflow_core-2.1.16.dist-info/entry_points.txt,sha256=ppj_EIbYrJJwCPg1kfdsZk5q1N-Ejfis1neYrnjhO8o,117
+dataflow_core-2.1.16.dist-info/top_level.txt,sha256=-gGoIBh-bUMtzOdna2jsDOdkE2CRu3w3aA7yEFCmNAI,35
+dataflow_core-2.1.16.dist-info/RECORD,,