dataflow-conda-plugin 0.1.0__tar.gz

dataflow_conda_plugin-0.1.0/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: dataflow-conda-plugin
+Version: 0.1.0

dataflow_conda_plugin-0.1.0/README.md

@@ -0,0 +1,2 @@
+# dataflow-conda-plugin
+Conda plugin which install dataflow dependencies in every newly created conda environment.

dataflow_conda_plugin-0.1.0/dataflow_conda_plugin.egg-info/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.4
+Name: dataflow-conda-plugin
+Version: 0.1.0

dataflow_conda_plugin-0.1.0/dataflow_conda_plugin.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+setup.py
+dataflow_conda_plugin.egg-info/PKG-INFO
+dataflow_conda_plugin.egg-info/SOURCES.txt
+dataflow_conda_plugin.egg-info/dependency_links.txt
+dataflow_conda_plugin.egg-info/entry_points.txt
+dataflow_conda_plugin.egg-info/top_level.txt
+plugin/__init__.py
+plugin/plugin.py
+plugin/scripts/install_dataflow_deps.sh

dataflow_conda_plugin-0.1.0/dataflow_conda_plugin.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

dataflow_conda_plugin-0.1.0/dataflow_conda_plugin.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[conda]
+dataflow-conda-plugin = plugin.plugin

dataflow_conda_plugin-0.1.0/dataflow_conda_plugin.egg-info/top_level.txt

@@ -0,0 +1 @@
+plugin

dataflow_conda_plugin-0.1.0/plugin/plugin.py

@@ -0,0 +1,42 @@
+import subprocess, sys, pkg_resources, os
+from conda import plugins
+from conda.base.context import context
+
+def install_deps(command: str):
+    """Install dataflow dependencies."""
+    target_prefix = context.target_prefix
+    args = context._argparse_args
+    try:
+        # if cloning, skip the install
+        if (args.get('clone') is not None):
+            return
+        
+        install_dataflow_deps = pkg_resources.resource_filename('plugin', 'scripts/install_dataflow_deps.sh')
+        process = subprocess.Popen(
+            ["bash", install_dataflow_deps, target_prefix],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1
+        )
+        
+        for line in iter(process.stdout.readline, ''):
+            print(line, end='')
+            sys.stdout.flush()
+        
+        return_code = process.wait()
+        if return_code != 0:
+            print(f"Error in creating environment!!")
+        
+    except Exception as e:
+        print(f"An unexpected error occurred: {str(e)}")
+
+
+    
+@plugins.hookimpl
+def conda_post_commands():
+    yield plugins.CondaPostCommand(
+        name=f"install_deps_post_command",
+        action=install_deps,
+        run_for={"create"},
+    )

dataflow_conda_plugin-0.1.0/plugin/scripts/install_dataflow_deps.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+conda_env_path=$1
+
+py_version=$(${conda_env_path}/bin/python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
+                      
+${conda_env_path}/bin/pip install dash dash-renderer plotly flask flask-appbuilder typing jupyterhub oauthenticator jupyter-server-proxy jupyter streamlit ipython ipykernel ipython-sql jupysql psycopg2-binary dataflow-core dataflow-dbt    
+
+# 3. Install Dataflow Airflow to a separate path in environment 
+${conda_env_path}/bin/pip install \
+    --force-reinstall --root-user-action ignore \
+    --no-warn-conflicts dataflow-airflow==2.10.5 \
+    --target ${conda_env_path}/bin/airflow-libraries/
+
+files=(
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/config/profile.py 
+    ${conda_env_path}/lib/python${py_version}/site-packages/dbt/task/debug.py
+)
+for file in ${files[@]}
+do      
+    awk '{gsub("from dbt.clients.yaml_helper import load_yaml_text", "from dbt.dataflow_config.secrets_manager import load_yaml_text"); print}' $file > temp 
+    mv temp $file
+done
+
+echo "Environment Creation Successful"

dataflow_conda_plugin-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

dataflow_conda_plugin-0.1.0/setup.py

@@ -0,0 +1,24 @@
+from setuptools import setup, find_packages
+from setuptools.command.install import install
+import os
+
+class PostInstall(install):
+    """Post-installation script to set executable permissions for scripts."""
+    def run(self):
+        install.run(self)
+        install_dir = os.path.join(self.install_lib, 'plugin', 'scripts')
+        for filename in os.listdir(install_dir):
+            if filename.endswith('.sh'):
+                filepath = os.path.join(install_dir, filename)
+                os.chmod(filepath, 0o755)
+
+setup(
+    name="dataflow-conda-plugin",
+    version="0.1.0",
+    entry_points={"conda": ["dataflow-conda-plugin = plugin.plugin"]},
+    packages=find_packages(include=["plugin"]),
+    package_data={'plugin': ['scripts/*.sh']},
+    cmdclass={
+        'install': PostInstall,
+    },
+)