terrafox-datalake 0.1.0__tar.gz

terrafox_datalake-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: terrafox-datalake
+Version: 0.1.0
+Summary: Automated connector wrapper for streaming data securely from a private MinIO Data Lake
+Author: Sethu Gopalan
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: s3fs
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Terrafox Data Lake Connector
+A simple, secure wrapper module to stream files out of your private data lake into remote notebook runtimes seamlessly.

terrafox_datalake-0.1.0/README.md

@@ -0,0 +1,2 @@
+# Terrafox Data Lake Connector
+A simple, secure wrapper module to stream files out of your private data lake into remote notebook runtimes seamlessly.

terrafox_datalake-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

terrafox_datalake-0.1.0/setup.py

@@ -0,0 +1,21 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="terrafox-datalake", 
+    version="0.1.0",
+    author="Sethu Gopalan",
+    description="Automated connector wrapper for streaming data securely from a private MinIO Data Lake",
+    long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
+    packages=find_packages(),
+    install_requires=[
+        "pandas",
+        "s3fs"
+    ],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires=">=3.7",
+)

terrafox_datalake-0.1.0/terrafox_datalake/__init__.py

@@ -0,0 +1,39 @@
+import os
+import pandas as pd
+from functools import partial
+
+ENDPOINT = "https://minio.terrafoxai.com"
+
+def connect(user=None, password=None):
+    user = user or os.environ.get("MINIO_USER")
+    password = password or os.environ.get("MINIO_PASSWORD")
+    
+    if not user or not password:
+        import getpass
+        print("🔐 Data Lake credentials not found in environment settings.")
+        user = user or input("Enter MinIO Username: ")
+        password = password or getpass.getpass("Enter MinIO Password: ")
+
+    os.environ["AWS_ACCESS_KEY_ID"] = user
+    os.environ["AWS_SECRET_ACCESS_KEY"] = password
+    
+    pd.read_csv = partial(pd.read_csv, storage_options={
+        "key": user, 
+        "secret": password, 
+        "client_kwargs": {"endpoint_url": ENDPOINT}
+    })
+    
+    print("🚀 Terrafox Data Lake Connected Natively via Background Environment!")
+
+def get_spark(app_name="Terrafox-DataLake"):
+    from pyspark.sql import SparkSession
+    user = os.environ.get("AWS_ACCESS_KEY_ID")
+    password = os.environ.get("AWS_SECRET_ACCESS_KEY")
+    return SparkSession.builder \
+        .appName(app_name) \
+        .config("spark.hadoop.fs.s3a.endpoint", ENDPOINT) \
+        .config("spark.hadoop.fs.s3a.access.key", user) \
+        .config("spark.hadoop.fs.s3a.secret.key", password) \
+        .config("spark.hadoop.fs.s3a.path.style.access", "true") \
+        .config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") \
+        .getOrCreate()

terrafox_datalake-0.1.0/terrafox_datalake.egg-info/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: terrafox-datalake
+Version: 0.1.0
+Summary: Automated connector wrapper for streaming data securely from a private MinIO Data Lake
+Author: Sethu Gopalan
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: pandas
+Requires-Dist: s3fs
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Terrafox Data Lake Connector
+A simple, secure wrapper module to stream files out of your private data lake into remote notebook runtimes seamlessly.

terrafox_datalake-0.1.0/terrafox_datalake.egg-info/SOURCES.txt

@@ -0,0 +1,8 @@
+README.md
+setup.py
+terrafox_datalake/__init__.py
+terrafox_datalake.egg-info/PKG-INFO
+terrafox_datalake.egg-info/SOURCES.txt
+terrafox_datalake.egg-info/dependency_links.txt
+terrafox_datalake.egg-info/requires.txt
+terrafox_datalake.egg-info/top_level.txt

terrafox_datalake-0.1.0/terrafox_datalake.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

terrafox_datalake-0.1.0/terrafox_datalake.egg-info/requires.txt

@@ -0,0 +1,2 @@
+pandas
+s3fs

terrafox_datalake-0.1.0/terrafox_datalake.egg-info/top_level.txt

@@ -0,0 +1 @@
+terrafox_datalake