demo_uc_setup 0.1.0__tar.gz

demo_uc_setup-0.1.0/.gitignore

@@ -0,0 +1,62 @@
+# Created by https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=macos,visualstudiocode
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/macos,visualstudiocode
+
+.databricks
+*.pdf
+__pycache__
+.env
+.venv

demo_uc_setup-0.1.0/.python-version

@@ -0,0 +1 @@
+3.11

demo_uc_setup-0.1.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.4
+Name: demo_uc_setup
+Version: 0.1.0
+Summary: A reusable task-based framework for managing Unity Catalog in Databricks
+Requires-Python: >=3.11
+Requires-Dist: databricks-connect>=16.1.1
+Requires-Dist: databricks-sdk>=0.44.1
+Requires-Dist: pydantic-settings>=2.8.0
+Requires-Dist: pydantic>=2.10.6
+Description-Content-Type: text/markdown
+
+# Databricks Unity Catalog Setup Demo
+
+A Python package that demonstrates automated setup and teardown of Databricks Unity Catalog resources using the Databricks SDK. This package provides a reusable framework for managing Unity Catalog resources programmatically, both from local environments and within Databricks notebooks.
+
+## Features
+
+- Automated creation of Unity Catalog resources:
+  - Catalogs
+  - Schemas
+  - Volumes
+- Configurable resource naming via environment variables
+- Support for both local execution and Databricks notebook execution
+- Type-safe configuration management using Pydantic
+- Clean teardown functionality
+
+## Prerequisites
+
+- Python 3.8+
+- A Databricks workspace with Unity Catalog enabled
+- Appropriate permissions to create/manage Unity Catalog resources
+
+## Installation
+
+```bash
+pip install demo-uc-setup
+```
+
+## Configuration
+
+The package uses environment variables for configuration. You can set these either in your environment or in a `.env` file:
+
+```env
+# Required for local execution (optional in Databricks notebooks)
+DATABRICKS_HOST=your-workspace-url
+DATABRICKS_TOKEN=your-pat-token
+
+# Optional - override default resource names
+DEMO_CATALOG_NAME=custom_catalog_name
+DEMO_SCHEMAS=["schema1", "schema2"]
+DEMO_VOLUME_NAME=custom_volume_name
+```
+
+## Usage
+
+### Local Execution
+
+```python
+from demo_uc_setup.unity_catalog_setup import UnityCatalogSetupTask
+from demo_uc_setup.unity_catalog_teardown import UnityCatalogTeardownTask
+
+# Setup Unity Catalog resources
+UnityCatalogSetupTask.entrypoint()
+
+# Teardown Unity Catalog resources
+UnityCatalogTeardownTask.entrypoint()
+```
+
+### Databricks Notebook Execution
+
+```python
+%pip install demo-uc-setup
+
+from demo_uc_setup.unity_catalog_setup import UnityCatalogSetupTask
+UnityCatalogSetupTask.entrypoint()
+```
+
+## Default Resource Names
+
+If not overridden via environment variables, the package will create:
+- A catalog named `demo_catalog`
+- Two schemas: `demo_schema_1` and `demo_schema_2`
+- A volume named `demo_volume` in each schema
+
+## Extending the Framework
+
+The package provides a reusable `Task` base class that can be extended for custom Unity Catalog operations:
+
+```python
+from demo_uc_setup.common import Task
+from demo_uc_setup.config import Config
+
+class CustomTask(Task[Config]):
+    def run(self):
+        self.logger.info("Starting custom task...")
+        # Your custom logic here
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

demo_uc_setup-0.1.0/README.md

@@ -0,0 +1,94 @@
+# Databricks Unity Catalog Setup Demo
+
+A Python package that demonstrates automated setup and teardown of Databricks Unity Catalog resources using the Databricks SDK. This package provides a reusable framework for managing Unity Catalog resources programmatically, both from local environments and within Databricks notebooks.
+
+## Features
+
+- Automated creation of Unity Catalog resources:
+  - Catalogs
+  - Schemas
+  - Volumes
+- Configurable resource naming via environment variables
+- Support for both local execution and Databricks notebook execution
+- Type-safe configuration management using Pydantic
+- Clean teardown functionality
+
+## Prerequisites
+
+- Python 3.8+
+- A Databricks workspace with Unity Catalog enabled
+- Appropriate permissions to create/manage Unity Catalog resources
+
+## Installation
+
+```bash
+pip install demo-uc-setup
+```
+
+## Configuration
+
+The package uses environment variables for configuration. You can set these either in your environment or in a `.env` file:
+
+```env
+# Required for local execution (optional in Databricks notebooks)
+DATABRICKS_HOST=your-workspace-url
+DATABRICKS_TOKEN=your-pat-token
+
+# Optional - override default resource names
+DEMO_CATALOG_NAME=custom_catalog_name
+DEMO_SCHEMAS=["schema1", "schema2"]
+DEMO_VOLUME_NAME=custom_volume_name
+```
+
+## Usage
+
+### Local Execution
+
+```python
+from demo_uc_setup.unity_catalog_setup import UnityCatalogSetupTask
+from demo_uc_setup.unity_catalog_teardown import UnityCatalogTeardownTask
+
+# Setup Unity Catalog resources
+UnityCatalogSetupTask.entrypoint()
+
+# Teardown Unity Catalog resources
+UnityCatalogTeardownTask.entrypoint()
+```
+
+### Databricks Notebook Execution
+
+```python
+%pip install demo-uc-setup
+
+from demo_uc_setup.unity_catalog_setup import UnityCatalogSetupTask
+UnityCatalogSetupTask.entrypoint()
+```
+
+## Default Resource Names
+
+If not overridden via environment variables, the package will create:
+- A catalog named `demo_catalog`
+- Two schemas: `demo_schema_1` and `demo_schema_2`
+- A volume named `demo_volume` in each schema
+
+## Extending the Framework
+
+The package provides a reusable `Task` base class that can be extended for custom Unity Catalog operations:
+
+```python
+from demo_uc_setup.common import Task
+from demo_uc_setup.config import Config
+
+class CustomTask(Task[Config]):
+    def run(self):
+        self.logger.info("Starting custom task...")
+        # Your custom logic here
+```
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

demo_uc_setup-0.1.0/configuration_approach.txt

@@ -0,0 +1,38 @@
+# Pythonic Config Management Approach
+Type variables are a key concept in generic programming, allowing for the creation of flexible and reusable code that can work with multiple data types while maintaining type safety. In the context of Python configuration management, type variables play a crucial role in implementing generic classes and methods, as demonstrated in the common.py file of the chatten project, where they are used to create a versatile Task class that can work with different configuration types.
+
+## Pythonic Configuration with Pydantic
+Pydantic's BaseSettings class forms the foundation of a flexible, type-safe configuration management system in the chatten project. This approach centralizes configuration parameters, leveraging Pydantic's automatic type validation and easy integration with environment variables and command-line arguments[1][2]. The configuration can be seamlessly imported into various tasks, as demonstrated in the loader.py and indexer.py files, while the databricks.yml file showcases parameterization for cloud deployment[3]. This method offers a balance between flexibility and type safety, particularly suitable for Python projects in cloud environments like Databricks, though it may not be ideal for scenarios requiring language-agnostic configurations.
+
+Citations:
+[1] https://docs.pydantic.dev/latest/api/pydantic_settings/
+[2] https://docs.pydantic.dev/2.4/concepts/pydantic_settings/
+[3] https://dzone.com/articles/order-in-chaos-python-configuration-management-for
+
+## Task Class Structure Explained
+The `Task` class in `common.py` serves as a foundation for various tasks within the `chatten_rag` package, reducing boilerplate code and providing a reusable structure. It utilizes a generic type `T`, bound to the `Config` class, allowing each task to have its own specific configuration while maintaining type safety. The class initializes common components such as SparkSession, logger, and Databricks WorkspaceClient[1]. A key feature is the dynamic creation of configuration instances in the `__init__` method, where `self.config: T = self.config_class()` instantiates the task-specific configuration[1]. The `entrypoint` class method offers a standardized way to execute tasks, handling logging, configuration setup, and Spark environment initialization[1].
+
+Citations:
+[1] https://github.com/renardeinside/chatten/blob/main/packages/chatten_rag/chatten_rag/common.py
+
+## Understanding Type Variables in Python
+Type variables serve as placeholders for specific types in generic programming, allowing for the creation of flexible and reusable code. They are typically denoted by single uppercase letters like T, U, or V, and can represent any non-primitive type, including class types, interface types, array types, or even other type variables[1][2]. In Python, type variables are defined using the TypeVar construct from the typing module, enabling the specification of generic types in type hints[3].
+
+* Enhance code reusability by enabling functions or classes to work with multiple types
+* Provide compile-time type checking, reducing runtime errors
+* Eliminate the need for type casting, potentially improving performance
+* Distinct from type parameters, which are formal declarations in class or method signatures[4]
+
+Citations:
+[1] https://stackoverflow.com/questions/42847287/what-is-type-variable-in-haskell-java
+[2] https://docs.oracle.com/javase/tutorial/java/generics/types.html
+[3] https://realpython.com/python-variables/
+[4] https://stackoverflow.com/questions/7075363/definition-of-type-variable-and-parameter
+
+## Example of Configuration Management in the Chatten Project
+The chatten project demonstrates an efficient approach to configuration management using Pydantic's BaseSettings. In the `config.py` file, a `Config` class is defined that inherits from `BaseSettings`, allowing for easy configuration sharing across multiple workflows and applications[1]. This class includes various settings such as database configurations, model parameters, and API keys, all with type annotations for improved safety and clarity.
+
+The configuration is then seamlessly integrated into task files like `loader.py` and `indexer.py`[1]. These tasks import the `Config` class and utilize its properties, demonstrating how easily the shared configuration can be referenced and used across different components of the project. This approach not only centralizes configuration management but also leverages Pydantic's built-in validation and environment variable integration, making it a flexible and maintainable solution for complex Python projects, particularly those deployed in cloud environments like Databricks.
+
+Citations:
+[1] https://gist.github.com/renardeinside

demo_uc_setup-0.1.0/demo_uc_setup/common.py

@@ -0,0 +1,49 @@
+import logging
+from typing import TypeVar, Generic
+from databricks.sdk import WorkspaceClient
+
+from demo_uc_setup.config import Config
+
+# Example of a type variable bound to our Config class
+T = TypeVar("T", bound=Config)
+
+class Task(Generic[T]):
+    """
+    A reusable Task base class that works both locally and in Databricks notebooks.
+    When running locally, requires databricks_host and databricks_token.
+    When running in a notebook, these parameters are optional.
+    """
+
+    def __init__(self, config_class: type[T]):
+        # Instantiate the typed configuration
+        self.config: T = config_class()
+        # Setup a basic logger
+        self.logger = logging.getLogger(self.__class__.__name__)
+        logging.basicConfig(level=logging.INFO)
+        
+        # Create a Databricks workspace client with or without credentials
+        if self.config.databricks_host and self.config.databricks_token:
+            # Local execution with credentials
+            self.workspace_client = WorkspaceClient(
+                host=self.config.databricks_host,
+                token=self.config.databricks_token
+            )
+        else:
+            # Notebook execution - no credentials needed
+            self.workspace_client = WorkspaceClient()
+
+    @classmethod
+    def entrypoint(cls, *args, **kwargs):
+        """
+        Creates an instance of the task and runs it. If you
+        want a consistent run pattern, place it here.
+        """
+        instance = cls(*args, **kwargs)
+        instance.run()
+
+    def run(self):
+        """
+        The main entrypoint for the task's execution.
+        Override this in subclasses to implement custom logic.
+        """
+        self.logger.info("Base Task run method. Override in subclasses.") 

demo_uc_setup-0.1.0/demo_uc_setup/config.py

@@ -0,0 +1,23 @@
+from pydantic_settings import BaseSettings
+from typing import Optional
+
+class Config(BaseSettings):
+    """
+    Configuration class using Pydantic BaseSettings.
+    By default, each field can be overridden by environment
+    variables matching the field name (in uppercase).
+    For example, DATABRICKS_HOST, DATABRICKS_TOKEN, etc.
+    """
+
+    # Databricks connection settings - optional for notebook execution
+    databricks_host: Optional[str] = None
+    databricks_token: Optional[str] = None
+
+    # Default names for Unity Catalog demo objects
+    demo_catalog_name: str = "demo_catalog"
+    demo_schemas: list[str] = ["demo_schema_1", "demo_schema_2"]  # List of schemas
+    demo_volume_name: str = "demo_volume"  # This could also be a list if needed
+
+    class Config:
+        env_file = ".env"  # or any custom file, if desired
+        env_file_encoding = "utf-8" 

demo_uc_setup-0.1.0/demo_uc_setup/unity_catalog_setup.py

@@ -0,0 +1,82 @@
+from typing import Type
+from databricks.sdk.service import catalog
+from demo_uc_setup.common import Task, T
+from demo_uc_setup.config import Config
+
+class UnityCatalogSetupTask(Task[Config]):
+    """
+    A task to ensure catalogs, schemas, and volumes exist
+    in the Databricks workspace. Uses typed config for
+    resource names, credentials, etc.
+    """
+
+    def __init__(self, config_class: Type[T] = Config):
+        super().__init__(config_class)
+
+    def run(self):
+        self.logger.info("Starting Unity Catalog setup...")
+
+        # 1) Ensure the catalog exists
+        catalog_name = self.config.demo_catalog_name
+        self.logger.info(f"Ensuring catalog '{catalog_name}'")
+        try:
+            self.workspace_client.catalogs.get(name=catalog_name)
+            self.logger.info(f"Catalog '{catalog_name}' already exists.")
+        except Exception:
+            self.logger.info(f"Catalog '{catalog_name}' not found; creating it.")
+            self.workspace_client.catalogs.create(
+                name=catalog_name,
+                comment="Demo Catalog for Databricks demos"
+            )
+
+        # 2) Ensure all schemas exist and create volumes within each schema
+        for schema_name in self.config.demo_schemas:
+            # Create schema
+            self.logger.info(f"Ensuring schema '{catalog_name}.{schema_name}'")
+            try:
+                self.workspace_client.schemas.get(
+                    name=schema_name,
+                    catalog_name=catalog_name
+                )
+                self.logger.info(f"Schema '{catalog_name}.{schema_name}' already exists.")
+            except Exception:
+                try:
+                    self.logger.info(f"Schema '{catalog_name}.{schema_name}' not found; creating it.")
+                    self.workspace_client.schemas.create(
+                        name=schema_name,
+                        catalog_name=catalog_name,
+                        comment=f"Demo Schema {schema_name} for Databricks demos"
+                    )
+                except Exception as e:
+                    if "already exists" in str(e):
+                        self.logger.info(f"Schema '{catalog_name}.{schema_name}' already exists (caught during creation).")
+                    else:
+                        raise e
+
+            # Create volume within this schema
+            volume_name = self.config.demo_volume_name
+            self.logger.info(f"Ensuring volume '{catalog_name}.{schema_name}.{volume_name}'")
+            try:
+                self.workspace_client.volumes.get(
+                    name=volume_name,
+                    catalog_name=catalog_name,
+                    schema_name=schema_name
+                )
+                self.logger.info(f"Volume '{catalog_name}.{schema_name}.{volume_name}' already exists.")
+            except Exception:
+                try:
+                    self.logger.info(f"Volume '{catalog_name}.{schema_name}.{volume_name}' not found; creating it.")
+                    self.workspace_client.volumes.create(
+                        name=volume_name,
+                        catalog_name=catalog_name,
+                        schema_name=schema_name,
+                        volume_type=catalog.VolumeType.MANAGED,
+                        comment=f"Demo Volume for schema {schema_name}"
+                    )
+                except Exception as e:
+                    if "already exists" in str(e):
+                        self.logger.info(f"Volume '{catalog_name}.{schema_name}.{volume_name}' already exists (caught during creation).")
+                    else:
+                        raise e
+
+        self.logger.info("Unity Catalog setup complete!") 

demo_uc_setup-0.1.0/demo_uc_setup/unity_catalog_teardown.py

@@ -0,0 +1,28 @@
+from typing import Type
+from databricks.sdk import WorkspaceClient
+from demo_uc_setup.common import Task, T
+from demo_uc_setup.config import Config
+
+class UnityCatalogTeardownTask(Task[Config]):
+    """
+    A task to delete (teardown) the Unity Catalog resources in
+    the configured Databricks workspace. Uses typed config for
+    resource names, credentials, etc.
+    """
+
+    def __init__(self, config_class: Type[T] = Config):
+        super().__init__(config_class)
+
+    def run(self):
+        self.logger.info("Starting teardown of Unity Catalog resources...")
+
+        catalog_name = self.config.demo_catalog_name
+        self.logger.info(f"Deleting catalog '{catalog_name}' and its dependencies (force=True).")
+
+        try:
+            self.workspace_client.catalogs.delete(name=catalog_name, force=True)
+            self.logger.info(f"Catalog '{catalog_name}' (and its contents) successfully deleted.")
+        except Exception as e:
+            self.logger.error(f"Failed to delete catalog '{catalog_name}'. Reason: {e}")
+
+        self.logger.info("Unity Catalog teardown complete!") 

demo_uc_setup-0.1.0/main.py

@@ -0,0 +1,31 @@
+"""
+Main entrypoint to run the Unity Catalog resource creation using
+the Databricks Python SDK and the flexible Pydantic-based config.
+
+Usage:
+    python main.py setup     # Run setup
+    python main.py teardown  # Run teardown
+"""
+
+import sys
+from demo_uc_setup.unity_catalog_setup import UnityCatalogSetupTask
+from demo_uc_setup.unity_catalog_teardown import UnityCatalogTeardownTask
+
+def print_usage():
+    print("Usage: python main.py [setup|teardown]")
+    print("  setup    - Run Unity Catalog setup")
+    print("  teardown - Run Unity Catalog teardown")
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2 or sys.argv[1] not in ["setup", "teardown"]:
+        print_usage()
+        sys.exit(1)
+
+    if sys.argv[1] == "setup":
+        UnityCatalogSetupTask.entrypoint()
+    else:
+        UnityCatalogTeardownTask.entrypoint()
+
+    # Option B) Or instantiate directly:
+    # task = UnityCatalogSetupTask()
+    # task.run() 

demo_uc_setup-0.1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "demo_uc_setup"
+version = "0.1.0"
+description = "A reusable task-based framework for managing Unity Catalog in Databricks"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "databricks-connect>=16.1.1",
+    "databricks-sdk>=0.44.1",
+    "pydantic>=2.10.6",
+    "pydantic-settings>=2.8.0",
+]
+
+[dependency-groups]
+dev = [
+    "hatch>=1.14.0",
+]

demo_uc_setup-0.1.0/setup.py

@@ -0,0 +1,7 @@
+"""
+Script to run the Unity Catalog setup process.
+"""
+from demo_uc_setup.unity_catalog_setup import UnityCatalogSetupTask
+
+if __name__ == "__main__":
+    UnityCatalogSetupTask.entrypoint() 

demo_uc_setup-0.1.0/teardown.py

@@ -0,0 +1,7 @@
+"""
+Script to run the Unity Catalog teardown process.
+"""
+from demo_uc_setup.unity_catalog_teardown import UnityCatalogTeardownTask
+
+if __name__ == "__main__":
+    UnityCatalogTeardownTask.entrypoint()