prefect-databricks 0.2.3__tar.gz → 0.2.4__tar.gz

prefect_databricks-0.2.4/PKG-INFO

@@ -0,0 +1,210 @@
+Metadata-Version: 2.1
+Name: prefect-databricks
+Version: 0.2.4
+Summary: Prefect integrations with Databricks
+Author-email: "Prefect Technologies, Inc." <help@prefect.io>
+License: Apache License 2.0
+Project-URL: Homepage, https://github.com/PrefectHQ/prefect/tree/main/src/integrations/prefect-databricks
+Keywords: prefect
+Classifier: Natural Language :: English
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: prefect>=2.14.10
+Provides-Extra: dev
+Requires-Dist: coverage; extra == "dev"
+Requires-Dist: interrogate; extra == "dev"
+Requires-Dist: mkdocs-gen-files; extra == "dev"
+Requires-Dist: mkdocs-material; extra == "dev"
+Requires-Dist: mkdocs; extra == "dev"
+Requires-Dist: mkdocstrings[python]; extra == "dev"
+Requires-Dist: mock; python_version < "3.8" and extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: pillow; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pytest-asyncio; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-xdist; extra == "dev"
+Requires-Dist: respx; extra == "dev"
+
+# prefect-databricks
+
+Visit the full docs [here](https://PrefectHQ.github.io/prefect-databricks) to see additional examples and the API reference.
+ 
+<p align="center">
+    <a href="https://pypi.python.org/pypi/prefect-databricks/" alt="PyPI version">
+        <img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-databricks?color=0052FF&labelColor=090422"></a>
+    <a href="https://pepy.tech/badge/prefect-databricks/" alt="Downloads">
+        <img src="https://img.shields.io/pypi/dm/prefect-databricks?color=0052FF&labelColor=090422" /></a>
+    <a href="https://github.com/PrefectHQ/prefect-databricks/pulse" alt="Activity">
+</p>
+
+## Welcome!
+
+Prefect integrations for interacting with Databricks
+
+The tasks within this collection were created by a code generator using the service's OpenAPI spec.
+
+The service's REST API documentation can be found [here](https://docs.databricks.com/dev-tools/api/latest/index.html).
+
+## Getting Started
+
+### Python setup
+
+Requires an installation of Python 3.7+.
+
+We recommend using a Python virtual environment manager such as pipenv, conda or virtualenv.
+
+These tasks are designed to work with Prefect 2. For more information about how to use Prefect, please refer to the [Prefect documentation](https://orion-docs.prefect.io/).
+
+### Installation
+
+Install `prefect-databricks` with `pip`:
+
+```bash
+pip install prefect-databricks
+```
+
+A list of available blocks in `prefect-databricks` and their setup instructions can be found [here](https://PrefectHQ.github.io/prefect-databricks/#blocks-catalog).
+
+### Lists jobs on the Databricks instance
+
+```python
+from prefect import flow
+from prefect_databricks import DatabricksCredentials
+from prefect_databricks.jobs import jobs_list
+
+
+@flow
+def example_execute_endpoint_flow():
+    databricks_credentials = DatabricksCredentials.load("my-block")
+    jobs = jobs_list(
+        databricks_credentials,
+        limit=5
+    )
+    return jobs
+
+example_execute_endpoint_flow()
+```
+
+### Use `with_options` to customize options on any existing task or flow
+
+```python
+custom_example_execute_endpoint_flow = example_execute_endpoint_flow.with_options(
+    name="My custom flow name",
+    retries=2,
+    retry_delay_seconds=10,
+)
+```
+
+### Launch a new cluster and run a Databricks notebook
+
+Notebook named `example.ipynb` on Databricks which accepts a name parameter:
+
+```python
+name = dbutils.widgets.get("name")
+message = f"Don't worry {name}, I got your request! Welcome to prefect-databricks!"
+print(message)
+```
+
+Prefect flow that launches a new cluster to run `example.ipynb`:
+
+```python
+from prefect import flow
+from prefect_databricks import DatabricksCredentials
+from prefect_databricks.jobs import jobs_runs_submit
+from prefect_databricks.models.jobs import (
+    AutoScale,
+    AwsAttributes,
+    JobTaskSettings,
+    NotebookTask,
+    NewCluster,
+)
+
+
+@flow
+def jobs_runs_submit_flow(notebook_path, **base_parameters):
+    databricks_credentials = DatabricksCredentials.load("my-block")
+
+    # specify new cluster settings
+    aws_attributes = AwsAttributes(
+        availability="SPOT",
+        zone_id="us-west-2a",
+        ebs_volume_type="GENERAL_PURPOSE_SSD",
+        ebs_volume_count=3,
+        ebs_volume_size=100,
+    )
+    auto_scale = AutoScale(min_workers=1, max_workers=2)
+    new_cluster = NewCluster(
+        aws_attributes=aws_attributes,
+        autoscale=auto_scale,
+        node_type_id="m4.large",
+        spark_version="10.4.x-scala2.12",
+        spark_conf={"spark.speculation": True},
+    )
+
+    # specify notebook to use and parameters to pass
+    notebook_task = NotebookTask(
+        notebook_path=notebook_path,
+        base_parameters=base_parameters,
+    )
+
+    # compile job task settings
+    job_task_settings = JobTaskSettings(
+        new_cluster=new_cluster,
+        notebook_task=notebook_task,
+        task_key="prefect-task"
+    )
+
+    run = jobs_runs_submit(
+        databricks_credentials=databricks_credentials,
+        run_name="prefect-job",
+        tasks=[job_task_settings]
+    )
+
+    return run
+
+
+jobs_runs_submit_flow("/Users/username@gmail.com/example.ipynb", name="Marvin")
+```
+
+Note, instead of using the built-in models, you may also input valid JSON. For example, `AutoScale(min_workers=1, max_workers=2)` is equivalent to `{"min_workers": 1, "max_workers": 2}`.
+
+For more tips on how to use tasks and flows in a Collection, check out [Using Collections](https://orion-docs.prefect.io/collections/usage/)!
+
+## Resources
+
+If you encounter any bugs while using `prefect-databricks`, feel free to open an issue in the [prefect-databricks](https://github.com/PrefectHQ/prefect-databricks) repository.
+
+If you have any questions or issues while using `prefect-databricks`, you can find help in either the [Prefect Discourse forum](https://discourse.prefect.io/) or the [Prefect Slack community](https://prefect.io/slack).
+
+Feel free to star or watch [`prefect-databricks`](https://github.com/PrefectHQ/prefect-databricks) for updates too!
+
+## Contributing
+
+If you'd like to help contribute to fix an issue or add a feature to `prefect-databricks`, please [propose changes through a pull request from a fork of the repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork).
+
+Here are the steps:
+1. [Fork the repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository)
+2. [Clone the forked repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo#cloning-your-forked-repository)
+3. Install the repository and its dependencies:
+```
+pip install -e ".[dev]"
+```
+4. Make desired changes
+5. Add tests
+6. Insert an entry to [CHANGELOG.md](https://github.com/PrefectHQ/prefect-databricks/blob/main/CHANGELOG.md)
+7. Install `pre-commit` to perform quality checks prior to commit:
+```
+pre-commit install
+```
+8. `git commit`, `git push`, and create a pull request

prefect_databricks-0.2.4/README.md

@@ -0,0 +1,172 @@
+# prefect-databricks
+
+Visit the full docs [here](https://PrefectHQ.github.io/prefect-databricks) to see additional examples and the API reference.
+ 
+<p align="center">
+    <a href="https://pypi.python.org/pypi/prefect-databricks/" alt="PyPI version">
+        <img alt="PyPI" src="https://img.shields.io/pypi/v/prefect-databricks?color=0052FF&labelColor=090422"></a>
+    <a href="https://pepy.tech/badge/prefect-databricks/" alt="Downloads">
+        <img src="https://img.shields.io/pypi/dm/prefect-databricks?color=0052FF&labelColor=090422" /></a>
+    <a href="https://github.com/PrefectHQ/prefect-databricks/pulse" alt="Activity">
+</p>
+
+## Welcome!
+
+Prefect integrations for interacting with Databricks
+
+The tasks within this collection were created by a code generator using the service's OpenAPI spec.
+
+The service's REST API documentation can be found [here](https://docs.databricks.com/dev-tools/api/latest/index.html).
+
+## Getting Started
+
+### Python setup
+
+Requires an installation of Python 3.7+.
+
+We recommend using a Python virtual environment manager such as pipenv, conda or virtualenv.
+
+These tasks are designed to work with Prefect 2. For more information about how to use Prefect, please refer to the [Prefect documentation](https://orion-docs.prefect.io/).
+
+### Installation
+
+Install `prefect-databricks` with `pip`:
+
+```bash
+pip install prefect-databricks
+```
+
+A list of available blocks in `prefect-databricks` and their setup instructions can be found [here](https://PrefectHQ.github.io/prefect-databricks/#blocks-catalog).
+
+### Lists jobs on the Databricks instance
+
+```python
+from prefect import flow
+from prefect_databricks import DatabricksCredentials
+from prefect_databricks.jobs import jobs_list
+
+
+@flow
+def example_execute_endpoint_flow():
+    databricks_credentials = DatabricksCredentials.load("my-block")
+    jobs = jobs_list(
+        databricks_credentials,
+        limit=5
+    )
+    return jobs
+
+example_execute_endpoint_flow()
+```
+
+### Use `with_options` to customize options on any existing task or flow
+
+```python
+custom_example_execute_endpoint_flow = example_execute_endpoint_flow.with_options(
+    name="My custom flow name",
+    retries=2,
+    retry_delay_seconds=10,
+)
+```
+
+### Launch a new cluster and run a Databricks notebook
+
+Notebook named `example.ipynb` on Databricks which accepts a name parameter:
+
+```python
+name = dbutils.widgets.get("name")
+message = f"Don't worry {name}, I got your request! Welcome to prefect-databricks!"
+print(message)
+```
+
+Prefect flow that launches a new cluster to run `example.ipynb`:
+
+```python
+from prefect import flow
+from prefect_databricks import DatabricksCredentials
+from prefect_databricks.jobs import jobs_runs_submit
+from prefect_databricks.models.jobs import (
+    AutoScale,
+    AwsAttributes,
+    JobTaskSettings,
+    NotebookTask,
+    NewCluster,
+)
+
+
+@flow
+def jobs_runs_submit_flow(notebook_path, **base_parameters):
+    databricks_credentials = DatabricksCredentials.load("my-block")
+
+    # specify new cluster settings
+    aws_attributes = AwsAttributes(
+        availability="SPOT",
+        zone_id="us-west-2a",
+        ebs_volume_type="GENERAL_PURPOSE_SSD",
+        ebs_volume_count=3,
+        ebs_volume_size=100,
+    )
+    auto_scale = AutoScale(min_workers=1, max_workers=2)
+    new_cluster = NewCluster(
+        aws_attributes=aws_attributes,
+        autoscale=auto_scale,
+        node_type_id="m4.large",
+        spark_version="10.4.x-scala2.12",
+        spark_conf={"spark.speculation": True},
+    )
+
+    # specify notebook to use and parameters to pass
+    notebook_task = NotebookTask(
+        notebook_path=notebook_path,
+        base_parameters=base_parameters,
+    )
+
+    # compile job task settings
+    job_task_settings = JobTaskSettings(
+        new_cluster=new_cluster,
+        notebook_task=notebook_task,
+        task_key="prefect-task"
+    )
+
+    run = jobs_runs_submit(
+        databricks_credentials=databricks_credentials,
+        run_name="prefect-job",
+        tasks=[job_task_settings]
+    )
+
+    return run
+
+
+jobs_runs_submit_flow("/Users/username@gmail.com/example.ipynb", name="Marvin")
+```
+
+Note, instead of using the built-in models, you may also input valid JSON. For example, `AutoScale(min_workers=1, max_workers=2)` is equivalent to `{"min_workers": 1, "max_workers": 2}`.
+
+For more tips on how to use tasks and flows in a Collection, check out [Using Collections](https://orion-docs.prefect.io/collections/usage/)!
+
+## Resources
+
+If you encounter any bugs while using `prefect-databricks`, feel free to open an issue in the [prefect-databricks](https://github.com/PrefectHQ/prefect-databricks) repository.
+
+If you have any questions or issues while using `prefect-databricks`, you can find help in either the [Prefect Discourse forum](https://discourse.prefect.io/) or the [Prefect Slack community](https://prefect.io/slack).
+
+Feel free to star or watch [`prefect-databricks`](https://github.com/PrefectHQ/prefect-databricks) for updates too!
+
+## Contributing
+
+If you'd like to help contribute to fix an issue or add a feature to `prefect-databricks`, please [propose changes through a pull request from a fork of the repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork).
+
+Here are the steps:
+1. [Fork the repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo#forking-a-repository)
+2. [Clone the forked repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo#cloning-your-forked-repository)
+3. Install the repository and its dependencies:
+```
+pip install -e ".[dev]"
+```
+4. Make desired changes
+5. Add tests
+6. Insert an entry to [CHANGELOG.md](https://github.com/PrefectHQ/prefect-databricks/blob/main/CHANGELOG.md)
+7. Install `pre-commit` to perform quality checks prior to commit:
+```
+pre-commit install
+```
+8. `git commit`, `git push`, and create a pull request

prefect_databricks-0.2.4/prefect_databricks/__init__.py

@@ -0,0 +1 @@
+from .credentials import DatabricksCredentials  # noqa

prefect_databricks-0.2.4/prefect_databricks/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.2.4'
+__version_tuple__ = version_tuple = (0, 2, 4)

{prefect-databricks-0.2.3 → prefect_databricks-0.2.4}/prefect_databricks/credentials.py

@@ -3,14 +3,16 @@
 from typing import Any, Dict, Optional
 
 from httpx import AsyncClient
-from prefect.blocks.core import Block
-from pydantic import VERSION as PYDANTIC_VERSION
 
-if PYDANTIC_VERSION.startswith("2."):
+from prefect._internal.pydantic import HAS_PYDANTIC_V2
+
+if HAS_PYDANTIC_V2:
     from pydantic.v1 import Field, SecretStr
 else:
     from pydantic import Field, SecretStr
 
+from prefect.blocks.core import Block
+
 
 class DatabricksCredentials(Block):
     """
@@ -31,8 +33,7 @@ class DatabricksCredentials(Block):
     """
 
     _block_type_name = "Databricks Credentials"
-    _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/ff9a2573c23954bedd27b0f420465a55b1a99dfd-250x250.png"  # noqa
-    _documentation_url = "https://prefecthq.github.io/prefect-databricks/credentials/#prefect_databricks.credentials.DatabricksCredentials"  # noqa
+    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/5GTHI1PH2dTiantfps6Fnc/1c750fab7f4c14ea1b93a62b9fea6a94/databricks_logo_icon_170295.png?h=250"  # noqa
 
     databricks_instance: str = Field(
         default=...,