airflow-toolkit 2.1.0__tar.gz → 2.3.0__tar.gz

{airflow_toolkit-2.1.0/src/airflow_toolkit.egg-info → airflow_toolkit-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: airflow-toolkit
-Version: 2.1.0
+Version: 2.3.0
 Summary: A toolkit of operators, hooks and utilities for Apache Airflow 3
 Author-email: Biel Llobera <biel_llobera@dkl.digital>
 Requires-Python: <3.15,>=3.11
@@ -23,8 +23,6 @@ Provides-Extra: azure
 Requires-Dist: apache-airflow-providers-microsoft-azure>=8; extra == "azure"
 Provides-Extra: sftp
 Requires-Dist: apache-airflow-providers-sftp>=5.3; extra == "sftp"
-Provides-Extra: slack
-Requires-Dist: apache-airflow-providers-slack>=9.3; extra == "slack"
 Provides-Extra: http
 Requires-Dist: apache-airflow-providers-http>=5.6.4; extra == "http"
 Requires-Dist: requests>=2.31.0; extra == "http"
@@ -34,6 +32,10 @@ Provides-Extra: duckdb
 Requires-Dist: airflow-provider-duckdb>=0.1.2; extra == "duckdb"
 Provides-Extra: sqlite
 Requires-Dist: apache-airflow-providers-sqlite; extra == "sqlite"
+Provides-Extra: excel
+Requires-Dist: openpyxl>=3.1; extra == "excel"
+Provides-Extra: avro
+Requires-Dist: fastavro>=1.9; extra == "avro"
 Provides-Extra: airflow3-full
 Requires-Dist: apache-airflow<4,>=3; extra == "airflow3-full"
 Requires-Dist: apache-airflow-providers-fab>=3.0.0; extra == "airflow3-full"
@@ -46,12 +48,13 @@ Requires-Dist: apache-airflow-providers-amazon>=9.15.0; extra == "airflow3-full"
 Requires-Dist: apache-airflow-providers-google>=18; extra == "airflow3-full"
 Requires-Dist: apache-airflow-providers-microsoft-azure>=8; extra == "airflow3-full"
 Requires-Dist: apache-airflow-providers-sftp>=5.3; extra == "airflow3-full"
-Requires-Dist: apache-airflow-providers-slack>=9.3; extra == "airflow3-full"
 Requires-Dist: apache-airflow-providers-http>=5.6.4; extra == "airflow3-full"
 Requires-Dist: requests>=2.31.0; extra == "airflow3-full"
 Requires-Dist: jmespath<2,>=1.0.1; extra == "airflow3-full"
 Requires-Dist: airflow-provider-duckdb>=0.1.2; extra == "airflow3-full"
 Requires-Dist: apache-airflow-providers-sqlite; extra == "airflow3-full"
+Requires-Dist: openpyxl>=3.1; extra == "airflow3-full"
+Requires-Dist: fastavro>=1.9; extra == "airflow3-full"
 Dynamic: license-file
 
 # Airflow Toolkit
@@ -139,10 +142,11 @@ pip install "airflow-toolkit[airflow3-full]"
 | `google` | `providers-google` | GCS filesystem backend |
 | `azure` | `providers-microsoft-azure` | Azure Blob / ADLS filesystem backend |
 | `sftp` | `providers-sftp` | SFTP filesystem backend |
-| `slack` | `providers-slack` | Slack failure notifications |
 | `http` | `providers-http`, `requests`, `jmespath`, `pandas` | `HttpToFilesystem`, `MultiHttpToFilesystem` |
 | `duckdb` | `airflow-provider-duckdb` | `DuckdbToDeltalake` operator |
 | `sqlite` | `providers-sqlite` | SQLite as source or destination |
+| `excel` | `openpyxl` | Excel (`.xlsx` / `.xls`) support in `FilesystemToDatabase` |
+| `avro` | `fastavro` | Avro support in `FilesystemToDatabase` |
 | `airflow3-full` | all of the above | Quick start / development |
 
 ---
@@ -316,7 +320,9 @@ FilesystemToFilesystem(
 
 ### FilesystemToDatabase
 
-Reads files (CSV, JSON, or Parquet) from any filesystem and loads them into any SQLAlchemy-compatible database. Handles schema drift automatically: columns present in the file but missing from the table are added; columns present in the table but missing from the file are filled with `NULL`.
+Reads files from any filesystem and loads them into any SQLAlchemy-compatible database. Handles schema drift automatically: columns present in the file but missing from the table are added; columns present in the table but missing from the file are filled with `NULL`.
+
+**Supported formats:** `csv`, `json`, `parquet`, `excel`, `avro`, `fixed_width`.
 
 ```python
 from airflow_toolkit.providers.deltalake.operators.filesystem_to_database import FilesystemToDatabaseOperator
@@ -328,7 +334,7 @@ FilesystemToDatabaseOperator(
     filesystem_path='raw/orders/{{ ds }}/',
     db_schema='public',
     db_table='orders',
-    source_format='csv',
+    source_format='csv',                   # 'csv' | 'json' | 'parquet' | 'excel' | 'avro' | 'fixed_width'
     table_aggregation_type='append',       # 'append' | 'replace' | 'fail'
     metadata={
         '_ds':          '{{ ds }}',
@@ -338,6 +344,52 @@ FilesystemToDatabaseOperator(
 )
 ```
 
+**Excel** (requires the `[excel]` extra):
+
+```python
+FilesystemToDatabaseOperator(
+    task_id='load_excel_report',
+    filesystem_conn_id='my_data_lake',
+    database_conn_id='my_postgres',
+    filesystem_path='raw/reports/{{ ds }}/',
+    db_table='monthly_report',
+    source_format='excel',
+    source_format_options={'sheet_name': 'Data'},
+)
+```
+
+**Avro** (requires the `[avro]` extra):
+
+```python
+FilesystemToDatabaseOperator(
+    task_id='load_avro_events',
+    filesystem_conn_id='my_data_lake',
+    database_conn_id='my_postgres',
+    filesystem_path='raw/events/{{ ds }}/',
+    db_table='events',
+    source_format='avro',
+)
+```
+
+**Fixed-width** (no extra required — pandas native):
+
+```python
+FilesystemToDatabaseOperator(
+    task_id='load_fixed_width',
+    filesystem_conn_id='my_data_lake',
+    database_conn_id='my_postgres',
+    filesystem_path='raw/exports/{{ ds }}/',
+    db_table='transactions',
+    source_format='fixed_width',
+    source_format_options={
+        'colspecs': [(0, 10), (10, 25), (25, 35)],
+        'names': ['date', 'description', 'amount'],
+    },
+)
+```
+
+Each format is matched by file extension: `.csv`/`.csv.gz`, `.json`/`.json.gz`, `.parquet`/`.parquet.gz`, `.xlsx`/`.xls`, `.avro`, `.fwf`/`.txt`/`.dat`. Files with other extensions in the same prefix are silently skipped.
+
 ### DuckdbToDeltalake
 
 Executes a DuckDB SQL query and writes the result directly to a Delta Lake table on Azure storage. Useful for in-process transformations that land results as an open table format.
@@ -437,74 +489,143 @@ Because `AzureDatabricksSqlHook` implements `DbApiHook`, it can be used as `sour
 
 ## Notifications
 
-### Slack (incoming webhook)
+Send rich failure notifications to Slack, email, Microsoft Teams, and Discord from a single call. The notification system is built around three ideas:
 
-Send DAG or task failure alerts to a Slack channel using `dag_failure_slack_notification_webhook`. Requires a Slack App with Incoming Webhooks enabled.
+1. **Context builder** — extracts DAG run metadata (run ID, logical date, schedule, interval, duration, environment) from the Airflow callback context once, and makes it available to all channels.
+2. **Channel formatters** — each channel (Slack Block Kit, HTML email, Teams Adaptive Card, Discord embed) formats the same context into the right payload for that platform.
+3. **Two usage patterns** — as an `on_failure_callback` (invisible to the graph) or as an explicit `@task` node in the DAG graph.
 
-Create an Airflow connection named `SLACK_WEBHOOK_NOTIFICATION_CONN` (or set `AIRFLOW_CONN_SLACK_WEBHOOK_NOTIFICATION_CONN`).
+### Pattern A — `on_failure_callback`
 
-#### DAG-level notification
+The callback fires automatically when any task in the DAG fails. Nothing appears in the task graph.
 
 ```python
-from datetime import datetime, timedelta
-from airflow.sdk import DAG
-from airflow.providers.standard.operators.bash import BashOperator
-from airflow_toolkit.notifications.slack.webhook import dag_failure_slack_notification_webhook
+from airflow_toolkit.notifications import dag_failure_notification
 
 with DAG(
     'my_pipeline',
-    schedule=timedelta(days=1),
-    start_date=datetime(2024, 1, 1),
-    catchup=False,
-    on_failure_callback=dag_failure_slack_notification_webhook(),
-) as dag:
-
-    t = BashOperator(task_id='run', bash_command='python my_script.py')
+    schedule='0 6 * * *',
+    on_failure_callback=dag_failure_notification(
+        channels=['slack', 'email'],
+        environment='PROD',
+        slack_webhook_url='https://hooks.slack.com/services/...',
+        email_to=['data-team@example.com'],
+    ),
+):
+    ...
 ```
 
-#### Task-level notification
+### Pattern B — explicit task in the graph
+
+`get_failure_notification_task` returns an Airflow task with `trigger_rule='one_failed'`. It fires when any upstream task fails and is **skipped** when all tasks succeed. The notification step is visible in the Airflow UI, has its own logs, and appears in the task history.
 
 ```python
-BashOperator(
-    task_id='run',
-    bash_command='python my_script.py',
-    on_failure_callback=dag_failure_slack_notification_webhook(source='TASK'),
-)
+from airflow_toolkit.notifications import get_failure_notification_task
+
+with DAG('my_pipeline', schedule='0 6 * * *'):
+    extract = ...
+    load    = ...
+
+    notify = get_failure_notification_task(
+        channels=['slack', 'email'],
+        environment='PROD',
+        slack_webhook_url='https://hooks.slack.com/services/...',
+        email_to=['data-team@example.com'],
+    )
+
+    [extract, load] >> notify
 ```
 
-#### Custom message
+### Supported channels
+
+| Channel | Parameter | Requires |
+|---|---|---|
+| `slack` | `slack_webhook_url` | — |
+| `email` | `email_to: list[str]`, `email_from` (optional) | Airflow SMTP configured |
+| `teams` | `teams_webhook_url` | — |
+| `discord` | `discord_webhook_url` | — |
+
+Any combination of channels can be used in a single call. Channels are delivered sequentially in the order listed.
+
+### All parameters
 
 ```python
-on_failure_callback=dag_failure_slack_notification_webhook(
-    text='Pipeline {{ dag.dag_id }} failed on {{ ds }}',
-    include_blocks=False,
+dag_failure_notification(
+    channels=['slack', 'email', 'teams', 'discord'],
+
+    # Environment label shown in every notification (DEV / STG / PROD)
+    environment='PROD',
+
+    # Slack
+    slack_webhook_url='https://hooks.slack.com/services/...',
+
+    # Email
+    email_to=['ops@example.com'],
+    email_from=None,                 # uses Airflow SMTP default if omitted
+
+    # Teams
+    teams_webhook_url='https://outlook.office.com/webhook/...',
+
+    # Discord
+    discord_webhook_url='https://discord.com/api/webhooks/...',
 )
 ```
 
-#### Custom Slack blocks
+`get_failure_notification_task` accepts the same parameters.
+
+### Environment colours
+
+Each environment maps to a distinct colour across all channels so alerts are recognisable at a glance:
+
+| Environment | Slack | Teams | Discord |
+|---|---|---|---|
+| `PROD` | 🔴 red | Attention (red) | #ED4245 |
+| `STG` | 🟡 yellow | Warning (orange) | #FF8C00 |
+| `DEV` | 🟢 green | Good (green) | #57F287 |
+
+---
+
+## Testing Utilities
+
+### MockFilesystem
+
+`MockFilesystem` is an in-memory implementation of `FilesystemProtocol` for unit testing. It requires no Docker, no cloud credentials, and no network — all files are stored in a plain Python dict.
 
 ```python
-on_failure_callback=dag_failure_slack_notification_webhook(
-    blocks={
-        'type': 'section',
-        'text': {'type': 'mrkdwn', 'text': '*Pipeline failed* — check the logs.'},
-    }
-)
-```
+from airflow_toolkit.testing import MockFilesystem
+
+# Pre-load files at construction time
+fs = MockFilesystem({
+    "raw/orders/2024-01-01/data.csv": b"id,amount\n1,100\n2,200",
+})
 
-Default notification format:
+# Or write files programmatically
+fs.write(b"id,amount\n3,300", "raw/orders/2024-01-02/data.csv")
 
-![image](https://github.com/DeepKernelLabs/airflow-toolkit/assets/152852247/52a5bf95-21bc-4c3b-8093-79953c0c5d61)
+# Inspect the result in assertions
+assert fs.check_file("raw/orders/2024-01-01/data.csv")
+assert len(fs.list_files("raw/orders/")) == 2
+assert fs.files["raw/orders/2024-01-01/data.csv"] == b"id,amount\n1,100\n2,200"
+```
 
-**Parameters:**
+Use it to patch `FilesystemFactory.get_data_lake_filesystem` in your operator tests:
 
-| Parameter | Type | Description |
-|---|---|---|
-| `text` | `str` (optional) | Plain-text message. Overrides blocks if provided. |
-| `blocks` | `dict` (optional) | Custom Slack Block Kit payload. |
-| `include_blocks` | `bool` (optional) | Whether to include the default formatted block. |
-| `source` | `'DAG'` \| `'TASK'` (optional) | Source of the failure. Default: `'DAG'`. |
-| `image_url` | `str` (optional) | Accessory image URL. Can also be set via `AIRFLOW_TOOLKIT__SLACK_NOTIFICATION_IMG_URL`. |
+```python
+from unittest.mock import patch
+from airflow_toolkit.testing import MockFilesystem
+
+def test_my_pipeline(tmp_path):
+    fs = MockFilesystem({"data/file.csv": b"id,name\n1,Alice"})
+
+    with patch(
+        "airflow_toolkit.filesystems.filesystem_factory.FilesystemFactory.get_data_lake_filesystem",
+        return_value=fs,
+    ):
+        # run your operator or task here
+        ...
+```
+
+`MockFilesystem` implements the full `FilesystemProtocol`: `read`, `write`, `delete_file`, `create_prefix`, `delete_prefix`, `check_file`, `check_prefix`, `list_files`.
 
 ---
 

{airflow_toolkit-2.1.0 → airflow_toolkit-2.3.0}/README.md

@@ -83,10 +83,11 @@ pip install "airflow-toolkit[airflow3-full]"
 | `google` | `providers-google` | GCS filesystem backend |
 | `azure` | `providers-microsoft-azure` | Azure Blob / ADLS filesystem backend |
 | `sftp` | `providers-sftp` | SFTP filesystem backend |
-| `slack` | `providers-slack` | Slack failure notifications |
 | `http` | `providers-http`, `requests`, `jmespath`, `pandas` | `HttpToFilesystem`, `MultiHttpToFilesystem` |
 | `duckdb` | `airflow-provider-duckdb` | `DuckdbToDeltalake` operator |
 | `sqlite` | `providers-sqlite` | SQLite as source or destination |
+| `excel` | `openpyxl` | Excel (`.xlsx` / `.xls`) support in `FilesystemToDatabase` |
+| `avro` | `fastavro` | Avro support in `FilesystemToDatabase` |
 | `airflow3-full` | all of the above | Quick start / development |
 
 ---
@@ -260,7 +261,9 @@ FilesystemToFilesystem(
 
 ### FilesystemToDatabase
 
-Reads files (CSV, JSON, or Parquet) from any filesystem and loads them into any SQLAlchemy-compatible database. Handles schema drift automatically: columns present in the file but missing from the table are added; columns present in the table but missing from the file are filled with `NULL`.
+Reads files from any filesystem and loads them into any SQLAlchemy-compatible database. Handles schema drift automatically: columns present in the file but missing from the table are added; columns present in the table but missing from the file are filled with `NULL`.
+
+**Supported formats:** `csv`, `json`, `parquet`, `excel`, `avro`, `fixed_width`.
 
 ```python
 from airflow_toolkit.providers.deltalake.operators.filesystem_to_database import FilesystemToDatabaseOperator
@@ -272,7 +275,7 @@ FilesystemToDatabaseOperator(
     filesystem_path='raw/orders/{{ ds }}/',
     db_schema='public',
     db_table='orders',
-    source_format='csv',
+    source_format='csv',                   # 'csv' | 'json' | 'parquet' | 'excel' | 'avro' | 'fixed_width'
     table_aggregation_type='append',       # 'append' | 'replace' | 'fail'
     metadata={
         '_ds':          '{{ ds }}',
@@ -282,6 +285,52 @@ FilesystemToDatabaseOperator(
 )
 ```
 
+**Excel** (requires the `[excel]` extra):
+
+```python
+FilesystemToDatabaseOperator(
+    task_id='load_excel_report',
+    filesystem_conn_id='my_data_lake',
+    database_conn_id='my_postgres',
+    filesystem_path='raw/reports/{{ ds }}/',
+    db_table='monthly_report',
+    source_format='excel',
+    source_format_options={'sheet_name': 'Data'},
+)
+```
+
+**Avro** (requires the `[avro]` extra):
+
+```python
+FilesystemToDatabaseOperator(
+    task_id='load_avro_events',
+    filesystem_conn_id='my_data_lake',
+    database_conn_id='my_postgres',
+    filesystem_path='raw/events/{{ ds }}/',
+    db_table='events',
+    source_format='avro',
+)
+```
+
+**Fixed-width** (no extra required — pandas native):
+
+```python
+FilesystemToDatabaseOperator(
+    task_id='load_fixed_width',
+    filesystem_conn_id='my_data_lake',
+    database_conn_id='my_postgres',
+    filesystem_path='raw/exports/{{ ds }}/',
+    db_table='transactions',
+    source_format='fixed_width',
+    source_format_options={
+        'colspecs': [(0, 10), (10, 25), (25, 35)],
+        'names': ['date', 'description', 'amount'],
+    },
+)
+```
+
+Each format is matched by file extension: `.csv`/`.csv.gz`, `.json`/`.json.gz`, `.parquet`/`.parquet.gz`, `.xlsx`/`.xls`, `.avro`, `.fwf`/`.txt`/`.dat`. Files with other extensions in the same prefix are silently skipped.
+
 ### DuckdbToDeltalake
 
 Executes a DuckDB SQL query and writes the result directly to a Delta Lake table on Azure storage. Useful for in-process transformations that land results as an open table format.
@@ -381,74 +430,143 @@ Because `AzureDatabricksSqlHook` implements `DbApiHook`, it can be used as `sour
 
 ## Notifications
 
-### Slack (incoming webhook)
+Send rich failure notifications to Slack, email, Microsoft Teams, and Discord from a single call. The notification system is built around three ideas:
 
-Send DAG or task failure alerts to a Slack channel using `dag_failure_slack_notification_webhook`. Requires a Slack App with Incoming Webhooks enabled.
+1. **Context builder** — extracts DAG run metadata (run ID, logical date, schedule, interval, duration, environment) from the Airflow callback context once, and makes it available to all channels.
+2. **Channel formatters** — each channel (Slack Block Kit, HTML email, Teams Adaptive Card, Discord embed) formats the same context into the right payload for that platform.
+3. **Two usage patterns** — as an `on_failure_callback` (invisible to the graph) or as an explicit `@task` node in the DAG graph.
 
-Create an Airflow connection named `SLACK_WEBHOOK_NOTIFICATION_CONN` (or set `AIRFLOW_CONN_SLACK_WEBHOOK_NOTIFICATION_CONN`).
+### Pattern A — `on_failure_callback`
 
-#### DAG-level notification
+The callback fires automatically when any task in the DAG fails. Nothing appears in the task graph.
 
 ```python
-from datetime import datetime, timedelta
-from airflow.sdk import DAG
-from airflow.providers.standard.operators.bash import BashOperator
-from airflow_toolkit.notifications.slack.webhook import dag_failure_slack_notification_webhook
+from airflow_toolkit.notifications import dag_failure_notification
 
 with DAG(
     'my_pipeline',
-    schedule=timedelta(days=1),
-    start_date=datetime(2024, 1, 1),
-    catchup=False,
-    on_failure_callback=dag_failure_slack_notification_webhook(),
-) as dag:
-
-    t = BashOperator(task_id='run', bash_command='python my_script.py')
+    schedule='0 6 * * *',
+    on_failure_callback=dag_failure_notification(
+        channels=['slack', 'email'],
+        environment='PROD',
+        slack_webhook_url='https://hooks.slack.com/services/...',
+        email_to=['data-team@example.com'],
+    ),
+):
+    ...
 ```
 
-#### Task-level notification
+### Pattern B — explicit task in the graph
+
+`get_failure_notification_task` returns an Airflow task with `trigger_rule='one_failed'`. It fires when any upstream task fails and is **skipped** when all tasks succeed. The notification step is visible in the Airflow UI, has its own logs, and appears in the task history.
 
 ```python
-BashOperator(
-    task_id='run',
-    bash_command='python my_script.py',
-    on_failure_callback=dag_failure_slack_notification_webhook(source='TASK'),
-)
+from airflow_toolkit.notifications import get_failure_notification_task
+
+with DAG('my_pipeline', schedule='0 6 * * *'):
+    extract = ...
+    load    = ...
+
+    notify = get_failure_notification_task(
+        channels=['slack', 'email'],
+        environment='PROD',
+        slack_webhook_url='https://hooks.slack.com/services/...',
+        email_to=['data-team@example.com'],
+    )
+
+    [extract, load] >> notify
 ```
 
-#### Custom message
+### Supported channels
+
+| Channel | Parameter | Requires |
+|---|---|---|
+| `slack` | `slack_webhook_url` | — |
+| `email` | `email_to: list[str]`, `email_from` (optional) | Airflow SMTP configured |
+| `teams` | `teams_webhook_url` | — |
+| `discord` | `discord_webhook_url` | — |
+
+Any combination of channels can be used in a single call. Channels are delivered sequentially in the order listed.
+
+### All parameters
 
 ```python
-on_failure_callback=dag_failure_slack_notification_webhook(
-    text='Pipeline {{ dag.dag_id }} failed on {{ ds }}',
-    include_blocks=False,
+dag_failure_notification(
+    channels=['slack', 'email', 'teams', 'discord'],
+
+    # Environment label shown in every notification (DEV / STG / PROD)
+    environment='PROD',
+
+    # Slack
+    slack_webhook_url='https://hooks.slack.com/services/...',
+
+    # Email
+    email_to=['ops@example.com'],
+    email_from=None,                 # uses Airflow SMTP default if omitted
+
+    # Teams
+    teams_webhook_url='https://outlook.office.com/webhook/...',
+
+    # Discord
+    discord_webhook_url='https://discord.com/api/webhooks/...',
 )
 ```
 
-#### Custom Slack blocks
+`get_failure_notification_task` accepts the same parameters.
+
+### Environment colours
+
+Each environment maps to a distinct colour across all channels so alerts are recognisable at a glance:
+
+| Environment | Slack | Teams | Discord |
+|---|---|---|---|
+| `PROD` | 🔴 red | Attention (red) | #ED4245 |
+| `STG` | 🟡 yellow | Warning (orange) | #FF8C00 |
+| `DEV` | 🟢 green | Good (green) | #57F287 |
+
+---
+
+## Testing Utilities
+
+### MockFilesystem
+
+`MockFilesystem` is an in-memory implementation of `FilesystemProtocol` for unit testing. It requires no Docker, no cloud credentials, and no network — all files are stored in a plain Python dict.
 
 ```python
-on_failure_callback=dag_failure_slack_notification_webhook(
-    blocks={
-        'type': 'section',
-        'text': {'type': 'mrkdwn', 'text': '*Pipeline failed* — check the logs.'},
-    }
-)
-```
+from airflow_toolkit.testing import MockFilesystem
+
+# Pre-load files at construction time
+fs = MockFilesystem({
+    "raw/orders/2024-01-01/data.csv": b"id,amount\n1,100\n2,200",
+})
 
-Default notification format:
+# Or write files programmatically
+fs.write(b"id,amount\n3,300", "raw/orders/2024-01-02/data.csv")
 
-![image](https://github.com/DeepKernelLabs/airflow-toolkit/assets/152852247/52a5bf95-21bc-4c3b-8093-79953c0c5d61)
+# Inspect the result in assertions
+assert fs.check_file("raw/orders/2024-01-01/data.csv")
+assert len(fs.list_files("raw/orders/")) == 2
+assert fs.files["raw/orders/2024-01-01/data.csv"] == b"id,amount\n1,100\n2,200"
+```
 
-**Parameters:**
+Use it to patch `FilesystemFactory.get_data_lake_filesystem` in your operator tests:
 
-| Parameter | Type | Description |
-|---|---|---|
-| `text` | `str` (optional) | Plain-text message. Overrides blocks if provided. |
-| `blocks` | `dict` (optional) | Custom Slack Block Kit payload. |
-| `include_blocks` | `bool` (optional) | Whether to include the default formatted block. |
-| `source` | `'DAG'` \| `'TASK'` (optional) | Source of the failure. Default: `'DAG'`. |
-| `image_url` | `str` (optional) | Accessory image URL. Can also be set via `AIRFLOW_TOOLKIT__SLACK_NOTIFICATION_IMG_URL`. |
+```python
+from unittest.mock import patch
+from airflow_toolkit.testing import MockFilesystem
+
+def test_my_pipeline(tmp_path):
+    fs = MockFilesystem({"data/file.csv": b"id,name\n1,Alice"})
+
+    with patch(
+        "airflow_toolkit.filesystems.filesystem_factory.FilesystemFactory.get_data_lake_filesystem",
+        return_value=fs,
+    ):
+        # run your operator or task here
+        ...
+```
+
+`MockFilesystem` implements the full `FilesystemProtocol`: `read`, `write`, `delete_file`, `create_prefix`, `delete_prefix`, `check_file`, `check_prefix`, `list_files`.
 
 ---
 

{airflow_toolkit-2.1.0 → airflow_toolkit-2.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "airflow-toolkit"
-version = "2.1.0"
+version = "2.3.0"
 description = "A toolkit of operators, hooks and utilities for Apache Airflow 3"
 authors = [{ name = "Biel Llobera", email = "biel_llobera@dkl.digital" }]
 requires-python = ">=3.11,<3.15"
@@ -37,9 +37,6 @@ azure = [
 sftp = [
     "apache-airflow-providers-sftp>=5.3",
 ]
-slack = [
-    "apache-airflow-providers-slack>=9.3",
-]
 http = [
     "apache-airflow-providers-http>=5.6.4",
     "requests>=2.31.0",
@@ -52,6 +49,12 @@ duckdb = [
 sqlite = [
     "apache-airflow-providers-sqlite",
 ]
+excel = [
+    "openpyxl>=3.1",
+]
+avro = [
+    "fastavro>=1.9",
+]
 airflow3-full = [
     "apache-airflow>=3,<4",
     "apache-airflow-providers-fab>=3.0.0",
@@ -64,12 +67,13 @@ airflow3-full = [
     "apache-airflow-providers-google>=18",
     "apache-airflow-providers-microsoft-azure>=8",
     "apache-airflow-providers-sftp>=5.3",
-    "apache-airflow-providers-slack>=9.3",
     "apache-airflow-providers-http>=5.6.4",
     "requests>=2.31.0",
     "jmespath>=1.0.1,<2",
     "airflow-provider-duckdb>=0.1.2",
     "apache-airflow-providers-sqlite",
+    "openpyxl>=3.1",
+    "fastavro>=1.9",
 ]
 
 [dependency-groups]

{airflow_toolkit-2.1.0 → airflow_toolkit-2.3.0}/src/airflow_toolkit/compression_utils.py

@@ -1,10 +1,10 @@
 import gzip
 import zipfile
 from io import BytesIO
-from typing import Literal, Union
+
+from airflow_toolkit.types import CompressionOptions
 
 DEFAULT_ZIP_FILENAME = "file.zip"
-CompressionOptions = Union[Literal["infer", "gzip", "bz2", "zip", "xz", "zstd"], None]
 
 
 def gzip_data(data: bytes) -> bytes: