flowsense-engine 0.5.0__tar.gz → 0.5.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/CHANGELOG.md +20 -0
- {flowsense_engine-0.5.0/src/flowsense_engine.egg-info → flowsense_engine-0.5.1}/PKG-INFO +6 -2
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/README.md +5 -1
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/grafana.md +20 -1
- flowsense_engine-0.5.1/docs/release-0.5.1.md +21 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/releasing.md +4 -4
- flowsense_engine-0.5.1/docs/showcase.md +79 -0
- flowsense_engine-0.5.1/examples/dags/flowsense_showcase.py +91 -0
- flowsense_engine-0.5.1/examples/python/showcase.py +290 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/pyproject.toml +1 -1
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/client.py +2 -2
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1/src/flowsense_engine.egg-info}/PKG-INFO +6 -2
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense_engine.egg-info/SOURCES.txt +5 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_client.py +35 -4
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_package_metadata.py +1 -1
- flowsense_engine-0.5.1/tests/test_showcase.py +50 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/LICENSE +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/MANIFEST.in +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/alerting.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/architecture.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/authentication.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/benchmarks.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/migrating-to-0.3.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/migrating-to-0.5.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/production-quickstart.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/prometheus.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/python-api.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/docs/release-0.5.0.md +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/examples/dags/flowsense_demo.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/examples/python/production_batch.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/setup.cfg +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/analysis_engine.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/analyzer.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/batch.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/client.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/output.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/pipeline.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/policy_document.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/ports.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/request.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/serialization.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/application/use_cases.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/cli/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/cli/doctor.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/cli/main.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/cli/policy.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/cli/report.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/collector/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/collector/airflow_client.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/config.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/domain/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/domain/enums.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/domain/exceptions.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/domain/models.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/domain/policy.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/domain/results.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/analyzer.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/change_point.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/drift.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/history.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/impact.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/propagation.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/root_cause.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/timing.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/trend.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/engine/validation.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/auth.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/config.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/dto.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/exceptions.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/factory.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/mapper.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/mcp/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/mcp/server.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/models/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/models/dag_analysis.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/models/task_run.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/observability/__init__.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/observability/prometheus.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/observability/service.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/py.typed +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/version.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense_engine.egg-info/dependency_links.txt +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense_engine.egg-info/entry_points.txt +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense_engine.egg-info/requires.txt +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense_engine.egg-info/top_level.txt +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_auth.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_client_integration.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_compatibility.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_contract.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_factory.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_mapper.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_airflow_resilience.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_analysis_golden.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_analysis_output.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_analysis_pipeline.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_analysis_policy_document.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_analysis_request.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_analysis_use_case.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_analyzer.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_architecture.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_batch_analysis_output.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_benchmark_analysis.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_change_point.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_cli.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_cli_policy.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_cli_report.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_dag_summary.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_deprecations.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_doctor.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_domain.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_drift.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_examples.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_history.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_impact.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_library_client.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_mcp_integration.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_mcp_server.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_observability_assets.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_observation_validation.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_policy.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_prometheus.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_propagation.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_public_api.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_root_cause.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_timing.py +0 -0
- {flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/tests/test_trend.py +0 -0
|
@@ -11,6 +11,26 @@ All notable changes to FlowSense are documented in this file. The project uses
|
|
|
11
11
|
|
|
12
12
|
### Fixed
|
|
13
13
|
|
|
14
|
+
## [0.5.1] - 2026-09-25
|
|
15
|
+
|
|
16
|
+
### Added
|
|
17
|
+
|
|
18
|
+
- Six controlled Airflow demo DAGs, each with 20 seeded runs, bounded live
|
|
19
|
+
execution waves, and a read-only execution-evidence verifier.
|
|
20
|
+
- Dedicated multi-DAG showcase dashboard with per-pipeline results and separate
|
|
21
|
+
queued/running activity metrics.
|
|
22
|
+
- FlowSense Engine wordmark in the README and PyPI package description.
|
|
23
|
+
|
|
24
|
+
### Changed
|
|
25
|
+
|
|
26
|
+
- Overview dashboard now uses instant summary queries, clearer multi-DAG labels,
|
|
27
|
+
and a shorter default history window.
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
|
|
31
|
+
- Corrected Airflow 3 GET state filters from `states` to `state`, preventing
|
|
32
|
+
running runs from shrinking the bounded successful-run analysis window.
|
|
33
|
+
|
|
14
34
|
## [0.5.0] - 2026-09-23
|
|
15
35
|
|
|
16
36
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flowsense-engine
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: Temporal drift and anomaly detection for Apache Airflow
|
|
5
5
|
Author: Omer Cengiz
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -36,9 +36,13 @@ Dynamic: license-file
|
|
|
36
36
|
|
|
37
37
|
# FlowSense Engine
|
|
38
38
|
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="https://raw.githubusercontent.com/omercengiz/flowsense-engine/27f7db5e55bfefbba6e355c53c7c84ff34e6f37b/assets/branding/flowsense-engine-logo-v2.png" alt="FlowSense Engine logo" width="640">
|
|
41
|
+
</p>
|
|
42
|
+
|
|
39
43
|
Temporal drift, anomaly detection, and dependency-aware propagation analysis for Apache Airflow.
|
|
40
44
|
|
|
41
|
-
Current package release: `0.5.
|
|
45
|
+
Current package release: `0.5.1`. See [CHANGELOG.md](CHANGELOG.md) for release
|
|
42
46
|
notes, [docs/architecture.md](docs/architecture.md) for the system architecture,
|
|
43
47
|
[docs/benchmarks.md](docs/benchmarks.md) for performance measurement, and
|
|
44
48
|
[docs/production-quickstart.md](docs/production-quickstart.md) for a production
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
# FlowSense Engine
|
|
2
2
|
|
|
3
|
+
<p align="center">
|
|
4
|
+
<img src="https://raw.githubusercontent.com/omercengiz/flowsense-engine/27f7db5e55bfefbba6e355c53c7c84ff34e6f37b/assets/branding/flowsense-engine-logo-v2.png" alt="FlowSense Engine logo" width="640">
|
|
5
|
+
</p>
|
|
6
|
+
|
|
3
7
|
Temporal drift, anomaly detection, and dependency-aware propagation analysis for Apache Airflow.
|
|
4
8
|
|
|
5
|
-
Current package release: `0.5.
|
|
9
|
+
Current package release: `0.5.1`. See [CHANGELOG.md](CHANGELOG.md) for release
|
|
6
10
|
notes, [docs/architecture.md](docs/architecture.md) for the system architecture,
|
|
7
11
|
[docs/benchmarks.md](docs/benchmarks.md) for performance measurement, and
|
|
8
12
|
[docs/production-quickstart.md](docs/production-quickstart.md) for a production
|
|
@@ -56,7 +56,21 @@ The dashboard provides:
|
|
|
56
56
|
- currently firing FlowSense alerts.
|
|
57
57
|
|
|
58
58
|
Use the DAG and task variables at the top of the dashboard to narrow the view.
|
|
59
|
-
The default time range is
|
|
59
|
+
The default time range is 15 minutes and the dashboard refreshes every 30 seconds.
|
|
60
|
+
Summary cards use instant queries, so they remain populated independently of
|
|
61
|
+
the selected history window. Current task comparisons include both DAG and
|
|
62
|
+
task names to distinguish tasks across multiple DAGs.
|
|
63
|
+
|
|
64
|
+
The top row summarizes the selected DAGs: monitored count, highest severity,
|
|
65
|
+
analysis health (healthy only when every selected DAG succeeded), mean coverage,
|
|
66
|
+
and total anomalous and affected tasks. Task comparisons and history appear
|
|
67
|
+
below; collection diagnostics and active alerts follow.
|
|
68
|
+
|
|
69
|
+
Prometheus history starts when the metrics service is scraped; historical
|
|
70
|
+
Airflow runs are analyzed but are not backfilled into these charts. After
|
|
71
|
+
starting the exporter, use a recent time range and allow a few scrapes. An
|
|
72
|
+
existing browser URL can retain an older time range; select Last 15 minutes
|
|
73
|
+
manually in that case.
|
|
60
74
|
See [alerting.md](alerting.md) for rule behavior, validation, and production
|
|
61
75
|
notification configuration.
|
|
62
76
|
|
|
@@ -68,3 +82,8 @@ docker compose -f deploy/observability/docker-compose.yml down
|
|
|
68
82
|
|
|
69
83
|
Prometheus and Grafana data remain in named volumes. To explicitly remove those
|
|
70
84
|
local volumes, add `--volumes` to the command.
|
|
85
|
+
|
|
86
|
+
## Multi-DAG demo with execution evidence
|
|
87
|
+
|
|
88
|
+
For six real Airflow workloads, 20 seeded runs per DAG, continued live activity,
|
|
89
|
+
and a dedicated presentation dashboard, see [showcase.md](showcase.md).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# FlowSense 0.5.1
|
|
2
|
+
|
|
3
|
+
This patch fixes Airflow 3 state filtering and adds a reproducible multi-DAG
|
|
4
|
+
showcase and updated project branding.
|
|
5
|
+
|
|
6
|
+
## Highlights
|
|
7
|
+
|
|
8
|
+
- Correct `state` filters for Airflow 3 DAG runs and task instances. A running
|
|
9
|
+
run no longer consumes a slot in the bounded successful-run history window.
|
|
10
|
+
- Six controlled demo DAGs, four tasks each, with 20 seeded runs per DAG and
|
|
11
|
+
optional bounded live execution waves.
|
|
12
|
+
- Read-only verification of 120 successful seed runs and 480 task executions.
|
|
13
|
+
- Dedicated Grafana showcase with per-DAG analyses, severity history, task
|
|
14
|
+
deviations, and separately collected running/queued activity.
|
|
15
|
+
- Improved overview dashboard and a wordmark in the README/PyPI description.
|
|
16
|
+
|
|
17
|
+
Demo workloads use controlled delays. Drift analysis evaluates completed
|
|
18
|
+
successful runs; live activity monitoring is separate. Output schemas and
|
|
19
|
+
public API contracts are unchanged.
|
|
20
|
+
|
|
21
|
+
See [the showcase guide](showcase.md) for setup and verification.
|
|
@@ -33,8 +33,8 @@ After the release pull request is merged into `master` and CI passes:
|
|
|
33
33
|
```bash
|
|
34
34
|
git switch master
|
|
35
35
|
git pull --ff-only origin master
|
|
36
|
-
git tag -a v0.5.
|
|
37
|
-
git push origin v0.5.
|
|
36
|
+
git tag -a v0.5.1 -m "FlowSense 0.5.1"
|
|
37
|
+
git push origin v0.5.1
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
Create and publish the corresponding GitHub Release after pushing the tag. The
|
|
@@ -42,7 +42,7 @@ release workflow builds that exact tag and publishes it through the protected
|
|
|
42
42
|
`pypi` environment. Package publication should run only after the built artifacts
|
|
43
43
|
have been verified.
|
|
44
44
|
|
|
45
|
-
Use `FlowSense 0.5.
|
|
46
|
-
stable release as a pre-release. Use [release-0.5.
|
|
45
|
+
Use `FlowSense 0.5.1` as the release title, select `Latest`, and do not mark a
|
|
46
|
+
stable release as a pre-release. Use [release-0.5.1.md](release-0.5.1.md) as the
|
|
47
47
|
GitHub release notes; the matching `CHANGELOG.md` section remains the canonical
|
|
48
48
|
change inventory.
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Live multi-DAG showcase
|
|
2
|
+
|
|
3
|
+
This is a controlled demo, executed by real Airflow workers. It is not production
|
|
4
|
+
traffic or a scalability benchmark. No run states, task durations, or Prometheus
|
|
5
|
+
history are fabricated or backdated.
|
|
6
|
+
|
|
7
|
+
The six DAGs in `examples/dags/flowsense_showcase.py` each contain four tasks:
|
|
8
|
+
orders, payments, inventory, analytics, ML features, and customer 360. The first
|
|
9
|
+
14 runs establish a baseline; the next six introduce different controlled sleep
|
|
10
|
+
patterns. Worker and scheduler overhead are included in the measured durations,
|
|
11
|
+
so actual severities can vary. The orders workload has no injected slowdown.
|
|
12
|
+
|
|
13
|
+
## Run locally
|
|
14
|
+
|
|
15
|
+
The repository's Airflow Compose configuration mounts `examples/dags`. If an
|
|
16
|
+
existing installation uses a different DAG mount, copy the file to its actual
|
|
17
|
+
DAG folder. Wait for all six `flowsense_showcase_*` DAGs to be parsed.
|
|
18
|
+
|
|
19
|
+
With Airflow credentials in environment variables or the local `.env`:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
PYTHONPATH=src .venv/bin/python examples/python/showcase.py \
|
|
23
|
+
--seed --live-cycles 30 --host 0.0.0.0
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Only one exporter can use port 9108 at a time; stop an existing `serve-metrics`
|
|
27
|
+
process before starting this one. The showcase seeds 20 real executions per DAG
|
|
28
|
+
(120 runs, 480 task executions), skipping seed run IDs that already exist. It
|
|
29
|
+
unpauses only the six showcase DAGs. Failed seed runs are reported, not silently
|
|
30
|
+
marked successful or retried. Once each DAG has at least 20 successes, up to 30
|
|
31
|
+
additional waves are submitted, one wave per minute. Live waves triple injected
|
|
32
|
+
slow task delays so running activity is visible across scrape intervals; the
|
|
33
|
+
unchanged orders workload remains a control. Run counts can therefore
|
|
34
|
+
exceed 20; each analysis still uses the latest 20 successful runs. After the last
|
|
35
|
+
wave, collection continues without submitting additional work. Ctrl-C stops the
|
|
36
|
+
collector; already submitted Airflow runs continue to completion.
|
|
37
|
+
|
|
38
|
+
Start the observability stack as described in [grafana.md](grafana.md), then open
|
|
39
|
+
[FlowSense · Live Multi-DAG Showcase](http://localhost:3000/d/flowsense-showcase?from=now-5m&to=now&kiosk).
|
|
40
|
+
The dashboard is automatically provisioned alongside FlowSense Overview.
|
|
41
|
+
|
|
42
|
+
## What the dashboard proves
|
|
43
|
+
|
|
44
|
+
- `flowsense_dag_runs_analyzed`: actual successful runs consumed by the engine,
|
|
45
|
+
per DAG; the sum is the current analysis-window total, not lifetime throughput.
|
|
46
|
+
- `flowsense_task_*`: measured duration deviations, robust z-scores, and severity.
|
|
47
|
+
- DAG-level affected tasks, change points, trends, and handoff anomalies are
|
|
48
|
+
computed by the existing FlowSense engine.
|
|
49
|
+
- `airflow_showcase_runs`: queued, running, successful, and failed run counts read
|
|
50
|
+
separately from the Airflow REST API by the demo collector.
|
|
51
|
+
|
|
52
|
+
**Running-run counts are activity monitoring, not in-flight drift analysis.**
|
|
53
|
+
The engine analyzes completed successful runs. Both activity and analysis update
|
|
54
|
+
on each collection loop, followed by the normal Prometheus scrape interval.
|
|
55
|
+
Brief runs can finish between scrapes. Time-series charts show actual observations
|
|
56
|
+
since collection began, not all historical Airflow run timestamps.
|
|
57
|
+
|
|
58
|
+
`logs/showcase-evidence/runs.json` records Airflow run IDs, states, timestamps,
|
|
59
|
+
and demo configuration. The sibling per-DAG JSON documents contain full FlowSense
|
|
60
|
+
analyses, including the current run ID, baseline, deviations, and root cause.
|
|
61
|
+
These are continuously updated local evidence files, not committed credentials.
|
|
62
|
+
For a durable report, copy a snapshot into `artifacts/showcase-evidence` alongside
|
|
63
|
+
the screenshot, recording the observation time.
|
|
64
|
+
|
|
65
|
+
A defensible description for sharing is: “FlowSense analyzed a rolling window
|
|
66
|
+
of 20 successful executions across each of six controlled Airflow pipelines
|
|
67
|
+
(120 runs, 24 task definitions), while separately monitoring live run activity.”
|
|
68
|
+
Use the captured evidence for anomaly counts, which change between observations.
|
|
69
|
+
|
|
70
|
+
Re-run the read-only execution proof against Airflow:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
PYTHONPATH=src .venv/bin/python examples/python/showcase.py \
|
|
74
|
+
--verify-seed --evidence-dir artifacts/showcase-evidence
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
This fails if a seed run is missing/unsuccessful or a run lacks four successful
|
|
78
|
+
task instances with start/end timestamps. It writes `seed-execution-proof.json`
|
|
79
|
+
only after validating all six DAGs.
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Six controlled workloads; real task executions, deliberately varied latency."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
|
|
8
|
+
from airflow.sdk import DAG, get_current_context, task
|
|
9
|
+
|
|
10
|
+
SCENARIOS = {
|
|
11
|
+
"orders": (
|
|
12
|
+
"extract_orders",
|
|
13
|
+
"validate_orders",
|
|
14
|
+
"aggregate_orders",
|
|
15
|
+
"publish_orders",
|
|
16
|
+
),
|
|
17
|
+
"payments": ("fetch_payments", "validate_payments", "reconcile", "settle"),
|
|
18
|
+
"inventory": ("read_stock", "normalize_stock", "forecast_stock", "sync_stock"),
|
|
19
|
+
"analytics": (
|
|
20
|
+
"collect_events",
|
|
21
|
+
"enrich_events",
|
|
22
|
+
"build_sessions",
|
|
23
|
+
"publish_metrics",
|
|
24
|
+
),
|
|
25
|
+
"ml_features": (
|
|
26
|
+
"read_events",
|
|
27
|
+
"clean_features",
|
|
28
|
+
"compute_features",
|
|
29
|
+
"write_features",
|
|
30
|
+
),
|
|
31
|
+
"customer360": (
|
|
32
|
+
"fetch_profiles",
|
|
33
|
+
"deduplicate",
|
|
34
|
+
"join_profiles",
|
|
35
|
+
"publish_profiles",
|
|
36
|
+
),
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def build_dag(name: str, task_names: tuple[str, ...]) -> DAG:
|
|
41
|
+
with DAG(
|
|
42
|
+
dag_id=f"flowsense_showcase_{name}",
|
|
43
|
+
start_date=datetime(2026, 1, 1, tzinfo=UTC),
|
|
44
|
+
schedule=None,
|
|
45
|
+
catchup=False,
|
|
46
|
+
max_active_runs=3,
|
|
47
|
+
max_active_tasks=6,
|
|
48
|
+
tags=["flowsense", "controlled-demo", name],
|
|
49
|
+
doc_md="Controlled latency demo. Real Airflow executions; no production data.",
|
|
50
|
+
) as dag:
|
|
51
|
+
|
|
52
|
+
@task
|
|
53
|
+
def work(stage: int, scenario: str) -> dict:
|
|
54
|
+
run = get_current_context()["dag_run"]
|
|
55
|
+
conf = run.conf or {}
|
|
56
|
+
index = int(conf.get("sample", 0))
|
|
57
|
+
# Small deterministic baseline variation, then different incident patterns.
|
|
58
|
+
delay = 0.35 + ((index * 5 + stage * 3) % 7) * 0.04
|
|
59
|
+
if index >= 14:
|
|
60
|
+
if scenario == "payments" and stage == 2:
|
|
61
|
+
delay += 3.0
|
|
62
|
+
elif scenario == "inventory" and stage == 2:
|
|
63
|
+
delay += (index - 13) * 0.45
|
|
64
|
+
elif scenario == "analytics" and stage in (1, 2):
|
|
65
|
+
delay += 1.8 if index % 3 else 0.2
|
|
66
|
+
elif scenario == "ml_features" and stage == 2:
|
|
67
|
+
delay += 4.0
|
|
68
|
+
elif scenario == "customer360" and stage == 1:
|
|
69
|
+
delay += 1.2
|
|
70
|
+
# Live waves intensify the incident so activity spans multiple scrapes.
|
|
71
|
+
if "_live_" in run.run_id and delay > 1.0:
|
|
72
|
+
delay *= 3
|
|
73
|
+
time.sleep(delay)
|
|
74
|
+
return {
|
|
75
|
+
"scenario": scenario,
|
|
76
|
+
"sample": index,
|
|
77
|
+
"stage": stage,
|
|
78
|
+
"sleep_seconds": delay,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
previous = None
|
|
82
|
+
for stage, task_name in enumerate(task_names):
|
|
83
|
+
current = work.override(task_id=task_name)(stage, name)
|
|
84
|
+
if previous is not None:
|
|
85
|
+
previous >> current
|
|
86
|
+
previous = current
|
|
87
|
+
return dag
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
for scenario, names in SCENARIOS.items():
|
|
91
|
+
globals()[f"flowsense_showcase_{scenario}"] = build_dag(scenario, names)
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
"""Run and observe six real Airflow demo workloads (local Airflow 3 only).
|
|
2
|
+
|
|
3
|
+
The analysis engine evaluates completed runs. A separate, explicitly named
|
|
4
|
+
activity metric reports queued/running runs; it is not in-flight drift detection.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import shlex
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
from collections import Counter
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
from flowsense.application import AnalysisRequest, AnalyzeDAG
|
|
20
|
+
from flowsense.application.serialization import serialize_analysis
|
|
21
|
+
from flowsense.infrastructure.airflow.client import AirflowClient
|
|
22
|
+
from flowsense.infrastructure.airflow.config import load_airflow_config
|
|
23
|
+
from flowsense.infrastructure.airflow.factory import create_airflow_data_source_factory
|
|
24
|
+
from flowsense.observability.prometheus import PrometheusExporter, create_metrics_server
|
|
25
|
+
|
|
26
|
+
DAGS = [
|
|
27
|
+
f"flowsense_showcase_{name}"
|
|
28
|
+
for name in (
|
|
29
|
+
"orders",
|
|
30
|
+
"payments",
|
|
31
|
+
"inventory",
|
|
32
|
+
"analytics",
|
|
33
|
+
"ml_features",
|
|
34
|
+
"customer360",
|
|
35
|
+
)
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ShowcaseExporter(PrometheusExporter):
|
|
40
|
+
"""FlowSense analysis plus separately labelled Airflow demo activity."""
|
|
41
|
+
|
|
42
|
+
def __init__(self):
|
|
43
|
+
super().__init__()
|
|
44
|
+
self.activity = ""
|
|
45
|
+
|
|
46
|
+
def render(self):
|
|
47
|
+
return super().render() + self.activity
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def verify_seed_runs(client: AirflowClient, evidence_dir: Path) -> dict:
|
|
51
|
+
"""Verify real successful seed runs and task timestamps, without mutations."""
|
|
52
|
+
proof = {
|
|
53
|
+
"observed_at": datetime.now(UTC).isoformat(),
|
|
54
|
+
"controlled_demo": True,
|
|
55
|
+
"source": "Airflow REST API",
|
|
56
|
+
"dags": {},
|
|
57
|
+
}
|
|
58
|
+
expected = {f"flowsense_showcase_seed_{index:02d}" for index in range(20)}
|
|
59
|
+
for dag in DAGS:
|
|
60
|
+
seeds = [
|
|
61
|
+
run
|
|
62
|
+
for run in client.get_dag_runs(dag)["dag_runs"]
|
|
63
|
+
if run["dag_run_id"] in expected
|
|
64
|
+
]
|
|
65
|
+
if {r["dag_run_id"] for r in seeds} != expected or any(
|
|
66
|
+
r["state"] != "success" for r in seeds
|
|
67
|
+
):
|
|
68
|
+
raise ValueError(f"{dag}: expected 20 successful seed runs")
|
|
69
|
+
tasks = []
|
|
70
|
+
for run in seeds:
|
|
71
|
+
instances = client.get_task_instances(dag, run["dag_run_id"])[
|
|
72
|
+
"task_instances"
|
|
73
|
+
]
|
|
74
|
+
if len(instances) != 4 or any(
|
|
75
|
+
t["state"] != "success"
|
|
76
|
+
or not t.get("start_date")
|
|
77
|
+
or not t.get("end_date")
|
|
78
|
+
for t in instances
|
|
79
|
+
):
|
|
80
|
+
raise ValueError(
|
|
81
|
+
f"{dag}/{run['dag_run_id']}: expected four successful timed tasks"
|
|
82
|
+
)
|
|
83
|
+
tasks.extend(
|
|
84
|
+
{
|
|
85
|
+
key: task.get(key)
|
|
86
|
+
for key in (
|
|
87
|
+
"dag_id",
|
|
88
|
+
"dag_run_id",
|
|
89
|
+
"task_id",
|
|
90
|
+
"state",
|
|
91
|
+
"start_date",
|
|
92
|
+
"end_date",
|
|
93
|
+
"duration",
|
|
94
|
+
"try_number",
|
|
95
|
+
)
|
|
96
|
+
}
|
|
97
|
+
for task in instances
|
|
98
|
+
)
|
|
99
|
+
proof["dags"][dag] = {
|
|
100
|
+
"successful_seed_runs": len(seeds),
|
|
101
|
+
"successful_task_executions": len(tasks),
|
|
102
|
+
"task_instances": tasks,
|
|
103
|
+
}
|
|
104
|
+
evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
(evidence_dir / "seed-execution-proof.json").write_text(
|
|
106
|
+
json.dumps(proof, indent=2) + "\n"
|
|
107
|
+
)
|
|
108
|
+
return proof
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def main():
|
|
112
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
113
|
+
parser.add_argument(
|
|
114
|
+
"--seed",
|
|
115
|
+
action="store_true",
|
|
116
|
+
help="Create 20 real runs per demo DAG, idempotently.",
|
|
117
|
+
)
|
|
118
|
+
parser.add_argument(
|
|
119
|
+
"--live-cycles",
|
|
120
|
+
type=int,
|
|
121
|
+
default=0,
|
|
122
|
+
help="Additional waves after seeding; one wave per minute.",
|
|
123
|
+
)
|
|
124
|
+
parser.add_argument(
|
|
125
|
+
"--verify-seed",
|
|
126
|
+
action="store_true",
|
|
127
|
+
help="Verify 120 seed runs and 480 task executions without triggering work.",
|
|
128
|
+
)
|
|
129
|
+
parser.add_argument("--port", type=int, default=9108)
|
|
130
|
+
parser.add_argument("--host", default="127.0.0.1")
|
|
131
|
+
parser.add_argument(
|
|
132
|
+
"--evidence-dir", type=Path, default=Path("logs/showcase-evidence")
|
|
133
|
+
)
|
|
134
|
+
args = parser.parse_args()
|
|
135
|
+
if args.live_cycles < 0 or (args.verify_seed and (args.seed or args.live_cycles)):
|
|
136
|
+
parser.error(
|
|
137
|
+
"Use nonnegative live cycles; verification cannot be combined with execution."
|
|
138
|
+
)
|
|
139
|
+
# Parse values as data, never execute the .env file as shell code.
|
|
140
|
+
if Path(".env").exists():
|
|
141
|
+
for line in Path(".env").read_text().splitlines():
|
|
142
|
+
key, sep, value = line.strip().removeprefix("export ").partition("=")
|
|
143
|
+
if sep and key.startswith("AIRFLOW_"):
|
|
144
|
+
os.environ.setdefault(key, " ".join(shlex.split(value, comments=True)))
|
|
145
|
+
config = load_airflow_config()
|
|
146
|
+
client = AirflowClient(config)
|
|
147
|
+
if args.verify_seed:
|
|
148
|
+
try:
|
|
149
|
+
verify_seed_runs(client, args.evidence_dir)
|
|
150
|
+
print(
|
|
151
|
+
"Verified 120 successful seed runs and 480 successful task executions."
|
|
152
|
+
)
|
|
153
|
+
finally:
|
|
154
|
+
client.close()
|
|
155
|
+
return
|
|
156
|
+
|
|
157
|
+
def api(method, path, **kwargs):
|
|
158
|
+
response = client._authenticated_request(
|
|
159
|
+
method, client._api_url(path), **kwargs
|
|
160
|
+
)
|
|
161
|
+
response.raise_for_status()
|
|
162
|
+
return response.json()
|
|
163
|
+
|
|
164
|
+
def trigger(dag, run_id, sample):
|
|
165
|
+
api(
|
|
166
|
+
"POST",
|
|
167
|
+
f"/dags/{dag}/dagRuns",
|
|
168
|
+
json={
|
|
169
|
+
"dag_run_id": run_id,
|
|
170
|
+
"logical_date": None,
|
|
171
|
+
"conf": {"sample": sample, "controlled_demo": True},
|
|
172
|
+
"note": "FlowSense showcase: real execution, controlled latency workload.",
|
|
173
|
+
},
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
args.evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
177
|
+
exporter = ShowcaseExporter()
|
|
178
|
+
server = create_metrics_server(exporter, host=args.host, port=args.port)
|
|
179
|
+
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
180
|
+
analyze = AnalyzeDAG(create_airflow_data_source_factory(config)).execute
|
|
181
|
+
if args.seed:
|
|
182
|
+
for dag in DAGS:
|
|
183
|
+
api("PATCH", f"/dags/{dag}", json={"is_paused": False})
|
|
184
|
+
existing = {r["dag_run_id"] for r in client.get_dag_runs(dag)["dag_runs"]}
|
|
185
|
+
for index in range(20):
|
|
186
|
+
run_id = f"flowsense_showcase_seed_{index:02d}"
|
|
187
|
+
if run_id not in existing:
|
|
188
|
+
trigger(dag, run_id, index)
|
|
189
|
+
print(f"{dag}: 20 seed runs submitted", flush=True)
|
|
190
|
+
|
|
191
|
+
cycles = 0
|
|
192
|
+
last_wave = 0.0
|
|
193
|
+
session = datetime.now(UTC).strftime("%Y%m%dT%H%M%S")
|
|
194
|
+
try:
|
|
195
|
+
while True:
|
|
196
|
+
snapshot = {
|
|
197
|
+
"observed_at": datetime.now(UTC).isoformat(),
|
|
198
|
+
"controlled_demo": True,
|
|
199
|
+
"dags": {},
|
|
200
|
+
}
|
|
201
|
+
activity = [
|
|
202
|
+
"# HELP airflow_showcase_runs Real Airflow run counts for controlled demo DAGs.",
|
|
203
|
+
"# TYPE airflow_showcase_runs gauge",
|
|
204
|
+
]
|
|
205
|
+
ready = True
|
|
206
|
+
for dag in DAGS:
|
|
207
|
+
started = time.monotonic()
|
|
208
|
+
try:
|
|
209
|
+
runs = client.get_dag_runs(dag)["dag_runs"]
|
|
210
|
+
counts = Counter(r["state"] for r in runs)
|
|
211
|
+
ready = ready and counts["success"] >= 20
|
|
212
|
+
for state in ("queued", "running", "success", "failed"):
|
|
213
|
+
activity.append(
|
|
214
|
+
f'airflow_showcase_runs{{dag_id="{dag}",state="{state}"}} {counts[state]}'
|
|
215
|
+
)
|
|
216
|
+
snapshot["dags"][dag] = {
|
|
217
|
+
"states": dict(counts),
|
|
218
|
+
"runs": [
|
|
219
|
+
{
|
|
220
|
+
k: r.get(k)
|
|
221
|
+
for k in (
|
|
222
|
+
"dag_run_id",
|
|
223
|
+
"state",
|
|
224
|
+
"start_date",
|
|
225
|
+
"end_date",
|
|
226
|
+
"conf",
|
|
227
|
+
)
|
|
228
|
+
}
|
|
229
|
+
for r in runs
|
|
230
|
+
],
|
|
231
|
+
}
|
|
232
|
+
if counts["success"] >= 6:
|
|
233
|
+
analysis = analyze(
|
|
234
|
+
AnalysisRequest(dag_id=dag, history_run_limit=20)
|
|
235
|
+
)
|
|
236
|
+
exporter.record_success(
|
|
237
|
+
analysis, duration_seconds=time.monotonic() - started
|
|
238
|
+
)
|
|
239
|
+
output = serialize_analysis(analysis)
|
|
240
|
+
(args.evidence_dir / f"{dag}.json").write_text(
|
|
241
|
+
json.dumps(output, indent=2) + "\n"
|
|
242
|
+
)
|
|
243
|
+
snapshot["dags"][dag]["analysis"] = {
|
|
244
|
+
"runs_analyzed": analysis.runs_analyzed,
|
|
245
|
+
"severity": str(analysis.overall_severity),
|
|
246
|
+
"anomalous_tasks": analysis.summary.anomalous_tasks,
|
|
247
|
+
}
|
|
248
|
+
except Exception as exc:
|
|
249
|
+
ready = False
|
|
250
|
+
exporter.record_failure(
|
|
251
|
+
dag, duration_seconds=time.monotonic() - started
|
|
252
|
+
)
|
|
253
|
+
snapshot["dags"][dag] = {"error": type(exc).__name__}
|
|
254
|
+
print(f"{dag}: {type(exc).__name__}: {exc}", flush=True)
|
|
255
|
+
exporter.activity = "\n".join(activity) + "\n"
|
|
256
|
+
(args.evidence_dir / "runs.json").write_text(
|
|
257
|
+
json.dumps(snapshot, indent=2) + "\n"
|
|
258
|
+
)
|
|
259
|
+
print(
|
|
260
|
+
json.dumps(
|
|
261
|
+
{
|
|
262
|
+
dag: value.get("states", value)
|
|
263
|
+
for dag, value in snapshot["dags"].items()
|
|
264
|
+
}
|
|
265
|
+
),
|
|
266
|
+
flush=True,
|
|
267
|
+
)
|
|
268
|
+
if (
|
|
269
|
+
ready
|
|
270
|
+
and cycles < args.live_cycles
|
|
271
|
+
and time.monotonic() - last_wave >= 60
|
|
272
|
+
):
|
|
273
|
+
for dag in DAGS:
|
|
274
|
+
trigger(
|
|
275
|
+
dag,
|
|
276
|
+
f"flowsense_showcase_live_{session}_{cycles:02d}",
|
|
277
|
+
14 + cycles % 6,
|
|
278
|
+
)
|
|
279
|
+
cycles += 1
|
|
280
|
+
last_wave = time.monotonic()
|
|
281
|
+
print(f"Live wave {cycles}/{args.live_cycles} submitted", flush=True)
|
|
282
|
+
time.sleep(2)
|
|
283
|
+
finally:
|
|
284
|
+
server.shutdown()
|
|
285
|
+
server.server_close()
|
|
286
|
+
client.close()
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
if __name__ == "__main__":
|
|
290
|
+
main()
|
{flowsense_engine-0.5.0 → flowsense_engine-0.5.1}/src/flowsense/infrastructure/airflow/client.py
RENAMED
|
@@ -381,7 +381,7 @@ class AirflowClient:
|
|
|
381
381
|
return self._get_paginated(
|
|
382
382
|
self._api_url(f"/dags/{dag_id}/dagRuns"),
|
|
383
383
|
"dag_runs",
|
|
384
|
-
params={"
|
|
384
|
+
params={"state": ["success"], "order_by": "-run_after"},
|
|
385
385
|
item_limit=self.history_run_limit,
|
|
386
386
|
)
|
|
387
387
|
except AirflowApiError as exc:
|
|
@@ -408,7 +408,7 @@ class AirflowClient:
|
|
|
408
408
|
return self._get_paginated(
|
|
409
409
|
self._api_url(f"/dags/{dag_id}/dagRuns/~/taskInstances"),
|
|
410
410
|
"task_instances",
|
|
411
|
-
params={"dag_run_ids": dag_run_ids, "
|
|
411
|
+
params={"dag_run_ids": dag_run_ids, "state": ["success"]},
|
|
412
412
|
)
|
|
413
413
|
except AirflowApiError as exc:
|
|
414
414
|
if exc.status_code not in UNSUPPORTED_FILTER_STATUS_CODES:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flowsense-engine
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.1
|
|
4
4
|
Summary: Temporal drift and anomaly detection for Apache Airflow
|
|
5
5
|
Author: Omer Cengiz
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -36,9 +36,13 @@ Dynamic: license-file
|
|
|
36
36
|
|
|
37
37
|
# FlowSense Engine
|
|
38
38
|
|
|
39
|
+
<p align="center">
|
|
40
|
+
<img src="https://raw.githubusercontent.com/omercengiz/flowsense-engine/27f7db5e55bfefbba6e355c53c7c84ff34e6f37b/assets/branding/flowsense-engine-logo-v2.png" alt="FlowSense Engine logo" width="640">
|
|
41
|
+
</p>
|
|
42
|
+
|
|
39
43
|
Temporal drift, anomaly detection, and dependency-aware propagation analysis for Apache Airflow.
|
|
40
44
|
|
|
41
|
-
Current package release: `0.5.
|
|
45
|
+
Current package release: `0.5.1`. See [CHANGELOG.md](CHANGELOG.md) for release
|
|
42
46
|
notes, [docs/architecture.md](docs/architecture.md) for the system architecture,
|
|
43
47
|
[docs/benchmarks.md](docs/benchmarks.md) for performance measurement, and
|
|
44
48
|
[docs/production-quickstart.md](docs/production-quickstart.md) for a production
|
|
@@ -14,9 +14,13 @@ docs/production-quickstart.md
|
|
|
14
14
|
docs/prometheus.md
|
|
15
15
|
docs/python-api.md
|
|
16
16
|
docs/release-0.5.0.md
|
|
17
|
+
docs/release-0.5.1.md
|
|
17
18
|
docs/releasing.md
|
|
19
|
+
docs/showcase.md
|
|
18
20
|
examples/dags/flowsense_demo.py
|
|
21
|
+
examples/dags/flowsense_showcase.py
|
|
19
22
|
examples/python/production_batch.py
|
|
23
|
+
examples/python/showcase.py
|
|
20
24
|
src/flowsense/__init__.py
|
|
21
25
|
src/flowsense/config.py
|
|
22
26
|
src/flowsense/py.typed
|
|
@@ -121,5 +125,6 @@ tests/test_prometheus.py
|
|
|
121
125
|
tests/test_propagation.py
|
|
122
126
|
tests/test_public_api.py
|
|
123
127
|
tests/test_root_cause.py
|
|
128
|
+
tests/test_showcase.py
|
|
124
129
|
tests/test_timing.py
|
|
125
130
|
tests/test_trend.py
|