flowsense-engine 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/CHANGELOG.md +32 -0
- flowsense_engine-0.3.0/MANIFEST.in +2 -0
- {flowsense_engine-0.2.1/src/flowsense_engine.egg-info → flowsense_engine-0.3.0}/PKG-INFO +15 -9
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/README.md +14 -8
- flowsense_engine-0.3.0/docs/architecture.md +122 -0
- flowsense_engine-0.3.0/docs/benchmarks.md +37 -0
- flowsense_engine-0.3.0/docs/migrating-to-0.3.md +66 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/docs/releasing.md +6 -2
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/pyproject.toml +4 -3
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/__init__.py +10 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/application/__init__.py +11 -1
- flowsense_engine-0.3.0/src/flowsense/application/analysis_engine.py +71 -0
- flowsense_engine-0.3.0/src/flowsense/application/analyzer.py +21 -0
- flowsense_engine-0.3.0/src/flowsense/application/ports.py +23 -0
- flowsense_engine-0.3.0/src/flowsense/application/use_cases.py +36 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/cli/main.py +6 -11
- flowsense_engine-0.3.0/src/flowsense/collector/__init__.py +9 -0
- flowsense_engine-0.3.0/src/flowsense/config.py +22 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/domain/__init__.py +2 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/domain/exceptions.py +6 -0
- flowsense_engine-0.3.0/src/flowsense/domain/models.py +28 -0
- flowsense_engine-0.3.0/src/flowsense/engine/analyzer.py +23 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/change_point.py +3 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/drift.py +3 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/root_cause.py +3 -1
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/trend.py +3 -0
- flowsense_engine-0.3.0/src/flowsense/engine/validation.py +12 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/infrastructure/airflow/__init__.py +5 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/infrastructure/airflow/client.py +12 -29
- {flowsense_engine-0.2.1/src/flowsense → flowsense_engine-0.3.0/src/flowsense/infrastructure/airflow}/config.py +2 -1
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/infrastructure/airflow/dto.py +4 -1
- flowsense_engine-0.3.0/src/flowsense/infrastructure/airflow/factory.py +12 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/mcp/server.py +3 -11
- flowsense_engine-0.3.0/src/flowsense/models/__init__.py +17 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0/src/flowsense_engine.egg-info}/PKG-INFO +15 -9
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense_engine.egg-info/SOURCES.txt +16 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_airflow_client.py +22 -31
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_airflow_client_integration.py +2 -2
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_airflow_compatibility.py +25 -11
- flowsense_engine-0.3.0/tests/test_airflow_contract.py +95 -0
- flowsense_engine-0.3.0/tests/test_airflow_factory.py +46 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_airflow_resilience.py +7 -8
- flowsense_engine-0.3.0/tests/test_analysis_golden.py +111 -0
- flowsense_engine-0.3.0/tests/test_analysis_use_case.py +68 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_analyzer.py +1 -2
- flowsense_engine-0.3.0/tests/test_architecture.py +78 -0
- flowsense_engine-0.3.0/tests/test_benchmark_analysis.py +56 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_cli.py +36 -30
- flowsense_engine-0.3.0/tests/test_deprecations.py +26 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_domain.py +39 -9
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_history.py +1 -1
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_impact.py +1 -1
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_mcp_server.py +15 -13
- flowsense_engine-0.3.0/tests/test_observation_validation.py +27 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_package_metadata.py +1 -1
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_policy.py +1 -2
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_propagation.py +1 -1
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_public_api.py +28 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_root_cause.py +22 -3
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_timing.py +1 -2
- flowsense_engine-0.2.1/MANIFEST.in +0 -2
- flowsense_engine-0.2.1/src/flowsense/application/analyzer.py +0 -68
- flowsense_engine-0.2.1/src/flowsense/application/ports.py +0 -11
- flowsense_engine-0.2.1/src/flowsense/domain/models.py +0 -19
- flowsense_engine-0.2.1/src/flowsense/engine/analyzer.py +0 -17
- flowsense_engine-0.2.1/src/flowsense/mcp/__init__.py +0 -0
- flowsense_engine-0.2.1/src/flowsense/models/__init__.py +0 -7
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/LICENSE +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/setup.cfg +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/application/output.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/application/pipeline.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/application/request.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/application/serialization.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/cli/__init__.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/cli/report.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/collector/airflow_client.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/domain/enums.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/domain/policy.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/domain/results.py +0 -0
- {flowsense_engine-0.2.1/src/flowsense/collector → flowsense_engine-0.3.0/src/flowsense/engine}/__init__.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/history.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/impact.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/propagation.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/engine/timing.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/infrastructure/__init__.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/infrastructure/airflow/exceptions.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/infrastructure/airflow/mapper.py +0 -0
- {flowsense_engine-0.2.1/src/flowsense/engine → flowsense_engine-0.3.0/src/flowsense/mcp}/__init__.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/models/dag_analysis.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/models/task_run.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/py.typed +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense/version.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense_engine.egg-info/dependency_links.txt +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense_engine.egg-info/entry_points.txt +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense_engine.egg-info/requires.txt +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/src/flowsense_engine.egg-info/top_level.txt +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_airflow_mapper.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_analysis_output.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_analysis_pipeline.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_analysis_request.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_change_point.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_cli_report.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_dag_summary.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_drift.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_mcp_integration.py +0 -0
- {flowsense_engine-0.2.1 → flowsense_engine-0.3.0}/tests/test_trend.py +0 -0
|
@@ -3,6 +3,38 @@
|
|
|
3
3
|
All notable changes to FlowSense are documented in this file. The project uses
|
|
4
4
|
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
5
5
|
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
## [0.3.0] - 2026-09-20
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Added `AnalyzeDAG` as the shared application use-case boundary for CLI, MCP,
|
|
13
|
+
and library integrations.
|
|
14
|
+
- Added the `DAGAnalysisEngine` extension port and injectable default analysis
|
|
15
|
+
engine.
|
|
16
|
+
- Added versioned golden regression coverage for the complete analysis output.
|
|
17
|
+
- Added deterministic performance benchmarks with machine-readable results and
|
|
18
|
+
a manually triggered GitHub Actions workflow.
|
|
19
|
+
- Added an Airflow 2 and Airflow 3 REST contract test matrix covering
|
|
20
|
+
authentication, routing, validation, mapping, and end-to-end analysis.
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
|
|
24
|
+
- Separated environment loading and Airflow client construction through explicit
|
|
25
|
+
`AirflowConfig` and infrastructure factories.
|
|
26
|
+
- Made the `TaskRun` domain entity immutable and framework-independent while
|
|
27
|
+
keeping Pydantic at external DTO and output-contract boundaries.
|
|
28
|
+
- Added automated enforcement preventing the domain layer from importing
|
|
29
|
+
Pydantic.
|
|
30
|
+
- Added architecture dependency guardrails and documented supported extension
|
|
31
|
+
points and compatibility boundaries.
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- Rejected non-finite and negative analysis inputs at the domain boundary.
|
|
36
|
+
- Made root-cause selection deterministic when candidates have equal scores.
|
|
37
|
+
|
|
6
38
|
## [0.2.1] - 2026-09-10
|
|
7
39
|
|
|
8
40
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flowsense-engine
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Temporal drift and anomaly detection for Apache Airflow
|
|
5
5
|
Author: Omer Cengiz
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -37,8 +37,11 @@ Dynamic: license-file
|
|
|
37
37
|
|
|
38
38
|
Temporal drift, anomaly detection, and dependency-aware propagation analysis for Apache Airflow.
|
|
39
39
|
|
|
40
|
-
Current package release: `0.
|
|
41
|
-
notes
|
|
40
|
+
Current package release: `0.3.0`. See [CHANGELOG.md](CHANGELOG.md) for release
|
|
41
|
+
notes, [docs/architecture.md](docs/architecture.md) for the system architecture,
|
|
42
|
+
[docs/benchmarks.md](docs/benchmarks.md) for performance measurement, and
|
|
43
|
+
[docs/migrating-to-0.3.md](docs/migrating-to-0.3.md) for migration guidance.
|
|
44
|
+
Maintainers can use [docs/releasing.md](docs/releasing.md) as the release checklist.
|
|
42
45
|
|
|
43
46
|
FlowSense analyzes historical DAG executions to identify abnormal task behavior and trace how anomalies propagate through downstream dependencies.
|
|
44
47
|
|
|
@@ -189,6 +192,10 @@ Stable REST API deployments. Airflow 3.x uses the `v2` API and typically uses
|
|
|
189
192
|
token authentication. Authentication still depends on the API auth backend
|
|
190
193
|
configured in the Airflow deployment.
|
|
191
194
|
|
|
195
|
+
CI verifies both integrations through versioned Airflow 2 and Airflow 3 REST
|
|
196
|
+
contract fixtures. These boundary tests cover authentication, endpoint routing,
|
|
197
|
+
response validation, domain mapping, and the complete analysis use case.
|
|
198
|
+
|
|
192
199
|
Transient transport failures and HTTP `429`, `502`, `503`, and `504` responses
|
|
193
200
|
are retried with exponential backoff. `Retry-After` is honored when Airflow
|
|
194
201
|
provides it. Connect/read timeouts, retry count, and base backoff can be tuned
|
|
@@ -276,13 +283,12 @@ FlowSense can also be used as a Python library through its supported top-level
|
|
|
276
283
|
API:
|
|
277
284
|
|
|
278
285
|
```python
|
|
279
|
-
from flowsense import
|
|
286
|
+
from flowsense import AnalysisRequest, AnalyzeDAG
|
|
287
|
+
from flowsense.infrastructure.airflow import create_airflow_data_source
|
|
280
288
|
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
source=source,
|
|
285
|
-
)
|
|
289
|
+
analysis = AnalyzeDAG(create_airflow_data_source).execute(
|
|
290
|
+
AnalysisRequest(dag_id="flowsense_demo")
|
|
291
|
+
)
|
|
286
292
|
|
|
287
293
|
print(analysis.overall_severity)
|
|
288
294
|
print(analysis.primary_origin)
|
|
@@ -2,8 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
Temporal drift, anomaly detection, and dependency-aware propagation analysis for Apache Airflow.
|
|
4
4
|
|
|
5
|
-
Current package release: `0.
|
|
6
|
-
notes
|
|
5
|
+
Current package release: `0.3.0`. See [CHANGELOG.md](CHANGELOG.md) for release
|
|
6
|
+
notes, [docs/architecture.md](docs/architecture.md) for the system architecture,
|
|
7
|
+
[docs/benchmarks.md](docs/benchmarks.md) for performance measurement, and
|
|
8
|
+
[docs/migrating-to-0.3.md](docs/migrating-to-0.3.md) for migration guidance.
|
|
9
|
+
Maintainers can use [docs/releasing.md](docs/releasing.md) as the release checklist.
|
|
7
10
|
|
|
8
11
|
FlowSense analyzes historical DAG executions to identify abnormal task behavior and trace how anomalies propagate through downstream dependencies.
|
|
9
12
|
|
|
@@ -154,6 +157,10 @@ Stable REST API deployments. Airflow 3.x uses the `v2` API and typically uses
|
|
|
154
157
|
token authentication. Authentication still depends on the API auth backend
|
|
155
158
|
configured in the Airflow deployment.
|
|
156
159
|
|
|
160
|
+
CI verifies both integrations through versioned Airflow 2 and Airflow 3 REST
|
|
161
|
+
contract fixtures. These boundary tests cover authentication, endpoint routing,
|
|
162
|
+
response validation, domain mapping, and the complete analysis use case.
|
|
163
|
+
|
|
157
164
|
Transient transport failures and HTTP `429`, `502`, `503`, and `504` responses
|
|
158
165
|
are retried with exponential backoff. `Retry-After` is honored when Airflow
|
|
159
166
|
provides it. Connect/read timeouts, retry count, and base backoff can be tuned
|
|
@@ -241,13 +248,12 @@ FlowSense can also be used as a Python library through its supported top-level
|
|
|
241
248
|
API:
|
|
242
249
|
|
|
243
250
|
```python
|
|
244
|
-
from flowsense import
|
|
251
|
+
from flowsense import AnalysisRequest, AnalyzeDAG
|
|
252
|
+
from flowsense.infrastructure.airflow import create_airflow_data_source
|
|
245
253
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
source=source,
|
|
250
|
-
)
|
|
254
|
+
analysis = AnalyzeDAG(create_airflow_data_source).execute(
|
|
255
|
+
AnalysisRequest(dag_id="flowsense_demo")
|
|
256
|
+
)
|
|
251
257
|
|
|
252
258
|
print(analysis.overall_severity)
|
|
253
259
|
print(analysis.primary_origin)
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
# FlowSense Architecture
|
|
2
|
+
|
|
3
|
+
FlowSense uses an inward-facing layered architecture. Delivery mechanisms and
|
|
4
|
+
external systems depend on application contracts; the analysis domain does not
|
|
5
|
+
depend on Airflow, CLI, MCP, or environment configuration.
|
|
6
|
+
|
|
7
|
+
## Dependency direction
|
|
8
|
+
|
|
9
|
+
```text
|
|
10
|
+
CLI / MCP
|
|
11
|
+
|
|
|
12
|
+
v
|
|
13
|
+
Application use cases -----> Domain models and policies
|
|
14
|
+
| ^
|
|
15
|
+
v |
|
|
16
|
+
Application ports <----- Infrastructure adapters
|
|
17
|
+
|
|
|
18
|
+
v
|
|
19
|
+
Apache Airflow
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Dependencies must point toward the application and domain layers. The
|
|
23
|
+
composition roots in the CLI and MCP adapters are responsible for selecting
|
|
24
|
+
concrete infrastructure implementations.
|
|
25
|
+
|
|
26
|
+
## Layers
|
|
27
|
+
|
|
28
|
+
### Domain
|
|
29
|
+
|
|
30
|
+
`flowsense.domain` contains the analysis vocabulary: task runs, policies,
|
|
31
|
+
severity and impact classifications, results, and expected domain failures.
|
|
32
|
+
It must not import application, infrastructure, CLI, MCP, or compatibility
|
|
33
|
+
modules. Domain entities and results use standard-library dataclasses; they do
|
|
34
|
+
not depend on validation or transport frameworks.
|
|
35
|
+
|
|
36
|
+
`flowsense.engine` currently contains pure analysis services such as drift,
|
|
37
|
+
trend, handoff, propagation, and root-cause calculations. These services may
|
|
38
|
+
depend on the domain but not on delivery or infrastructure code.
|
|
39
|
+
|
|
40
|
+
### Application
|
|
41
|
+
|
|
42
|
+
`flowsense.application` coordinates use cases without knowing how Airflow data
|
|
43
|
+
is retrieved or how results are displayed.
|
|
44
|
+
|
|
45
|
+
`AnalysisRequest` is the validated input contract. `AnalyzeDAG.execute()` is the
|
|
46
|
+
primary use-case boundary. `DAGDataSource` and `DAGDataSourceFactory` are ports
|
|
47
|
+
implemented by infrastructure adapters. The existing functional
|
|
48
|
+
`analyze_dag(dag_id, source, policy)` API remains available for callers that
|
|
49
|
+
already own a data source.
|
|
50
|
+
|
|
51
|
+
`DAGAnalysisEngine` is the coarse-grained algorithm extension port. The default
|
|
52
|
+
implementation owns the complete statistical workflow after data collection.
|
|
53
|
+
Alternative engines can be injected into `AnalyzeDAG` without changing CLI,
|
|
54
|
+
MCP, data-source, or output-contract code. Individual detector interfaces are
|
|
55
|
+
intentionally avoided until independent detector replacement is required.
|
|
56
|
+
|
|
57
|
+
### Infrastructure
|
|
58
|
+
|
|
59
|
+
`flowsense.infrastructure.airflow` implements Airflow HTTP access. Pydantic DTOs
|
|
60
|
+
validate external API payloads and mappers translate them into domain objects.
|
|
61
|
+
Pydantic is intentionally restricted to infrastructure DTOs and application
|
|
62
|
+
output contracts, where runtime boundary validation and JSON Schema generation
|
|
63
|
+
are required.
|
|
64
|
+
|
|
65
|
+
`AirflowClient` receives an explicit `AirflowConfig` and never reads environment
|
|
66
|
+
variables. `load_airflow_config()` and `create_airflow_data_source()` belong to
|
|
67
|
+
the infrastructure composition boundary.
|
|
68
|
+
|
|
69
|
+
### Delivery adapters
|
|
70
|
+
|
|
71
|
+
`flowsense.cli` and `flowsense.mcp` translate user input into an
|
|
72
|
+
`AnalysisRequest`, invoke `AnalyzeDAG`, and translate the result into their own
|
|
73
|
+
output mechanism. Business analysis and data-collection orchestration must not
|
|
74
|
+
be duplicated in these adapters.
|
|
75
|
+
|
|
76
|
+
## Main analysis flow
|
|
77
|
+
|
|
78
|
+
```text
|
|
79
|
+
Input options
|
|
80
|
+
-> AnalysisRequest
|
|
81
|
+
-> AnalyzeDAG
|
|
82
|
+
-> DAGDataSourceFactory
|
|
83
|
+
-> DAGDataSource
|
|
84
|
+
-> task and handoff histories
|
|
85
|
+
-> drift / change-point / trend analysis
|
|
86
|
+
-> impact and propagation analysis
|
|
87
|
+
-> primary root-cause selection
|
|
88
|
+
-> DAGAnalysis
|
|
89
|
+
-> versioned AnalysisDocument
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Extension points
|
|
93
|
+
|
|
94
|
+
- Add another data source by implementing `DAGDataSource` and providing a
|
|
95
|
+
`DAGDataSourceFactory`.
|
|
96
|
+
- Replace the complete analysis workflow by implementing `DAGAnalysisEngine`
|
|
97
|
+
and injecting it into `AnalyzeDAG`.
|
|
98
|
+
- Add a delivery mechanism by creating an adapter that builds an
|
|
99
|
+
`AnalysisRequest` and invokes `AnalyzeDAG`.
|
|
100
|
+
- Extend output consumers through `AnalysisDocument`; incompatible contract
|
|
101
|
+
changes require a schema-version change.
|
|
102
|
+
- Keep external payload models inside their infrastructure adapter and map them
|
|
103
|
+
into domain models before analysis.
|
|
104
|
+
|
|
105
|
+
## Compatibility policy
|
|
106
|
+
|
|
107
|
+
The following pre-layered import paths remain temporarily available:
|
|
108
|
+
|
|
109
|
+
- `flowsense.models` -> `flowsense.domain`
|
|
110
|
+
- `flowsense.collector` -> `flowsense.infrastructure.airflow`
|
|
111
|
+
- `flowsense.config` -> `flowsense.infrastructure.airflow`
|
|
112
|
+
- `flowsense.engine.analyzer` -> `flowsense.AnalyzeDAG`
|
|
113
|
+
|
|
114
|
+
They emit `DeprecationWarning` and must not be used by new internal code. They
|
|
115
|
+
will be removed only in an explicitly announced breaking release.
|
|
116
|
+
|
|
117
|
+
## Automated guardrails
|
|
118
|
+
|
|
119
|
+
`tests/test_architecture.py` parses internal imports and fails when a layer
|
|
120
|
+
introduces a forbidden outward dependency. The compatibility analyzer is the
|
|
121
|
+
only documented exception. New exceptions require an architectural decision and
|
|
122
|
+
must not be added merely to make the test pass.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Performance benchmarks
|
|
2
|
+
|
|
3
|
+
FlowSense includes a deterministic synthetic benchmark for measuring the
|
|
4
|
+
default analysis engine independently from Airflow and network latency.
|
|
5
|
+
|
|
6
|
+
The benchmark creates an ordered chain DAG, generates repeatable task timings,
|
|
7
|
+
injects anomalies into the latest run, and reports:
|
|
8
|
+
|
|
9
|
+
- analysis execution time;
|
|
10
|
+
- total time including versioned output serialization;
|
|
11
|
+
- peak Python memory observed through `tracemalloc`;
|
|
12
|
+
- scenario dimensions and analyzed task count.
|
|
13
|
+
|
|
14
|
+
Run the default scenario:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
uv run python -m benchmarks.benchmark_analysis
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Run a larger scenario and save machine-readable results:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
uv run python -m benchmarks.benchmark_analysis \
|
|
24
|
+
--tasks 500 \
|
|
25
|
+
--runs 100 \
|
|
26
|
+
--iterations 5 \
|
|
27
|
+
--output benchmark-results.json
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
The GitHub `Analysis benchmark` workflow provides the same parameters and
|
|
31
|
+
uploads the JSON result as a workflow artifact.
|
|
32
|
+
|
|
33
|
+
Benchmark results are observational and do not use hard pass/fail time limits.
|
|
34
|
+
Shared CI runners have variable performance, so fixed thresholds would create
|
|
35
|
+
flaky checks. Compare runs produced with the same scenario, Python version, and
|
|
36
|
+
similar hardware. Introduce performance gates only after a stable baseline has
|
|
37
|
+
been collected on controlled runners.
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Migrating to FlowSense 0.3
|
|
2
|
+
|
|
3
|
+
FlowSense 0.3 strengthens the library architecture without changing the
|
|
4
|
+
versioned analysis output contract. The schema remains at version `1.1`, so
|
|
5
|
+
existing CLI JSON and MCP consumers do not need a response migration.
|
|
6
|
+
|
|
7
|
+
## Supported imports
|
|
8
|
+
|
|
9
|
+
Library integrations should import public contracts directly from `flowsense`:
|
|
10
|
+
|
|
11
|
+
```python
|
|
12
|
+
from flowsense import AnalysisRequest, AnalyzeDAG, TaskRun
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The following compatibility namespaces still work in 0.3 but emit
|
|
16
|
+
`DeprecationWarning` and are planned for removal in a future breaking release:
|
|
17
|
+
|
|
18
|
+
| Deprecated import | Replacement |
|
|
19
|
+
| --- | --- |
|
|
20
|
+
| `flowsense.models` | `flowsense.domain` or public names from `flowsense` |
|
|
21
|
+
| `flowsense.collector` | `flowsense.infrastructure.airflow` |
|
|
22
|
+
| `flowsense.config` | `flowsense.infrastructure.airflow` |
|
|
23
|
+
| `flowsense.engine.analyzer` | `flowsense.AnalyzeDAG` |
|
|
24
|
+
|
|
25
|
+
Applications should migrate these imports now; no compatibility namespace is
|
|
26
|
+
used by FlowSense internals.
|
|
27
|
+
|
|
28
|
+
## Analysis entry point
|
|
29
|
+
|
|
30
|
+
Use `AnalyzeDAG` when FlowSense owns data-source creation:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from flowsense import AnalysisRequest, AnalyzeDAG
|
|
34
|
+
from flowsense.infrastructure.airflow import create_airflow_data_source
|
|
35
|
+
|
|
36
|
+
analysis = AnalyzeDAG(create_airflow_data_source).execute(
|
|
37
|
+
AnalysisRequest(dag_id="example_dag")
|
|
38
|
+
)
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Existing integrations that already own a `DAGDataSource` can continue using
|
|
42
|
+
the public `analyze_dag` function. Custom analysis implementations can implement
|
|
43
|
+
`DAGAnalysisEngine` and inject that engine into `AnalyzeDAG`.
|
|
44
|
+
|
|
45
|
+
## Airflow configuration
|
|
46
|
+
|
|
47
|
+
`AirflowClient` now receives an explicit immutable `AirflowConfig`. Environment
|
|
48
|
+
access belongs to `load_airflow_config()` or `create_airflow_data_source()`:
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from flowsense.infrastructure.airflow import AirflowClient, load_airflow_config
|
|
52
|
+
|
|
53
|
+
client = AirflowClient(load_airflow_config())
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
CLI and MCP users can keep using the existing `AIRFLOW_*` environment variables.
|
|
57
|
+
For Airflow 2 use `AIRFLOW_API_VERSION=v1` with the authentication mode provided
|
|
58
|
+
by the deployment, commonly `basic`. Airflow 3 normally uses
|
|
59
|
+
`AIRFLOW_API_VERSION=v2` and `token` authentication.
|
|
60
|
+
|
|
61
|
+
## Domain model behavior
|
|
62
|
+
|
|
63
|
+
`TaskRun` is now an immutable standard-library dataclass. Code that mutated a
|
|
64
|
+
task run after construction must instead create a new value, for example with
|
|
65
|
+
`dataclasses.replace`. Invalid negative or non-finite durations are rejected at
|
|
66
|
+
construction time.
|
|
@@ -33,11 +33,15 @@ After the release pull request is merged into `master` and CI passes:
|
|
|
33
33
|
```bash
|
|
34
34
|
git switch master
|
|
35
35
|
git pull --ff-only origin master
|
|
36
|
-
git tag -a v0.
|
|
37
|
-
git push origin v0.
|
|
36
|
+
git tag -a v0.3.0 -m "FlowSense 0.3.0"
|
|
37
|
+
git push origin v0.3.0
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
Create and publish the corresponding GitHub Release after pushing the tag. The
|
|
41
41
|
release workflow builds that exact tag and publishes it through the protected
|
|
42
42
|
`pypi` environment. Package publication should run only after the built artifacts
|
|
43
43
|
have been verified.
|
|
44
|
+
|
|
45
|
+
Use `FlowSense 0.3.0` as the release title, select `Latest`, and do not mark a
|
|
46
|
+
stable release as a pre-release. Copy the `0.3.0` section from `CHANGELOG.md` into
|
|
47
|
+
the release notes.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "flowsense-engine"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Temporal drift and anomaly detection for Apache Airflow"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -68,12 +68,13 @@ where = ["src"]
|
|
|
68
68
|
flowsense = ["py.typed"]
|
|
69
69
|
|
|
70
70
|
[tool.pyright]
|
|
71
|
-
include = ["src"]
|
|
71
|
+
include = ["src", "benchmarks"]
|
|
72
72
|
typeCheckingMode = "standard"
|
|
73
73
|
pythonVersion = "3.12"
|
|
74
74
|
|
|
75
75
|
[tool.pytest.ini_options]
|
|
76
|
-
pythonpath = ["src"]
|
|
76
|
+
pythonpath = ["src", "."]
|
|
77
77
|
markers = [
|
|
78
|
+
"airflow_contract: Airflow REST API boundary contract tests",
|
|
78
79
|
"integration: tests that require a running Airflow instance",
|
|
79
80
|
]
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
from flowsense.application import (
|
|
2
2
|
ANALYSIS_SCHEMA_VERSION,
|
|
3
|
+
DEFAULT_DAG_ANALYSIS_ENGINE,
|
|
3
4
|
AnalysisDocument,
|
|
4
5
|
AnalysisRequest,
|
|
6
|
+
AnalyzeDAG,
|
|
7
|
+
DAGAnalysisEngine,
|
|
5
8
|
DAGDataSource,
|
|
9
|
+
DefaultDAGAnalysisEngine,
|
|
6
10
|
analysis_json_schema,
|
|
7
11
|
analyze_dag,
|
|
8
12
|
build_analysis_document,
|
|
@@ -21,6 +25,7 @@ from flowsense.domain import (
|
|
|
21
25
|
FlowSenseError,
|
|
22
26
|
ImpactClassification,
|
|
23
27
|
InsufficientHistoryError,
|
|
28
|
+
InvalidObservationError,
|
|
24
29
|
InvalidTaskTimingError,
|
|
25
30
|
MappedTaskAggregation,
|
|
26
31
|
PropagationResult,
|
|
@@ -42,6 +47,7 @@ from flowsense.version import __version__
|
|
|
42
47
|
__all__ = [
|
|
43
48
|
"ANALYSIS_SCHEMA_VERSION",
|
|
44
49
|
"DEFAULT_ANALYSIS_POLICY",
|
|
50
|
+
"DEFAULT_DAG_ANALYSIS_ENGINE",
|
|
45
51
|
"AirflowApiError",
|
|
46
52
|
"AirflowClient",
|
|
47
53
|
"AirflowDagRunNotFoundError",
|
|
@@ -50,16 +56,20 @@ __all__ = [
|
|
|
50
56
|
"AnalysisDocument",
|
|
51
57
|
"AnalysisPolicy",
|
|
52
58
|
"AnalysisRequest",
|
|
59
|
+
"AnalyzeDAG",
|
|
53
60
|
"ChangeDirection",
|
|
54
61
|
"ChangePointResult",
|
|
55
62
|
"ConfigurationError",
|
|
56
63
|
"DAGAnalysis",
|
|
64
|
+
"DAGAnalysisEngine",
|
|
57
65
|
"DAGAnalysisSummary",
|
|
58
66
|
"DAGDataSource",
|
|
67
|
+
"DefaultDAGAnalysisEngine",
|
|
59
68
|
"DriftResult",
|
|
60
69
|
"FlowSenseError",
|
|
61
70
|
"ImpactClassification",
|
|
62
71
|
"InsufficientHistoryError",
|
|
72
|
+
"InvalidObservationError",
|
|
63
73
|
"InvalidTaskTimingError",
|
|
64
74
|
"MappedTaskAggregation",
|
|
65
75
|
"PropagationResult",
|
|
@@ -1,18 +1,28 @@
|
|
|
1
|
+
from flowsense.application.analysis_engine import (
|
|
2
|
+
DEFAULT_DAG_ANALYSIS_ENGINE,
|
|
3
|
+
DefaultDAGAnalysisEngine,
|
|
4
|
+
)
|
|
1
5
|
from flowsense.application.analyzer import analyze_dag
|
|
2
6
|
from flowsense.application.output import ANALYSIS_SCHEMA_VERSION, AnalysisDocument
|
|
3
|
-
from flowsense.application.ports import DAGDataSource
|
|
7
|
+
from flowsense.application.ports import DAGAnalysisEngine, DAGDataSource
|
|
4
8
|
from flowsense.application.request import AnalysisRequest
|
|
5
9
|
from flowsense.application.serialization import (
|
|
6
10
|
analysis_json_schema,
|
|
7
11
|
build_analysis_document,
|
|
8
12
|
serialize_analysis,
|
|
9
13
|
)
|
|
14
|
+
from flowsense.application.use_cases import AnalyzeDAG, DAGDataSourceFactory
|
|
10
15
|
|
|
11
16
|
__all__ = [
|
|
12
17
|
"ANALYSIS_SCHEMA_VERSION",
|
|
18
|
+
"DEFAULT_DAG_ANALYSIS_ENGINE",
|
|
13
19
|
"AnalysisDocument",
|
|
14
20
|
"AnalysisRequest",
|
|
21
|
+
"AnalyzeDAG",
|
|
22
|
+
"DAGAnalysisEngine",
|
|
15
23
|
"DAGDataSource",
|
|
24
|
+
"DAGDataSourceFactory",
|
|
25
|
+
"DefaultDAGAnalysisEngine",
|
|
16
26
|
"analysis_json_schema",
|
|
17
27
|
"analyze_dag",
|
|
18
28
|
"build_analysis_document",
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
from flowsense.application.pipeline import (
|
|
2
|
+
analyze_handoff_histories,
|
|
3
|
+
analyze_task_histories,
|
|
4
|
+
classify_task_impacts,
|
|
5
|
+
determine_overall_severity,
|
|
6
|
+
)
|
|
7
|
+
from flowsense.domain import AnalysisPolicy, DAGAnalysis, TaskRun
|
|
8
|
+
from flowsense.engine.history import build_duration_history
|
|
9
|
+
from flowsense.engine.propagation import analyze_propagation
|
|
10
|
+
from flowsense.engine.root_cause import select_primary_origin
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class DefaultDAGAnalysisEngine:
|
|
14
|
+
"""Default stateless implementation of the DAG analysis domain workflow."""
|
|
15
|
+
|
|
16
|
+
def analyze(
|
|
17
|
+
self,
|
|
18
|
+
dag_id: str,
|
|
19
|
+
task_runs: list[TaskRun],
|
|
20
|
+
dependencies: dict[str, list[str]],
|
|
21
|
+
policy: AnalysisPolicy,
|
|
22
|
+
) -> DAGAnalysis:
|
|
23
|
+
duration_history = build_duration_history(
|
|
24
|
+
task_runs,
|
|
25
|
+
aggregation=policy.mapped_task_aggregation,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
task_analysis = analyze_task_histories(duration_history, policy)
|
|
29
|
+
handoff_analysis = analyze_handoff_histories(task_runs, dependencies, policy)
|
|
30
|
+
task_impacts = classify_task_impacts(
|
|
31
|
+
task_analysis.drift_results,
|
|
32
|
+
handoff_analysis.drift_results,
|
|
33
|
+
)
|
|
34
|
+
propagation_results = analyze_propagation(
|
|
35
|
+
drift_results=task_analysis.drift_results,
|
|
36
|
+
dependencies=dependencies,
|
|
37
|
+
)
|
|
38
|
+
primary_origin = select_primary_origin(
|
|
39
|
+
drift_results=task_analysis.drift_results,
|
|
40
|
+
task_impacts=task_impacts,
|
|
41
|
+
dependencies=dependencies,
|
|
42
|
+
propagation_results=propagation_results,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
return DAGAnalysis(
|
|
46
|
+
dag_id=dag_id,
|
|
47
|
+
runs_analyzed=len({run.dag_run_id for run in task_runs}),
|
|
48
|
+
current_dag_run_id=(task_runs[-1].dag_run_id if task_runs else None),
|
|
49
|
+
overall_severity=determine_overall_severity(
|
|
50
|
+
task_analysis.drift_results,
|
|
51
|
+
handoff_analysis.drift_results,
|
|
52
|
+
),
|
|
53
|
+
primary_origin=primary_origin,
|
|
54
|
+
drift_results=task_analysis.drift_results,
|
|
55
|
+
handoff_drift_results=handoff_analysis.drift_results,
|
|
56
|
+
task_impacts=task_impacts,
|
|
57
|
+
propagation_results=propagation_results,
|
|
58
|
+
dependencies=dependencies,
|
|
59
|
+
diagnostics=[
|
|
60
|
+
*task_analysis.diagnostics,
|
|
61
|
+
*handoff_analysis.diagnostics,
|
|
62
|
+
],
|
|
63
|
+
policy=policy,
|
|
64
|
+
change_point_results=task_analysis.change_point_results,
|
|
65
|
+
handoff_change_point_results=handoff_analysis.change_point_results,
|
|
66
|
+
trend_results=task_analysis.trend_results,
|
|
67
|
+
handoff_trend_results=handoff_analysis.trend_results,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
DEFAULT_DAG_ANALYSIS_ENGINE = DefaultDAGAnalysisEngine()
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from flowsense.application.analysis_engine import DEFAULT_DAG_ANALYSIS_ENGINE
|
|
4
|
+
from flowsense.application.ports import DAGAnalysisEngine, DAGDataSource
|
|
5
|
+
from flowsense.domain import DEFAULT_ANALYSIS_POLICY, AnalysisPolicy, DAGAnalysis
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def analyze_dag(
|
|
9
|
+
dag_id: str,
|
|
10
|
+
source: DAGDataSource,
|
|
11
|
+
policy: AnalysisPolicy = DEFAULT_ANALYSIS_POLICY,
|
|
12
|
+
analysis_engine: DAGAnalysisEngine = DEFAULT_DAG_ANALYSIS_ENGINE,
|
|
13
|
+
) -> DAGAnalysis:
|
|
14
|
+
task_runs = source.collect_task_runs(dag_id)
|
|
15
|
+
dependencies = source.get_dag_dependencies(dag_id)
|
|
16
|
+
return analysis_engine.analyze(
|
|
17
|
+
dag_id=dag_id,
|
|
18
|
+
task_runs=task_runs,
|
|
19
|
+
dependencies=dependencies,
|
|
20
|
+
policy=policy,
|
|
21
|
+
)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from typing import Protocol
|
|
2
|
+
|
|
3
|
+
from flowsense.domain import AnalysisPolicy, DAGAnalysis, TaskRun
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DAGDataSource(Protocol):
|
|
7
|
+
def collect_task_runs(self, dag_id: str) -> list[TaskRun]:
|
|
8
|
+
"""Return task runs ordered from oldest DAG run to newest."""
|
|
9
|
+
...
|
|
10
|
+
|
|
11
|
+
def get_dag_dependencies(self, dag_id: str) -> dict[str, list[str]]: ...
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class DAGAnalysisEngine(Protocol):
|
|
15
|
+
"""Analyze already collected domain data without performing I/O."""
|
|
16
|
+
|
|
17
|
+
def analyze(
|
|
18
|
+
self,
|
|
19
|
+
dag_id: str,
|
|
20
|
+
task_runs: list[TaskRun],
|
|
21
|
+
dependencies: dict[str, list[str]],
|
|
22
|
+
policy: AnalysisPolicy,
|
|
23
|
+
) -> DAGAnalysis: ...
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from collections.abc import Callable
|
|
2
|
+
from contextlib import AbstractContextManager
|
|
3
|
+
|
|
4
|
+
from flowsense.application.analysis_engine import DEFAULT_DAG_ANALYSIS_ENGINE
|
|
5
|
+
from flowsense.application.ports import DAGAnalysisEngine, DAGDataSource
|
|
6
|
+
from flowsense.application.request import AnalysisRequest
|
|
7
|
+
from flowsense.domain import DAGAnalysis
|
|
8
|
+
|
|
9
|
+
DAGDataSourceFactory = Callable[
|
|
10
|
+
[AnalysisRequest],
|
|
11
|
+
AbstractContextManager[DAGDataSource],
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class AnalyzeDAG:
|
|
16
|
+
"""Application use case coordinating data collection and DAG analysis."""
|
|
17
|
+
|
|
18
|
+
def __init__(
|
|
19
|
+
self,
|
|
20
|
+
source_factory: DAGDataSourceFactory,
|
|
21
|
+
analysis_engine: DAGAnalysisEngine = DEFAULT_DAG_ANALYSIS_ENGINE,
|
|
22
|
+
) -> None:
|
|
23
|
+
self._source_factory = source_factory
|
|
24
|
+
self._analysis_engine = analysis_engine
|
|
25
|
+
|
|
26
|
+
def execute(self, request: AnalysisRequest) -> DAGAnalysis:
|
|
27
|
+
with self._source_factory(request) as source:
|
|
28
|
+
task_runs = source.collect_task_runs(request.dag_id)
|
|
29
|
+
dependencies = source.get_dag_dependencies(request.dag_id)
|
|
30
|
+
|
|
31
|
+
return self._analysis_engine.analyze(
|
|
32
|
+
dag_id=request.dag_id,
|
|
33
|
+
task_runs=task_runs,
|
|
34
|
+
dependencies=dependencies,
|
|
35
|
+
policy=request.policy,
|
|
36
|
+
)
|