phlo-core-plugins 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_core_plugins-0.1.0/PKG-INFO +17 -0
- phlo_core_plugins-0.1.0/README.md +76 -0
- phlo_core_plugins-0.1.0/pyproject.toml +45 -0
- phlo_core_plugins-0.1.0/setup.cfg +4 -0
- phlo_core_plugins-0.1.0/src/phlo_core/__init__.py +16 -0
- phlo_core_plugins-0.1.0/src/phlo_core/quality/__init__.py +13 -0
- phlo_core_plugins-0.1.0/src/phlo_core/quality/freshness_check.py +32 -0
- phlo_core_plugins-0.1.0/src/phlo_core/quality/null_check.py +21 -0
- phlo_core_plugins-0.1.0/src/phlo_core/quality/schema_check.py +23 -0
- phlo_core_plugins-0.1.0/src/phlo_core/quality/uniqueness_check.py +21 -0
- phlo_core_plugins-0.1.0/src/phlo_core/sources/__init__.py +5 -0
- phlo_core_plugins-0.1.0/src/phlo_core/sources/rest_api.py +57 -0
- phlo_core_plugins-0.1.0/src/phlo_core_plugins.egg-info/PKG-INFO +17 -0
- phlo_core_plugins-0.1.0/src/phlo_core_plugins.egg-info/SOURCES.txt +19 -0
- phlo_core_plugins-0.1.0/src/phlo_core_plugins.egg-info/dependency_links.txt +1 -0
- phlo_core_plugins-0.1.0/src/phlo_core_plugins.egg-info/entry_points.txt +8 -0
- phlo_core_plugins-0.1.0/src/phlo_core_plugins.egg-info/requires.txt +8 -0
- phlo_core_plugins-0.1.0/src/phlo_core_plugins.egg-info/top_level.txt +1 -0
- phlo_core_plugins-0.1.0/tests/test_integration_core_plugins.py +250 -0
- phlo_core_plugins-0.1.0/tests/test_quality_plugins.py +37 -0
- phlo_core_plugins-0.1.0/tests/test_rest_api_plugin.py +40 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phlo-core-plugins
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Core plugin package for Phlo
|
|
5
|
+
Author-email: Phlo Team <team@phlo.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/plain
|
|
9
|
+
Requires-Dist: phlo>=0.1.0
|
|
10
|
+
Requires-Dist: phlo-quality>=0.1.0
|
|
11
|
+
Requires-Dist: requests>=2.28.0
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
14
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
15
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
16
|
+
|
|
17
|
+
Core plugin package for Phlo.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# phlo-core-plugins
|
|
2
|
+
|
|
3
|
+
Core plugins for Phlo (quality checks and source connectors).
|
|
4
|
+
|
|
5
|
+
## Description
|
|
6
|
+
|
|
7
|
+
Provides built-in quality check plugins and source connectors. These are registered via entry points and auto-discovered at runtime.
|
|
8
|
+
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install phlo-core-plugins
|
|
13
|
+
# or included with phlo by default
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Auto-Configuration
|
|
17
|
+
|
|
18
|
+
This package is **fully auto-configured**:
|
|
19
|
+
|
|
20
|
+
| Feature | How It Works |
|
|
21
|
+
| --------------------- | ------------------------------------------------------- |
|
|
22
|
+
| **Quality Checks** | Auto-registered via `phlo.plugins.quality` entry points |
|
|
23
|
+
| **Source Connectors** | Auto-registered via `phlo.plugins.sources` entry points |
|
|
24
|
+
|
|
25
|
+
## Quality Check Plugins
|
|
26
|
+
|
|
27
|
+
| Check | Entry Point | Description |
|
|
28
|
+
| ------------------ | ---------------------- | ------------------------ |
|
|
29
|
+
| `null_check` | `phlo.plugins.quality` | Validates no NULL values |
|
|
30
|
+
| `uniqueness_check` | `phlo.plugins.quality` | Validates unique values |
|
|
31
|
+
| `range_check` | `phlo.plugins.quality` | Validates value ranges |
|
|
32
|
+
| `regex_check` | `phlo.plugins.quality` | Validates regex patterns |
|
|
33
|
+
| `freshness_check` | `phlo.plugins.quality` | Validates data freshness |
|
|
34
|
+
|
|
35
|
+
## Source Connector Plugins
|
|
36
|
+
|
|
37
|
+
| Connector | Entry Point | Description |
|
|
38
|
+
| ---------- | ---------------------- | -------------------------- |
|
|
39
|
+
| `rest_api` | `phlo.plugins.sources` | Generic REST API connector |
|
|
40
|
+
|
|
41
|
+
## Usage
|
|
42
|
+
|
|
43
|
+
### Quality Checks
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from phlo_quality.checks import null_check, uniqueness_check
|
|
47
|
+
|
|
48
|
+
@phlo_quality(
|
|
49
|
+
asset="bronze.users",
|
|
50
|
+
checks=[
|
|
51
|
+
null_check(column="id"),
|
|
52
|
+
uniqueness_check(column="email"),
|
|
53
|
+
]
|
|
54
|
+
)
|
|
55
|
+
def validate_users():
|
|
56
|
+
pass
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Source Connectors
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from phlo.ingestion import phlo_ingestion
|
|
63
|
+
|
|
64
|
+
@phlo_ingestion(
|
|
65
|
+
name="api_data",
|
|
66
|
+
source="rest_api", # Uses rest_api connector
|
|
67
|
+
destination="bronze.api_data"
|
|
68
|
+
)
|
|
69
|
+
def ingest_api():
|
|
70
|
+
return {"client": {"base_url": "https://api.example.com"}}
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Entry Points
|
|
74
|
+
|
|
75
|
+
- `phlo.plugins.quality` - Quality check plugins
|
|
76
|
+
- `phlo.plugins.sources` - Source connector plugins
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=45", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "phlo-core-plugins"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Core plugin package for Phlo"
|
|
9
|
+
readme = {text = "Core plugin package for Phlo.", content-type = "text/plain"}
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
authors = [
|
|
12
|
+
{name = "Phlo Team", email = "team@phlo.dev"},
|
|
13
|
+
]
|
|
14
|
+
license = {text = "MIT"}
|
|
15
|
+
dependencies = [
|
|
16
|
+
"phlo>=0.1.0",
|
|
17
|
+
"phlo-quality>=0.1.0",
|
|
18
|
+
"requests>=2.28.0",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
dev = [
|
|
23
|
+
"pytest>=7.0",
|
|
24
|
+
"pytest-cov>=4.0",
|
|
25
|
+
"ruff>=0.1.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.entry-points."phlo.plugins.quality"]
|
|
29
|
+
null_check = "phlo_core.quality.null_check:NullCheckPlugin"
|
|
30
|
+
uniqueness_check = "phlo_core.quality.uniqueness_check:UniquenessCheckPlugin"
|
|
31
|
+
freshness_check = "phlo_core.quality.freshness_check:FreshnessCheckPlugin"
|
|
32
|
+
schema_check = "phlo_core.quality.schema_check:SchemaCheckPlugin"
|
|
33
|
+
|
|
34
|
+
[project.entry-points."phlo.plugins.sources"]
|
|
35
|
+
rest_api = "phlo_core.sources.rest_api:RestAPIPlugin"
|
|
36
|
+
|
|
37
|
+
[tool.setuptools]
|
|
38
|
+
package-dir = {"" = "src"}
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.packages.find]
|
|
41
|
+
where = ["src"]
|
|
42
|
+
|
|
43
|
+
[tool.ruff]
|
|
44
|
+
line-length = 100
|
|
45
|
+
target-version = "py311"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Core plugins for Phlo."""
|
|
2
|
+
|
|
3
|
+
from phlo_core.quality.freshness_check import FreshnessCheckPlugin
|
|
4
|
+
from phlo_core.quality.null_check import NullCheckPlugin
|
|
5
|
+
from phlo_core.quality.schema_check import SchemaCheckPlugin
|
|
6
|
+
from phlo_core.quality.uniqueness_check import UniquenessCheckPlugin
|
|
7
|
+
from phlo_core.sources.rest_api import RestAPIPlugin
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"NullCheckPlugin",
|
|
11
|
+
"UniquenessCheckPlugin",
|
|
12
|
+
"FreshnessCheckPlugin",
|
|
13
|
+
"SchemaCheckPlugin",
|
|
14
|
+
"RestAPIPlugin",
|
|
15
|
+
]
|
|
16
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Quality check plugins bundled with Phlo."""
|
|
2
|
+
|
|
3
|
+
from phlo_core.quality.freshness_check import FreshnessCheckPlugin
|
|
4
|
+
from phlo_core.quality.null_check import NullCheckPlugin
|
|
5
|
+
from phlo_core.quality.schema_check import SchemaCheckPlugin
|
|
6
|
+
from phlo_core.quality.uniqueness_check import UniquenessCheckPlugin
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"NullCheckPlugin",
|
|
10
|
+
"UniquenessCheckPlugin",
|
|
11
|
+
"FreshnessCheckPlugin",
|
|
12
|
+
"SchemaCheckPlugin",
|
|
13
|
+
]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Freshness check plugin."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
|
|
5
|
+
from phlo.plugins import PluginMetadata, QualityCheckPlugin
|
|
6
|
+
from phlo_quality.checks import FreshnessCheck
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class FreshnessCheckPlugin(QualityCheckPlugin[FreshnessCheck]):
|
|
10
|
+
"""Plugin for freshness checks."""
|
|
11
|
+
|
|
12
|
+
@property
|
|
13
|
+
def metadata(self) -> PluginMetadata:
|
|
14
|
+
return PluginMetadata(
|
|
15
|
+
name="freshness_check",
|
|
16
|
+
version="0.1.0",
|
|
17
|
+
description="Freshness checks for timestamped data",
|
|
18
|
+
author="Phlo Team",
|
|
19
|
+
tags=["quality", "freshness"],
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
def create_check(
|
|
23
|
+
self,
|
|
24
|
+
timestamp_column: str,
|
|
25
|
+
max_age_hours: float,
|
|
26
|
+
reference_time: datetime | None = None,
|
|
27
|
+
) -> FreshnessCheck:
|
|
28
|
+
return FreshnessCheck(
|
|
29
|
+
timestamp_column=timestamp_column,
|
|
30
|
+
max_age_hours=max_age_hours,
|
|
31
|
+
reference_time=reference_time,
|
|
32
|
+
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Null check plugin."""
|
|
2
|
+
|
|
3
|
+
from phlo.plugins import PluginMetadata, QualityCheckPlugin
|
|
4
|
+
from phlo_quality.checks import NullCheck
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class NullCheckPlugin(QualityCheckPlugin[NullCheck]):
|
|
8
|
+
"""Plugin for NullCheck quality checks."""
|
|
9
|
+
|
|
10
|
+
@property
|
|
11
|
+
def metadata(self) -> PluginMetadata:
|
|
12
|
+
return PluginMetadata(
|
|
13
|
+
name="null_check",
|
|
14
|
+
version="0.1.0",
|
|
15
|
+
description="Null checks for column completeness",
|
|
16
|
+
author="Phlo Team",
|
|
17
|
+
tags=["quality", "nulls"],
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
def create_check(self, columns: list[str], allow_threshold: float = 0.0) -> NullCheck:
|
|
21
|
+
return NullCheck(columns=columns, allow_threshold=allow_threshold)
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Schema check plugin."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from phlo.plugins import PluginMetadata, QualityCheckPlugin
|
|
6
|
+
from phlo_quality.checks_extra import SchemaCheck
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class SchemaCheckPlugin(QualityCheckPlugin[SchemaCheck]):
|
|
10
|
+
"""Plugin for schema checks."""
|
|
11
|
+
|
|
12
|
+
@property
|
|
13
|
+
def metadata(self) -> PluginMetadata:
|
|
14
|
+
return PluginMetadata(
|
|
15
|
+
name="schema_check",
|
|
16
|
+
version="0.1.0",
|
|
17
|
+
description="Schema validation for expected columns and types",
|
|
18
|
+
author="Phlo Team",
|
|
19
|
+
tags=["quality", "schema"],
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
def create_check(self, schema: Any, lazy: bool = True) -> SchemaCheck:
|
|
23
|
+
return SchemaCheck(schema=schema, lazy=lazy)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Uniqueness check plugin."""
|
|
2
|
+
|
|
3
|
+
from phlo.plugins import PluginMetadata, QualityCheckPlugin
|
|
4
|
+
from phlo_quality.checks import UniqueCheck
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class UniquenessCheckPlugin(QualityCheckPlugin[UniqueCheck]):
|
|
8
|
+
"""Plugin for uniqueness checks."""
|
|
9
|
+
|
|
10
|
+
@property
|
|
11
|
+
def metadata(self) -> PluginMetadata:
|
|
12
|
+
return PluginMetadata(
|
|
13
|
+
name="uniqueness_check",
|
|
14
|
+
version="0.1.0",
|
|
15
|
+
description="Uniqueness validation for primary keys",
|
|
16
|
+
author="Phlo Team",
|
|
17
|
+
tags=["quality", "uniqueness"],
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
def create_check(self, columns: list[str], allow_threshold: float = 0.0) -> UniqueCheck:
|
|
21
|
+
return UniqueCheck(columns=columns, allow_threshold=allow_threshold)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""REST API source connector plugin."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
from phlo.plugins import PluginMetadata, SourceConnectorPlugin
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class RestAPIPlugin(SourceConnectorPlugin):
|
|
11
|
+
"""Generic REST API source connector."""
|
|
12
|
+
|
|
13
|
+
@property
|
|
14
|
+
def metadata(self) -> PluginMetadata:
|
|
15
|
+
return PluginMetadata(
|
|
16
|
+
name="rest_api",
|
|
17
|
+
version="0.1.0",
|
|
18
|
+
description="Generic REST API source connector",
|
|
19
|
+
author="Phlo Team",
|
|
20
|
+
tags=["source", "api"],
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
def fetch_data(self, config: dict[str, Any]):
|
|
24
|
+
url = config["url"]
|
|
25
|
+
headers = config.get("headers", {})
|
|
26
|
+
params = config.get("params", {})
|
|
27
|
+
timeout = config.get("timeout", 30)
|
|
28
|
+
records_path = config.get("records_path")
|
|
29
|
+
|
|
30
|
+
response = requests.get(url, headers=headers, params=params, timeout=timeout)
|
|
31
|
+
response.raise_for_status()
|
|
32
|
+
|
|
33
|
+
payload = response.json()
|
|
34
|
+
records = _extract_records(payload, records_path)
|
|
35
|
+
for record in records:
|
|
36
|
+
yield record
|
|
37
|
+
|
|
38
|
+
def get_schema(self, config: dict[str, Any]) -> dict[str, str] | None:
|
|
39
|
+
return config.get("schema")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _extract_records(payload: Any, records_path: str | None) -> list[dict[str, Any]]:
|
|
43
|
+
if records_path:
|
|
44
|
+
current = payload
|
|
45
|
+
for key in records_path.split("."):
|
|
46
|
+
if isinstance(current, dict) and key in current:
|
|
47
|
+
current = current[key]
|
|
48
|
+
else:
|
|
49
|
+
raise ValueError(f"records_path '{records_path}' not found in payload")
|
|
50
|
+
payload = current
|
|
51
|
+
|
|
52
|
+
if isinstance(payload, list):
|
|
53
|
+
return payload
|
|
54
|
+
if isinstance(payload, dict):
|
|
55
|
+
return [payload]
|
|
56
|
+
|
|
57
|
+
raise ValueError("Unsupported payload shape for REST API response")
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phlo-core-plugins
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Core plugin package for Phlo
|
|
5
|
+
Author-email: Phlo Team <team@phlo.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/plain
|
|
9
|
+
Requires-Dist: phlo>=0.1.0
|
|
10
|
+
Requires-Dist: phlo-quality>=0.1.0
|
|
11
|
+
Requires-Dist: requests>=2.28.0
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
14
|
+
Requires-Dist: pytest-cov>=4.0; extra == "dev"
|
|
15
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
16
|
+
|
|
17
|
+
Core plugin package for Phlo.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
src/phlo_core/__init__.py
|
|
4
|
+
src/phlo_core/quality/__init__.py
|
|
5
|
+
src/phlo_core/quality/freshness_check.py
|
|
6
|
+
src/phlo_core/quality/null_check.py
|
|
7
|
+
src/phlo_core/quality/schema_check.py
|
|
8
|
+
src/phlo_core/quality/uniqueness_check.py
|
|
9
|
+
src/phlo_core/sources/__init__.py
|
|
10
|
+
src/phlo_core/sources/rest_api.py
|
|
11
|
+
src/phlo_core_plugins.egg-info/PKG-INFO
|
|
12
|
+
src/phlo_core_plugins.egg-info/SOURCES.txt
|
|
13
|
+
src/phlo_core_plugins.egg-info/dependency_links.txt
|
|
14
|
+
src/phlo_core_plugins.egg-info/entry_points.txt
|
|
15
|
+
src/phlo_core_plugins.egg-info/requires.txt
|
|
16
|
+
src/phlo_core_plugins.egg-info/top_level.txt
|
|
17
|
+
tests/test_integration_core_plugins.py
|
|
18
|
+
tests/test_quality_plugins.py
|
|
19
|
+
tests/test_rest_api_plugin.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
[phlo.plugins.quality]
|
|
2
|
+
freshness_check = phlo_core.quality.freshness_check:FreshnessCheckPlugin
|
|
3
|
+
null_check = phlo_core.quality.null_check:NullCheckPlugin
|
|
4
|
+
schema_check = phlo_core.quality.schema_check:SchemaCheckPlugin
|
|
5
|
+
uniqueness_check = phlo_core.quality.uniqueness_check:UniquenessCheckPlugin
|
|
6
|
+
|
|
7
|
+
[phlo.plugins.sources]
|
|
8
|
+
rest_api = phlo_core.sources.rest_api:RestAPIPlugin
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
phlo_core
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
"""Comprehensive integration tests for phlo-core-plugins.
|
|
2
|
+
|
|
3
|
+
Per TEST_STRATEGY.md:
|
|
4
|
+
- Plugin Loading: Verify discovery mechanism
|
|
5
|
+
- Plugin Lifecycle: Registration, conflict resolution, metadata validation
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
pytestmark = pytest.mark.integration
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# =============================================================================
|
|
14
|
+
# Plugin Base Classes Tests
|
|
15
|
+
# =============================================================================
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TestPluginMetadata:
|
|
19
|
+
"""Test PluginMetadata functionality."""
|
|
20
|
+
|
|
21
|
+
def test_plugin_metadata_creation(self):
|
|
22
|
+
"""Test creating PluginMetadata."""
|
|
23
|
+
from phlo.plugins.base import PluginMetadata
|
|
24
|
+
|
|
25
|
+
metadata = PluginMetadata(name="test-plugin", version="1.0.0", description="A test plugin")
|
|
26
|
+
|
|
27
|
+
assert metadata.name == "test-plugin"
|
|
28
|
+
assert metadata.version == "1.0.0"
|
|
29
|
+
assert metadata.description == "A test plugin"
|
|
30
|
+
|
|
31
|
+
def test_plugin_metadata_with_optional_fields(self):
|
|
32
|
+
"""Test PluginMetadata with optional fields."""
|
|
33
|
+
from phlo.plugins.base import PluginMetadata
|
|
34
|
+
|
|
35
|
+
metadata = PluginMetadata(
|
|
36
|
+
name="full-plugin",
|
|
37
|
+
version="2.0.0",
|
|
38
|
+
description="Full featured plugin",
|
|
39
|
+
author="Phlo Team",
|
|
40
|
+
tags=["data", "pipeline"],
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
assert metadata.author == "Phlo Team"
|
|
44
|
+
assert "data" in metadata.tags
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# =============================================================================
|
|
48
|
+
# ServicePlugin Tests
|
|
49
|
+
# =============================================================================
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class TestServicePlugin:
|
|
53
|
+
"""Test ServicePlugin base class."""
|
|
54
|
+
|
|
55
|
+
def test_service_plugin_importable(self):
|
|
56
|
+
"""Test ServicePlugin is importable."""
|
|
57
|
+
from phlo.plugins import ServicePlugin
|
|
58
|
+
|
|
59
|
+
assert ServicePlugin is not None
|
|
60
|
+
|
|
61
|
+
def test_service_plugin_is_abstract(self):
|
|
62
|
+
"""Test ServicePlugin requires implementation."""
|
|
63
|
+
from phlo.plugins import ServicePlugin, PluginMetadata
|
|
64
|
+
|
|
65
|
+
# Attempting to instantiate directly should fail or have abstract methods
|
|
66
|
+
class ConcretePlugin(ServicePlugin):
|
|
67
|
+
@property
|
|
68
|
+
def metadata(self):
|
|
69
|
+
return PluginMetadata("test", "1.0.0", "test")
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def service_definition(self):
|
|
73
|
+
return {"services": {}}
|
|
74
|
+
|
|
75
|
+
plugin = ConcretePlugin()
|
|
76
|
+
assert plugin.metadata.name == "test"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# =============================================================================
|
|
80
|
+
# HookPlugin Tests
|
|
81
|
+
# =============================================================================
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class TestHookPlugin:
|
|
85
|
+
"""Test HookPlugin base class."""
|
|
86
|
+
|
|
87
|
+
def test_hook_plugin_importable(self):
|
|
88
|
+
"""Test HookPlugin is importable."""
|
|
89
|
+
from phlo.plugins.hooks import HookPlugin
|
|
90
|
+
|
|
91
|
+
assert HookPlugin is not None
|
|
92
|
+
|
|
93
|
+
def test_hook_registration_structure(self):
|
|
94
|
+
"""Test HookRegistration structure."""
|
|
95
|
+
from phlo.plugins.hooks import HookRegistration, HookFilter
|
|
96
|
+
|
|
97
|
+
registration = HookRegistration(
|
|
98
|
+
hook_name="test_hook",
|
|
99
|
+
handler=lambda x: x,
|
|
100
|
+
filters=HookFilter(event_types={"test.event"}),
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
assert registration.hook_name == "test_hook"
|
|
104
|
+
assert callable(registration.handler)
|
|
105
|
+
|
|
106
|
+
def test_hook_filter_creation(self):
|
|
107
|
+
"""Test HookFilter creation."""
|
|
108
|
+
from phlo.plugins.hooks import HookFilter
|
|
109
|
+
|
|
110
|
+
filter = HookFilter(event_types={"ingestion.start", "ingestion.end"})
|
|
111
|
+
|
|
112
|
+
assert filter.event_types is not None
|
|
113
|
+
assert "ingestion.start" in filter.event_types
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# =============================================================================
|
|
117
|
+
# Plugin Discovery Tests
|
|
118
|
+
# =============================================================================
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class TestPluginDiscovery:
|
|
122
|
+
"""Test plugin discovery mechanism."""
|
|
123
|
+
|
|
124
|
+
def test_discover_plugins_function_exists(self):
|
|
125
|
+
"""Test discover_plugins function is importable."""
|
|
126
|
+
from phlo.plugins.discovery import discover_plugins
|
|
127
|
+
|
|
128
|
+
assert callable(discover_plugins)
|
|
129
|
+
|
|
130
|
+
def test_discover_plugins_returns_dict(self):
|
|
131
|
+
"""Test discover_plugins returns a dictionary."""
|
|
132
|
+
from phlo.plugins.discovery import discover_plugins
|
|
133
|
+
|
|
134
|
+
# May return empty dict if no plugins installed
|
|
135
|
+
result = discover_plugins(plugin_type="services", auto_register=False)
|
|
136
|
+
|
|
137
|
+
assert isinstance(result, dict)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# =============================================================================
|
|
141
|
+
# CatalogPlugin Tests
|
|
142
|
+
# =============================================================================
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class TestCatalogPlugin:
|
|
146
|
+
"""Test CatalogPlugin base class."""
|
|
147
|
+
|
|
148
|
+
def test_catalog_plugin_importable(self):
|
|
149
|
+
"""Test CatalogPlugin is importable."""
|
|
150
|
+
from phlo.plugins.base import CatalogPlugin
|
|
151
|
+
|
|
152
|
+
assert CatalogPlugin is not None
|
|
153
|
+
|
|
154
|
+
def test_catalog_plugin_interface(self):
|
|
155
|
+
"""Test CatalogPlugin interface."""
|
|
156
|
+
from phlo.plugins.base import CatalogPlugin, PluginMetadata
|
|
157
|
+
|
|
158
|
+
class MockCatalog(CatalogPlugin):
|
|
159
|
+
@property
|
|
160
|
+
def metadata(self):
|
|
161
|
+
return PluginMetadata("mock", "1.0.0", "Mock catalog")
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def targets(self) -> list[str]:
|
|
165
|
+
return ["trino"]
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def catalog_name(self):
|
|
169
|
+
return "mock"
|
|
170
|
+
|
|
171
|
+
def get_properties(self):
|
|
172
|
+
return {"connector.name": "mock"}
|
|
173
|
+
|
|
174
|
+
catalog = MockCatalog()
|
|
175
|
+
assert catalog.catalog_name == "mock"
|
|
176
|
+
assert catalog.get_properties()["connector.name"] == "mock"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# =============================================================================
|
|
180
|
+
# CLI Plugin Tests
|
|
181
|
+
# =============================================================================
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class TestCliPlugin:
|
|
185
|
+
"""Test CLI plugin functionality."""
|
|
186
|
+
|
|
187
|
+
def test_cli_plugin_base_importable(self):
|
|
188
|
+
"""Test CliCommandPlugin base is importable."""
|
|
189
|
+
from phlo.plugins.base import CliCommandPlugin
|
|
190
|
+
|
|
191
|
+
assert CliCommandPlugin is not None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# =============================================================================
|
|
195
|
+
# Plugin Integration Tests
|
|
196
|
+
# =============================================================================
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class TestPluginIntegration:
|
|
200
|
+
"""Test plugin integration scenarios."""
|
|
201
|
+
|
|
202
|
+
def test_multiple_plugin_types_coexist(self):
|
|
203
|
+
"""Test multiple plugin types can be discovered together."""
|
|
204
|
+
from phlo.plugins import ServicePlugin
|
|
205
|
+
from phlo.plugins.hooks import HookPlugin
|
|
206
|
+
|
|
207
|
+
# Both should be importable
|
|
208
|
+
assert ServicePlugin is not None
|
|
209
|
+
assert HookPlugin is not None
|
|
210
|
+
|
|
211
|
+
def test_plugin_metadata_serialization(self):
|
|
212
|
+
"""Test plugin metadata can be converted to dict."""
|
|
213
|
+
from phlo.plugins.base import PluginMetadata
|
|
214
|
+
|
|
215
|
+
metadata = PluginMetadata(name="test", version="1.0.0", description="Test")
|
|
216
|
+
|
|
217
|
+
# Should have __dict__ or similar
|
|
218
|
+
assert hasattr(metadata, "name")
|
|
219
|
+
assert hasattr(metadata, "version")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# =============================================================================
|
|
223
|
+
# Export Tests
|
|
224
|
+
# =============================================================================
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class TestCorePluginsExports:
|
|
228
|
+
"""Test core plugins exports."""
|
|
229
|
+
|
|
230
|
+
def test_phlo_plugins_exports(self):
|
|
231
|
+
"""Test phlo.plugins exports expected classes."""
|
|
232
|
+
from phlo.plugins import ServicePlugin, PluginMetadata
|
|
233
|
+
|
|
234
|
+
assert ServicePlugin is not None
|
|
235
|
+
assert PluginMetadata is not None
|
|
236
|
+
|
|
237
|
+
def test_phlo_plugins_base_exports(self):
|
|
238
|
+
"""Test phlo.plugins.base exports expected classes."""
|
|
239
|
+
from phlo.plugins.base import CatalogPlugin, PluginMetadata
|
|
240
|
+
|
|
241
|
+
assert PluginMetadata is not None
|
|
242
|
+
assert CatalogPlugin is not None
|
|
243
|
+
|
|
244
|
+
def test_phlo_plugins_hooks_exports(self):
|
|
245
|
+
"""Test phlo.plugins.hooks exports expected classes."""
|
|
246
|
+
from phlo.plugins.hooks import HookPlugin, HookRegistration, HookFilter
|
|
247
|
+
|
|
248
|
+
assert HookPlugin is not None
|
|
249
|
+
assert HookRegistration is not None
|
|
250
|
+
assert HookFilter is not None
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Tests for core quality check plugins."""
|
|
2
|
+
|
|
3
|
+
from phlo_core.quality.freshness_check import FreshnessCheckPlugin
|
|
4
|
+
from phlo_core.quality.null_check import NullCheckPlugin
|
|
5
|
+
from phlo_core.quality.schema_check import SchemaCheckPlugin
|
|
6
|
+
from phlo_core.quality.uniqueness_check import UniquenessCheckPlugin
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def test_null_check_plugin():
|
|
10
|
+
plugin = NullCheckPlugin()
|
|
11
|
+
check = plugin.create_check(columns=["id"])
|
|
12
|
+
assert check.name == "null_check_id"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_uniqueness_check_plugin():
|
|
16
|
+
plugin = UniquenessCheckPlugin()
|
|
17
|
+
check = plugin.create_check(columns=["id"])
|
|
18
|
+
assert check.name == "unique_check_id"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_freshness_check_plugin():
|
|
22
|
+
plugin = FreshnessCheckPlugin()
|
|
23
|
+
check = plugin.create_check(timestamp_column="ts", max_age_hours=2)
|
|
24
|
+
assert check.name == "freshness_check_ts"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_schema_check_plugin():
|
|
28
|
+
plugin = SchemaCheckPlugin()
|
|
29
|
+
|
|
30
|
+
class DummySchema:
|
|
31
|
+
__name__ = "DummySchema"
|
|
32
|
+
|
|
33
|
+
def validate(self, df, lazy=True):
|
|
34
|
+
return df
|
|
35
|
+
|
|
36
|
+
check = plugin.create_check(schema=DummySchema)
|
|
37
|
+
assert check.name == "schema_check_DummySchema"
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Tests for REST API plugin."""
|
|
2
|
+
|
|
3
|
+
from phlo_core.sources.rest_api import RestAPIPlugin
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class DummyResponse:
|
|
7
|
+
def __init__(self, payload):
|
|
8
|
+
self._payload = payload
|
|
9
|
+
|
|
10
|
+
def raise_for_status(self):
|
|
11
|
+
return None
|
|
12
|
+
|
|
13
|
+
def json(self):
|
|
14
|
+
return self._payload
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_rest_api_plugin_fetches_records(monkeypatch):
|
|
18
|
+
plugin = RestAPIPlugin()
|
|
19
|
+
payload = [{"id": 1}, {"id": 2}]
|
|
20
|
+
|
|
21
|
+
def dummy_get(url, headers=None, params=None, timeout=30):
|
|
22
|
+
return DummyResponse(payload)
|
|
23
|
+
|
|
24
|
+
monkeypatch.setattr("phlo_core.sources.rest_api.requests.get", dummy_get)
|
|
25
|
+
records = list(plugin.fetch_data({"url": "https://example.com"}))
|
|
26
|
+
|
|
27
|
+
assert records == payload
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_rest_api_plugin_records_path(monkeypatch):
|
|
31
|
+
plugin = RestAPIPlugin()
|
|
32
|
+
payload = {"data": {"items": [{"id": 3}]}}
|
|
33
|
+
|
|
34
|
+
def dummy_get(url, headers=None, params=None, timeout=30):
|
|
35
|
+
return DummyResponse(payload)
|
|
36
|
+
|
|
37
|
+
monkeypatch.setattr("phlo_core.sources.rest_api.requests.get", dummy_get)
|
|
38
|
+
records = list(plugin.fetch_data({"url": "https://example.com", "records_path": "data.items"}))
|
|
39
|
+
|
|
40
|
+
assert records == [{"id": 3}]
|