phlo-testing 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_testing/__init__.py +297 -0
- phlo_testing/conftest_template.py +65 -0
- phlo_testing/execution.py +405 -0
- phlo_testing/fixtures.py +447 -0
- phlo_testing/hooks.py +135 -0
- phlo_testing/local_mode.py +366 -0
- phlo_testing/mock_dlt.py +287 -0
- phlo_testing/mock_iceberg.py +435 -0
- phlo_testing/mock_trino.py +486 -0
- phlo_testing/placeholders.py +830 -0
- phlo_testing/utils.py +20 -0
- phlo_testing-0.1.0.dist-info/METADATA +22 -0
- phlo_testing-0.1.0.dist-info/RECORD +15 -0
- phlo_testing-0.1.0.dist-info/WHEEL +5 -0
- phlo_testing-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,830 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Testing Utilities
|
|
3
|
+
|
|
4
|
+
Provides testing utilities for Phlo workflows including mock DLT sources,
|
|
5
|
+
mock Iceberg catalog, fixture management, and test helpers.
|
|
6
|
+
|
|
7
|
+
Status:
|
|
8
|
+
- MockDLTSource: ✅ Implemented
|
|
9
|
+
- MockIcebergCatalog: ✅ Implemented (DuckDB backend)
|
|
10
|
+
- Fixture management: ✅ Implemented
|
|
11
|
+
- test_asset_execution: ✅ Implemented (basic version)
|
|
12
|
+
|
|
13
|
+
For comprehensive testing guide, see: docs/TESTING_GUIDE.md
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from contextlib import contextmanager
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Callable, Dict, Iterator, List, Optional, Union
|
|
20
|
+
|
|
21
|
+
import pandas as pd
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
import duckdb
|
|
25
|
+
import pyarrow as pa
|
|
26
|
+
from pyiceberg.schema import Schema
|
|
27
|
+
from pyiceberg.types import (
|
|
28
|
+
BinaryType,
|
|
29
|
+
BooleanType,
|
|
30
|
+
DateType,
|
|
31
|
+
DoubleType,
|
|
32
|
+
FloatType,
|
|
33
|
+
IntegerType,
|
|
34
|
+
LongType,
|
|
35
|
+
StringType,
|
|
36
|
+
TimestamptzType,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
ICEBERG_DEPS_AVAILABLE = True
|
|
40
|
+
except ImportError:
|
|
41
|
+
ICEBERG_DEPS_AVAILABLE = False
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class MockDLTSource:
|
|
45
|
+
"""
|
|
46
|
+
Mock DLT source for testing ingestion assets without API calls.
|
|
47
|
+
|
|
48
|
+
Creates a DLT-compatible source from test data, allowing you to test
|
|
49
|
+
schema validation, data transformations, and asset logic without
|
|
50
|
+
requiring actual API connections.
|
|
51
|
+
|
|
52
|
+
Status: ✅ Fully implemented
|
|
53
|
+
|
|
54
|
+
Usage:
|
|
55
|
+
# Direct instantiation
|
|
56
|
+
test_data = [
|
|
57
|
+
{"id": "1", "city": "London", "temp": 15.5},
|
|
58
|
+
{"id": "2", "city": "Paris", "temp": 12.3},
|
|
59
|
+
]
|
|
60
|
+
source = MockDLTSource(data=test_data, resource_name="weather")
|
|
61
|
+
|
|
62
|
+
# Use in asset for testing
|
|
63
|
+
@phlo_ingestion(...)
|
|
64
|
+
def my_asset(partition_date: str):
|
|
65
|
+
if os.getenv("TESTING"):
|
|
66
|
+
return MockDLTSource(data=[...], resource_name="observations")
|
|
67
|
+
else:
|
|
68
|
+
return rest_api({...}) # Real source
|
|
69
|
+
|
|
70
|
+
# Or use context manager
|
|
71
|
+
with mock_dlt_source(data=test_data, resource_name="weather") as source:
|
|
72
|
+
# Test your asset logic
|
|
73
|
+
result = process_data(source)
|
|
74
|
+
assert len(result) == 2
|
|
75
|
+
|
|
76
|
+
Example with Pandera validation:
|
|
77
|
+
from workflows.schemas.weather import RawWeatherData
|
|
78
|
+
|
|
79
|
+
test_data = [{"id": "1", "city": "London", "temp": 15.5}]
|
|
80
|
+
df = pd.DataFrame(test_data)
|
|
81
|
+
|
|
82
|
+
# Validate schema works with test data
|
|
83
|
+
validated = RawWeatherData.validate(df)
|
|
84
|
+
assert len(validated) == 1
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
def __init__(
|
|
88
|
+
self,
|
|
89
|
+
data: Union[List[Dict[str, Any]], pd.DataFrame],
|
|
90
|
+
resource_name: str = "mock_resource",
|
|
91
|
+
):
|
|
92
|
+
"""
|
|
93
|
+
Initialize mock DLT source.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
data: Either list of dictionaries or pandas DataFrame
|
|
97
|
+
resource_name: Name of the mock DLT resource
|
|
98
|
+
"""
|
|
99
|
+
if isinstance(data, pd.DataFrame):
|
|
100
|
+
self.data = data.to_dict("records")
|
|
101
|
+
self._dataframe = data
|
|
102
|
+
else:
|
|
103
|
+
self.data = data
|
|
104
|
+
self._dataframe = None
|
|
105
|
+
|
|
106
|
+
self.resource_name = resource_name
|
|
107
|
+
|
|
108
|
+
def __iter__(self) -> Iterator[Dict[str, Any]]:
|
|
109
|
+
"""Iterate over mock data rows."""
|
|
110
|
+
for row in self.data:
|
|
111
|
+
yield row
|
|
112
|
+
|
|
113
|
+
def __call__(self):
|
|
114
|
+
"""Make the source callable like a DLT resource."""
|
|
115
|
+
return self
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def name(self) -> str:
|
|
119
|
+
"""Return the resource name."""
|
|
120
|
+
return self.resource_name
|
|
121
|
+
|
|
122
|
+
def to_pandas(self) -> pd.DataFrame:
|
|
123
|
+
"""Convert mock data to pandas DataFrame."""
|
|
124
|
+
if self._dataframe is not None:
|
|
125
|
+
return self._dataframe
|
|
126
|
+
return pd.DataFrame(self.data)
|
|
127
|
+
|
|
128
|
+
def __len__(self) -> int:
|
|
129
|
+
"""Return number of rows."""
|
|
130
|
+
return len(self.data)
|
|
131
|
+
|
|
132
|
+
def __repr__(self) -> str:
|
|
133
|
+
"""String representation."""
|
|
134
|
+
return f"MockDLTSource(resource_name='{self.resource_name}', rows={len(self.data)})"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@contextmanager
|
|
138
|
+
def mock_dlt_source(
|
|
139
|
+
data: Union[List[Dict[str, Any]], pd.DataFrame],
|
|
140
|
+
resource_name: str = "mock_resource",
|
|
141
|
+
):
|
|
142
|
+
"""
|
|
143
|
+
Context manager for mocking DLT sources.
|
|
144
|
+
|
|
145
|
+
Status: ✅ Fully implemented
|
|
146
|
+
|
|
147
|
+
Args:
|
|
148
|
+
data: Either list of dictionaries or pandas DataFrame
|
|
149
|
+
resource_name: Name of the mock DLT resource
|
|
150
|
+
|
|
151
|
+
Yields:
|
|
152
|
+
MockDLTSource instance
|
|
153
|
+
|
|
154
|
+
Example:
|
|
155
|
+
def test_my_asset():
|
|
156
|
+
test_data = [
|
|
157
|
+
{"id": "1", "value": 42},
|
|
158
|
+
{"id": "2", "value": 84},
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
with mock_dlt_source(data=test_data, resource_name="test") as source:
|
|
162
|
+
# Test your asset logic
|
|
163
|
+
result = my_asset_function(source)
|
|
164
|
+
assert result is not None
|
|
165
|
+
assert len(source) == 2
|
|
166
|
+
|
|
167
|
+
def test_with_dataframe():
|
|
168
|
+
test_df = pd.DataFrame([
|
|
169
|
+
{"id": "1", "value": 42},
|
|
170
|
+
])
|
|
171
|
+
|
|
172
|
+
with mock_dlt_source(data=test_df, resource_name="test") as source:
|
|
173
|
+
# Validate schema
|
|
174
|
+
validated = MySchema.validate(source.to_pandas())
|
|
175
|
+
assert len(validated) == 1
|
|
176
|
+
"""
|
|
177
|
+
source = MockDLTSource(data, resource_name)
|
|
178
|
+
try:
|
|
179
|
+
yield source
|
|
180
|
+
finally:
|
|
181
|
+
pass
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class MockIcebergTable:
|
|
185
|
+
"""Mock Iceberg table backed by DuckDB."""
|
|
186
|
+
|
|
187
|
+
def __init__(self, name: str, schema: Schema, conn: "duckdb.DuckDBPyConnection"):
|
|
188
|
+
"""Initialize mock table."""
|
|
189
|
+
self.name = name
|
|
190
|
+
self.schema = schema
|
|
191
|
+
self.conn = conn
|
|
192
|
+
self._create_duckdb_table()
|
|
193
|
+
|
|
194
|
+
def _iceberg_type_to_duckdb(self, iceberg_type: Any) -> str:
|
|
195
|
+
"""Convert PyIceberg type to DuckDB type."""
|
|
196
|
+
if isinstance(iceberg_type, StringType):
|
|
197
|
+
return "VARCHAR"
|
|
198
|
+
elif isinstance(iceberg_type, IntegerType):
|
|
199
|
+
return "INTEGER"
|
|
200
|
+
elif isinstance(iceberg_type, LongType):
|
|
201
|
+
return "BIGINT"
|
|
202
|
+
elif isinstance(iceberg_type, FloatType):
|
|
203
|
+
return "FLOAT"
|
|
204
|
+
elif isinstance(iceberg_type, DoubleType):
|
|
205
|
+
return "DOUBLE"
|
|
206
|
+
elif isinstance(iceberg_type, BooleanType):
|
|
207
|
+
return "BOOLEAN"
|
|
208
|
+
elif isinstance(iceberg_type, TimestamptzType):
|
|
209
|
+
return "TIMESTAMP WITH TIME ZONE"
|
|
210
|
+
elif isinstance(iceberg_type, DateType):
|
|
211
|
+
return "DATE"
|
|
212
|
+
elif isinstance(iceberg_type, BinaryType):
|
|
213
|
+
return "BLOB"
|
|
214
|
+
else:
|
|
215
|
+
# Default to VARCHAR for unknown types
|
|
216
|
+
return "VARCHAR"
|
|
217
|
+
|
|
218
|
+
def _create_duckdb_table(self) -> None:
|
|
219
|
+
"""Create DuckDB table from Iceberg schema."""
|
|
220
|
+
# Build CREATE TABLE statement
|
|
221
|
+
columns = []
|
|
222
|
+
for field in self.schema.fields:
|
|
223
|
+
duckdb_type = self._iceberg_type_to_duckdb(field.field_type)
|
|
224
|
+
nullable = "NULL" if not field.required else "NOT NULL"
|
|
225
|
+
columns.append(f'"{field.name}" {duckdb_type} {nullable}')
|
|
226
|
+
|
|
227
|
+
create_sql = f"CREATE TABLE IF NOT EXISTS {self.name} ({', '.join(columns)})"
|
|
228
|
+
self.conn.execute(create_sql)
|
|
229
|
+
|
|
230
|
+
def append(self, data: Union[pd.DataFrame, pa.Table]) -> None:
|
|
231
|
+
"""
|
|
232
|
+
Append data to table.
|
|
233
|
+
|
|
234
|
+
Args:
|
|
235
|
+
data: Pandas DataFrame or PyArrow Table
|
|
236
|
+
"""
|
|
237
|
+
if isinstance(data, pa.Table):
|
|
238
|
+
data = data.to_pandas()
|
|
239
|
+
|
|
240
|
+
# Insert into DuckDB table (data is now a pandas DataFrame)
|
|
241
|
+
self.conn.execute(f"INSERT INTO {self.name} SELECT * FROM data")
|
|
242
|
+
|
|
243
|
+
def scan(self) -> "MockTableScan":
|
|
244
|
+
"""Return a table scan for querying."""
|
|
245
|
+
return MockTableScan(self.name, self.conn)
|
|
246
|
+
|
|
247
|
+
def to_pandas(self) -> pd.DataFrame:
|
|
248
|
+
"""Read entire table as pandas DataFrame."""
|
|
249
|
+
return self.conn.execute(f"SELECT * FROM {self.name}").df()
|
|
250
|
+
|
|
251
|
+
def to_arrow(self) -> pa.Table:
|
|
252
|
+
"""Read entire table as PyArrow Table."""
|
|
253
|
+
return self.conn.execute(f"SELECT * FROM {self.name}").arrow()
|
|
254
|
+
|
|
255
|
+
def count(self) -> int:
|
|
256
|
+
"""Return number of rows in table."""
|
|
257
|
+
result = self.conn.execute(f"SELECT COUNT(*) FROM {self.name}").fetchone()
|
|
258
|
+
return result[0] if result else 0
|
|
259
|
+
|
|
260
|
+
def delete_all(self) -> None:
|
|
261
|
+
"""Delete all rows from table."""
|
|
262
|
+
self.conn.execute(f"DELETE FROM {self.name}")
|
|
263
|
+
|
|
264
|
+
def drop(self) -> None:
|
|
265
|
+
"""Drop the table."""
|
|
266
|
+
self.conn.execute(f"DROP TABLE IF EXISTS {self.name}")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class MockTableScan:
|
|
270
|
+
"""Mock Iceberg table scan."""
|
|
271
|
+
|
|
272
|
+
def __init__(self, table_name: str, conn: "duckdb.DuckDBPyConnection"):
|
|
273
|
+
"""Initialize table scan."""
|
|
274
|
+
self.table_name = table_name
|
|
275
|
+
self.conn = conn
|
|
276
|
+
self._filter_expr: Optional[str] = None
|
|
277
|
+
self._limit: Optional[int] = None
|
|
278
|
+
|
|
279
|
+
def filter(self, expr: str) -> "MockTableScan":
|
|
280
|
+
"""Add WHERE clause filter (SQL syntax)."""
|
|
281
|
+
self._filter_expr = expr
|
|
282
|
+
return self
|
|
283
|
+
|
|
284
|
+
def limit(self, n: int) -> "MockTableScan":
|
|
285
|
+
"""Limit number of rows."""
|
|
286
|
+
self._limit = n
|
|
287
|
+
return self
|
|
288
|
+
|
|
289
|
+
def to_pandas(self) -> pd.DataFrame:
|
|
290
|
+
"""Execute scan and return pandas DataFrame."""
|
|
291
|
+
query = f"SELECT * FROM {self.table_name}"
|
|
292
|
+
if self._filter_expr:
|
|
293
|
+
query += f" WHERE {self._filter_expr}"
|
|
294
|
+
if self._limit:
|
|
295
|
+
query += f" LIMIT {self._limit}"
|
|
296
|
+
return self.conn.execute(query).df()
|
|
297
|
+
|
|
298
|
+
def to_arrow(self) -> pa.Table:
|
|
299
|
+
"""Execute scan and return PyArrow Table."""
|
|
300
|
+
query = f"SELECT * FROM {self.table_name}"
|
|
301
|
+
if self._filter_expr:
|
|
302
|
+
query += f" WHERE {self._filter_expr}"
|
|
303
|
+
if self._limit:
|
|
304
|
+
query += f" LIMIT {self._limit}"
|
|
305
|
+
return self.conn.execute(query).arrow()
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
class MockIcebergCatalog:
|
|
309
|
+
"""
|
|
310
|
+
Mock Iceberg catalog for testing without Docker.
|
|
311
|
+
|
|
312
|
+
Status: ✅ Fully implemented (using DuckDB backend)
|
|
313
|
+
|
|
314
|
+
Provides a PyIceberg-compatible API backed by in-memory DuckDB.
|
|
315
|
+
Perfect for fast unit tests without Docker infrastructure.
|
|
316
|
+
|
|
317
|
+
Features:
|
|
318
|
+
- In-memory DuckDB backend (no persistence)
|
|
319
|
+
- PyIceberg schema support
|
|
320
|
+
- Create/drop tables
|
|
321
|
+
- Append data (DataFrame or Arrow)
|
|
322
|
+
- Scan with filters and limits
|
|
323
|
+
- < 5 second test execution
|
|
324
|
+
|
|
325
|
+
Limitations:
|
|
326
|
+
- No actual Iceberg format files (uses DuckDB tables)
|
|
327
|
+
- No time travel/snapshots
|
|
328
|
+
- No partitioning
|
|
329
|
+
- Schema evolution not implemented
|
|
330
|
+
- Good for unit tests, not production
|
|
331
|
+
|
|
332
|
+
Usage:
|
|
333
|
+
with mock_iceberg_catalog() as catalog:
|
|
334
|
+
# Create table from PyIceberg schema
|
|
335
|
+
table = catalog.create_table("test.my_table", schema=my_schema)
|
|
336
|
+
|
|
337
|
+
# Append data
|
|
338
|
+
df = pd.DataFrame([{"id": "1", "value": 42}])
|
|
339
|
+
table.append(df)
|
|
340
|
+
|
|
341
|
+
# Query data
|
|
342
|
+
result = table.scan().to_pandas()
|
|
343
|
+
assert len(result) == 1
|
|
344
|
+
|
|
345
|
+
# Query with filters
|
|
346
|
+
filtered = table.scan().filter("value > 40").to_pandas()
|
|
347
|
+
|
|
348
|
+
Example with Phlo schema:
|
|
349
|
+
from phlo_dlt.converter import pandera_to_iceberg
|
|
350
|
+
from workflows.schemas.weather import RawWeatherData
|
|
351
|
+
|
|
352
|
+
# Convert Pandera to Iceberg schema
|
|
353
|
+
iceberg_schema = pandera_to_iceberg(RawWeatherData)
|
|
354
|
+
|
|
355
|
+
with mock_iceberg_catalog() as catalog:
|
|
356
|
+
table = catalog.create_table("raw.weather", schema=iceberg_schema)
|
|
357
|
+
|
|
358
|
+
test_data = pd.DataFrame([
|
|
359
|
+
{"city": "London", "temp": 15.5, "timestamp": "2024-01-15"},
|
|
360
|
+
])
|
|
361
|
+
|
|
362
|
+
# Validate with Pandera first
|
|
363
|
+
validated = RawWeatherData.validate(test_data)
|
|
364
|
+
|
|
365
|
+
# Then append to mock Iceberg
|
|
366
|
+
table.append(validated)
|
|
367
|
+
|
|
368
|
+
# Query back
|
|
369
|
+
result = table.scan().to_pandas()
|
|
370
|
+
assert len(result) == 1
|
|
371
|
+
"""
|
|
372
|
+
|
|
373
|
+
def __init__(self):
|
|
374
|
+
"""Initialize mock Iceberg catalog with in-memory DuckDB."""
|
|
375
|
+
if not ICEBERG_DEPS_AVAILABLE:
|
|
376
|
+
raise ImportError(
|
|
377
|
+
"DuckDB and PyArrow are required for MockIcebergCatalog. "
|
|
378
|
+
"Install with: pip install duckdb pyarrow"
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
# Create in-memory DuckDB connection
|
|
382
|
+
self.conn = duckdb.connect(":memory:")
|
|
383
|
+
self.tables: Dict[str, MockIcebergTable] = {}
|
|
384
|
+
|
|
385
|
+
def create_table(
|
|
386
|
+
self,
|
|
387
|
+
name: str,
|
|
388
|
+
schema: Schema,
|
|
389
|
+
if_not_exists: bool = False,
|
|
390
|
+
) -> MockIcebergTable:
|
|
391
|
+
"""
|
|
392
|
+
Create a new table.
|
|
393
|
+
|
|
394
|
+
Args:
|
|
395
|
+
name: Table name (can include namespace like "raw.table_name")
|
|
396
|
+
schema: PyIceberg Schema object
|
|
397
|
+
if_not_exists: If True, don't error if table exists
|
|
398
|
+
|
|
399
|
+
Returns:
|
|
400
|
+
MockIcebergTable instance
|
|
401
|
+
|
|
402
|
+
Example:
|
|
403
|
+
from pyiceberg.schema import Schema
|
|
404
|
+
from pyiceberg.types import NestedField, StringType, IntegerType
|
|
405
|
+
|
|
406
|
+
schema = Schema(
|
|
407
|
+
NestedField(1, "id", StringType(), required=True),
|
|
408
|
+
NestedField(2, "value", IntegerType(), required=False),
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
table = catalog.create_table("test.my_table", schema)
|
|
412
|
+
"""
|
|
413
|
+
# Sanitize table name for DuckDB (replace dots with underscores)
|
|
414
|
+
duckdb_name = name.replace(".", "_")
|
|
415
|
+
|
|
416
|
+
if duckdb_name in self.tables:
|
|
417
|
+
if if_not_exists:
|
|
418
|
+
return self.tables[duckdb_name]
|
|
419
|
+
raise ValueError(f"Table {name} already exists")
|
|
420
|
+
|
|
421
|
+
table = MockIcebergTable(duckdb_name, schema, self.conn)
|
|
422
|
+
self.tables[duckdb_name] = table
|
|
423
|
+
return table
|
|
424
|
+
|
|
425
|
+
def load_table(self, name: str) -> MockIcebergTable:
|
|
426
|
+
"""
|
|
427
|
+
Load an existing table.
|
|
428
|
+
|
|
429
|
+
Args:
|
|
430
|
+
name: Table name
|
|
431
|
+
|
|
432
|
+
Returns:
|
|
433
|
+
MockIcebergTable instance
|
|
434
|
+
|
|
435
|
+
Raises:
|
|
436
|
+
KeyError: If table doesn't exist
|
|
437
|
+
"""
|
|
438
|
+
duckdb_name = name.replace(".", "_")
|
|
439
|
+
if duckdb_name not in self.tables:
|
|
440
|
+
raise KeyError(f"Table {name} not found. Available tables: {list(self.tables.keys())}")
|
|
441
|
+
return self.tables[duckdb_name]
|
|
442
|
+
|
|
443
|
+
def list_tables(self) -> List[str]:
|
|
444
|
+
"""List all tables in catalog."""
|
|
445
|
+
return list(self.tables.keys())
|
|
446
|
+
|
|
447
|
+
def drop_table(self, name: str) -> None:
|
|
448
|
+
"""
|
|
449
|
+
Drop a table.
|
|
450
|
+
|
|
451
|
+
Args:
|
|
452
|
+
name: Table name
|
|
453
|
+
"""
|
|
454
|
+
duckdb_name = name.replace(".", "_")
|
|
455
|
+
if duckdb_name in self.tables:
|
|
456
|
+
self.tables[duckdb_name].drop()
|
|
457
|
+
del self.tables[duckdb_name]
|
|
458
|
+
|
|
459
|
+
def close(self) -> None:
|
|
460
|
+
"""Close DuckDB connection."""
|
|
461
|
+
self.conn.close()
|
|
462
|
+
|
|
463
|
+
def __enter__(self):
|
|
464
|
+
"""Context manager entry."""
|
|
465
|
+
return self
|
|
466
|
+
|
|
467
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
468
|
+
"""Context manager exit - cleanup."""
|
|
469
|
+
self.close()
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
@contextmanager
|
|
473
|
+
def mock_iceberg_catalog():
|
|
474
|
+
"""
|
|
475
|
+
Context manager for mocking Iceberg catalog.
|
|
476
|
+
|
|
477
|
+
Status: ✅ Fully implemented
|
|
478
|
+
|
|
479
|
+
Creates an in-memory Iceberg catalog backed by DuckDB.
|
|
480
|
+
Perfect for fast unit tests without Docker infrastructure.
|
|
481
|
+
|
|
482
|
+
Yields:
|
|
483
|
+
MockIcebergCatalog instance
|
|
484
|
+
|
|
485
|
+
Example:
|
|
486
|
+
from phlo_dlt.converter import pandera_to_iceberg
|
|
487
|
+
from workflows.schemas.weather import RawWeatherData
|
|
488
|
+
|
|
489
|
+
iceberg_schema = pandera_to_iceberg(RawWeatherData)
|
|
490
|
+
|
|
491
|
+
with mock_iceberg_catalog() as catalog:
|
|
492
|
+
table = catalog.create_table("raw.weather", schema=iceberg_schema)
|
|
493
|
+
|
|
494
|
+
test_data = pd.DataFrame([
|
|
495
|
+
{"city": "London", "temp": 15.5, "timestamp": "2024-01-15"},
|
|
496
|
+
])
|
|
497
|
+
|
|
498
|
+
validated = RawWeatherData.validate(test_data)
|
|
499
|
+
table.append(validated)
|
|
500
|
+
|
|
501
|
+
result = table.scan().to_pandas()
|
|
502
|
+
assert len(result) == 1
|
|
503
|
+
assert result["city"][0] == "London"
|
|
504
|
+
"""
|
|
505
|
+
catalog = MockIcebergCatalog()
|
|
506
|
+
try:
|
|
507
|
+
yield catalog
|
|
508
|
+
finally:
|
|
509
|
+
catalog.close()
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
class TestAssetResult:
|
|
513
|
+
"""Result from test_asset_execution."""
|
|
514
|
+
|
|
515
|
+
def __init__(
|
|
516
|
+
self,
|
|
517
|
+
success: bool,
|
|
518
|
+
data: Optional[pd.DataFrame] = None,
|
|
519
|
+
error: Optional[Exception] = None,
|
|
520
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
521
|
+
):
|
|
522
|
+
"""Initialize test result."""
|
|
523
|
+
self.success = success
|
|
524
|
+
self.data = data
|
|
525
|
+
self.error = error
|
|
526
|
+
self.metadata = metadata or {}
|
|
527
|
+
|
|
528
|
+
def __repr__(self) -> str:
|
|
529
|
+
"""String representation."""
|
|
530
|
+
status = "SUCCESS" if self.success else "FAILED"
|
|
531
|
+
rows = len(self.data) if self.data is not None else 0
|
|
532
|
+
return f"TestAssetResult(status={status}, rows={rows})"
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def test_asset_execution(
|
|
536
|
+
asset_fn: Callable[..., Any],
|
|
537
|
+
partition: str,
|
|
538
|
+
mock_data: Optional[Union[List[Dict[str, Any]], pd.DataFrame]] = None,
|
|
539
|
+
validation_schema: Optional[Any] = None,
|
|
540
|
+
) -> TestAssetResult:
|
|
541
|
+
"""
|
|
542
|
+
Test asset execution with mocked dependencies.
|
|
543
|
+
|
|
544
|
+
Status: ✅ Implemented (basic version)
|
|
545
|
+
|
|
546
|
+
Executes an asset function with mock data and optional schema validation.
|
|
547
|
+
Does not require Docker or Dagster infrastructure.
|
|
548
|
+
|
|
549
|
+
This is a simplified testing helper that:
|
|
550
|
+
- Executes the asset function directly (bypasses Dagster)
|
|
551
|
+
- Uses MockDLTSource if mock_data provided
|
|
552
|
+
- Validates with Pandera schema if provided
|
|
553
|
+
- Returns success/failure with data
|
|
554
|
+
|
|
555
|
+
Limitations:
|
|
556
|
+
- Does not execute within Dagster context
|
|
557
|
+
- Does not write to actual Iceberg tables
|
|
558
|
+
- Does not test Dagster-specific features (retries, metadata, etc.)
|
|
559
|
+
- Good for testing asset logic, not full pipeline
|
|
560
|
+
|
|
561
|
+
Args:
|
|
562
|
+
asset_fn: The asset function to test (NOT the decorated asset, the original function)
|
|
563
|
+
partition: Partition date (e.g., "2024-01-15")
|
|
564
|
+
mock_data: Test data to use (if None, asset must fetch real data)
|
|
565
|
+
validation_schema: Pandera schema for validation (optional)
|
|
566
|
+
|
|
567
|
+
Returns:
|
|
568
|
+
TestAssetResult with success status, data, and any errors
|
|
569
|
+
|
|
570
|
+
Example - Test with mock data:
|
|
571
|
+
def my_asset_logic(partition_date: str):
|
|
572
|
+
# This is the function WITHOUT the decorator
|
|
573
|
+
return rest_api({...}) # Returns DLT source
|
|
574
|
+
|
|
575
|
+
# Test with mock data
|
|
576
|
+
test_data = [{"id": "1", "city": "London", "temp": 15.5}]
|
|
577
|
+
|
|
578
|
+
result = test_asset_execution(
|
|
579
|
+
asset_fn=my_asset_logic,
|
|
580
|
+
partition="2024-01-15",
|
|
581
|
+
mock_data=test_data,
|
|
582
|
+
validation_schema=RawWeatherData,
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
assert result.success
|
|
586
|
+
assert len(result.data) == 1
|
|
587
|
+
assert result.data["city"][0] == "London"
|
|
588
|
+
|
|
589
|
+
Example - Test actual API call:
|
|
590
|
+
# Test without mock data (makes real API call)
|
|
591
|
+
result = test_asset_execution(
|
|
592
|
+
asset_fn=my_asset_logic,
|
|
593
|
+
partition="2024-01-15",
|
|
594
|
+
validation_schema=RawWeatherData,
|
|
595
|
+
)
|
|
596
|
+
|
|
597
|
+
assert result.success
|
|
598
|
+
assert len(result.data) > 0
|
|
599
|
+
|
|
600
|
+
Example - Full test with validation:
|
|
601
|
+
def test_my_asset():
|
|
602
|
+
test_data = [
|
|
603
|
+
{"id": "1", "city": "London", "temp": 15.5, "timestamp": "2024-01-15"},
|
|
604
|
+
]
|
|
605
|
+
|
|
606
|
+
result = test_asset_execution(
|
|
607
|
+
asset_fn=weather_observations,
|
|
608
|
+
partition="2024-01-15",
|
|
609
|
+
mock_data=test_data,
|
|
610
|
+
validation_schema=RawWeatherData,
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
# Assertions
|
|
614
|
+
assert result.success, f"Asset execution failed: {result.error}"
|
|
615
|
+
assert len(result.data) == 1
|
|
616
|
+
assert result.data["city"].iloc[0] == "London"
|
|
617
|
+
assert result.data["temp"].iloc[0] == 15.5
|
|
618
|
+
"""
|
|
619
|
+
try:
|
|
620
|
+
# Execute asset function
|
|
621
|
+
if mock_data is not None:
|
|
622
|
+
# Use mock data - wrap in MockDLTSource if it's not already a source
|
|
623
|
+
if isinstance(mock_data, (list, pd.DataFrame)):
|
|
624
|
+
source = MockDLTSource(data=mock_data)
|
|
625
|
+
# Asset functions typically return a DLT source, not the data
|
|
626
|
+
# So we simulate this by returning the mock source
|
|
627
|
+
result_source = source
|
|
628
|
+
else:
|
|
629
|
+
result_source = mock_data
|
|
630
|
+
else:
|
|
631
|
+
# No mock data - execute actual asset function
|
|
632
|
+
result_source = asset_fn(partition)
|
|
633
|
+
|
|
634
|
+
# Convert source to DataFrame
|
|
635
|
+
if hasattr(result_source, "to_pandas"):
|
|
636
|
+
df = result_source.to_pandas()
|
|
637
|
+
elif hasattr(result_source, "__iter__"):
|
|
638
|
+
# DLT source is iterable
|
|
639
|
+
data_list = list(result_source)
|
|
640
|
+
df = pd.DataFrame(data_list)
|
|
641
|
+
elif isinstance(result_source, pd.DataFrame):
|
|
642
|
+
df = result_source
|
|
643
|
+
else:
|
|
644
|
+
raise ValueError(
|
|
645
|
+
f"Asset function returned unexpected type: {type(result_source)}. "
|
|
646
|
+
"Expected DLT source, DataFrame, or iterable."
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
# Validate with schema if provided
|
|
650
|
+
if validation_schema is not None:
|
|
651
|
+
try:
|
|
652
|
+
df = validation_schema.validate(df)
|
|
653
|
+
metadata = {"validation": "passed"}
|
|
654
|
+
except Exception as e:
|
|
655
|
+
return TestAssetResult(
|
|
656
|
+
success=False,
|
|
657
|
+
data=df,
|
|
658
|
+
error=e,
|
|
659
|
+
metadata={"validation": "failed", "error": str(e)},
|
|
660
|
+
)
|
|
661
|
+
else:
|
|
662
|
+
metadata = {"validation": "skipped"}
|
|
663
|
+
|
|
664
|
+
return TestAssetResult(
|
|
665
|
+
success=True,
|
|
666
|
+
data=df,
|
|
667
|
+
error=None,
|
|
668
|
+
metadata=metadata,
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
except Exception as e:
|
|
672
|
+
return TestAssetResult(
|
|
673
|
+
success=False,
|
|
674
|
+
data=None,
|
|
675
|
+
error=e,
|
|
676
|
+
metadata={"error": str(e)},
|
|
677
|
+
)
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
# Fixture Management - ✅ Implemented
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def load_fixture(
|
|
684
|
+
path: Union[str, Path],
|
|
685
|
+
) -> Union[pd.DataFrame, List[Dict[str, Any]], Dict[str, Any]]:
|
|
686
|
+
"""
|
|
687
|
+
Load test fixture from file.
|
|
688
|
+
|
|
689
|
+
Status: ✅ Fully implemented
|
|
690
|
+
|
|
691
|
+
Supports JSON, CSV, and Parquet files. Automatically detects format
|
|
692
|
+
from file extension.
|
|
693
|
+
|
|
694
|
+
Args:
|
|
695
|
+
path: Path to fixture file (.json, .csv, or .parquet)
|
|
696
|
+
|
|
697
|
+
Returns:
|
|
698
|
+
Loaded data as DataFrame, dict, or list of dicts depending on format
|
|
699
|
+
|
|
700
|
+
Example:
|
|
701
|
+
# Load JSON fixture
|
|
702
|
+
test_data = load_fixture("tests/fixtures/weather_data.json")
|
|
703
|
+
|
|
704
|
+
# Use in test
|
|
705
|
+
with mock_dlt_source(data=test_data) as source:
|
|
706
|
+
result = my_asset_function(source)
|
|
707
|
+
|
|
708
|
+
# Load CSV fixture
|
|
709
|
+
test_df = load_fixture("tests/fixtures/sample_data.csv")
|
|
710
|
+
validated = MySchema.validate(test_df)
|
|
711
|
+
|
|
712
|
+
Raises:
|
|
713
|
+
FileNotFoundError: If file doesn't exist
|
|
714
|
+
ValueError: If file format is not supported
|
|
715
|
+
"""
|
|
716
|
+
path = Path(path)
|
|
717
|
+
|
|
718
|
+
if not path.exists():
|
|
719
|
+
raise FileNotFoundError(f"Fixture file not found: {path}")
|
|
720
|
+
|
|
721
|
+
suffix = path.suffix.lower()
|
|
722
|
+
|
|
723
|
+
if suffix == ".json":
|
|
724
|
+
with open(path, "r") as f:
|
|
725
|
+
data = json.load(f)
|
|
726
|
+
# If it's a list of dicts, return as-is for easy use with MockDLTSource
|
|
727
|
+
# If it's a dict, return as-is
|
|
728
|
+
return data
|
|
729
|
+
|
|
730
|
+
elif suffix == ".csv":
|
|
731
|
+
return pd.read_csv(path)
|
|
732
|
+
|
|
733
|
+
elif suffix == ".parquet":
|
|
734
|
+
return pd.read_parquet(path)
|
|
735
|
+
|
|
736
|
+
else:
|
|
737
|
+
raise ValueError(
|
|
738
|
+
f"Unsupported fixture format: {suffix}. Supported formats: .json, .csv, .parquet"
|
|
739
|
+
)
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def save_fixture(
|
|
743
|
+
data: Union[pd.DataFrame, List[Dict[str, Any]], Dict[str, Any]],
|
|
744
|
+
path: Union[str, Path],
|
|
745
|
+
pretty: bool = True,
|
|
746
|
+
) -> None:
|
|
747
|
+
"""
|
|
748
|
+
Save test data as fixture file.
|
|
749
|
+
|
|
750
|
+
Status: ✅ Fully implemented
|
|
751
|
+
|
|
752
|
+
Automatically determines format from file extension.
|
|
753
|
+
Creates parent directories if they don't exist.
|
|
754
|
+
|
|
755
|
+
Args:
|
|
756
|
+
data: Data to save (DataFrame, dict, or list of dicts)
|
|
757
|
+
path: Path to save fixture file (.json, .csv, or .parquet)
|
|
758
|
+
pretty: If True, format JSON with indentation (default: True)
|
|
759
|
+
|
|
760
|
+
Example:
|
|
761
|
+
# Save test data for reuse
|
|
762
|
+
test_data = [
|
|
763
|
+
{"id": "1", "value": 42},
|
|
764
|
+
{"id": "2", "value": 84},
|
|
765
|
+
]
|
|
766
|
+
save_fixture(test_data, "tests/fixtures/sample_data.json")
|
|
767
|
+
|
|
768
|
+
# Save DataFrame
|
|
769
|
+
df = pd.DataFrame(test_data)
|
|
770
|
+
save_fixture(df, "tests/fixtures/sample_data.csv")
|
|
771
|
+
|
|
772
|
+
# Later, load it in tests
|
|
773
|
+
loaded = load_fixture("tests/fixtures/sample_data.json")
|
|
774
|
+
|
|
775
|
+
Raises:
|
|
776
|
+
ValueError: If file format is not supported
|
|
777
|
+
"""
|
|
778
|
+
path = Path(path)
|
|
779
|
+
|
|
780
|
+
# Create parent directories if needed
|
|
781
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
782
|
+
|
|
783
|
+
suffix = path.suffix.lower()
|
|
784
|
+
|
|
785
|
+
if suffix == ".json":
|
|
786
|
+
with open(path, "w") as f:
|
|
787
|
+
if pretty:
|
|
788
|
+
json.dump(data, f, indent=2, default=str)
|
|
789
|
+
else:
|
|
790
|
+
json.dump(data, f, default=str)
|
|
791
|
+
|
|
792
|
+
elif suffix == ".csv":
|
|
793
|
+
if isinstance(data, pd.DataFrame):
|
|
794
|
+
data.to_csv(path, index=False)
|
|
795
|
+
else:
|
|
796
|
+
# Convert to DataFrame first
|
|
797
|
+
df: pd.DataFrame = (
|
|
798
|
+
pd.DataFrame(data) if isinstance(data, list) else pd.DataFrame([data])
|
|
799
|
+
)
|
|
800
|
+
df.to_csv(path, index=False)
|
|
801
|
+
|
|
802
|
+
elif suffix == ".parquet":
|
|
803
|
+
if isinstance(data, pd.DataFrame):
|
|
804
|
+
data.to_parquet(path, index=False)
|
|
805
|
+
else:
|
|
806
|
+
# Convert to DataFrame first
|
|
807
|
+
df: pd.DataFrame = (
|
|
808
|
+
pd.DataFrame(data) if isinstance(data, list) else pd.DataFrame([data])
|
|
809
|
+
)
|
|
810
|
+
df.to_parquet(path, index=False)
|
|
811
|
+
|
|
812
|
+
else:
|
|
813
|
+
raise ValueError(
|
|
814
|
+
f"Unsupported fixture format: {suffix}. Supported formats: .json, .csv, .parquet"
|
|
815
|
+
)
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
# Implementation Roadmap
|
|
819
|
+
# =====================
|
|
820
|
+
#
|
|
821
|
+
# ✅ Phase 1 (Complete): MockDLTSource and fixture management
|
|
822
|
+
# ⚠️ Phase 2 (Planned - 20h): Mock Iceberg catalog using DuckDB
|
|
823
|
+
# ⚠️ Phase 3 (Planned - 30h): Test asset execution framework
|
|
824
|
+
# ⚠️ Phase 4 (Planned - 10h): Test coverage reporting
|
|
825
|
+
#
|
|
826
|
+
# Total estimated: ~60 hours remaining
|
|
827
|
+
#
|
|
828
|
+
# Priority: HIGH (significantly improves developer experience)
|
|
829
|
+
#
|
|
830
|
+
# See: docs/audit/testing_experience_audit.md for full requirements
|