focus-data-toolkit 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- focus_data_toolkit/__init__.py +69 -0
- focus_data_toolkit/__main__.py +6 -0
- focus_data_toolkit/_version.py +8 -0
- focus_data_toolkit/cli.py +968 -0
- focus_data_toolkit/context/__init__.py +88 -0
- focus_data_toolkit/context/billing.py +54 -0
- focus_data_toolkit/context/provider.py +90 -0
- focus_data_toolkit/convert/__init__.py +708 -0
- focus_data_toolkit/convert/billing_period.py +65 -0
- focus_data_toolkit/convert/contract_applied.py +235 -0
- focus_data_toolkit/convert/contract_commitment.py +182 -0
- focus_data_toolkit/convert/cost_and_usage.py +179 -0
- focus_data_toolkit/convert/detect.py +39 -0
- focus_data_toolkit/convert/invoice_detail.py +199 -0
- focus_data_toolkit/convert/streaming.py +1030 -0
- focus_data_toolkit/errors.py +145 -0
- focus_data_toolkit/focus_json.py +68 -0
- focus_data_toolkit/generators/__init__.py +61 -0
- focus_data_toolkit/generators/_shim.py +43 -0
- focus_data_toolkit/generators/engine/__init__.py +14 -0
- focus_data_toolkit/generators/engine/context.py +12 -0
- focus_data_toolkit/generators/engine/determinism.py +117 -0
- focus_data_toolkit/generators/engine/json_focus.py +63 -0
- focus_data_toolkit/generators/engine/ladder.py +71 -0
- focus_data_toolkit/generators/engine/scenarios_core.py +380 -0
- focus_data_toolkit/generators/engine/serialize.py +151 -0
- focus_data_toolkit/generators/generate_aws_focus_1_2.py +19 -0
- focus_data_toolkit/generators/generate_aws_focus_1_3.py +20 -0
- focus_data_toolkit/generators/generate_azure_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_azure_focus_1_3.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_3.py +17 -0
- focus_data_toolkit/generators/providers/__init__.py +29 -0
- focus_data_toolkit/generators/providers/aws.py +186 -0
- focus_data_toolkit/generators/providers/azure.py +191 -0
- focus_data_toolkit/generators/providers/gcp.py +194 -0
- focus_data_toolkit/generators/providers/profile.py +123 -0
- focus_data_toolkit/generators/scenarios.py +178 -0
- focus_data_toolkit/generators/versions/__init__.py +17 -0
- focus_data_toolkit/generators/versions/adapter.py +41 -0
- focus_data_toolkit/generators/versions/v1_2.py +111 -0
- focus_data_toolkit/generators/versions/v1_3.py +154 -0
- focus_data_toolkit/io/__init__.py +1 -0
- focus_data_toolkit/io/atomic_writer.py +462 -0
- focus_data_toolkit/io/csv_io.py +128 -0
- focus_data_toolkit/io/parquet_io.py +528 -0
- focus_data_toolkit/io/records.py +92 -0
- focus_data_toolkit/io/row_source.py +117 -0
- focus_data_toolkit/lifecycle.py +342 -0
- focus_data_toolkit/manifest.py +114 -0
- focus_data_toolkit/model/__init__.py +43 -0
- focus_data_toolkit/model/capabilities.py +66 -0
- focus_data_toolkit/model/focus_1_4_decimal_scale.json +10 -0
- focus_data_toolkit/model/focus_1_4_model.json +1913 -0
- focus_data_toolkit/model/focus_1_4_servicesubcategory.json +84 -0
- focus_data_toolkit/model/focus_json_keys.py +112 -0
- focus_data_toolkit/model/iso_4217_currencies.json +23 -0
- focus_data_toolkit/model/json_schema_check.py +205 -0
- focus_data_toolkit/model/json_schemas/allocatedmethoddetailsobjectschema.json +82 -0
- focus_data_toolkit/model/json_schemas/commitmentprogrameligibilitydetailsobjectschema.json +41 -0
- focus_data_toolkit/model/json_schemas/contractappliedobjectschema.json +104 -0
- focus_data_toolkit/model/json_schemas/contractcommitmentapplicabilityobjectschema.json +290 -0
- focus_data_toolkit/model/json_schemas/json_schemas_provenance.json +38 -0
- focus_data_toolkit/model/model_provenance.json +58 -0
- focus_data_toolkit/model/validator.py +498 -0
- focus_data_toolkit/modes.py +18 -0
- focus_data_toolkit/official_validator.py +61 -0
- focus_data_toolkit/progress.py +89 -0
- focus_data_toolkit/provenance.py +106 -0
- focus_data_toolkit/py.typed +1 -0
- focus_data_toolkit/runtime.py +243 -0
- focus_data_toolkit/schema/__init__.py +17 -0
- focus_data_toolkit/schema/detection.py +274 -0
- focus_data_toolkit/schema/registry.py +127 -0
- focus_data_toolkit/storage/__init__.py +1 -0
- focus_data_toolkit/storage/external_index.py +99 -0
- focus_data_toolkit/storage/spill.py +150 -0
- focus_data_toolkit/studio/__init__.py +19 -0
- focus_data_toolkit/studio/app.py +467 -0
- focus_data_toolkit/studio/config.py +42 -0
- focus_data_toolkit/studio/frontend/app.js +214 -0
- focus_data_toolkit/studio/frontend/index.html +101 -0
- focus_data_toolkit/studio/frontend/style.css +60 -0
- focus_data_toolkit/studio/jobs.py +142 -0
- focus_data_toolkit/studio/preview.py +32 -0
- focus_data_toolkit/studio/security.py +125 -0
- focus_data_toolkit/studio/server.py +71 -0
- focus_data_toolkit/supplement/__init__.py +50 -0
- focus_data_toolkit/supplement/adapters/__init__.py +21 -0
- focus_data_toolkit/supplement/adapters/adapters_provenance.json +39 -0
- focus_data_toolkit/supplement/adapters/aws_invoice_summary.json +24 -0
- focus_data_toolkit/supplement/adapters/aws_savings_plans.json +31 -0
- focus_data_toolkit/supplement/adapters/azure_invoice.json +25 -0
- focus_data_toolkit/supplement/adapters/gcp_compute_commitments.json +28 -0
- focus_data_toolkit/supplement/adapters/registry.py +215 -0
- focus_data_toolkit/supplement/apply.py +318 -0
- focus_data_toolkit/supplement/gaps.py +219 -0
- focus_data_toolkit/supplement/kinds.py +118 -0
- focus_data_toolkit/supplement/loader.py +409 -0
- focus_data_toolkit/supplement/spec.py +74 -0
- focus_data_toolkit/supplement/validate.py +215 -0
- focus_data_toolkit/validate/__init__.py +15 -0
- focus_data_toolkit/validate/allocation.py +333 -0
- focus_data_toolkit/validate/bundle.py +254 -0
- focus_data_toolkit/validate/codes.py +93 -0
- focus_data_toolkit/validate/corrections.py +245 -0
- focus_data_toolkit/validate/reconciliation.py +98 -0
- focus_data_toolkit/validate/referential.py +289 -0
- focus_data_toolkit-0.11.0.dist-info/METADATA +519 -0
- focus_data_toolkit-0.11.0.dist-info/RECORD +116 -0
- focus_data_toolkit-0.11.0.dist-info/WHEEL +5 -0
- focus_data_toolkit-0.11.0.dist-info/entry_points.txt +2 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSE +21 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSES/CC-BY-4.0.txt +156 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/NOTICE +60 -0
- focus_data_toolkit-0.11.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
"""Parquet reader/writer with exact decimal columnar I/O (P1.6).
|
|
2
|
+
|
|
3
|
+
PyArrow is an **optional** dependency (``pip install 'focus-data-toolkit[parquet]'``); importing
|
|
4
|
+
this module without it raises a clear, actionable error rather than an opaque ``ImportError``.
|
|
5
|
+
|
|
6
|
+
Type mapping (driven by the FOCUS 1.4 model + the committed decimal-scale registry):
|
|
7
|
+
|
|
8
|
+
* ``Decimal`` → :func:`pyarrow.decimal128` at the column's ``(precision, scale)`` — **never**
|
|
9
|
+
a binary float, so financial values keep exact decimal semantics. A value with more
|
|
10
|
+
fractional digits than the column scale raises (line-numbered) instead of rounding silently.
|
|
11
|
+
* ``Date/Time`` → ``timestamp('us', tz=UTC)``.
|
|
12
|
+
* ``JSON`` / ``String`` → ``string`` (the JSON text is preserved verbatim).
|
|
13
|
+
* Nulls: an empty string ``""`` maps to a null (validity bit 0) and reads back as ``""``.
|
|
14
|
+
|
|
15
|
+
Exactness contract: CSV output is exact **at the literal** (byte-for-byte); Parquet output is
|
|
16
|
+
exact **in decimal value** — ``decimal128`` normalises the representation (``35.2`` is stored
|
|
17
|
+
and re-read as ``35.200000000000``), so equivalence tests compare Parquet by ``Decimal`` value,
|
|
18
|
+
not by string. Reading is bounded-memory (batched row groups); writing flushes row groups so
|
|
19
|
+
memory does not scale with row count.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import json
|
|
25
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
26
|
+
from datetime import UTC, datetime
|
|
27
|
+
from decimal import Decimal
|
|
28
|
+
from functools import cache
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from urllib.parse import quote
|
|
31
|
+
|
|
32
|
+
from focus_data_toolkit.io.records import DatasetSchema, MalformedRecordError, Record
|
|
33
|
+
from focus_data_toolkit.model import column_spec, load_model, resolve_dataset
|
|
34
|
+
|
|
35
|
+
# Reuse the linter's FOCUS format predicates so Parquet coercion refuses exactly what the CSV
|
|
36
|
+
# lint gate refuses. Otherwise Decimal()/fromisoformat() would accept non-FOCUS literals (`+1`,
|
|
37
|
+
# `.5`, `1E+7`, an offset/naive datetime), normalise them to decimal128/UTC, and the read-back
|
|
38
|
+
# lint would never see the original defect.
|
|
39
|
+
from focus_data_toolkit.model.validator import _decimal_or_none, _is_utc_datetime
|
|
40
|
+
|
|
41
|
+
_PARQUET_HINT = (
|
|
42
|
+
"Parquet support requires PyArrow. Install it with: pip install 'focus-data-toolkit[parquet]'"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _require_pyarrow():
|
|
47
|
+
"""Import PyArrow or raise a clear install hint (kept out of the core import graph)."""
|
|
48
|
+
try:
|
|
49
|
+
import pyarrow as pa # noqa: PLC0415
|
|
50
|
+
import pyarrow.parquet as pq # noqa: PLC0415
|
|
51
|
+
except ModuleNotFoundError as exc: # pragma: no cover - exercised via monkeypatch
|
|
52
|
+
raise MalformedRecordError(_PARQUET_HINT) from exc
|
|
53
|
+
return pa, pq
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# Rows per row group when writing / batch when reading (bounds memory).
|
|
57
|
+
_BATCH = 10_000
|
|
58
|
+
# Timestamps are stored at microsecond precision (covers FOCUS millisecond timestamps exactly).
|
|
59
|
+
_DECIMAL_SCALE_FILE = "focus_1_4_decimal_scale.json"
|
|
60
|
+
|
|
61
|
+
# Parquet compression codecs the CLI/API accept ("none" -> uncompressed).
|
|
62
|
+
COMPRESSIONS = ("snappy", "zstd", "gzip", "none")
|
|
63
|
+
# Partition columns must be low-cardinality identifiers, not measures/JSON. String and Date/Time
|
|
64
|
+
# both round-trip losslessly through Hive directory names (reconstructed as strings on read).
|
|
65
|
+
_PARTITIONABLE_TYPES = frozenset({"String", "Date/Time"})
|
|
66
|
+
# Empty partition values are written as an empty segment (``COL=``), which reads back as "".
|
|
67
|
+
# The reader disables Hive's null convention with a sentinel that percent-encoding can never
|
|
68
|
+
# produce, so a *real* value equal to Hive's ``__HIVE_DEFAULT_PARTITION__`` token is not
|
|
69
|
+
# silently turned into null. (Percent-encoded values only ever contain [A-Za-z0-9_.~%-].)
|
|
70
|
+
_NO_HIVE_NULL = "\x00__fdt_never_null__"
|
|
71
|
+
# High-cardinality guards: warn past the soft threshold, refuse past the hard cap (each distinct
|
|
72
|
+
# partition holds an open writer, so an unbounded key would exhaust file handles / memory).
|
|
73
|
+
PARTITION_WARN_THRESHOLD = 100
|
|
74
|
+
MAX_PARTITIONS = 1000
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def partitionable_columns(dataset: str, columns: Sequence[str]) -> list[str]:
|
|
78
|
+
"""Return the requested partition columns that are NOT valid partition keys (for an error).
|
|
79
|
+
|
|
80
|
+
A valid key exists in ``dataset`` and is String or Date/Time typed; Decimal/JSON measures and
|
|
81
|
+
unknown columns are rejected so partitioning stays on low-cardinality identifiers.
|
|
82
|
+
"""
|
|
83
|
+
dataset = resolve_dataset(dataset)
|
|
84
|
+
model_cols = load_model()["datasets"][dataset]["columns"]
|
|
85
|
+
bad = []
|
|
86
|
+
for col in columns:
|
|
87
|
+
spec = model_cols.get(col)
|
|
88
|
+
if spec is None or spec.get("data_type") not in _PARTITIONABLE_TYPES:
|
|
89
|
+
bad.append(col)
|
|
90
|
+
return bad
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _hive_segment(column: str, value: str) -> str:
|
|
94
|
+
"""Render one ``column=value`` Hive path segment (percent-encoded; empty -> ``COL=``).
|
|
95
|
+
|
|
96
|
+
An empty value becomes an empty segment rather than Hive's ``__HIVE_DEFAULT_PARTITION__``
|
|
97
|
+
sentinel, so a real value that happens to equal that sentinel can never collide with it.
|
|
98
|
+
"""
|
|
99
|
+
return f"{column}={quote(value, safe='')}"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@cache
|
|
103
|
+
def _decimal_scale_registry() -> dict:
|
|
104
|
+
path = Path(__file__).resolve().parent.parent / "model" / _DECIMAL_SCALE_FILE
|
|
105
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def decimal_precision_scale(column: str) -> tuple[int, int]:
|
|
109
|
+
"""Return the ``(precision, scale)`` for a Decimal column (registry override or default)."""
|
|
110
|
+
reg = _decimal_scale_registry()
|
|
111
|
+
spec = reg["columns"].get(column, reg["default"])
|
|
112
|
+
return int(spec["precision"]), int(spec["scale"])
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def arrow_schema(dataset: str, columns: Sequence[str]):
|
|
116
|
+
"""Build the PyArrow schema for ``columns`` of ``dataset`` from the FOCUS model types."""
|
|
117
|
+
pa, _ = _require_pyarrow()
|
|
118
|
+
dataset = resolve_dataset(dataset)
|
|
119
|
+
fields = []
|
|
120
|
+
for col in columns:
|
|
121
|
+
data_type = column_spec(dataset, col)["data_type"]
|
|
122
|
+
if data_type == "Decimal":
|
|
123
|
+
precision, scale = decimal_precision_scale(col)
|
|
124
|
+
arrow_type = pa.decimal128(precision, scale)
|
|
125
|
+
elif data_type == "Date/Time":
|
|
126
|
+
arrow_type = pa.timestamp("us", tz="UTC")
|
|
127
|
+
else: # String, JSON -> UTF-8 string (JSON text preserved verbatim)
|
|
128
|
+
arrow_type = pa.string()
|
|
129
|
+
fields.append(pa.field(col, arrow_type, nullable=True))
|
|
130
|
+
return pa.schema(fields)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _parse_timestamp(value: str, column: str, line: int) -> datetime | None:
|
|
134
|
+
text = value.strip()
|
|
135
|
+
if not text:
|
|
136
|
+
return None
|
|
137
|
+
# Enforce the FOCUS Date/Time format (ISO-8601 UTC with a 'Z') *before* normalising, so an
|
|
138
|
+
# offset (`+00:00`) or naive value is refused here exactly as the CSV lint gate refuses it.
|
|
139
|
+
if not _is_utc_datetime(text):
|
|
140
|
+
raise MalformedRecordError(
|
|
141
|
+
f"column {column!r}: {value!r} is not a FOCUS Date/Time (ISO-8601 UTC, ...Z)",
|
|
142
|
+
line_number=line,
|
|
143
|
+
)
|
|
144
|
+
return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(UTC)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _to_decimal(value: str, column: str, line: int) -> Decimal | None:
|
|
148
|
+
text = value.strip()
|
|
149
|
+
if not text:
|
|
150
|
+
return None
|
|
151
|
+
# Enforce the FOCUS NumericFormat before coercion: `+1`, `.5`, `1E+7`, NaN/Inf are parseable
|
|
152
|
+
# by Decimal but violate the spec and would pass a read-back lint once normalised.
|
|
153
|
+
parsed = _decimal_or_none(text)
|
|
154
|
+
if parsed is None:
|
|
155
|
+
raise MalformedRecordError(
|
|
156
|
+
f"column {column!r}: {value!r} is not a FOCUS numeric literal", line_number=line
|
|
157
|
+
)
|
|
158
|
+
return parsed
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _column_arrays(pa, dataset: str, columns: Sequence[str], rows: list[Mapping[str, str]], base_line: int):
|
|
162
|
+
"""Convert a batch of string rows into typed Arrow arrays (one per column)."""
|
|
163
|
+
arrays = []
|
|
164
|
+
for col in columns:
|
|
165
|
+
data_type = column_spec(dataset, col)["data_type"]
|
|
166
|
+
if data_type == "Decimal":
|
|
167
|
+
precision, scale = decimal_precision_scale(col)
|
|
168
|
+
arrow_type = pa.decimal128(precision, scale)
|
|
169
|
+
# `values` is reused by the Date/Time and String branches below; a broad element
|
|
170
|
+
# type keeps mypy happy across the mutually-exclusive branches (Arrow validates it).
|
|
171
|
+
values: list = [_to_decimal(r.get(col, ""), col, base_line + i) for i, r in enumerate(rows)]
|
|
172
|
+
try:
|
|
173
|
+
arrays.append(pa.array(values, type=arrow_type))
|
|
174
|
+
except pa.lib.ArrowInvalid:
|
|
175
|
+
# Pinpoint the offending row: a value needing more scale than the column allows
|
|
176
|
+
# must fail loudly with its line, never round silently.
|
|
177
|
+
for i, v in enumerate(values):
|
|
178
|
+
if v is None:
|
|
179
|
+
continue
|
|
180
|
+
try:
|
|
181
|
+
pa.array([v], type=arrow_type)
|
|
182
|
+
except pa.lib.ArrowInvalid as exc:
|
|
183
|
+
raise MalformedRecordError(
|
|
184
|
+
f"column {col!r}: {v} exceeds decimal128({precision},{scale}) scale",
|
|
185
|
+
line_number=base_line + i,
|
|
186
|
+
) from exc
|
|
187
|
+
raise
|
|
188
|
+
elif data_type == "Date/Time":
|
|
189
|
+
values = [_parse_timestamp(r.get(col, ""), col, base_line + i) for i, r in enumerate(rows)]
|
|
190
|
+
arrays.append(pa.array(values, type=pa.timestamp("us", tz="UTC")))
|
|
191
|
+
else:
|
|
192
|
+
values = [(r.get(col, "") or None) for r in rows]
|
|
193
|
+
arrays.append(pa.array(values, type=pa.string()))
|
|
194
|
+
return arrays
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _stringify(dataset: str, column: str, value) -> str:
|
|
198
|
+
"""Render a typed Arrow value back to the toolkit's canonical string form."""
|
|
199
|
+
if value is None:
|
|
200
|
+
return ""
|
|
201
|
+
if isinstance(value, str):
|
|
202
|
+
# Already textual (e.g. a client export storing every column as strings): pass it
|
|
203
|
+
# through untouched — downstream validation judges the format, not the reader.
|
|
204
|
+
return value
|
|
205
|
+
data_type = column_spec(dataset, column)["data_type"]
|
|
206
|
+
if data_type == "Date/Time":
|
|
207
|
+
dt = value if value.tzinfo else value.replace(tzinfo=UTC)
|
|
208
|
+
dt = dt.astimezone(UTC)
|
|
209
|
+
text = dt.isoformat()
|
|
210
|
+
return text[:-6] + "Z" if text.endswith("+00:00") else text
|
|
211
|
+
return str(value)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def dataset_metadata(
|
|
215
|
+
dataset: str,
|
|
216
|
+
*,
|
|
217
|
+
target_version: str,
|
|
218
|
+
source_version: str,
|
|
219
|
+
mode: str,
|
|
220
|
+
conformance: str,
|
|
221
|
+
tool_version: str,
|
|
222
|
+
) -> dict:
|
|
223
|
+
"""Business (deterministic) file metadata; operational keys live in a separate namespace.
|
|
224
|
+
|
|
225
|
+
``target_version`` is the FOCUS version the file conforms to (1.4); ``source_version`` is the
|
|
226
|
+
version it was converted from. Keeping them distinct stops a Parquet-metadata reader from
|
|
227
|
+
mistaking the source version for the file's own FOCUS version.
|
|
228
|
+
"""
|
|
229
|
+
return {
|
|
230
|
+
"focus.dataset": resolve_dataset(dataset),
|
|
231
|
+
"focus.target_version": target_version,
|
|
232
|
+
"focus.source_version": source_version,
|
|
233
|
+
"focus.mode": mode,
|
|
234
|
+
"focus.conformance": conformance,
|
|
235
|
+
"focus.toolkit_version": tool_version,
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class ParquetRowWriter:
|
|
240
|
+
"""Write rows to a Parquet file in ``schema.columns`` order, flushing bounded row groups."""
|
|
241
|
+
|
|
242
|
+
def __init__(
|
|
243
|
+
self,
|
|
244
|
+
path: str | Path,
|
|
245
|
+
schema: DatasetSchema,
|
|
246
|
+
*,
|
|
247
|
+
metadata: Mapping[str, str] | None = None,
|
|
248
|
+
compression: str = "snappy",
|
|
249
|
+
) -> None:
|
|
250
|
+
self._pa, self._pq = _require_pyarrow()
|
|
251
|
+
self._dataset = resolve_dataset(schema.dataset)
|
|
252
|
+
self._columns = tuple(schema.columns)
|
|
253
|
+
arrow = arrow_schema(self._dataset, self._columns)
|
|
254
|
+
if metadata:
|
|
255
|
+
arrow = arrow.with_metadata({k: str(v) for k, v in metadata.items()})
|
|
256
|
+
self._arrow_schema = arrow
|
|
257
|
+
self._writer = self._pq.ParquetWriter(str(path), arrow, compression=compression)
|
|
258
|
+
self._buffer: list[Mapping[str, str]] = []
|
|
259
|
+
self._written = 0
|
|
260
|
+
|
|
261
|
+
@property
|
|
262
|
+
def buffered(self) -> int:
|
|
263
|
+
"""Rows currently buffered (not yet flushed to a row group)."""
|
|
264
|
+
return len(self._buffer)
|
|
265
|
+
|
|
266
|
+
def write(self, values: Mapping[str, str]) -> None:
|
|
267
|
+
self._buffer.append(values)
|
|
268
|
+
if len(self._buffer) >= _BATCH:
|
|
269
|
+
self._flush()
|
|
270
|
+
|
|
271
|
+
def flush(self) -> None:
|
|
272
|
+
"""Flush buffered rows to a row group (public; used to bound memory across writers)."""
|
|
273
|
+
self._flush()
|
|
274
|
+
|
|
275
|
+
def _flush(self) -> None:
|
|
276
|
+
if not self._buffer:
|
|
277
|
+
return
|
|
278
|
+
arrays = _column_arrays(
|
|
279
|
+
self._pa, self._dataset, self._columns, self._buffer, base_line=self._written + 1
|
|
280
|
+
)
|
|
281
|
+
batch = self._pa.record_batch(arrays, schema=self._arrow_schema)
|
|
282
|
+
self._writer.write_table(self._pa.Table.from_batches([batch], self._arrow_schema))
|
|
283
|
+
self._written += len(self._buffer)
|
|
284
|
+
self._buffer = []
|
|
285
|
+
|
|
286
|
+
def close(self) -> None:
|
|
287
|
+
self._flush()
|
|
288
|
+
self._writer.close()
|
|
289
|
+
|
|
290
|
+
def __enter__(self) -> ParquetRowWriter:
|
|
291
|
+
return self
|
|
292
|
+
|
|
293
|
+
def __exit__(self, *exc: object) -> None:
|
|
294
|
+
self.close()
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
class ParquetRowReader:
|
|
298
|
+
"""Stream ``Record``s from a Parquet file, rendering typed values back to strings."""
|
|
299
|
+
|
|
300
|
+
def __init__(self, path: str | Path, *, dataset: str | None = None) -> None:
|
|
301
|
+
self._pa, self._pq = _require_pyarrow()
|
|
302
|
+
self._path = Path(path)
|
|
303
|
+
self._file = self._pq.ParquetFile(str(path))
|
|
304
|
+
self.source_columns: tuple[str, ...] = tuple(self._file.schema_arrow.names)
|
|
305
|
+
self._dataset = self._resolve_dataset(dataset)
|
|
306
|
+
|
|
307
|
+
@property
|
|
308
|
+
def expected_rows(self) -> int | None:
|
|
309
|
+
"""Total rows, read cheaply from the Parquet footer (no data load) — for progress."""
|
|
310
|
+
try:
|
|
311
|
+
return int(self._file.metadata.num_rows)
|
|
312
|
+
except (OSError, ValueError, AttributeError):
|
|
313
|
+
return None
|
|
314
|
+
|
|
315
|
+
def _resolve_dataset(self, dataset: str | None) -> str:
|
|
316
|
+
if dataset is not None:
|
|
317
|
+
return resolve_dataset(dataset)
|
|
318
|
+
# Infer the dataset from the column set so values can be rendered by their model types.
|
|
319
|
+
cols = set(self.source_columns)
|
|
320
|
+
best, best_overlap = "Cost and Usage", -1
|
|
321
|
+
for name in load_model()["datasets"]:
|
|
322
|
+
overlap = len(cols & set(load_model()["datasets"][name]["columns"]))
|
|
323
|
+
if overlap > best_overlap:
|
|
324
|
+
best, best_overlap = name, overlap
|
|
325
|
+
return best
|
|
326
|
+
|
|
327
|
+
def __iter__(self) -> Iterator[Record]:
|
|
328
|
+
model_cols = set(load_model()["datasets"][self._dataset]["columns"])
|
|
329
|
+
line = 0
|
|
330
|
+
for batch in self._file.iter_batches(batch_size=_BATCH):
|
|
331
|
+
columns = batch.schema.names
|
|
332
|
+
pydata = {name: batch.column(i).to_pylist() for i, name in enumerate(columns)}
|
|
333
|
+
for r in range(batch.num_rows):
|
|
334
|
+
line += 1
|
|
335
|
+
values = {}
|
|
336
|
+
for name in columns:
|
|
337
|
+
raw = pydata[name][r]
|
|
338
|
+
values[name] = (
|
|
339
|
+
_stringify(self._dataset, name, raw) if name in model_cols
|
|
340
|
+
else ("" if raw is None else str(raw))
|
|
341
|
+
)
|
|
342
|
+
yield Record(values, line)
|
|
343
|
+
|
|
344
|
+
def close(self) -> None:
|
|
345
|
+
pass # ParquetFile has no long-lived handle to close in this usage
|
|
346
|
+
|
|
347
|
+
def __enter__(self) -> ParquetRowReader:
|
|
348
|
+
return self
|
|
349
|
+
|
|
350
|
+
def __exit__(self, *exc: object) -> None:
|
|
351
|
+
self.close()
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
class PartitionTooWideError(MalformedRecordError):
|
|
355
|
+
"""Raised when a ``--partition-by`` key produces more partitions than :data:`MAX_PARTITIONS`."""
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class PartitionedParquetWriter:
|
|
359
|
+
"""Write a dataset as a Hive-partitioned Parquet tree under ``base_dir``.
|
|
360
|
+
|
|
361
|
+
Rows are routed to ``base_dir/COL=value/.../part-N.parquet`` by their partition-column values;
|
|
362
|
+
the partition columns are **omitted** from the part files (standard Hive — a reader
|
|
363
|
+
reconstructs them from the path). Open file handles scale with the *number of partitions*
|
|
364
|
+
(hence the hard :data:`MAX_PARTITIONS` cap), but **buffered rows are bounded globally**: once
|
|
365
|
+
the total unflushed rows across all partitions reaches :data:`_BATCH`, every partition writer
|
|
366
|
+
is flushed — so memory stays bounded no matter how the rows interleave across partition keys.
|
|
367
|
+
When ``target_file_size`` is set, a partition rolls to a new part file once its (approximate,
|
|
368
|
+
uncompressed) running size crosses the threshold.
|
|
369
|
+
"""
|
|
370
|
+
|
|
371
|
+
def __init__(
|
|
372
|
+
self,
|
|
373
|
+
base_dir: str | Path,
|
|
374
|
+
schema: DatasetSchema,
|
|
375
|
+
partition_by: Sequence[str],
|
|
376
|
+
*,
|
|
377
|
+
metadata: Mapping[str, str] | None = None,
|
|
378
|
+
compression: str = "snappy",
|
|
379
|
+
target_file_size: int | None = None,
|
|
380
|
+
) -> None:
|
|
381
|
+
if target_file_size is not None and target_file_size <= 0:
|
|
382
|
+
raise ValueError(f"target_file_size must be positive, got {target_file_size}")
|
|
383
|
+
self._base = Path(base_dir)
|
|
384
|
+
self._dataset = resolve_dataset(schema.dataset)
|
|
385
|
+
self._partition_by = tuple(partition_by)
|
|
386
|
+
# Part files carry every column except the partition columns (Hive stores those in path).
|
|
387
|
+
self._file_columns = tuple(c for c in schema.columns if c not in set(self._partition_by))
|
|
388
|
+
self._metadata = metadata
|
|
389
|
+
self._compression = compression
|
|
390
|
+
self._target = target_file_size
|
|
391
|
+
# Per partition: (writer, part_index, running_byte_estimate).
|
|
392
|
+
self._writers: dict[tuple[str, ...], list] = {}
|
|
393
|
+
self._buffered = 0 # total rows buffered across all partition writers
|
|
394
|
+
|
|
395
|
+
def _partition_dir(self, values: Mapping[str, str]) -> tuple[tuple[str, ...], Path]:
|
|
396
|
+
key = tuple((values.get(c) or "") for c in self._partition_by)
|
|
397
|
+
rel = Path(*[_hive_segment(c, v) for c, v in zip(self._partition_by, key, strict=True)])
|
|
398
|
+
return key, self._base / rel
|
|
399
|
+
|
|
400
|
+
def _open_part(self, directory: Path, index: int):
|
|
401
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
402
|
+
return ParquetRowWriter(
|
|
403
|
+
directory / f"part-{index}.parquet",
|
|
404
|
+
DatasetSchema(self._dataset, self._file_columns),
|
|
405
|
+
metadata=self._metadata,
|
|
406
|
+
compression=self._compression,
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
def write(self, values: Mapping[str, str]) -> None:
|
|
410
|
+
key, directory = self._partition_dir(values)
|
|
411
|
+
state = self._writers.get(key)
|
|
412
|
+
if state is None:
|
|
413
|
+
if len(self._writers) >= MAX_PARTITIONS:
|
|
414
|
+
raise PartitionTooWideError(
|
|
415
|
+
f"--partition-by produced more than {MAX_PARTITIONS} partitions; choose a "
|
|
416
|
+
"lower-cardinality key"
|
|
417
|
+
)
|
|
418
|
+
state = [self._open_part(directory, 0), 0, 0]
|
|
419
|
+
self._writers[key] = state
|
|
420
|
+
if self._target is not None and state[2] >= self._target:
|
|
421
|
+
state[0].close()
|
|
422
|
+
state[1] += 1
|
|
423
|
+
state[2] = 0
|
|
424
|
+
state[0] = self._open_part(directory, state[1])
|
|
425
|
+
state[0].write(values)
|
|
426
|
+
self._buffered += 1
|
|
427
|
+
if self._target is not None:
|
|
428
|
+
state[2] += sum(len(values.get(c) or "") for c in self._file_columns) + len(
|
|
429
|
+
self._file_columns
|
|
430
|
+
)
|
|
431
|
+
# Bound total buffered rows across ALL partitions (not per-partition): flush everything
|
|
432
|
+
# once the global buffer fills, so memory can't grow with the number of partition keys.
|
|
433
|
+
if self._buffered >= _BATCH:
|
|
434
|
+
for st in self._writers.values():
|
|
435
|
+
st[0].flush()
|
|
436
|
+
self._buffered = 0
|
|
437
|
+
|
|
438
|
+
def partition_count(self) -> int:
|
|
439
|
+
return len(self._writers)
|
|
440
|
+
|
|
441
|
+
def close(self) -> None:
|
|
442
|
+
for state in self._writers.values():
|
|
443
|
+
state[0].close()
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
class PartitionedParquetReader:
|
|
447
|
+
"""Read a Hive-partitioned Parquet dataset back as ``Record``s (partition columns rebuilt).
|
|
448
|
+
|
|
449
|
+
Used by the lint gate: the partition columns (omitted from the part files) are reconstructed
|
|
450
|
+
from the directory names via an explicit **string** Hive schema, so a Date/Time or numeric-
|
|
451
|
+
looking partition value is never re-typed and reads back exactly as written.
|
|
452
|
+
"""
|
|
453
|
+
|
|
454
|
+
def __init__(self, base_dir: str | Path, dataset: str, partition_by: Sequence[str]) -> None:
|
|
455
|
+
self._pa, _ = _require_pyarrow()
|
|
456
|
+
import pyarrow.dataset as pds # noqa: PLC0415
|
|
457
|
+
|
|
458
|
+
self._dataset = resolve_dataset(dataset)
|
|
459
|
+
self._partition_cols = set(partition_by)
|
|
460
|
+
schema = self._pa.schema([(c, self._pa.string()) for c in partition_by])
|
|
461
|
+
partitioning = pds.HivePartitioning(
|
|
462
|
+
schema, null_fallback=_NO_HIVE_NULL, segment_encoding="uri"
|
|
463
|
+
)
|
|
464
|
+
self._ds = pds.dataset(str(base_dir), format="parquet", partitioning=partitioning)
|
|
465
|
+
self.source_columns: tuple[str, ...] = tuple(self._ds.schema.names)
|
|
466
|
+
|
|
467
|
+
@property
|
|
468
|
+
def expected_rows(self) -> int | None:
|
|
469
|
+
"""Total rows across the partition tree, from footer metadata — for progress."""
|
|
470
|
+
try:
|
|
471
|
+
return int(self._ds.count_rows())
|
|
472
|
+
except (OSError, ValueError, AttributeError):
|
|
473
|
+
return None
|
|
474
|
+
|
|
475
|
+
def __iter__(self) -> Iterator[Record]:
|
|
476
|
+
model_cols = set(load_model()["datasets"][self._dataset]["columns"])
|
|
477
|
+
line = 0
|
|
478
|
+
for batch in self._ds.to_batches(batch_size=_BATCH):
|
|
479
|
+
columns = batch.schema.names
|
|
480
|
+
pydata = {name: batch.column(i).to_pylist() for i, name in enumerate(columns)}
|
|
481
|
+
for r in range(batch.num_rows):
|
|
482
|
+
line += 1
|
|
483
|
+
values = {}
|
|
484
|
+
for name in columns:
|
|
485
|
+
raw = pydata[name][r]
|
|
486
|
+
if name in self._partition_cols:
|
|
487
|
+
# Reconstructed from the path as a string already — pass it through.
|
|
488
|
+
values[name] = "" if raw is None else str(raw)
|
|
489
|
+
elif name in model_cols:
|
|
490
|
+
values[name] = _stringify(self._dataset, name, raw)
|
|
491
|
+
else:
|
|
492
|
+
values[name] = "" if raw is None else str(raw)
|
|
493
|
+
yield Record(values, line)
|
|
494
|
+
|
|
495
|
+
def close(self) -> None:
|
|
496
|
+
pass
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def open_parquet_writer(
|
|
500
|
+
path: str | Path,
|
|
501
|
+
schema: DatasetSchema,
|
|
502
|
+
*,
|
|
503
|
+
metadata: Mapping[str, str] | None = None,
|
|
504
|
+
compression: str = "snappy",
|
|
505
|
+
):
|
|
506
|
+
"""Open ``path`` for Parquet writing and return ``(writer, writer)`` (handle == writer).
|
|
507
|
+
|
|
508
|
+
The tuple shape mirrors :func:`focus_data_toolkit.io.csv_io.open_csv_writer` so callers can
|
|
509
|
+
treat both formats uniformly; the Parquet writer owns its own file handle.
|
|
510
|
+
"""
|
|
511
|
+
writer = ParquetRowWriter(path, schema, metadata=metadata, compression=compression)
|
|
512
|
+
return writer, writer
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
__all__ = [
|
|
516
|
+
"COMPRESSIONS",
|
|
517
|
+
"MAX_PARTITIONS",
|
|
518
|
+
"PartitionedParquetReader",
|
|
519
|
+
"PartitionedParquetWriter",
|
|
520
|
+
"PartitionTooWideError",
|
|
521
|
+
"ParquetRowReader",
|
|
522
|
+
"ParquetRowWriter",
|
|
523
|
+
"arrow_schema",
|
|
524
|
+
"dataset_metadata",
|
|
525
|
+
"decimal_precision_scale",
|
|
526
|
+
"open_parquet_writer",
|
|
527
|
+
"partitionable_columns",
|
|
528
|
+
]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Streaming record abstraction shared by the CSV and Parquet I/O layers.
|
|
2
|
+
|
|
3
|
+
A :class:`Record` is one source row plus its physical line/record number (for actionable
|
|
4
|
+
errors). :class:`RowReader` / :class:`RowWriter` are the minimal streaming interfaces the
|
|
5
|
+
conversion engine consumes and produces; :class:`DatasetSchema` supplies the output column
|
|
6
|
+
order from the model registry (never from ``rows[0]``, which a stream cannot peek).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from collections.abc import Iterable, Iterator, Mapping
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Protocol, runtime_checkable
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class MalformedRecordError(ValueError):
|
|
17
|
+
"""A source record could not be parsed (e.g. wrong field count); carries the line number."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, message: str, *, line_number: int | None = None) -> None:
|
|
20
|
+
super().__init__(message)
|
|
21
|
+
self.line_number = line_number
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class Record:
|
|
26
|
+
"""One source row and its 1-based physical line/record number."""
|
|
27
|
+
|
|
28
|
+
values: Mapping[str, str]
|
|
29
|
+
line_number: int
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class DatasetSchema:
|
|
34
|
+
"""Output schema of a FOCUS dataset: the column names, in normative model order."""
|
|
35
|
+
|
|
36
|
+
dataset: str
|
|
37
|
+
columns: tuple[str, ...]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@runtime_checkable
|
|
41
|
+
class RowReader(Protocol):
|
|
42
|
+
"""A stream of :class:`Record`s whose header (``source_columns``) is known on open."""
|
|
43
|
+
|
|
44
|
+
source_columns: tuple[str, ...]
|
|
45
|
+
|
|
46
|
+
def __iter__(self) -> Iterator[Record]: ...
|
|
47
|
+
|
|
48
|
+
def close(self) -> None: ...
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@runtime_checkable
|
|
52
|
+
class RowWriter(Protocol):
|
|
53
|
+
"""A sink for output rows (values keyed by column name)."""
|
|
54
|
+
|
|
55
|
+
def write(self, values: Mapping[str, str]) -> None: ...
|
|
56
|
+
|
|
57
|
+
def close(self) -> None: ...
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class ListRowReader:
|
|
61
|
+
"""Adapt an in-memory ``list[dict]`` to the :class:`RowReader` interface.
|
|
62
|
+
|
|
63
|
+
Lets the small-volume ``convert_to_focus_1_4(rows)`` API run the exact same streaming
|
|
64
|
+
engine as ``convert_files`` (equivalence by construction, not by luck).
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def __init__(self, rows: Iterable[Mapping[str, str]], *, source_columns: tuple[str, ...] | None = None) -> None:
|
|
68
|
+
self._rows = list(rows)
|
|
69
|
+
if source_columns is not None:
|
|
70
|
+
self.source_columns = source_columns
|
|
71
|
+
else:
|
|
72
|
+
self.source_columns = tuple(self._rows[0].keys()) if self._rows else ()
|
|
73
|
+
|
|
74
|
+
def __iter__(self) -> Iterator[Record]:
|
|
75
|
+
for i, row in enumerate(self._rows, start=1):
|
|
76
|
+
yield Record(row, i)
|
|
77
|
+
|
|
78
|
+
def close(self) -> None:
|
|
79
|
+
pass
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class ListRowWriter:
|
|
83
|
+
"""Collect written rows into an in-memory list (the streaming counterpart of a sink)."""
|
|
84
|
+
|
|
85
|
+
def __init__(self) -> None:
|
|
86
|
+
self.rows: list[dict[str, str]] = []
|
|
87
|
+
|
|
88
|
+
def write(self, values: Mapping[str, str]) -> None:
|
|
89
|
+
self.rows.append(dict(values))
|
|
90
|
+
|
|
91
|
+
def close(self) -> None:
|
|
92
|
+
pass
|