focus-data-toolkit 0.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. focus_data_toolkit/__init__.py +69 -0
  2. focus_data_toolkit/__main__.py +6 -0
  3. focus_data_toolkit/_version.py +8 -0
  4. focus_data_toolkit/cli.py +968 -0
  5. focus_data_toolkit/context/__init__.py +88 -0
  6. focus_data_toolkit/context/billing.py +54 -0
  7. focus_data_toolkit/context/provider.py +90 -0
  8. focus_data_toolkit/convert/__init__.py +708 -0
  9. focus_data_toolkit/convert/billing_period.py +65 -0
  10. focus_data_toolkit/convert/contract_applied.py +235 -0
  11. focus_data_toolkit/convert/contract_commitment.py +182 -0
  12. focus_data_toolkit/convert/cost_and_usage.py +179 -0
  13. focus_data_toolkit/convert/detect.py +39 -0
  14. focus_data_toolkit/convert/invoice_detail.py +199 -0
  15. focus_data_toolkit/convert/streaming.py +1030 -0
  16. focus_data_toolkit/errors.py +145 -0
  17. focus_data_toolkit/focus_json.py +68 -0
  18. focus_data_toolkit/generators/__init__.py +61 -0
  19. focus_data_toolkit/generators/_shim.py +43 -0
  20. focus_data_toolkit/generators/engine/__init__.py +14 -0
  21. focus_data_toolkit/generators/engine/context.py +12 -0
  22. focus_data_toolkit/generators/engine/determinism.py +117 -0
  23. focus_data_toolkit/generators/engine/json_focus.py +63 -0
  24. focus_data_toolkit/generators/engine/ladder.py +71 -0
  25. focus_data_toolkit/generators/engine/scenarios_core.py +380 -0
  26. focus_data_toolkit/generators/engine/serialize.py +151 -0
  27. focus_data_toolkit/generators/generate_aws_focus_1_2.py +19 -0
  28. focus_data_toolkit/generators/generate_aws_focus_1_3.py +20 -0
  29. focus_data_toolkit/generators/generate_azure_focus_1_2.py +17 -0
  30. focus_data_toolkit/generators/generate_azure_focus_1_3.py +17 -0
  31. focus_data_toolkit/generators/generate_gcp_focus_1_2.py +17 -0
  32. focus_data_toolkit/generators/generate_gcp_focus_1_3.py +17 -0
  33. focus_data_toolkit/generators/providers/__init__.py +29 -0
  34. focus_data_toolkit/generators/providers/aws.py +186 -0
  35. focus_data_toolkit/generators/providers/azure.py +191 -0
  36. focus_data_toolkit/generators/providers/gcp.py +194 -0
  37. focus_data_toolkit/generators/providers/profile.py +123 -0
  38. focus_data_toolkit/generators/scenarios.py +178 -0
  39. focus_data_toolkit/generators/versions/__init__.py +17 -0
  40. focus_data_toolkit/generators/versions/adapter.py +41 -0
  41. focus_data_toolkit/generators/versions/v1_2.py +111 -0
  42. focus_data_toolkit/generators/versions/v1_3.py +154 -0
  43. focus_data_toolkit/io/__init__.py +1 -0
  44. focus_data_toolkit/io/atomic_writer.py +462 -0
  45. focus_data_toolkit/io/csv_io.py +128 -0
  46. focus_data_toolkit/io/parquet_io.py +528 -0
  47. focus_data_toolkit/io/records.py +92 -0
  48. focus_data_toolkit/io/row_source.py +117 -0
  49. focus_data_toolkit/lifecycle.py +342 -0
  50. focus_data_toolkit/manifest.py +114 -0
  51. focus_data_toolkit/model/__init__.py +43 -0
  52. focus_data_toolkit/model/capabilities.py +66 -0
  53. focus_data_toolkit/model/focus_1_4_decimal_scale.json +10 -0
  54. focus_data_toolkit/model/focus_1_4_model.json +1913 -0
  55. focus_data_toolkit/model/focus_1_4_servicesubcategory.json +84 -0
  56. focus_data_toolkit/model/focus_json_keys.py +112 -0
  57. focus_data_toolkit/model/iso_4217_currencies.json +23 -0
  58. focus_data_toolkit/model/json_schema_check.py +205 -0
  59. focus_data_toolkit/model/json_schemas/allocatedmethoddetailsobjectschema.json +82 -0
  60. focus_data_toolkit/model/json_schemas/commitmentprogrameligibilitydetailsobjectschema.json +41 -0
  61. focus_data_toolkit/model/json_schemas/contractappliedobjectschema.json +104 -0
  62. focus_data_toolkit/model/json_schemas/contractcommitmentapplicabilityobjectschema.json +290 -0
  63. focus_data_toolkit/model/json_schemas/json_schemas_provenance.json +38 -0
  64. focus_data_toolkit/model/model_provenance.json +58 -0
  65. focus_data_toolkit/model/validator.py +498 -0
  66. focus_data_toolkit/modes.py +18 -0
  67. focus_data_toolkit/official_validator.py +61 -0
  68. focus_data_toolkit/progress.py +89 -0
  69. focus_data_toolkit/provenance.py +106 -0
  70. focus_data_toolkit/py.typed +1 -0
  71. focus_data_toolkit/runtime.py +243 -0
  72. focus_data_toolkit/schema/__init__.py +17 -0
  73. focus_data_toolkit/schema/detection.py +274 -0
  74. focus_data_toolkit/schema/registry.py +127 -0
  75. focus_data_toolkit/storage/__init__.py +1 -0
  76. focus_data_toolkit/storage/external_index.py +99 -0
  77. focus_data_toolkit/storage/spill.py +150 -0
  78. focus_data_toolkit/studio/__init__.py +19 -0
  79. focus_data_toolkit/studio/app.py +467 -0
  80. focus_data_toolkit/studio/config.py +42 -0
  81. focus_data_toolkit/studio/frontend/app.js +214 -0
  82. focus_data_toolkit/studio/frontend/index.html +101 -0
  83. focus_data_toolkit/studio/frontend/style.css +60 -0
  84. focus_data_toolkit/studio/jobs.py +142 -0
  85. focus_data_toolkit/studio/preview.py +32 -0
  86. focus_data_toolkit/studio/security.py +125 -0
  87. focus_data_toolkit/studio/server.py +71 -0
  88. focus_data_toolkit/supplement/__init__.py +50 -0
  89. focus_data_toolkit/supplement/adapters/__init__.py +21 -0
  90. focus_data_toolkit/supplement/adapters/adapters_provenance.json +39 -0
  91. focus_data_toolkit/supplement/adapters/aws_invoice_summary.json +24 -0
  92. focus_data_toolkit/supplement/adapters/aws_savings_plans.json +31 -0
  93. focus_data_toolkit/supplement/adapters/azure_invoice.json +25 -0
  94. focus_data_toolkit/supplement/adapters/gcp_compute_commitments.json +28 -0
  95. focus_data_toolkit/supplement/adapters/registry.py +215 -0
  96. focus_data_toolkit/supplement/apply.py +318 -0
  97. focus_data_toolkit/supplement/gaps.py +219 -0
  98. focus_data_toolkit/supplement/kinds.py +118 -0
  99. focus_data_toolkit/supplement/loader.py +409 -0
  100. focus_data_toolkit/supplement/spec.py +74 -0
  101. focus_data_toolkit/supplement/validate.py +215 -0
  102. focus_data_toolkit/validate/__init__.py +15 -0
  103. focus_data_toolkit/validate/allocation.py +333 -0
  104. focus_data_toolkit/validate/bundle.py +254 -0
  105. focus_data_toolkit/validate/codes.py +93 -0
  106. focus_data_toolkit/validate/corrections.py +245 -0
  107. focus_data_toolkit/validate/reconciliation.py +98 -0
  108. focus_data_toolkit/validate/referential.py +289 -0
  109. focus_data_toolkit-0.11.0.dist-info/METADATA +519 -0
  110. focus_data_toolkit-0.11.0.dist-info/RECORD +116 -0
  111. focus_data_toolkit-0.11.0.dist-info/WHEEL +5 -0
  112. focus_data_toolkit-0.11.0.dist-info/entry_points.txt +2 -0
  113. focus_data_toolkit-0.11.0.dist-info/licenses/LICENSE +21 -0
  114. focus_data_toolkit-0.11.0.dist-info/licenses/LICENSES/CC-BY-4.0.txt +156 -0
  115. focus_data_toolkit-0.11.0.dist-info/licenses/NOTICE +60 -0
  116. focus_data_toolkit-0.11.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,528 @@
1
+ """Parquet reader/writer with exact decimal columnar I/O (P1.6).
2
+
3
+ PyArrow is an **optional** dependency (``pip install 'focus-data-toolkit[parquet]'``); importing
4
+ this module without it raises a clear, actionable error rather than an opaque ``ImportError``.
5
+
6
+ Type mapping (driven by the FOCUS 1.4 model + the committed decimal-scale registry):
7
+
8
+ * ``Decimal`` → :func:`pyarrow.decimal128` at the column's ``(precision, scale)`` — **never**
9
+ a binary float, so financial values keep exact decimal semantics. A value with more
10
+ fractional digits than the column scale raises (line-numbered) instead of rounding silently.
11
+ * ``Date/Time`` → ``timestamp('us', tz=UTC)``.
12
+ * ``JSON`` / ``String`` → ``string`` (the JSON text is preserved verbatim).
13
+ * Nulls: an empty string ``""`` maps to a null (validity bit 0) and reads back as ``""``.
14
+
15
+ Exactness contract: CSV output is exact **at the literal** (byte-for-byte); Parquet output is
16
+ exact **in decimal value** — ``decimal128`` normalises the representation (``35.2`` is stored
17
+ and re-read as ``35.200000000000``), so equivalence tests compare Parquet by ``Decimal`` value,
18
+ not by string. Reading is bounded-memory (batched row groups); writing flushes row groups so
19
+ memory does not scale with row count.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import json
25
+ from collections.abc import Iterator, Mapping, Sequence
26
+ from datetime import UTC, datetime
27
+ from decimal import Decimal
28
+ from functools import cache
29
+ from pathlib import Path
30
+ from urllib.parse import quote
31
+
32
+ from focus_data_toolkit.io.records import DatasetSchema, MalformedRecordError, Record
33
+ from focus_data_toolkit.model import column_spec, load_model, resolve_dataset
34
+
35
+ # Reuse the linter's FOCUS format predicates so Parquet coercion refuses exactly what the CSV
36
+ # lint gate refuses. Otherwise Decimal()/fromisoformat() would accept non-FOCUS literals (`+1`,
37
+ # `.5`, `1E+7`, an offset/naive datetime), normalise them to decimal128/UTC, and the read-back
38
+ # lint would never see the original defect.
39
+ from focus_data_toolkit.model.validator import _decimal_or_none, _is_utc_datetime
40
+
41
+ _PARQUET_HINT = (
42
+ "Parquet support requires PyArrow. Install it with: pip install 'focus-data-toolkit[parquet]'"
43
+ )
44
+
45
+
46
+ def _require_pyarrow():
47
+ """Import PyArrow or raise a clear install hint (kept out of the core import graph)."""
48
+ try:
49
+ import pyarrow as pa # noqa: PLC0415
50
+ import pyarrow.parquet as pq # noqa: PLC0415
51
+ except ModuleNotFoundError as exc: # pragma: no cover - exercised via monkeypatch
52
+ raise MalformedRecordError(_PARQUET_HINT) from exc
53
+ return pa, pq
54
+
55
+
56
+ # Rows per row group when writing / batch when reading (bounds memory).
57
+ _BATCH = 10_000
58
+ # Timestamps are stored at microsecond precision (covers FOCUS millisecond timestamps exactly).
59
+ _DECIMAL_SCALE_FILE = "focus_1_4_decimal_scale.json"
60
+
61
+ # Parquet compression codecs the CLI/API accept ("none" -> uncompressed).
62
+ COMPRESSIONS = ("snappy", "zstd", "gzip", "none")
63
+ # Partition columns must be low-cardinality identifiers, not measures/JSON. String and Date/Time
64
+ # both round-trip losslessly through Hive directory names (reconstructed as strings on read).
65
+ _PARTITIONABLE_TYPES = frozenset({"String", "Date/Time"})
66
+ # Empty partition values are written as an empty segment (``COL=``), which reads back as "".
67
+ # The reader disables Hive's null convention with a sentinel that percent-encoding can never
68
+ # produce, so a *real* value equal to Hive's ``__HIVE_DEFAULT_PARTITION__`` token is not
69
+ # silently turned into null. (Percent-encoded values only ever contain [A-Za-z0-9_.~%-].)
70
+ _NO_HIVE_NULL = "\x00__fdt_never_null__"
71
+ # High-cardinality guards: warn past the soft threshold, refuse past the hard cap (each distinct
72
+ # partition holds an open writer, so an unbounded key would exhaust file handles / memory).
73
+ PARTITION_WARN_THRESHOLD = 100
74
+ MAX_PARTITIONS = 1000
75
+
76
+
77
+ def partitionable_columns(dataset: str, columns: Sequence[str]) -> list[str]:
78
+ """Return the requested partition columns that are NOT valid partition keys (for an error).
79
+
80
+ A valid key exists in ``dataset`` and is String or Date/Time typed; Decimal/JSON measures and
81
+ unknown columns are rejected so partitioning stays on low-cardinality identifiers.
82
+ """
83
+ dataset = resolve_dataset(dataset)
84
+ model_cols = load_model()["datasets"][dataset]["columns"]
85
+ bad = []
86
+ for col in columns:
87
+ spec = model_cols.get(col)
88
+ if spec is None or spec.get("data_type") not in _PARTITIONABLE_TYPES:
89
+ bad.append(col)
90
+ return bad
91
+
92
+
93
+ def _hive_segment(column: str, value: str) -> str:
94
+ """Render one ``column=value`` Hive path segment (percent-encoded; empty -> ``COL=``).
95
+
96
+ An empty value becomes an empty segment rather than Hive's ``__HIVE_DEFAULT_PARTITION__``
97
+ sentinel, so a real value that happens to equal that sentinel can never collide with it.
98
+ """
99
+ return f"{column}={quote(value, safe='')}"
100
+
101
+
102
+ @cache
103
+ def _decimal_scale_registry() -> dict:
104
+ path = Path(__file__).resolve().parent.parent / "model" / _DECIMAL_SCALE_FILE
105
+ return json.loads(path.read_text(encoding="utf-8"))
106
+
107
+
108
+ def decimal_precision_scale(column: str) -> tuple[int, int]:
109
+ """Return the ``(precision, scale)`` for a Decimal column (registry override or default)."""
110
+ reg = _decimal_scale_registry()
111
+ spec = reg["columns"].get(column, reg["default"])
112
+ return int(spec["precision"]), int(spec["scale"])
113
+
114
+
115
+ def arrow_schema(dataset: str, columns: Sequence[str]):
116
+ """Build the PyArrow schema for ``columns`` of ``dataset`` from the FOCUS model types."""
117
+ pa, _ = _require_pyarrow()
118
+ dataset = resolve_dataset(dataset)
119
+ fields = []
120
+ for col in columns:
121
+ data_type = column_spec(dataset, col)["data_type"]
122
+ if data_type == "Decimal":
123
+ precision, scale = decimal_precision_scale(col)
124
+ arrow_type = pa.decimal128(precision, scale)
125
+ elif data_type == "Date/Time":
126
+ arrow_type = pa.timestamp("us", tz="UTC")
127
+ else: # String, JSON -> UTF-8 string (JSON text preserved verbatim)
128
+ arrow_type = pa.string()
129
+ fields.append(pa.field(col, arrow_type, nullable=True))
130
+ return pa.schema(fields)
131
+
132
+
133
+ def _parse_timestamp(value: str, column: str, line: int) -> datetime | None:
134
+ text = value.strip()
135
+ if not text:
136
+ return None
137
+ # Enforce the FOCUS Date/Time format (ISO-8601 UTC with a 'Z') *before* normalising, so an
138
+ # offset (`+00:00`) or naive value is refused here exactly as the CSV lint gate refuses it.
139
+ if not _is_utc_datetime(text):
140
+ raise MalformedRecordError(
141
+ f"column {column!r}: {value!r} is not a FOCUS Date/Time (ISO-8601 UTC, ...Z)",
142
+ line_number=line,
143
+ )
144
+ return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(UTC)
145
+
146
+
147
+ def _to_decimal(value: str, column: str, line: int) -> Decimal | None:
148
+ text = value.strip()
149
+ if not text:
150
+ return None
151
+ # Enforce the FOCUS NumericFormat before coercion: `+1`, `.5`, `1E+7`, NaN/Inf are parseable
152
+ # by Decimal but violate the spec and would pass a read-back lint once normalised.
153
+ parsed = _decimal_or_none(text)
154
+ if parsed is None:
155
+ raise MalformedRecordError(
156
+ f"column {column!r}: {value!r} is not a FOCUS numeric literal", line_number=line
157
+ )
158
+ return parsed
159
+
160
+
161
+ def _column_arrays(pa, dataset: str, columns: Sequence[str], rows: list[Mapping[str, str]], base_line: int):
162
+ """Convert a batch of string rows into typed Arrow arrays (one per column)."""
163
+ arrays = []
164
+ for col in columns:
165
+ data_type = column_spec(dataset, col)["data_type"]
166
+ if data_type == "Decimal":
167
+ precision, scale = decimal_precision_scale(col)
168
+ arrow_type = pa.decimal128(precision, scale)
169
+ # `values` is reused by the Date/Time and String branches below; a broad element
170
+ # type keeps mypy happy across the mutually-exclusive branches (Arrow validates it).
171
+ values: list = [_to_decimal(r.get(col, ""), col, base_line + i) for i, r in enumerate(rows)]
172
+ try:
173
+ arrays.append(pa.array(values, type=arrow_type))
174
+ except pa.lib.ArrowInvalid:
175
+ # Pinpoint the offending row: a value needing more scale than the column allows
176
+ # must fail loudly with its line, never round silently.
177
+ for i, v in enumerate(values):
178
+ if v is None:
179
+ continue
180
+ try:
181
+ pa.array([v], type=arrow_type)
182
+ except pa.lib.ArrowInvalid as exc:
183
+ raise MalformedRecordError(
184
+ f"column {col!r}: {v} exceeds decimal128({precision},{scale}) scale",
185
+ line_number=base_line + i,
186
+ ) from exc
187
+ raise
188
+ elif data_type == "Date/Time":
189
+ values = [_parse_timestamp(r.get(col, ""), col, base_line + i) for i, r in enumerate(rows)]
190
+ arrays.append(pa.array(values, type=pa.timestamp("us", tz="UTC")))
191
+ else:
192
+ values = [(r.get(col, "") or None) for r in rows]
193
+ arrays.append(pa.array(values, type=pa.string()))
194
+ return arrays
195
+
196
+
197
+ def _stringify(dataset: str, column: str, value) -> str:
198
+ """Render a typed Arrow value back to the toolkit's canonical string form."""
199
+ if value is None:
200
+ return ""
201
+ if isinstance(value, str):
202
+ # Already textual (e.g. a client export storing every column as strings): pass it
203
+ # through untouched — downstream validation judges the format, not the reader.
204
+ return value
205
+ data_type = column_spec(dataset, column)["data_type"]
206
+ if data_type == "Date/Time":
207
+ dt = value if value.tzinfo else value.replace(tzinfo=UTC)
208
+ dt = dt.astimezone(UTC)
209
+ text = dt.isoformat()
210
+ return text[:-6] + "Z" if text.endswith("+00:00") else text
211
+ return str(value)
212
+
213
+
214
+ def dataset_metadata(
215
+ dataset: str,
216
+ *,
217
+ target_version: str,
218
+ source_version: str,
219
+ mode: str,
220
+ conformance: str,
221
+ tool_version: str,
222
+ ) -> dict:
223
+ """Business (deterministic) file metadata; operational keys live in a separate namespace.
224
+
225
+ ``target_version`` is the FOCUS version the file conforms to (1.4); ``source_version`` is the
226
+ version it was converted from. Keeping them distinct stops a Parquet-metadata reader from
227
+ mistaking the source version for the file's own FOCUS version.
228
+ """
229
+ return {
230
+ "focus.dataset": resolve_dataset(dataset),
231
+ "focus.target_version": target_version,
232
+ "focus.source_version": source_version,
233
+ "focus.mode": mode,
234
+ "focus.conformance": conformance,
235
+ "focus.toolkit_version": tool_version,
236
+ }
237
+
238
+
239
+ class ParquetRowWriter:
240
+ """Write rows to a Parquet file in ``schema.columns`` order, flushing bounded row groups."""
241
+
242
+ def __init__(
243
+ self,
244
+ path: str | Path,
245
+ schema: DatasetSchema,
246
+ *,
247
+ metadata: Mapping[str, str] | None = None,
248
+ compression: str = "snappy",
249
+ ) -> None:
250
+ self._pa, self._pq = _require_pyarrow()
251
+ self._dataset = resolve_dataset(schema.dataset)
252
+ self._columns = tuple(schema.columns)
253
+ arrow = arrow_schema(self._dataset, self._columns)
254
+ if metadata:
255
+ arrow = arrow.with_metadata({k: str(v) for k, v in metadata.items()})
256
+ self._arrow_schema = arrow
257
+ self._writer = self._pq.ParquetWriter(str(path), arrow, compression=compression)
258
+ self._buffer: list[Mapping[str, str]] = []
259
+ self._written = 0
260
+
261
+ @property
262
+ def buffered(self) -> int:
263
+ """Rows currently buffered (not yet flushed to a row group)."""
264
+ return len(self._buffer)
265
+
266
+ def write(self, values: Mapping[str, str]) -> None:
267
+ self._buffer.append(values)
268
+ if len(self._buffer) >= _BATCH:
269
+ self._flush()
270
+
271
+ def flush(self) -> None:
272
+ """Flush buffered rows to a row group (public; used to bound memory across writers)."""
273
+ self._flush()
274
+
275
+ def _flush(self) -> None:
276
+ if not self._buffer:
277
+ return
278
+ arrays = _column_arrays(
279
+ self._pa, self._dataset, self._columns, self._buffer, base_line=self._written + 1
280
+ )
281
+ batch = self._pa.record_batch(arrays, schema=self._arrow_schema)
282
+ self._writer.write_table(self._pa.Table.from_batches([batch], self._arrow_schema))
283
+ self._written += len(self._buffer)
284
+ self._buffer = []
285
+
286
+ def close(self) -> None:
287
+ self._flush()
288
+ self._writer.close()
289
+
290
+ def __enter__(self) -> ParquetRowWriter:
291
+ return self
292
+
293
+ def __exit__(self, *exc: object) -> None:
294
+ self.close()
295
+
296
+
297
+ class ParquetRowReader:
298
+ """Stream ``Record``s from a Parquet file, rendering typed values back to strings."""
299
+
300
+ def __init__(self, path: str | Path, *, dataset: str | None = None) -> None:
301
+ self._pa, self._pq = _require_pyarrow()
302
+ self._path = Path(path)
303
+ self._file = self._pq.ParquetFile(str(path))
304
+ self.source_columns: tuple[str, ...] = tuple(self._file.schema_arrow.names)
305
+ self._dataset = self._resolve_dataset(dataset)
306
+
307
+ @property
308
+ def expected_rows(self) -> int | None:
309
+ """Total rows, read cheaply from the Parquet footer (no data load) — for progress."""
310
+ try:
311
+ return int(self._file.metadata.num_rows)
312
+ except (OSError, ValueError, AttributeError):
313
+ return None
314
+
315
+ def _resolve_dataset(self, dataset: str | None) -> str:
316
+ if dataset is not None:
317
+ return resolve_dataset(dataset)
318
+ # Infer the dataset from the column set so values can be rendered by their model types.
319
+ cols = set(self.source_columns)
320
+ best, best_overlap = "Cost and Usage", -1
321
+ for name in load_model()["datasets"]:
322
+ overlap = len(cols & set(load_model()["datasets"][name]["columns"]))
323
+ if overlap > best_overlap:
324
+ best, best_overlap = name, overlap
325
+ return best
326
+
327
+ def __iter__(self) -> Iterator[Record]:
328
+ model_cols = set(load_model()["datasets"][self._dataset]["columns"])
329
+ line = 0
330
+ for batch in self._file.iter_batches(batch_size=_BATCH):
331
+ columns = batch.schema.names
332
+ pydata = {name: batch.column(i).to_pylist() for i, name in enumerate(columns)}
333
+ for r in range(batch.num_rows):
334
+ line += 1
335
+ values = {}
336
+ for name in columns:
337
+ raw = pydata[name][r]
338
+ values[name] = (
339
+ _stringify(self._dataset, name, raw) if name in model_cols
340
+ else ("" if raw is None else str(raw))
341
+ )
342
+ yield Record(values, line)
343
+
344
+ def close(self) -> None:
345
+ pass # ParquetFile has no long-lived handle to close in this usage
346
+
347
+ def __enter__(self) -> ParquetRowReader:
348
+ return self
349
+
350
+ def __exit__(self, *exc: object) -> None:
351
+ self.close()
352
+
353
+
354
+ class PartitionTooWideError(MalformedRecordError):
355
+ """Raised when a ``--partition-by`` key produces more partitions than :data:`MAX_PARTITIONS`."""
356
+
357
+
358
+ class PartitionedParquetWriter:
359
+ """Write a dataset as a Hive-partitioned Parquet tree under ``base_dir``.
360
+
361
+ Rows are routed to ``base_dir/COL=value/.../part-N.parquet`` by their partition-column values;
362
+ the partition columns are **omitted** from the part files (standard Hive — a reader
363
+ reconstructs them from the path). Open file handles scale with the *number of partitions*
364
+ (hence the hard :data:`MAX_PARTITIONS` cap), but **buffered rows are bounded globally**: once
365
+ the total unflushed rows across all partitions reaches :data:`_BATCH`, every partition writer
366
+ is flushed — so memory stays bounded no matter how the rows interleave across partition keys.
367
+ When ``target_file_size`` is set, a partition rolls to a new part file once its (approximate,
368
+ uncompressed) running size crosses the threshold.
369
+ """
370
+
371
+ def __init__(
372
+ self,
373
+ base_dir: str | Path,
374
+ schema: DatasetSchema,
375
+ partition_by: Sequence[str],
376
+ *,
377
+ metadata: Mapping[str, str] | None = None,
378
+ compression: str = "snappy",
379
+ target_file_size: int | None = None,
380
+ ) -> None:
381
+ if target_file_size is not None and target_file_size <= 0:
382
+ raise ValueError(f"target_file_size must be positive, got {target_file_size}")
383
+ self._base = Path(base_dir)
384
+ self._dataset = resolve_dataset(schema.dataset)
385
+ self._partition_by = tuple(partition_by)
386
+ # Part files carry every column except the partition columns (Hive stores those in path).
387
+ self._file_columns = tuple(c for c in schema.columns if c not in set(self._partition_by))
388
+ self._metadata = metadata
389
+ self._compression = compression
390
+ self._target = target_file_size
391
+ # Per partition: (writer, part_index, running_byte_estimate).
392
+ self._writers: dict[tuple[str, ...], list] = {}
393
+ self._buffered = 0 # total rows buffered across all partition writers
394
+
395
+ def _partition_dir(self, values: Mapping[str, str]) -> tuple[tuple[str, ...], Path]:
396
+ key = tuple((values.get(c) or "") for c in self._partition_by)
397
+ rel = Path(*[_hive_segment(c, v) for c, v in zip(self._partition_by, key, strict=True)])
398
+ return key, self._base / rel
399
+
400
+ def _open_part(self, directory: Path, index: int):
401
+ directory.mkdir(parents=True, exist_ok=True)
402
+ return ParquetRowWriter(
403
+ directory / f"part-{index}.parquet",
404
+ DatasetSchema(self._dataset, self._file_columns),
405
+ metadata=self._metadata,
406
+ compression=self._compression,
407
+ )
408
+
409
+ def write(self, values: Mapping[str, str]) -> None:
410
+ key, directory = self._partition_dir(values)
411
+ state = self._writers.get(key)
412
+ if state is None:
413
+ if len(self._writers) >= MAX_PARTITIONS:
414
+ raise PartitionTooWideError(
415
+ f"--partition-by produced more than {MAX_PARTITIONS} partitions; choose a "
416
+ "lower-cardinality key"
417
+ )
418
+ state = [self._open_part(directory, 0), 0, 0]
419
+ self._writers[key] = state
420
+ if self._target is not None and state[2] >= self._target:
421
+ state[0].close()
422
+ state[1] += 1
423
+ state[2] = 0
424
+ state[0] = self._open_part(directory, state[1])
425
+ state[0].write(values)
426
+ self._buffered += 1
427
+ if self._target is not None:
428
+ state[2] += sum(len(values.get(c) or "") for c in self._file_columns) + len(
429
+ self._file_columns
430
+ )
431
+ # Bound total buffered rows across ALL partitions (not per-partition): flush everything
432
+ # once the global buffer fills, so memory can't grow with the number of partition keys.
433
+ if self._buffered >= _BATCH:
434
+ for st in self._writers.values():
435
+ st[0].flush()
436
+ self._buffered = 0
437
+
438
+ def partition_count(self) -> int:
439
+ return len(self._writers)
440
+
441
+ def close(self) -> None:
442
+ for state in self._writers.values():
443
+ state[0].close()
444
+
445
+
446
+ class PartitionedParquetReader:
447
+ """Read a Hive-partitioned Parquet dataset back as ``Record``s (partition columns rebuilt).
448
+
449
+ Used by the lint gate: the partition columns (omitted from the part files) are reconstructed
450
+ from the directory names via an explicit **string** Hive schema, so a Date/Time or numeric-
451
+ looking partition value is never re-typed and reads back exactly as written.
452
+ """
453
+
454
+ def __init__(self, base_dir: str | Path, dataset: str, partition_by: Sequence[str]) -> None:
455
+ self._pa, _ = _require_pyarrow()
456
+ import pyarrow.dataset as pds # noqa: PLC0415
457
+
458
+ self._dataset = resolve_dataset(dataset)
459
+ self._partition_cols = set(partition_by)
460
+ schema = self._pa.schema([(c, self._pa.string()) for c in partition_by])
461
+ partitioning = pds.HivePartitioning(
462
+ schema, null_fallback=_NO_HIVE_NULL, segment_encoding="uri"
463
+ )
464
+ self._ds = pds.dataset(str(base_dir), format="parquet", partitioning=partitioning)
465
+ self.source_columns: tuple[str, ...] = tuple(self._ds.schema.names)
466
+
467
+ @property
468
+ def expected_rows(self) -> int | None:
469
+ """Total rows across the partition tree, from footer metadata — for progress."""
470
+ try:
471
+ return int(self._ds.count_rows())
472
+ except (OSError, ValueError, AttributeError):
473
+ return None
474
+
475
+ def __iter__(self) -> Iterator[Record]:
476
+ model_cols = set(load_model()["datasets"][self._dataset]["columns"])
477
+ line = 0
478
+ for batch in self._ds.to_batches(batch_size=_BATCH):
479
+ columns = batch.schema.names
480
+ pydata = {name: batch.column(i).to_pylist() for i, name in enumerate(columns)}
481
+ for r in range(batch.num_rows):
482
+ line += 1
483
+ values = {}
484
+ for name in columns:
485
+ raw = pydata[name][r]
486
+ if name in self._partition_cols:
487
+ # Reconstructed from the path as a string already — pass it through.
488
+ values[name] = "" if raw is None else str(raw)
489
+ elif name in model_cols:
490
+ values[name] = _stringify(self._dataset, name, raw)
491
+ else:
492
+ values[name] = "" if raw is None else str(raw)
493
+ yield Record(values, line)
494
+
495
+ def close(self) -> None:
496
+ pass
497
+
498
+
499
+ def open_parquet_writer(
500
+ path: str | Path,
501
+ schema: DatasetSchema,
502
+ *,
503
+ metadata: Mapping[str, str] | None = None,
504
+ compression: str = "snappy",
505
+ ):
506
+ """Open ``path`` for Parquet writing and return ``(writer, writer)`` (handle == writer).
507
+
508
+ The tuple shape mirrors :func:`focus_data_toolkit.io.csv_io.open_csv_writer` so callers can
509
+ treat both formats uniformly; the Parquet writer owns its own file handle.
510
+ """
511
+ writer = ParquetRowWriter(path, schema, metadata=metadata, compression=compression)
512
+ return writer, writer
513
+
514
+
515
+ __all__ = [
516
+ "COMPRESSIONS",
517
+ "MAX_PARTITIONS",
518
+ "PartitionedParquetReader",
519
+ "PartitionedParquetWriter",
520
+ "PartitionTooWideError",
521
+ "ParquetRowReader",
522
+ "ParquetRowWriter",
523
+ "arrow_schema",
524
+ "dataset_metadata",
525
+ "decimal_precision_scale",
526
+ "open_parquet_writer",
527
+ "partitionable_columns",
528
+ ]
@@ -0,0 +1,92 @@
1
+ """Streaming record abstraction shared by the CSV and Parquet I/O layers.
2
+
3
+ A :class:`Record` is one source row plus its physical line/record number (for actionable
4
+ errors). :class:`RowReader` / :class:`RowWriter` are the minimal streaming interfaces the
5
+ conversion engine consumes and produces; :class:`DatasetSchema` supplies the output column
6
+ order from the model registry (never from ``rows[0]``, which a stream cannot peek).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from collections.abc import Iterable, Iterator, Mapping
12
+ from dataclasses import dataclass
13
+ from typing import Protocol, runtime_checkable
14
+
15
+
16
+ class MalformedRecordError(ValueError):
17
+ """A source record could not be parsed (e.g. wrong field count); carries the line number."""
18
+
19
+ def __init__(self, message: str, *, line_number: int | None = None) -> None:
20
+ super().__init__(message)
21
+ self.line_number = line_number
22
+
23
+
24
+ @dataclass(frozen=True)
25
+ class Record:
26
+ """One source row and its 1-based physical line/record number."""
27
+
28
+ values: Mapping[str, str]
29
+ line_number: int
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class DatasetSchema:
34
+ """Output schema of a FOCUS dataset: the column names, in normative model order."""
35
+
36
+ dataset: str
37
+ columns: tuple[str, ...]
38
+
39
+
40
+ @runtime_checkable
41
+ class RowReader(Protocol):
42
+ """A stream of :class:`Record`s whose header (``source_columns``) is known on open."""
43
+
44
+ source_columns: tuple[str, ...]
45
+
46
+ def __iter__(self) -> Iterator[Record]: ...
47
+
48
+ def close(self) -> None: ...
49
+
50
+
51
+ @runtime_checkable
52
+ class RowWriter(Protocol):
53
+ """A sink for output rows (values keyed by column name)."""
54
+
55
+ def write(self, values: Mapping[str, str]) -> None: ...
56
+
57
+ def close(self) -> None: ...
58
+
59
+
60
+ class ListRowReader:
61
+ """Adapt an in-memory ``list[dict]`` to the :class:`RowReader` interface.
62
+
63
+ Lets the small-volume ``convert_to_focus_1_4(rows)`` API run the exact same streaming
64
+ engine as ``convert_files`` (equivalence by construction, not by luck).
65
+ """
66
+
67
+ def __init__(self, rows: Iterable[Mapping[str, str]], *, source_columns: tuple[str, ...] | None = None) -> None:
68
+ self._rows = list(rows)
69
+ if source_columns is not None:
70
+ self.source_columns = source_columns
71
+ else:
72
+ self.source_columns = tuple(self._rows[0].keys()) if self._rows else ()
73
+
74
+ def __iter__(self) -> Iterator[Record]:
75
+ for i, row in enumerate(self._rows, start=1):
76
+ yield Record(row, i)
77
+
78
+ def close(self) -> None:
79
+ pass
80
+
81
+
82
+ class ListRowWriter:
83
+ """Collect written rows into an in-memory list (the streaming counterpart of a sink)."""
84
+
85
+ def __init__(self) -> None:
86
+ self.rows: list[dict[str, str]] = []
87
+
88
+ def write(self, values: Mapping[str, str]) -> None:
89
+ self.rows.append(dict(values))
90
+
91
+ def close(self) -> None:
92
+ pass