focus-data-toolkit 0.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. focus_data_toolkit/__init__.py +69 -0
  2. focus_data_toolkit/__main__.py +6 -0
  3. focus_data_toolkit/_version.py +8 -0
  4. focus_data_toolkit/cli.py +968 -0
  5. focus_data_toolkit/context/__init__.py +88 -0
  6. focus_data_toolkit/context/billing.py +54 -0
  7. focus_data_toolkit/context/provider.py +90 -0
  8. focus_data_toolkit/convert/__init__.py +708 -0
  9. focus_data_toolkit/convert/billing_period.py +65 -0
  10. focus_data_toolkit/convert/contract_applied.py +235 -0
  11. focus_data_toolkit/convert/contract_commitment.py +182 -0
  12. focus_data_toolkit/convert/cost_and_usage.py +179 -0
  13. focus_data_toolkit/convert/detect.py +39 -0
  14. focus_data_toolkit/convert/invoice_detail.py +199 -0
  15. focus_data_toolkit/convert/streaming.py +1030 -0
  16. focus_data_toolkit/errors.py +145 -0
  17. focus_data_toolkit/focus_json.py +68 -0
  18. focus_data_toolkit/generators/__init__.py +61 -0
  19. focus_data_toolkit/generators/_shim.py +43 -0
  20. focus_data_toolkit/generators/engine/__init__.py +14 -0
  21. focus_data_toolkit/generators/engine/context.py +12 -0
  22. focus_data_toolkit/generators/engine/determinism.py +117 -0
  23. focus_data_toolkit/generators/engine/json_focus.py +63 -0
  24. focus_data_toolkit/generators/engine/ladder.py +71 -0
  25. focus_data_toolkit/generators/engine/scenarios_core.py +380 -0
  26. focus_data_toolkit/generators/engine/serialize.py +151 -0
  27. focus_data_toolkit/generators/generate_aws_focus_1_2.py +19 -0
  28. focus_data_toolkit/generators/generate_aws_focus_1_3.py +20 -0
  29. focus_data_toolkit/generators/generate_azure_focus_1_2.py +17 -0
  30. focus_data_toolkit/generators/generate_azure_focus_1_3.py +17 -0
  31. focus_data_toolkit/generators/generate_gcp_focus_1_2.py +17 -0
  32. focus_data_toolkit/generators/generate_gcp_focus_1_3.py +17 -0
  33. focus_data_toolkit/generators/providers/__init__.py +29 -0
  34. focus_data_toolkit/generators/providers/aws.py +186 -0
  35. focus_data_toolkit/generators/providers/azure.py +191 -0
  36. focus_data_toolkit/generators/providers/gcp.py +194 -0
  37. focus_data_toolkit/generators/providers/profile.py +123 -0
  38. focus_data_toolkit/generators/scenarios.py +178 -0
  39. focus_data_toolkit/generators/versions/__init__.py +17 -0
  40. focus_data_toolkit/generators/versions/adapter.py +41 -0
  41. focus_data_toolkit/generators/versions/v1_2.py +111 -0
  42. focus_data_toolkit/generators/versions/v1_3.py +154 -0
  43. focus_data_toolkit/io/__init__.py +1 -0
  44. focus_data_toolkit/io/atomic_writer.py +462 -0
  45. focus_data_toolkit/io/csv_io.py +128 -0
  46. focus_data_toolkit/io/parquet_io.py +528 -0
  47. focus_data_toolkit/io/records.py +92 -0
  48. focus_data_toolkit/io/row_source.py +117 -0
  49. focus_data_toolkit/lifecycle.py +342 -0
  50. focus_data_toolkit/manifest.py +114 -0
  51. focus_data_toolkit/model/__init__.py +43 -0
  52. focus_data_toolkit/model/capabilities.py +66 -0
  53. focus_data_toolkit/model/focus_1_4_decimal_scale.json +10 -0
  54. focus_data_toolkit/model/focus_1_4_model.json +1913 -0
  55. focus_data_toolkit/model/focus_1_4_servicesubcategory.json +84 -0
  56. focus_data_toolkit/model/focus_json_keys.py +112 -0
  57. focus_data_toolkit/model/iso_4217_currencies.json +23 -0
  58. focus_data_toolkit/model/json_schema_check.py +205 -0
  59. focus_data_toolkit/model/json_schemas/allocatedmethoddetailsobjectschema.json +82 -0
  60. focus_data_toolkit/model/json_schemas/commitmentprogrameligibilitydetailsobjectschema.json +41 -0
  61. focus_data_toolkit/model/json_schemas/contractappliedobjectschema.json +104 -0
  62. focus_data_toolkit/model/json_schemas/contractcommitmentapplicabilityobjectschema.json +290 -0
  63. focus_data_toolkit/model/json_schemas/json_schemas_provenance.json +38 -0
  64. focus_data_toolkit/model/model_provenance.json +58 -0
  65. focus_data_toolkit/model/validator.py +498 -0
  66. focus_data_toolkit/modes.py +18 -0
  67. focus_data_toolkit/official_validator.py +61 -0
  68. focus_data_toolkit/progress.py +89 -0
  69. focus_data_toolkit/provenance.py +106 -0
  70. focus_data_toolkit/py.typed +1 -0
  71. focus_data_toolkit/runtime.py +243 -0
  72. focus_data_toolkit/schema/__init__.py +17 -0
  73. focus_data_toolkit/schema/detection.py +274 -0
  74. focus_data_toolkit/schema/registry.py +127 -0
  75. focus_data_toolkit/storage/__init__.py +1 -0
  76. focus_data_toolkit/storage/external_index.py +99 -0
  77. focus_data_toolkit/storage/spill.py +150 -0
  78. focus_data_toolkit/studio/__init__.py +19 -0
  79. focus_data_toolkit/studio/app.py +467 -0
  80. focus_data_toolkit/studio/config.py +42 -0
  81. focus_data_toolkit/studio/frontend/app.js +214 -0
  82. focus_data_toolkit/studio/frontend/index.html +101 -0
  83. focus_data_toolkit/studio/frontend/style.css +60 -0
  84. focus_data_toolkit/studio/jobs.py +142 -0
  85. focus_data_toolkit/studio/preview.py +32 -0
  86. focus_data_toolkit/studio/security.py +125 -0
  87. focus_data_toolkit/studio/server.py +71 -0
  88. focus_data_toolkit/supplement/__init__.py +50 -0
  89. focus_data_toolkit/supplement/adapters/__init__.py +21 -0
  90. focus_data_toolkit/supplement/adapters/adapters_provenance.json +39 -0
  91. focus_data_toolkit/supplement/adapters/aws_invoice_summary.json +24 -0
  92. focus_data_toolkit/supplement/adapters/aws_savings_plans.json +31 -0
  93. focus_data_toolkit/supplement/adapters/azure_invoice.json +25 -0
  94. focus_data_toolkit/supplement/adapters/gcp_compute_commitments.json +28 -0
  95. focus_data_toolkit/supplement/adapters/registry.py +215 -0
  96. focus_data_toolkit/supplement/apply.py +318 -0
  97. focus_data_toolkit/supplement/gaps.py +219 -0
  98. focus_data_toolkit/supplement/kinds.py +118 -0
  99. focus_data_toolkit/supplement/loader.py +409 -0
  100. focus_data_toolkit/supplement/spec.py +74 -0
  101. focus_data_toolkit/supplement/validate.py +215 -0
  102. focus_data_toolkit/validate/__init__.py +15 -0
  103. focus_data_toolkit/validate/allocation.py +333 -0
  104. focus_data_toolkit/validate/bundle.py +254 -0
  105. focus_data_toolkit/validate/codes.py +93 -0
  106. focus_data_toolkit/validate/corrections.py +245 -0
  107. focus_data_toolkit/validate/reconciliation.py +98 -0
  108. focus_data_toolkit/validate/referential.py +289 -0
  109. focus_data_toolkit-0.11.0.dist-info/METADATA +519 -0
  110. focus_data_toolkit-0.11.0.dist-info/RECORD +116 -0
  111. focus_data_toolkit-0.11.0.dist-info/WHEEL +5 -0
  112. focus_data_toolkit-0.11.0.dist-info/entry_points.txt +2 -0
  113. focus_data_toolkit-0.11.0.dist-info/licenses/LICENSE +21 -0
  114. focus_data_toolkit-0.11.0.dist-info/licenses/LICENSES/CC-BY-4.0.txt +156 -0
  115. focus_data_toolkit-0.11.0.dist-info/licenses/NOTICE +60 -0
  116. focus_data_toolkit-0.11.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,117 @@
1
+ """Format-agnostic row input: open a CSV or Parquet source behind one protocol.
2
+
3
+ Client exports arrive as CSV (possibly gzipped) or Parquet. :func:`open_row_source` detects
4
+ the physical format — by the ``PAR1`` magic bytes first, extension as a fallback for
5
+ unreadable paths — and returns the matching streaming reader. Both readers already share
6
+ the same shape (``source_columns``, iteration yielding ``Record``, ``close``); the
7
+ :class:`RowSource` protocol names that contract so conversion code can consume either
8
+ without caring which it got.
9
+
10
+ Parquet support requires the ``[parquet]`` extra; opening a Parquet file without it raises
11
+ the same actionable :class:`~focus_data_toolkit.io.records.MalformedRecordError` install
12
+ hint as Parquet output does.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import os
18
+ from collections.abc import Iterator
19
+ from contextlib import contextmanager
20
+ from pathlib import Path
21
+ from typing import Protocol, runtime_checkable
22
+
23
+ from focus_data_toolkit.io.csv_io import CsvRowReader
24
+ from focus_data_toolkit.io.records import MalformedRecordError, Record
25
+
26
+ _PARQUET_MAGIC = b"PAR1"
27
+
28
+
29
+ @runtime_checkable
30
+ class RowSource(Protocol):
31
+ """A streaming, closeable source of rows with a known header."""
32
+
33
+ source_columns: tuple[str, ...]
34
+
35
+ def __iter__(self) -> Iterator[Record]: ...
36
+
37
+ def close(self) -> None: ...
38
+
39
+
40
+ def is_parquet(path: str | os.PathLike[str]) -> bool:
41
+ """Whether ``path`` is a Parquet file (magic bytes; ``.parquet`` suffix as fallback)."""
42
+ p = Path(path)
43
+ try:
44
+ with open(p, "rb") as fh:
45
+ return fh.read(4) == _PARQUET_MAGIC
46
+ except OSError:
47
+ return p.suffix.lower() == ".parquet"
48
+
49
+
50
+ def _hive_partition_columns(base_dir: Path) -> tuple[str, ...]:
51
+ """Partition column names of a Hive-layout directory, walking ``COL=value`` segments."""
52
+ cols: list[str] = []
53
+ current = base_dir
54
+ while True:
55
+ subdirs = [e for e in current.iterdir() if e.is_dir() and "=" in e.name]
56
+ if not subdirs:
57
+ return tuple(cols)
58
+ cols.append(subdirs[0].name.split("=", 1)[0])
59
+ current = subdirs[0]
60
+
61
+
62
+ def open_row_source(path: str | os.PathLike[str], *, dataset: str | None = None) -> RowSource:
63
+ """Open ``path`` as a :class:`RowSource`, auto-detecting the physical format.
64
+
65
+ A file is sniffed as CSV (``.gz`` ok) or Parquet; a directory is read as a
66
+ Hive-partitioned Parquet dataset (partition columns inferred from the ``COL=value``
67
+ path segments). ``dataset`` optionally names the FOCUS dataset the file holds, letting
68
+ the Parquet readers render typed values by the right model columns instead of inferring
69
+ the dataset from the header overlap. CSV input ignores it (values are already strings).
70
+ """
71
+ p = Path(path)
72
+ if p.is_dir():
73
+ from focus_data_toolkit.io.parquet_io import PartitionedParquetReader
74
+
75
+ with _parquet_errors(p):
76
+ return PartitionedParquetReader(
77
+ p, dataset or "Cost and Usage", _hive_partition_columns(p)
78
+ )
79
+ if is_parquet(p):
80
+ from focus_data_toolkit.io.parquet_io import ParquetRowReader
81
+
82
+ with _parquet_errors(p):
83
+ return ParquetRowReader(p, dataset=dataset)
84
+ return CsvRowReader(p)
85
+
86
+
87
+ @contextmanager
88
+ def _parquet_errors(path: Path) -> Iterator[None]:
89
+ """Translate Arrow open/read failures into the CLI-friendly MalformedRecordError.
90
+
91
+ A missing-pyarrow MalformedRecordError (with its install hint) passes through untouched.
92
+ """
93
+ try:
94
+ yield
95
+ except MalformedRecordError:
96
+ raise
97
+ except (OSError, ValueError) as exc:
98
+ # pyarrow's ArrowInvalid/ArrowIOError derive from ValueError/OSError, so a corrupt
99
+ # or truncated Parquet file surfaces as a clean input error, never a traceback.
100
+ raise MalformedRecordError(f"cannot read Parquet source {path}: {exc}") from exc
101
+
102
+
103
+ def read_source_rows(
104
+ path: str | os.PathLike[str], *, dataset: str | None = None
105
+ ) -> list[dict[str, str]]:
106
+ """Materialise a CSV/Parquet source into a list of dict rows (eager-path input)."""
107
+ reader = open_row_source(path, dataset=dataset)
108
+ try:
109
+ if isinstance(reader, CsvRowReader):
110
+ return [dict(record.values) for record in reader]
111
+ with _parquet_errors(Path(path)): # iteration can hit mid-file corruption too
112
+ return [dict(record.values) for record in reader]
113
+ finally:
114
+ reader.close()
115
+
116
+
117
+ __all__ = ["RowSource", "is_parquet", "open_row_source", "read_source_rows"]
@@ -0,0 +1,342 @@
1
+ """Dataset-instance snapshots and billing/invoice lifecycle-chain rules (P1.10).
2
+
3
+ A **dataset instance** is one delivered snapshot of a FOCUS dataset for a subject (an invoice,
4
+ a billing period): who it is, when it was created / last updated, whether it is complete, the
5
+ period it covers, its lifecycle status, and which instance preceded it. Real client pipelines
6
+ *record* these (they are never fabricated for client data); the toolkit models them so a
7
+ sequence of snapshots can be checked for illegal status transitions — e.g. a ``Voided`` invoice
8
+ must not silently go back to ``Issued``, an ``Issued`` invoice must not revert to ``Open``.
9
+
10
+ The allowed transitions come straight from the FOCUS ``InvoiceIssueStatus`` (``Open`` →
11
+ ``Issued`` → ``Voided``) and ``BillingPeriodStatus`` (``Open`` → ``Closed``) value sets; a
12
+ correction is a *new* instance/line, never an in-place edit of a closed/issued one.
13
+
14
+ Beyond transitions, :func:`check_instance_chains` validates the chain structure itself —
15
+ ``instance_id`` / per-subject ``order`` uniqueness (FDT-LIFE-001/002),
16
+ ``previous_instance_id`` resolution (FDT-LIFE-003), cycle detection (FDT-LIFE-004),
17
+ ``last_updated`` monotonicity (FDT-LIFE-005), and closed-instance immutability
18
+ (FDT-LIFE-006) — all as structured diagnostics. :func:`check_dataset_instances` runs the
19
+ whole battery (chains + status transitions) in one call.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from collections.abc import Iterable, Sequence
25
+ from dataclasses import dataclass
26
+ from datetime import datetime
27
+
28
+ from focus_data_toolkit.errors import Diagnostic, Severity
29
+
30
+ # Allowed forward transitions (a status may also stay the same — an idempotent re-delivery).
31
+ INVOICE_TRANSITIONS: dict[str, frozenset[str]] = {
32
+ "Open": frozenset({"Open", "Issued", "Voided"}),
33
+ "Issued": frozenset({"Issued", "Voided"}),
34
+ "Voided": frozenset({"Voided"}),
35
+ }
36
+ BILLING_PERIOD_TRANSITIONS: dict[str, frozenset[str]] = {
37
+ "Open": frozenset({"Open", "Closed"}),
38
+ "Closed": frozenset({"Closed"}),
39
+ }
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class DatasetInstance:
44
+ """One recorded snapshot of a dataset for a subject (invoice / billing period)."""
45
+
46
+ instance_id: str
47
+ dataset: str
48
+ subject_id: str
49
+ order: int
50
+ created: str
51
+ last_updated: str
52
+ complete: bool
53
+ period_start: str
54
+ period_end: str
55
+ status: str
56
+ previous_instance_id: str | None = None
57
+
58
+
59
+ def _check_transitions(
60
+ instances: Sequence[DatasetInstance], allowed: dict[str, frozenset[str]]
61
+ ) -> list[Diagnostic]:
62
+ """Flag any status change not permitted by ``allowed`` between consecutive snapshots.
63
+
64
+ Snapshots are grouped by ``subject_id`` and ordered by ``order``; each adjacent pair is a
65
+ transition. An unknown status value is left to the per-dataset linter (allowed-value check).
66
+ """
67
+ by_subject: dict[str, list[DatasetInstance]] = {}
68
+ for inst in instances:
69
+ by_subject.setdefault(inst.subject_id, []).append(inst)
70
+
71
+ out: list[Diagnostic] = []
72
+ for subject, snaps in sorted(by_subject.items()):
73
+ ordered = sorted(snaps, key=lambda s: s.order)
74
+ for prev, cur in zip(ordered, ordered[1:], strict=False):
75
+ permitted = allowed.get(prev.status)
76
+ if permitted is None or cur.status not in permitted:
77
+ out.append(
78
+ Diagnostic(
79
+ code="FDT-CORR-004",
80
+ severity=Severity.ERROR,
81
+ message=f"illegal status transition {prev.status!r} -> {cur.status!r} "
82
+ f"for {subject!r} (instances {prev.instance_id} -> {cur.instance_id})",
83
+ datasets=(cur.dataset,),
84
+ dataset=cur.dataset,
85
+ record_keys={"subject_id": subject},
86
+ expected=f"one of {sorted(permitted)}" if permitted else "known status",
87
+ actual=cur.status,
88
+ context={"from_instance": prev.instance_id, "to_instance": cur.instance_id},
89
+ )
90
+ )
91
+ return out
92
+
93
+
94
+ def check_invoice_status_transitions(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
95
+ """Validate Invoice Detail status transitions across dataset instances (FDT-CORR-004)."""
96
+ invoices = [i for i in instances if i.dataset == "Invoice Detail"]
97
+ return _check_transitions(invoices, INVOICE_TRANSITIONS)
98
+
99
+
100
+ def check_billing_period_status_transitions(
101
+ instances: Iterable[DatasetInstance],
102
+ ) -> list[Diagnostic]:
103
+ """Validate Billing Period status transitions across dataset instances (FDT-CORR-004)."""
104
+ periods = [i for i in instances if i.dataset == "Billing Period"]
105
+ return _check_transitions(periods, BILLING_PERIOD_TRANSITIONS)
106
+
107
+
108
+ def check_status_transitions(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
109
+ """Validate both invoice and billing-period status transitions in one pass."""
110
+ instances = list(instances)
111
+ return check_invoice_status_transitions(instances) + check_billing_period_status_transitions(
112
+ instances
113
+ )
114
+
115
+
116
+ # Statuses after which a subject's recorded facts are frozen: later snapshots may re-deliver
117
+ # or move forward in status, but must not rewrite the period or un-complete the data.
118
+ _CLOSED_STATUSES: frozenset[str] = frozenset({"Closed", "Issued", "Voided"})
119
+
120
+
121
+ def _parse_dt(value: str) -> datetime | None:
122
+ """Parse a FOCUS Date/Time; naive or malformed values return None (the linter's job)."""
123
+ try:
124
+ parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
125
+ except ValueError:
126
+ return None
127
+ return parsed if parsed.tzinfo is not None else None
128
+
129
+
130
+ def _chain_diag(
131
+ code: str, message: str, inst: DatasetInstance, **extra: object
132
+ ) -> Diagnostic:
133
+ return Diagnostic(
134
+ code=code,
135
+ severity=Severity.ERROR,
136
+ message=message,
137
+ datasets=(inst.dataset,),
138
+ dataset=inst.dataset,
139
+ record_keys={"subject_id": inst.subject_id, "instance_id": inst.instance_id},
140
+ **extra, # type: ignore[arg-type]
141
+ )
142
+
143
+
144
+ def check_instance_chains(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
145
+ """Validate the structure of dataset-instance chains (FDT-LIFE-001..006).
146
+
147
+ * **FDT-LIFE-001** — ``instance_id`` is reused across snapshots.
148
+ * **FDT-LIFE-002** — two snapshots of one subject share the same ``order``.
149
+ * **FDT-LIFE-003** — ``previous_instance_id`` does not resolve to an existing earlier
150
+ snapshot (unknown id, self-reference, or non-decreasing order). Cross-subject and
151
+ cross-dataset links are legitimate delivery lineage (e.g. a replacement invoice
152
+ points at the voided one it replaces).
153
+ * **FDT-LIFE-004** — the ``previous_instance_id`` links form a cycle.
154
+ * **FDT-LIFE-005** — ``last_updated`` is before ``created``, or decreases between
155
+ consecutive snapshots of a subject (history must move forward).
156
+ * **FDT-LIFE-006** — a snapshot after a closed/issued/voided one rewrites the recorded
157
+ period, or flips ``complete`` back to ``False`` (closed instances are immutable;
158
+ a correction is a new line, never an in-place edit).
159
+
160
+ Unknown status values and malformed timestamps are left to the per-dataset linter.
161
+ """
162
+ snaps = list(instances)
163
+ out: list[Diagnostic] = []
164
+
165
+ by_id: dict[str, DatasetInstance] = {}
166
+ for inst in snaps:
167
+ if inst.instance_id in by_id:
168
+ out.append(
169
+ _chain_diag(
170
+ "FDT-LIFE-001",
171
+ f"instance_id {inst.instance_id!r} is reused "
172
+ f"(subjects {by_id[inst.instance_id].subject_id!r} and {inst.subject_id!r})",
173
+ inst,
174
+ value=inst.instance_id,
175
+ )
176
+ )
177
+ else:
178
+ by_id[inst.instance_id] = inst
179
+
180
+ by_subject: dict[tuple[str, str], list[DatasetInstance]] = {}
181
+ for inst in snaps:
182
+ by_subject.setdefault((inst.dataset, inst.subject_id), []).append(inst)
183
+
184
+ for (_dataset, subject), group in sorted(by_subject.items()):
185
+ seen_orders: dict[int, DatasetInstance] = {}
186
+ for inst in group:
187
+ if inst.order in seen_orders:
188
+ out.append(
189
+ _chain_diag(
190
+ "FDT-LIFE-002",
191
+ f"snapshots {seen_orders[inst.order].instance_id!r} and "
192
+ f"{inst.instance_id!r} of {subject!r} share order {inst.order}",
193
+ inst,
194
+ actual=str(inst.order),
195
+ )
196
+ )
197
+ else:
198
+ seen_orders[inst.order] = inst
199
+
200
+ ordered = sorted(group, key=lambda s: (s.order, s.instance_id))
201
+
202
+ # previous_instance_id resolution.
203
+ for inst in ordered:
204
+ prev_id = inst.previous_instance_id
205
+ if prev_id is None:
206
+ continue
207
+ prev = by_id.get(prev_id)
208
+ if prev is None or prev_id == inst.instance_id:
209
+ reason = "references itself" if prev_id == inst.instance_id else "is unknown"
210
+ out.append(
211
+ _chain_diag(
212
+ "FDT-LIFE-003",
213
+ f"previous_instance_id {prev_id!r} of {inst.instance_id!r} {reason}",
214
+ inst,
215
+ value=prev_id,
216
+ )
217
+ )
218
+ elif prev.order >= inst.order:
219
+ out.append(
220
+ _chain_diag(
221
+ "FDT-LIFE-003",
222
+ f"previous_instance_id {prev_id!r} of {inst.instance_id!r} does not "
223
+ f"precede it (order {prev.order} >= {inst.order})",
224
+ inst,
225
+ value=prev_id,
226
+ )
227
+ )
228
+
229
+ # last_updated monotonicity along the delivery sequence.
230
+ prev_updated: datetime | None = None
231
+ prev_inst: DatasetInstance | None = None
232
+ for inst in ordered:
233
+ created, updated = _parse_dt(inst.created), _parse_dt(inst.last_updated)
234
+ if created is not None and updated is not None and updated < created:
235
+ out.append(
236
+ _chain_diag(
237
+ "FDT-LIFE-005",
238
+ f"last_updated of {inst.instance_id!r} precedes its created",
239
+ inst,
240
+ expected=f">= {inst.created}",
241
+ actual=inst.last_updated,
242
+ )
243
+ )
244
+ if updated is not None:
245
+ if prev_updated is not None and updated < prev_updated:
246
+ assert prev_inst is not None
247
+ out.append(
248
+ _chain_diag(
249
+ "FDT-LIFE-005",
250
+ f"last_updated moves backwards between {prev_inst.instance_id!r} "
251
+ f"and {inst.instance_id!r} of {subject!r}",
252
+ inst,
253
+ expected=f">= {prev_inst.last_updated}",
254
+ actual=inst.last_updated,
255
+ )
256
+ )
257
+ prev_updated, prev_inst = updated, inst
258
+
259
+ # Closed-instance immutability: once closed/issued/voided, the recorded period is
260
+ # frozen and completeness must not regress.
261
+ frozen: DatasetInstance | None = None
262
+ for inst in ordered:
263
+ if frozen is not None:
264
+ if (inst.period_start, inst.period_end) != (
265
+ frozen.period_start,
266
+ frozen.period_end,
267
+ ):
268
+ out.append(
269
+ _chain_diag(
270
+ "FDT-LIFE-006",
271
+ f"{inst.instance_id!r} rewrites the period of {subject!r} recorded "
272
+ f"by closed snapshot {frozen.instance_id!r}",
273
+ inst,
274
+ expected=f"{frozen.period_start}/{frozen.period_end}",
275
+ actual=f"{inst.period_start}/{inst.period_end}",
276
+ )
277
+ )
278
+ if frozen.complete and not inst.complete:
279
+ out.append(
280
+ _chain_diag(
281
+ "FDT-LIFE-006",
282
+ f"{inst.instance_id!r} flips {subject!r} back to incomplete after "
283
+ f"closed snapshot {frozen.instance_id!r}",
284
+ inst,
285
+ expected="complete=True",
286
+ actual="complete=False",
287
+ )
288
+ )
289
+ if frozen is None and inst.status in _CLOSED_STATUSES:
290
+ frozen = inst
291
+
292
+ out.extend(_check_cycles(snaps, by_id))
293
+ return out
294
+
295
+
296
+ def _check_cycles(
297
+ snaps: Sequence[DatasetInstance], by_id: dict[str, DatasetInstance]
298
+ ) -> list[Diagnostic]:
299
+ """Detect cycles in the ``previous_instance_id`` graph (each reported once)."""
300
+ out: list[Diagnostic] = []
301
+ state: dict[str, int] = {} # 0 = visiting, 1 = done
302
+ for start in sorted(by_id):
303
+ if start in state:
304
+ continue
305
+ path: list[str] = []
306
+ current: str | None = start
307
+ while current is not None and current in by_id and current not in state:
308
+ state[current] = 0
309
+ path.append(current)
310
+ current = by_id[current].previous_instance_id
311
+ if current is not None and state.get(current) == 0:
312
+ cycle = path[path.index(current):]
313
+ inst = by_id[current]
314
+ out.append(
315
+ _chain_diag(
316
+ "FDT-LIFE-004",
317
+ "previous_instance_id links form a cycle: " + " -> ".join([*cycle, current]),
318
+ inst,
319
+ context={"cycle": ",".join(cycle)},
320
+ )
321
+ )
322
+ for node in path:
323
+ state[node] = 1
324
+ return out
325
+
326
+
327
+ def check_dataset_instances(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
328
+ """Run the full lifecycle battery: chain structure plus status transitions."""
329
+ snaps = list(instances)
330
+ return check_instance_chains(snaps) + check_status_transitions(snaps)
331
+
332
+
333
+ __all__ = [
334
+ "BILLING_PERIOD_TRANSITIONS",
335
+ "INVOICE_TRANSITIONS",
336
+ "DatasetInstance",
337
+ "check_billing_period_status_transitions",
338
+ "check_dataset_instances",
339
+ "check_instance_chains",
340
+ "check_invoice_status_transitions",
341
+ "check_status_transitions",
342
+ ]
@@ -0,0 +1,114 @@
1
+ """Deterministic, machine-readable conversion manifest.
2
+
3
+ The manifest records, per FOCUS 1.4 dataset, whether it was produced and — column by
4
+ column — how each value was obtained (see :mod:`focus_data_toolkit.provenance`). It is
5
+ built without a clock or RNG so a given conversion always yields the same manifest bytes.
6
+ Any produced column with ``ASSUMED`` lineage sets ``assumptions_present`` and prevents a
7
+ full-conformance claim.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+
14
+ from focus_data_toolkit.provenance import ColumnRule
15
+
16
+ MANIFEST_FILENAME = "focus_1_4_manifest.json"
17
+ TARGET_VERSION = "1.4"
18
+
19
+ # Dataset statuses.
20
+ PRODUCED = "PRODUCED"
21
+ PRODUCED_SYNTHETIC = "PRODUCED_SYNTHETIC"
22
+ NOT_PRODUCED = "NOT_PRODUCED"
23
+
24
+ # Conformance labels (never "fully conformant": see model/validator.py levels).
25
+ CONF_STRUCTURAL_LINT = "STRUCTURAL_LINT" # produced from factual lineage, passed the lint
26
+ CONF_LINT_FAILED = "LINT_FAILED" # produced from factual lineage, but the lint failed
27
+ CONF_NOT_VALIDATED = "NOT_VALIDATED" # produced but lint not run (validate=False)
28
+ CONF_SYNTHETIC = "SYNTHETIC" # produced with assumed values
29
+ CONF_INCOMPLETE = "INCOMPLETE" # not produced
30
+
31
+
32
+ def dataset_entry(
33
+ *,
34
+ status: str,
35
+ conformance: str,
36
+ provenance: dict[str, ColumnRule],
37
+ row_count: int | None = None,
38
+ reason: str | None = None,
39
+ blocking_columns: list[str] | None = None,
40
+ output_file: str | None = None,
41
+ partitioned_by: list[str] | None = None,
42
+ lineage_summary: dict[str, dict[str, int]] | None = None,
43
+ ) -> dict:
44
+ entry: dict = {
45
+ "status": status,
46
+ "conformance": conformance,
47
+ "columns": {col: provenance[col].as_dict() for col in sorted(provenance)},
48
+ }
49
+ if row_count is not None:
50
+ entry["row_count"] = row_count
51
+ if lineage_summary:
52
+ # Per-value lineage counts for columns whose rule varies by row: the headline
53
+ # column lineage above stays the weakest category, this shows the actual mix.
54
+ entry["lineage_summary"] = lineage_summary
55
+ if reason:
56
+ entry["reason"] = reason
57
+ if blocking_columns:
58
+ entry["blocking_columns"] = blocking_columns
59
+ if output_file:
60
+ entry["output_file"] = output_file
61
+ if partitioned_by:
62
+ # Present only for a Hive-partitioned Parquet dataset: output_file is the dataset
63
+ # directory, partitioned on these columns (a reader reconstructs them from the paths).
64
+ entry["partitioned_by"] = partitioned_by
65
+ return entry
66
+
67
+
68
+ def build_manifest(
69
+ *,
70
+ tool_version: str,
71
+ source_version: str,
72
+ mode: str,
73
+ datasets: dict[str, dict],
74
+ detection: dict | None = None,
75
+ contexts: dict | None = None,
76
+ diagnostics: list[dict] | None = None,
77
+ capability_profile: dict | None = None,
78
+ supplements: list[dict] | None = None,
79
+ ) -> dict:
80
+ assumptions_present = any(
81
+ entry["status"] in (PRODUCED, PRODUCED_SYNTHETIC)
82
+ and any(col.get("lineage") == "ASSUMED" for col in entry.get("columns", {}).values())
83
+ for entry in datasets.values()
84
+ )
85
+ manifest: dict = {
86
+ "tool_version": tool_version,
87
+ "source_version": source_version,
88
+ "target_version": TARGET_VERSION,
89
+ "mode": mode,
90
+ "assumptions_present": assumptions_present,
91
+ "datasets": datasets,
92
+ }
93
+ # Detection decision and per-row context summary are part of the deterministic business
94
+ # record (no clock/RNG). Diagnostics record non-fatal findings (e.g. ambiguous context).
95
+ if detection is not None:
96
+ manifest["detection"] = detection
97
+ if contexts is not None:
98
+ manifest["contexts"] = contexts
99
+ if diagnostics is not None:
100
+ manifest["diagnostics"] = diagnostics
101
+ if capability_profile is not None:
102
+ # The applicability conditions the lint actually enforced. Recorded even when
103
+ # empty ("none-declared"), so an unevaluated condition set is never silent.
104
+ manifest["capability_profile"] = capability_profile
105
+ if supplements:
106
+ # Client-supplied fact files (path, kind, sha256, row counts, declared
107
+ # provenance): the audit trail behind every ENRICHED column.
108
+ manifest["supplements"] = supplements
109
+ return manifest
110
+
111
+
112
+ def render(manifest: dict) -> str:
113
+ """Serialize the manifest to deterministic JSON bytes-as-text (sorted, trailing LF)."""
114
+ return json.dumps(manifest, indent=2, sort_keys=True) + "\n"
@@ -0,0 +1,43 @@
1
+ """FOCUS 1.4 data model (committed JSON artifact) and helpers.
2
+
3
+ The model is derived from the FinOps Foundation "FOCUS 1.4 Data Model"
4
+ workbook (https://focus.finops.org) by ``tools/extract_focus_1_4_model.py``;
5
+ the JSON is the artifact of record for this package.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from focus_data_toolkit.model.validator import (
11
+ lint_focus_1_4_structure,
12
+ load_model,
13
+ resolve_dataset,
14
+ validate_focus_1_4,
15
+ )
16
+
17
+ FOCUS_1_4_DATASETS: tuple[str, ...] = (
18
+ "Cost and Usage",
19
+ "Contract Commitment",
20
+ "Billing Period",
21
+ "Invoice Detail",
22
+ )
23
+
24
+
25
+ def dataset_columns(dataset: str) -> tuple[str, ...]:
26
+ """Return the FOCUS 1.4 column names of ``dataset``, in model order."""
27
+ return tuple(load_model()["datasets"][resolve_dataset(dataset)]["columns"])
28
+
29
+
30
+ def column_spec(dataset: str, column: str) -> dict:
31
+ """Return the model spec of one column (feature_level, allows_nulls, ...)."""
32
+ return load_model()["datasets"][resolve_dataset(dataset)]["columns"][column]
33
+
34
+
35
+ __all__ = [
36
+ "FOCUS_1_4_DATASETS",
37
+ "column_spec",
38
+ "dataset_columns",
39
+ "lint_focus_1_4_structure",
40
+ "load_model",
41
+ "resolve_dataset",
42
+ "validate_focus_1_4",
43
+ ]
@@ -0,0 +1,66 @@
1
+ """Capability profile — which FOCUS applicability conditions a source supports.
2
+
3
+ Several FOCUS columns are only *conditionally* required (e.g. ``SkuId`` /
4
+ ``SkuPriceId`` when the provider supports unit pricing). The linter can only
5
+ enforce those requirements when the caller declares which conditions apply; an
6
+ undeclared condition is **not evaluated**, never silently assumed either way.
7
+
8
+ A :class:`CapabilityProfile` makes that declaration explicit and auditable: the
9
+ conversion pipeline records the active profile in the manifest, so a clean lint
10
+ can always be read together with the exact set of conditions it enforced.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from dataclasses import dataclass, field
16
+
17
+ from focus_data_toolkit.model.validator import (
18
+ COND_MULTIPLE_PRICING_CATEGORIES,
19
+ COND_UNIT_PRICING,
20
+ )
21
+
22
+ # All applicability conditions the linter knows how to enforce.
23
+ KNOWN_CONDITIONS: frozenset[str] = frozenset(
24
+ {COND_MULTIPLE_PRICING_CATEGORIES, COND_UNIT_PRICING}
25
+ )
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class CapabilityProfile:
30
+ """An explicit, validated declaration of supported applicability conditions.
31
+
32
+ ``source`` documents where the declaration came from (e.g. ``"cli"``,
33
+ ``"api"``); the default profile declares nothing and is labelled
34
+ ``"none-declared"`` so an unevaluated condition set is visible, not silent.
35
+ """
36
+
37
+ supported_conditions: frozenset[str] = field(default_factory=frozenset)
38
+ source: str = "none-declared"
39
+
40
+ def __post_init__(self) -> None:
41
+ # Accept any iterable-of-str the caller passed and materialize it exactly once —
42
+ # a one-shot generator must not be exhausted by validation before it is frozen.
43
+ conditions = frozenset(self.supported_conditions)
44
+ object.__setattr__(self, "supported_conditions", conditions)
45
+ unknown = conditions - KNOWN_CONDITIONS
46
+ if unknown:
47
+ known = ", ".join(sorted(KNOWN_CONDITIONS))
48
+ raise ValueError(
49
+ f"unknown applicability condition(s) {sorted(unknown)}; known: {known}"
50
+ )
51
+
52
+ @classmethod
53
+ def none(cls) -> CapabilityProfile:
54
+ """The default profile: no condition declared (none enforced)."""
55
+ return cls()
56
+
57
+ @classmethod
58
+ def of(cls, *conditions: str, source: str = "api") -> CapabilityProfile:
59
+ return cls(frozenset(conditions), source)
60
+
61
+ def as_dict(self) -> dict:
62
+ """Deterministic manifest payload."""
63
+ return {
64
+ "supported_conditions": sorted(self.supported_conditions),
65
+ "source": self.source,
66
+ }