focus-data-toolkit 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- focus_data_toolkit/__init__.py +69 -0
- focus_data_toolkit/__main__.py +6 -0
- focus_data_toolkit/_version.py +8 -0
- focus_data_toolkit/cli.py +968 -0
- focus_data_toolkit/context/__init__.py +88 -0
- focus_data_toolkit/context/billing.py +54 -0
- focus_data_toolkit/context/provider.py +90 -0
- focus_data_toolkit/convert/__init__.py +708 -0
- focus_data_toolkit/convert/billing_period.py +65 -0
- focus_data_toolkit/convert/contract_applied.py +235 -0
- focus_data_toolkit/convert/contract_commitment.py +182 -0
- focus_data_toolkit/convert/cost_and_usage.py +179 -0
- focus_data_toolkit/convert/detect.py +39 -0
- focus_data_toolkit/convert/invoice_detail.py +199 -0
- focus_data_toolkit/convert/streaming.py +1030 -0
- focus_data_toolkit/errors.py +145 -0
- focus_data_toolkit/focus_json.py +68 -0
- focus_data_toolkit/generators/__init__.py +61 -0
- focus_data_toolkit/generators/_shim.py +43 -0
- focus_data_toolkit/generators/engine/__init__.py +14 -0
- focus_data_toolkit/generators/engine/context.py +12 -0
- focus_data_toolkit/generators/engine/determinism.py +117 -0
- focus_data_toolkit/generators/engine/json_focus.py +63 -0
- focus_data_toolkit/generators/engine/ladder.py +71 -0
- focus_data_toolkit/generators/engine/scenarios_core.py +380 -0
- focus_data_toolkit/generators/engine/serialize.py +151 -0
- focus_data_toolkit/generators/generate_aws_focus_1_2.py +19 -0
- focus_data_toolkit/generators/generate_aws_focus_1_3.py +20 -0
- focus_data_toolkit/generators/generate_azure_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_azure_focus_1_3.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_3.py +17 -0
- focus_data_toolkit/generators/providers/__init__.py +29 -0
- focus_data_toolkit/generators/providers/aws.py +186 -0
- focus_data_toolkit/generators/providers/azure.py +191 -0
- focus_data_toolkit/generators/providers/gcp.py +194 -0
- focus_data_toolkit/generators/providers/profile.py +123 -0
- focus_data_toolkit/generators/scenarios.py +178 -0
- focus_data_toolkit/generators/versions/__init__.py +17 -0
- focus_data_toolkit/generators/versions/adapter.py +41 -0
- focus_data_toolkit/generators/versions/v1_2.py +111 -0
- focus_data_toolkit/generators/versions/v1_3.py +154 -0
- focus_data_toolkit/io/__init__.py +1 -0
- focus_data_toolkit/io/atomic_writer.py +462 -0
- focus_data_toolkit/io/csv_io.py +128 -0
- focus_data_toolkit/io/parquet_io.py +528 -0
- focus_data_toolkit/io/records.py +92 -0
- focus_data_toolkit/io/row_source.py +117 -0
- focus_data_toolkit/lifecycle.py +342 -0
- focus_data_toolkit/manifest.py +114 -0
- focus_data_toolkit/model/__init__.py +43 -0
- focus_data_toolkit/model/capabilities.py +66 -0
- focus_data_toolkit/model/focus_1_4_decimal_scale.json +10 -0
- focus_data_toolkit/model/focus_1_4_model.json +1913 -0
- focus_data_toolkit/model/focus_1_4_servicesubcategory.json +84 -0
- focus_data_toolkit/model/focus_json_keys.py +112 -0
- focus_data_toolkit/model/iso_4217_currencies.json +23 -0
- focus_data_toolkit/model/json_schema_check.py +205 -0
- focus_data_toolkit/model/json_schemas/allocatedmethoddetailsobjectschema.json +82 -0
- focus_data_toolkit/model/json_schemas/commitmentprogrameligibilitydetailsobjectschema.json +41 -0
- focus_data_toolkit/model/json_schemas/contractappliedobjectschema.json +104 -0
- focus_data_toolkit/model/json_schemas/contractcommitmentapplicabilityobjectschema.json +290 -0
- focus_data_toolkit/model/json_schemas/json_schemas_provenance.json +38 -0
- focus_data_toolkit/model/model_provenance.json +58 -0
- focus_data_toolkit/model/validator.py +498 -0
- focus_data_toolkit/modes.py +18 -0
- focus_data_toolkit/official_validator.py +61 -0
- focus_data_toolkit/progress.py +89 -0
- focus_data_toolkit/provenance.py +106 -0
- focus_data_toolkit/py.typed +1 -0
- focus_data_toolkit/runtime.py +243 -0
- focus_data_toolkit/schema/__init__.py +17 -0
- focus_data_toolkit/schema/detection.py +274 -0
- focus_data_toolkit/schema/registry.py +127 -0
- focus_data_toolkit/storage/__init__.py +1 -0
- focus_data_toolkit/storage/external_index.py +99 -0
- focus_data_toolkit/storage/spill.py +150 -0
- focus_data_toolkit/studio/__init__.py +19 -0
- focus_data_toolkit/studio/app.py +467 -0
- focus_data_toolkit/studio/config.py +42 -0
- focus_data_toolkit/studio/frontend/app.js +214 -0
- focus_data_toolkit/studio/frontend/index.html +101 -0
- focus_data_toolkit/studio/frontend/style.css +60 -0
- focus_data_toolkit/studio/jobs.py +142 -0
- focus_data_toolkit/studio/preview.py +32 -0
- focus_data_toolkit/studio/security.py +125 -0
- focus_data_toolkit/studio/server.py +71 -0
- focus_data_toolkit/supplement/__init__.py +50 -0
- focus_data_toolkit/supplement/adapters/__init__.py +21 -0
- focus_data_toolkit/supplement/adapters/adapters_provenance.json +39 -0
- focus_data_toolkit/supplement/adapters/aws_invoice_summary.json +24 -0
- focus_data_toolkit/supplement/adapters/aws_savings_plans.json +31 -0
- focus_data_toolkit/supplement/adapters/azure_invoice.json +25 -0
- focus_data_toolkit/supplement/adapters/gcp_compute_commitments.json +28 -0
- focus_data_toolkit/supplement/adapters/registry.py +215 -0
- focus_data_toolkit/supplement/apply.py +318 -0
- focus_data_toolkit/supplement/gaps.py +219 -0
- focus_data_toolkit/supplement/kinds.py +118 -0
- focus_data_toolkit/supplement/loader.py +409 -0
- focus_data_toolkit/supplement/spec.py +74 -0
- focus_data_toolkit/supplement/validate.py +215 -0
- focus_data_toolkit/validate/__init__.py +15 -0
- focus_data_toolkit/validate/allocation.py +333 -0
- focus_data_toolkit/validate/bundle.py +254 -0
- focus_data_toolkit/validate/codes.py +93 -0
- focus_data_toolkit/validate/corrections.py +245 -0
- focus_data_toolkit/validate/reconciliation.py +98 -0
- focus_data_toolkit/validate/referential.py +289 -0
- focus_data_toolkit-0.11.0.dist-info/METADATA +519 -0
- focus_data_toolkit-0.11.0.dist-info/RECORD +116 -0
- focus_data_toolkit-0.11.0.dist-info/WHEEL +5 -0
- focus_data_toolkit-0.11.0.dist-info/entry_points.txt +2 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSE +21 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSES/CC-BY-4.0.txt +156 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/NOTICE +60 -0
- focus_data_toolkit-0.11.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Format-agnostic row input: open a CSV or Parquet source behind one protocol.
|
|
2
|
+
|
|
3
|
+
Client exports arrive as CSV (possibly gzipped) or Parquet. :func:`open_row_source` detects
|
|
4
|
+
the physical format — by the ``PAR1`` magic bytes first, extension as a fallback for
|
|
5
|
+
unreadable paths — and returns the matching streaming reader. Both readers already share
|
|
6
|
+
the same shape (``source_columns``, iteration yielding ``Record``, ``close``); the
|
|
7
|
+
:class:`RowSource` protocol names that contract so conversion code can consume either
|
|
8
|
+
without caring which it got.
|
|
9
|
+
|
|
10
|
+
Parquet support requires the ``[parquet]`` extra; opening a Parquet file without it raises
|
|
11
|
+
the same actionable :class:`~focus_data_toolkit.io.records.MalformedRecordError` install
|
|
12
|
+
hint as Parquet output does.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
from collections.abc import Iterator
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Protocol, runtime_checkable
|
|
22
|
+
|
|
23
|
+
from focus_data_toolkit.io.csv_io import CsvRowReader
|
|
24
|
+
from focus_data_toolkit.io.records import MalformedRecordError, Record
|
|
25
|
+
|
|
26
|
+
_PARQUET_MAGIC = b"PAR1"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@runtime_checkable
|
|
30
|
+
class RowSource(Protocol):
|
|
31
|
+
"""A streaming, closeable source of rows with a known header."""
|
|
32
|
+
|
|
33
|
+
source_columns: tuple[str, ...]
|
|
34
|
+
|
|
35
|
+
def __iter__(self) -> Iterator[Record]: ...
|
|
36
|
+
|
|
37
|
+
def close(self) -> None: ...
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def is_parquet(path: str | os.PathLike[str]) -> bool:
|
|
41
|
+
"""Whether ``path`` is a Parquet file (magic bytes; ``.parquet`` suffix as fallback)."""
|
|
42
|
+
p = Path(path)
|
|
43
|
+
try:
|
|
44
|
+
with open(p, "rb") as fh:
|
|
45
|
+
return fh.read(4) == _PARQUET_MAGIC
|
|
46
|
+
except OSError:
|
|
47
|
+
return p.suffix.lower() == ".parquet"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _hive_partition_columns(base_dir: Path) -> tuple[str, ...]:
|
|
51
|
+
"""Partition column names of a Hive-layout directory, walking ``COL=value`` segments."""
|
|
52
|
+
cols: list[str] = []
|
|
53
|
+
current = base_dir
|
|
54
|
+
while True:
|
|
55
|
+
subdirs = [e for e in current.iterdir() if e.is_dir() and "=" in e.name]
|
|
56
|
+
if not subdirs:
|
|
57
|
+
return tuple(cols)
|
|
58
|
+
cols.append(subdirs[0].name.split("=", 1)[0])
|
|
59
|
+
current = subdirs[0]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def open_row_source(path: str | os.PathLike[str], *, dataset: str | None = None) -> RowSource:
|
|
63
|
+
"""Open ``path`` as a :class:`RowSource`, auto-detecting the physical format.
|
|
64
|
+
|
|
65
|
+
A file is sniffed as CSV (``.gz`` ok) or Parquet; a directory is read as a
|
|
66
|
+
Hive-partitioned Parquet dataset (partition columns inferred from the ``COL=value``
|
|
67
|
+
path segments). ``dataset`` optionally names the FOCUS dataset the file holds, letting
|
|
68
|
+
the Parquet readers render typed values by the right model columns instead of inferring
|
|
69
|
+
the dataset from the header overlap. CSV input ignores it (values are already strings).
|
|
70
|
+
"""
|
|
71
|
+
p = Path(path)
|
|
72
|
+
if p.is_dir():
|
|
73
|
+
from focus_data_toolkit.io.parquet_io import PartitionedParquetReader
|
|
74
|
+
|
|
75
|
+
with _parquet_errors(p):
|
|
76
|
+
return PartitionedParquetReader(
|
|
77
|
+
p, dataset or "Cost and Usage", _hive_partition_columns(p)
|
|
78
|
+
)
|
|
79
|
+
if is_parquet(p):
|
|
80
|
+
from focus_data_toolkit.io.parquet_io import ParquetRowReader
|
|
81
|
+
|
|
82
|
+
with _parquet_errors(p):
|
|
83
|
+
return ParquetRowReader(p, dataset=dataset)
|
|
84
|
+
return CsvRowReader(p)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@contextmanager
|
|
88
|
+
def _parquet_errors(path: Path) -> Iterator[None]:
|
|
89
|
+
"""Translate Arrow open/read failures into the CLI-friendly MalformedRecordError.
|
|
90
|
+
|
|
91
|
+
A missing-pyarrow MalformedRecordError (with its install hint) passes through untouched.
|
|
92
|
+
"""
|
|
93
|
+
try:
|
|
94
|
+
yield
|
|
95
|
+
except MalformedRecordError:
|
|
96
|
+
raise
|
|
97
|
+
except (OSError, ValueError) as exc:
|
|
98
|
+
# pyarrow's ArrowInvalid/ArrowIOError derive from ValueError/OSError, so a corrupt
|
|
99
|
+
# or truncated Parquet file surfaces as a clean input error, never a traceback.
|
|
100
|
+
raise MalformedRecordError(f"cannot read Parquet source {path}: {exc}") from exc
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def read_source_rows(
|
|
104
|
+
path: str | os.PathLike[str], *, dataset: str | None = None
|
|
105
|
+
) -> list[dict[str, str]]:
|
|
106
|
+
"""Materialise a CSV/Parquet source into a list of dict rows (eager-path input)."""
|
|
107
|
+
reader = open_row_source(path, dataset=dataset)
|
|
108
|
+
try:
|
|
109
|
+
if isinstance(reader, CsvRowReader):
|
|
110
|
+
return [dict(record.values) for record in reader]
|
|
111
|
+
with _parquet_errors(Path(path)): # iteration can hit mid-file corruption too
|
|
112
|
+
return [dict(record.values) for record in reader]
|
|
113
|
+
finally:
|
|
114
|
+
reader.close()
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
__all__ = ["RowSource", "is_parquet", "open_row_source", "read_source_rows"]
|
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""Dataset-instance snapshots and billing/invoice lifecycle-chain rules (P1.10).
|
|
2
|
+
|
|
3
|
+
A **dataset instance** is one delivered snapshot of a FOCUS dataset for a subject (an invoice,
|
|
4
|
+
a billing period): who it is, when it was created / last updated, whether it is complete, the
|
|
5
|
+
period it covers, its lifecycle status, and which instance preceded it. Real client pipelines
|
|
6
|
+
*record* these (they are never fabricated for client data); the toolkit models them so a
|
|
7
|
+
sequence of snapshots can be checked for illegal status transitions — e.g. a ``Voided`` invoice
|
|
8
|
+
must not silently go back to ``Issued``, an ``Issued`` invoice must not revert to ``Open``.
|
|
9
|
+
|
|
10
|
+
The allowed transitions come straight from the FOCUS ``InvoiceIssueStatus`` (``Open`` →
|
|
11
|
+
``Issued`` → ``Voided``) and ``BillingPeriodStatus`` (``Open`` → ``Closed``) value sets; a
|
|
12
|
+
correction is a *new* instance/line, never an in-place edit of a closed/issued one.
|
|
13
|
+
|
|
14
|
+
Beyond transitions, :func:`check_instance_chains` validates the chain structure itself —
|
|
15
|
+
``instance_id`` / per-subject ``order`` uniqueness (FDT-LIFE-001/002),
|
|
16
|
+
``previous_instance_id`` resolution (FDT-LIFE-003), cycle detection (FDT-LIFE-004),
|
|
17
|
+
``last_updated`` monotonicity (FDT-LIFE-005), and closed-instance immutability
|
|
18
|
+
(FDT-LIFE-006) — all as structured diagnostics. :func:`check_dataset_instances` runs the
|
|
19
|
+
whole battery (chains + status transitions) in one call.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from collections.abc import Iterable, Sequence
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from datetime import datetime
|
|
27
|
+
|
|
28
|
+
from focus_data_toolkit.errors import Diagnostic, Severity
|
|
29
|
+
|
|
30
|
+
# Allowed forward transitions (a status may also stay the same — an idempotent re-delivery).
|
|
31
|
+
INVOICE_TRANSITIONS: dict[str, frozenset[str]] = {
|
|
32
|
+
"Open": frozenset({"Open", "Issued", "Voided"}),
|
|
33
|
+
"Issued": frozenset({"Issued", "Voided"}),
|
|
34
|
+
"Voided": frozenset({"Voided"}),
|
|
35
|
+
}
|
|
36
|
+
BILLING_PERIOD_TRANSITIONS: dict[str, frozenset[str]] = {
|
|
37
|
+
"Open": frozenset({"Open", "Closed"}),
|
|
38
|
+
"Closed": frozenset({"Closed"}),
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class DatasetInstance:
|
|
44
|
+
"""One recorded snapshot of a dataset for a subject (invoice / billing period)."""
|
|
45
|
+
|
|
46
|
+
instance_id: str
|
|
47
|
+
dataset: str
|
|
48
|
+
subject_id: str
|
|
49
|
+
order: int
|
|
50
|
+
created: str
|
|
51
|
+
last_updated: str
|
|
52
|
+
complete: bool
|
|
53
|
+
period_start: str
|
|
54
|
+
period_end: str
|
|
55
|
+
status: str
|
|
56
|
+
previous_instance_id: str | None = None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _check_transitions(
|
|
60
|
+
instances: Sequence[DatasetInstance], allowed: dict[str, frozenset[str]]
|
|
61
|
+
) -> list[Diagnostic]:
|
|
62
|
+
"""Flag any status change not permitted by ``allowed`` between consecutive snapshots.
|
|
63
|
+
|
|
64
|
+
Snapshots are grouped by ``subject_id`` and ordered by ``order``; each adjacent pair is a
|
|
65
|
+
transition. An unknown status value is left to the per-dataset linter (allowed-value check).
|
|
66
|
+
"""
|
|
67
|
+
by_subject: dict[str, list[DatasetInstance]] = {}
|
|
68
|
+
for inst in instances:
|
|
69
|
+
by_subject.setdefault(inst.subject_id, []).append(inst)
|
|
70
|
+
|
|
71
|
+
out: list[Diagnostic] = []
|
|
72
|
+
for subject, snaps in sorted(by_subject.items()):
|
|
73
|
+
ordered = sorted(snaps, key=lambda s: s.order)
|
|
74
|
+
for prev, cur in zip(ordered, ordered[1:], strict=False):
|
|
75
|
+
permitted = allowed.get(prev.status)
|
|
76
|
+
if permitted is None or cur.status not in permitted:
|
|
77
|
+
out.append(
|
|
78
|
+
Diagnostic(
|
|
79
|
+
code="FDT-CORR-004",
|
|
80
|
+
severity=Severity.ERROR,
|
|
81
|
+
message=f"illegal status transition {prev.status!r} -> {cur.status!r} "
|
|
82
|
+
f"for {subject!r} (instances {prev.instance_id} -> {cur.instance_id})",
|
|
83
|
+
datasets=(cur.dataset,),
|
|
84
|
+
dataset=cur.dataset,
|
|
85
|
+
record_keys={"subject_id": subject},
|
|
86
|
+
expected=f"one of {sorted(permitted)}" if permitted else "known status",
|
|
87
|
+
actual=cur.status,
|
|
88
|
+
context={"from_instance": prev.instance_id, "to_instance": cur.instance_id},
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
return out
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def check_invoice_status_transitions(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
|
|
95
|
+
"""Validate Invoice Detail status transitions across dataset instances (FDT-CORR-004)."""
|
|
96
|
+
invoices = [i for i in instances if i.dataset == "Invoice Detail"]
|
|
97
|
+
return _check_transitions(invoices, INVOICE_TRANSITIONS)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def check_billing_period_status_transitions(
|
|
101
|
+
instances: Iterable[DatasetInstance],
|
|
102
|
+
) -> list[Diagnostic]:
|
|
103
|
+
"""Validate Billing Period status transitions across dataset instances (FDT-CORR-004)."""
|
|
104
|
+
periods = [i for i in instances if i.dataset == "Billing Period"]
|
|
105
|
+
return _check_transitions(periods, BILLING_PERIOD_TRANSITIONS)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def check_status_transitions(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
|
|
109
|
+
"""Validate both invoice and billing-period status transitions in one pass."""
|
|
110
|
+
instances = list(instances)
|
|
111
|
+
return check_invoice_status_transitions(instances) + check_billing_period_status_transitions(
|
|
112
|
+
instances
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# Statuses after which a subject's recorded facts are frozen: later snapshots may re-deliver
|
|
117
|
+
# or move forward in status, but must not rewrite the period or un-complete the data.
|
|
118
|
+
_CLOSED_STATUSES: frozenset[str] = frozenset({"Closed", "Issued", "Voided"})
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _parse_dt(value: str) -> datetime | None:
|
|
122
|
+
"""Parse a FOCUS Date/Time; naive or malformed values return None (the linter's job)."""
|
|
123
|
+
try:
|
|
124
|
+
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
|
125
|
+
except ValueError:
|
|
126
|
+
return None
|
|
127
|
+
return parsed if parsed.tzinfo is not None else None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _chain_diag(
|
|
131
|
+
code: str, message: str, inst: DatasetInstance, **extra: object
|
|
132
|
+
) -> Diagnostic:
|
|
133
|
+
return Diagnostic(
|
|
134
|
+
code=code,
|
|
135
|
+
severity=Severity.ERROR,
|
|
136
|
+
message=message,
|
|
137
|
+
datasets=(inst.dataset,),
|
|
138
|
+
dataset=inst.dataset,
|
|
139
|
+
record_keys={"subject_id": inst.subject_id, "instance_id": inst.instance_id},
|
|
140
|
+
**extra, # type: ignore[arg-type]
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def check_instance_chains(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
|
|
145
|
+
"""Validate the structure of dataset-instance chains (FDT-LIFE-001..006).
|
|
146
|
+
|
|
147
|
+
* **FDT-LIFE-001** — ``instance_id`` is reused across snapshots.
|
|
148
|
+
* **FDT-LIFE-002** — two snapshots of one subject share the same ``order``.
|
|
149
|
+
* **FDT-LIFE-003** — ``previous_instance_id`` does not resolve to an existing earlier
|
|
150
|
+
snapshot (unknown id, self-reference, or non-decreasing order). Cross-subject and
|
|
151
|
+
cross-dataset links are legitimate delivery lineage (e.g. a replacement invoice
|
|
152
|
+
points at the voided one it replaces).
|
|
153
|
+
* **FDT-LIFE-004** — the ``previous_instance_id`` links form a cycle.
|
|
154
|
+
* **FDT-LIFE-005** — ``last_updated`` is before ``created``, or decreases between
|
|
155
|
+
consecutive snapshots of a subject (history must move forward).
|
|
156
|
+
* **FDT-LIFE-006** — a snapshot after a closed/issued/voided one rewrites the recorded
|
|
157
|
+
period, or flips ``complete`` back to ``False`` (closed instances are immutable;
|
|
158
|
+
a correction is a new line, never an in-place edit).
|
|
159
|
+
|
|
160
|
+
Unknown status values and malformed timestamps are left to the per-dataset linter.
|
|
161
|
+
"""
|
|
162
|
+
snaps = list(instances)
|
|
163
|
+
out: list[Diagnostic] = []
|
|
164
|
+
|
|
165
|
+
by_id: dict[str, DatasetInstance] = {}
|
|
166
|
+
for inst in snaps:
|
|
167
|
+
if inst.instance_id in by_id:
|
|
168
|
+
out.append(
|
|
169
|
+
_chain_diag(
|
|
170
|
+
"FDT-LIFE-001",
|
|
171
|
+
f"instance_id {inst.instance_id!r} is reused "
|
|
172
|
+
f"(subjects {by_id[inst.instance_id].subject_id!r} and {inst.subject_id!r})",
|
|
173
|
+
inst,
|
|
174
|
+
value=inst.instance_id,
|
|
175
|
+
)
|
|
176
|
+
)
|
|
177
|
+
else:
|
|
178
|
+
by_id[inst.instance_id] = inst
|
|
179
|
+
|
|
180
|
+
by_subject: dict[tuple[str, str], list[DatasetInstance]] = {}
|
|
181
|
+
for inst in snaps:
|
|
182
|
+
by_subject.setdefault((inst.dataset, inst.subject_id), []).append(inst)
|
|
183
|
+
|
|
184
|
+
for (_dataset, subject), group in sorted(by_subject.items()):
|
|
185
|
+
seen_orders: dict[int, DatasetInstance] = {}
|
|
186
|
+
for inst in group:
|
|
187
|
+
if inst.order in seen_orders:
|
|
188
|
+
out.append(
|
|
189
|
+
_chain_diag(
|
|
190
|
+
"FDT-LIFE-002",
|
|
191
|
+
f"snapshots {seen_orders[inst.order].instance_id!r} and "
|
|
192
|
+
f"{inst.instance_id!r} of {subject!r} share order {inst.order}",
|
|
193
|
+
inst,
|
|
194
|
+
actual=str(inst.order),
|
|
195
|
+
)
|
|
196
|
+
)
|
|
197
|
+
else:
|
|
198
|
+
seen_orders[inst.order] = inst
|
|
199
|
+
|
|
200
|
+
ordered = sorted(group, key=lambda s: (s.order, s.instance_id))
|
|
201
|
+
|
|
202
|
+
# previous_instance_id resolution.
|
|
203
|
+
for inst in ordered:
|
|
204
|
+
prev_id = inst.previous_instance_id
|
|
205
|
+
if prev_id is None:
|
|
206
|
+
continue
|
|
207
|
+
prev = by_id.get(prev_id)
|
|
208
|
+
if prev is None or prev_id == inst.instance_id:
|
|
209
|
+
reason = "references itself" if prev_id == inst.instance_id else "is unknown"
|
|
210
|
+
out.append(
|
|
211
|
+
_chain_diag(
|
|
212
|
+
"FDT-LIFE-003",
|
|
213
|
+
f"previous_instance_id {prev_id!r} of {inst.instance_id!r} {reason}",
|
|
214
|
+
inst,
|
|
215
|
+
value=prev_id,
|
|
216
|
+
)
|
|
217
|
+
)
|
|
218
|
+
elif prev.order >= inst.order:
|
|
219
|
+
out.append(
|
|
220
|
+
_chain_diag(
|
|
221
|
+
"FDT-LIFE-003",
|
|
222
|
+
f"previous_instance_id {prev_id!r} of {inst.instance_id!r} does not "
|
|
223
|
+
f"precede it (order {prev.order} >= {inst.order})",
|
|
224
|
+
inst,
|
|
225
|
+
value=prev_id,
|
|
226
|
+
)
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
# last_updated monotonicity along the delivery sequence.
|
|
230
|
+
prev_updated: datetime | None = None
|
|
231
|
+
prev_inst: DatasetInstance | None = None
|
|
232
|
+
for inst in ordered:
|
|
233
|
+
created, updated = _parse_dt(inst.created), _parse_dt(inst.last_updated)
|
|
234
|
+
if created is not None and updated is not None and updated < created:
|
|
235
|
+
out.append(
|
|
236
|
+
_chain_diag(
|
|
237
|
+
"FDT-LIFE-005",
|
|
238
|
+
f"last_updated of {inst.instance_id!r} precedes its created",
|
|
239
|
+
inst,
|
|
240
|
+
expected=f">= {inst.created}",
|
|
241
|
+
actual=inst.last_updated,
|
|
242
|
+
)
|
|
243
|
+
)
|
|
244
|
+
if updated is not None:
|
|
245
|
+
if prev_updated is not None and updated < prev_updated:
|
|
246
|
+
assert prev_inst is not None
|
|
247
|
+
out.append(
|
|
248
|
+
_chain_diag(
|
|
249
|
+
"FDT-LIFE-005",
|
|
250
|
+
f"last_updated moves backwards between {prev_inst.instance_id!r} "
|
|
251
|
+
f"and {inst.instance_id!r} of {subject!r}",
|
|
252
|
+
inst,
|
|
253
|
+
expected=f">= {prev_inst.last_updated}",
|
|
254
|
+
actual=inst.last_updated,
|
|
255
|
+
)
|
|
256
|
+
)
|
|
257
|
+
prev_updated, prev_inst = updated, inst
|
|
258
|
+
|
|
259
|
+
# Closed-instance immutability: once closed/issued/voided, the recorded period is
|
|
260
|
+
# frozen and completeness must not regress.
|
|
261
|
+
frozen: DatasetInstance | None = None
|
|
262
|
+
for inst in ordered:
|
|
263
|
+
if frozen is not None:
|
|
264
|
+
if (inst.period_start, inst.period_end) != (
|
|
265
|
+
frozen.period_start,
|
|
266
|
+
frozen.period_end,
|
|
267
|
+
):
|
|
268
|
+
out.append(
|
|
269
|
+
_chain_diag(
|
|
270
|
+
"FDT-LIFE-006",
|
|
271
|
+
f"{inst.instance_id!r} rewrites the period of {subject!r} recorded "
|
|
272
|
+
f"by closed snapshot {frozen.instance_id!r}",
|
|
273
|
+
inst,
|
|
274
|
+
expected=f"{frozen.period_start}/{frozen.period_end}",
|
|
275
|
+
actual=f"{inst.period_start}/{inst.period_end}",
|
|
276
|
+
)
|
|
277
|
+
)
|
|
278
|
+
if frozen.complete and not inst.complete:
|
|
279
|
+
out.append(
|
|
280
|
+
_chain_diag(
|
|
281
|
+
"FDT-LIFE-006",
|
|
282
|
+
f"{inst.instance_id!r} flips {subject!r} back to incomplete after "
|
|
283
|
+
f"closed snapshot {frozen.instance_id!r}",
|
|
284
|
+
inst,
|
|
285
|
+
expected="complete=True",
|
|
286
|
+
actual="complete=False",
|
|
287
|
+
)
|
|
288
|
+
)
|
|
289
|
+
if frozen is None and inst.status in _CLOSED_STATUSES:
|
|
290
|
+
frozen = inst
|
|
291
|
+
|
|
292
|
+
out.extend(_check_cycles(snaps, by_id))
|
|
293
|
+
return out
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _check_cycles(
|
|
297
|
+
snaps: Sequence[DatasetInstance], by_id: dict[str, DatasetInstance]
|
|
298
|
+
) -> list[Diagnostic]:
|
|
299
|
+
"""Detect cycles in the ``previous_instance_id`` graph (each reported once)."""
|
|
300
|
+
out: list[Diagnostic] = []
|
|
301
|
+
state: dict[str, int] = {} # 0 = visiting, 1 = done
|
|
302
|
+
for start in sorted(by_id):
|
|
303
|
+
if start in state:
|
|
304
|
+
continue
|
|
305
|
+
path: list[str] = []
|
|
306
|
+
current: str | None = start
|
|
307
|
+
while current is not None and current in by_id and current not in state:
|
|
308
|
+
state[current] = 0
|
|
309
|
+
path.append(current)
|
|
310
|
+
current = by_id[current].previous_instance_id
|
|
311
|
+
if current is not None and state.get(current) == 0:
|
|
312
|
+
cycle = path[path.index(current):]
|
|
313
|
+
inst = by_id[current]
|
|
314
|
+
out.append(
|
|
315
|
+
_chain_diag(
|
|
316
|
+
"FDT-LIFE-004",
|
|
317
|
+
"previous_instance_id links form a cycle: " + " -> ".join([*cycle, current]),
|
|
318
|
+
inst,
|
|
319
|
+
context={"cycle": ",".join(cycle)},
|
|
320
|
+
)
|
|
321
|
+
)
|
|
322
|
+
for node in path:
|
|
323
|
+
state[node] = 1
|
|
324
|
+
return out
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def check_dataset_instances(instances: Iterable[DatasetInstance]) -> list[Diagnostic]:
|
|
328
|
+
"""Run the full lifecycle battery: chain structure plus status transitions."""
|
|
329
|
+
snaps = list(instances)
|
|
330
|
+
return check_instance_chains(snaps) + check_status_transitions(snaps)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
__all__ = [
|
|
334
|
+
"BILLING_PERIOD_TRANSITIONS",
|
|
335
|
+
"INVOICE_TRANSITIONS",
|
|
336
|
+
"DatasetInstance",
|
|
337
|
+
"check_billing_period_status_transitions",
|
|
338
|
+
"check_dataset_instances",
|
|
339
|
+
"check_instance_chains",
|
|
340
|
+
"check_invoice_status_transitions",
|
|
341
|
+
"check_status_transitions",
|
|
342
|
+
]
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Deterministic, machine-readable conversion manifest.
|
|
2
|
+
|
|
3
|
+
The manifest records, per FOCUS 1.4 dataset, whether it was produced and — column by
|
|
4
|
+
column — how each value was obtained (see :mod:`focus_data_toolkit.provenance`). It is
|
|
5
|
+
built without a clock or RNG so a given conversion always yields the same manifest bytes.
|
|
6
|
+
Any produced column with ``ASSUMED`` lineage sets ``assumptions_present`` and prevents a
|
|
7
|
+
full-conformance claim.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
|
|
14
|
+
from focus_data_toolkit.provenance import ColumnRule
|
|
15
|
+
|
|
16
|
+
MANIFEST_FILENAME = "focus_1_4_manifest.json"
|
|
17
|
+
TARGET_VERSION = "1.4"
|
|
18
|
+
|
|
19
|
+
# Dataset statuses.
|
|
20
|
+
PRODUCED = "PRODUCED"
|
|
21
|
+
PRODUCED_SYNTHETIC = "PRODUCED_SYNTHETIC"
|
|
22
|
+
NOT_PRODUCED = "NOT_PRODUCED"
|
|
23
|
+
|
|
24
|
+
# Conformance labels (never "fully conformant": see model/validator.py levels).
|
|
25
|
+
CONF_STRUCTURAL_LINT = "STRUCTURAL_LINT" # produced from factual lineage, passed the lint
|
|
26
|
+
CONF_LINT_FAILED = "LINT_FAILED" # produced from factual lineage, but the lint failed
|
|
27
|
+
CONF_NOT_VALIDATED = "NOT_VALIDATED" # produced but lint not run (validate=False)
|
|
28
|
+
CONF_SYNTHETIC = "SYNTHETIC" # produced with assumed values
|
|
29
|
+
CONF_INCOMPLETE = "INCOMPLETE" # not produced
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def dataset_entry(
|
|
33
|
+
*,
|
|
34
|
+
status: str,
|
|
35
|
+
conformance: str,
|
|
36
|
+
provenance: dict[str, ColumnRule],
|
|
37
|
+
row_count: int | None = None,
|
|
38
|
+
reason: str | None = None,
|
|
39
|
+
blocking_columns: list[str] | None = None,
|
|
40
|
+
output_file: str | None = None,
|
|
41
|
+
partitioned_by: list[str] | None = None,
|
|
42
|
+
lineage_summary: dict[str, dict[str, int]] | None = None,
|
|
43
|
+
) -> dict:
|
|
44
|
+
entry: dict = {
|
|
45
|
+
"status": status,
|
|
46
|
+
"conformance": conformance,
|
|
47
|
+
"columns": {col: provenance[col].as_dict() for col in sorted(provenance)},
|
|
48
|
+
}
|
|
49
|
+
if row_count is not None:
|
|
50
|
+
entry["row_count"] = row_count
|
|
51
|
+
if lineage_summary:
|
|
52
|
+
# Per-value lineage counts for columns whose rule varies by row: the headline
|
|
53
|
+
# column lineage above stays the weakest category, this shows the actual mix.
|
|
54
|
+
entry["lineage_summary"] = lineage_summary
|
|
55
|
+
if reason:
|
|
56
|
+
entry["reason"] = reason
|
|
57
|
+
if blocking_columns:
|
|
58
|
+
entry["blocking_columns"] = blocking_columns
|
|
59
|
+
if output_file:
|
|
60
|
+
entry["output_file"] = output_file
|
|
61
|
+
if partitioned_by:
|
|
62
|
+
# Present only for a Hive-partitioned Parquet dataset: output_file is the dataset
|
|
63
|
+
# directory, partitioned on these columns (a reader reconstructs them from the paths).
|
|
64
|
+
entry["partitioned_by"] = partitioned_by
|
|
65
|
+
return entry
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def build_manifest(
|
|
69
|
+
*,
|
|
70
|
+
tool_version: str,
|
|
71
|
+
source_version: str,
|
|
72
|
+
mode: str,
|
|
73
|
+
datasets: dict[str, dict],
|
|
74
|
+
detection: dict | None = None,
|
|
75
|
+
contexts: dict | None = None,
|
|
76
|
+
diagnostics: list[dict] | None = None,
|
|
77
|
+
capability_profile: dict | None = None,
|
|
78
|
+
supplements: list[dict] | None = None,
|
|
79
|
+
) -> dict:
|
|
80
|
+
assumptions_present = any(
|
|
81
|
+
entry["status"] in (PRODUCED, PRODUCED_SYNTHETIC)
|
|
82
|
+
and any(col.get("lineage") == "ASSUMED" for col in entry.get("columns", {}).values())
|
|
83
|
+
for entry in datasets.values()
|
|
84
|
+
)
|
|
85
|
+
manifest: dict = {
|
|
86
|
+
"tool_version": tool_version,
|
|
87
|
+
"source_version": source_version,
|
|
88
|
+
"target_version": TARGET_VERSION,
|
|
89
|
+
"mode": mode,
|
|
90
|
+
"assumptions_present": assumptions_present,
|
|
91
|
+
"datasets": datasets,
|
|
92
|
+
}
|
|
93
|
+
# Detection decision and per-row context summary are part of the deterministic business
|
|
94
|
+
# record (no clock/RNG). Diagnostics record non-fatal findings (e.g. ambiguous context).
|
|
95
|
+
if detection is not None:
|
|
96
|
+
manifest["detection"] = detection
|
|
97
|
+
if contexts is not None:
|
|
98
|
+
manifest["contexts"] = contexts
|
|
99
|
+
if diagnostics is not None:
|
|
100
|
+
manifest["diagnostics"] = diagnostics
|
|
101
|
+
if capability_profile is not None:
|
|
102
|
+
# The applicability conditions the lint actually enforced. Recorded even when
|
|
103
|
+
# empty ("none-declared"), so an unevaluated condition set is never silent.
|
|
104
|
+
manifest["capability_profile"] = capability_profile
|
|
105
|
+
if supplements:
|
|
106
|
+
# Client-supplied fact files (path, kind, sha256, row counts, declared
|
|
107
|
+
# provenance): the audit trail behind every ENRICHED column.
|
|
108
|
+
manifest["supplements"] = supplements
|
|
109
|
+
return manifest
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def render(manifest: dict) -> str:
|
|
113
|
+
"""Serialize the manifest to deterministic JSON bytes-as-text (sorted, trailing LF)."""
|
|
114
|
+
return json.dumps(manifest, indent=2, sort_keys=True) + "\n"
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""FOCUS 1.4 data model (committed JSON artifact) and helpers.
|
|
2
|
+
|
|
3
|
+
The model is derived from the FinOps Foundation "FOCUS 1.4 Data Model"
|
|
4
|
+
workbook (https://focus.finops.org) by ``tools/extract_focus_1_4_model.py``;
|
|
5
|
+
the JSON is the artifact of record for this package.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from focus_data_toolkit.model.validator import (
|
|
11
|
+
lint_focus_1_4_structure,
|
|
12
|
+
load_model,
|
|
13
|
+
resolve_dataset,
|
|
14
|
+
validate_focus_1_4,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
FOCUS_1_4_DATASETS: tuple[str, ...] = (
|
|
18
|
+
"Cost and Usage",
|
|
19
|
+
"Contract Commitment",
|
|
20
|
+
"Billing Period",
|
|
21
|
+
"Invoice Detail",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def dataset_columns(dataset: str) -> tuple[str, ...]:
|
|
26
|
+
"""Return the FOCUS 1.4 column names of ``dataset``, in model order."""
|
|
27
|
+
return tuple(load_model()["datasets"][resolve_dataset(dataset)]["columns"])
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def column_spec(dataset: str, column: str) -> dict:
|
|
31
|
+
"""Return the model spec of one column (feature_level, allows_nulls, ...)."""
|
|
32
|
+
return load_model()["datasets"][resolve_dataset(dataset)]["columns"][column]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"FOCUS_1_4_DATASETS",
|
|
37
|
+
"column_spec",
|
|
38
|
+
"dataset_columns",
|
|
39
|
+
"lint_focus_1_4_structure",
|
|
40
|
+
"load_model",
|
|
41
|
+
"resolve_dataset",
|
|
42
|
+
"validate_focus_1_4",
|
|
43
|
+
]
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Capability profile — which FOCUS applicability conditions a source supports.
|
|
2
|
+
|
|
3
|
+
Several FOCUS columns are only *conditionally* required (e.g. ``SkuId`` /
|
|
4
|
+
``SkuPriceId`` when the provider supports unit pricing). The linter can only
|
|
5
|
+
enforce those requirements when the caller declares which conditions apply; an
|
|
6
|
+
undeclared condition is **not evaluated**, never silently assumed either way.
|
|
7
|
+
|
|
8
|
+
A :class:`CapabilityProfile` makes that declaration explicit and auditable: the
|
|
9
|
+
conversion pipeline records the active profile in the manifest, so a clean lint
|
|
10
|
+
can always be read together with the exact set of conditions it enforced.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
|
|
17
|
+
from focus_data_toolkit.model.validator import (
|
|
18
|
+
COND_MULTIPLE_PRICING_CATEGORIES,
|
|
19
|
+
COND_UNIT_PRICING,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
# All applicability conditions the linter knows how to enforce.
|
|
23
|
+
KNOWN_CONDITIONS: frozenset[str] = frozenset(
|
|
24
|
+
{COND_MULTIPLE_PRICING_CATEGORIES, COND_UNIT_PRICING}
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class CapabilityProfile:
|
|
30
|
+
"""An explicit, validated declaration of supported applicability conditions.
|
|
31
|
+
|
|
32
|
+
``source`` documents where the declaration came from (e.g. ``"cli"``,
|
|
33
|
+
``"api"``); the default profile declares nothing and is labelled
|
|
34
|
+
``"none-declared"`` so an unevaluated condition set is visible, not silent.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
supported_conditions: frozenset[str] = field(default_factory=frozenset)
|
|
38
|
+
source: str = "none-declared"
|
|
39
|
+
|
|
40
|
+
def __post_init__(self) -> None:
|
|
41
|
+
# Accept any iterable-of-str the caller passed and materialize it exactly once —
|
|
42
|
+
# a one-shot generator must not be exhausted by validation before it is frozen.
|
|
43
|
+
conditions = frozenset(self.supported_conditions)
|
|
44
|
+
object.__setattr__(self, "supported_conditions", conditions)
|
|
45
|
+
unknown = conditions - KNOWN_CONDITIONS
|
|
46
|
+
if unknown:
|
|
47
|
+
known = ", ".join(sorted(KNOWN_CONDITIONS))
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"unknown applicability condition(s) {sorted(unknown)}; known: {known}"
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
@classmethod
|
|
53
|
+
def none(cls) -> CapabilityProfile:
|
|
54
|
+
"""The default profile: no condition declared (none enforced)."""
|
|
55
|
+
return cls()
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
def of(cls, *conditions: str, source: str = "api") -> CapabilityProfile:
|
|
59
|
+
return cls(frozenset(conditions), source)
|
|
60
|
+
|
|
61
|
+
def as_dict(self) -> dict:
|
|
62
|
+
"""Deterministic manifest payload."""
|
|
63
|
+
return {
|
|
64
|
+
"supported_conditions": sorted(self.supported_conditions),
|
|
65
|
+
"source": self.source,
|
|
66
|
+
}
|