focus-data-toolkit 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- focus_data_toolkit/__init__.py +69 -0
- focus_data_toolkit/__main__.py +6 -0
- focus_data_toolkit/_version.py +8 -0
- focus_data_toolkit/cli.py +968 -0
- focus_data_toolkit/context/__init__.py +88 -0
- focus_data_toolkit/context/billing.py +54 -0
- focus_data_toolkit/context/provider.py +90 -0
- focus_data_toolkit/convert/__init__.py +708 -0
- focus_data_toolkit/convert/billing_period.py +65 -0
- focus_data_toolkit/convert/contract_applied.py +235 -0
- focus_data_toolkit/convert/contract_commitment.py +182 -0
- focus_data_toolkit/convert/cost_and_usage.py +179 -0
- focus_data_toolkit/convert/detect.py +39 -0
- focus_data_toolkit/convert/invoice_detail.py +199 -0
- focus_data_toolkit/convert/streaming.py +1030 -0
- focus_data_toolkit/errors.py +145 -0
- focus_data_toolkit/focus_json.py +68 -0
- focus_data_toolkit/generators/__init__.py +61 -0
- focus_data_toolkit/generators/_shim.py +43 -0
- focus_data_toolkit/generators/engine/__init__.py +14 -0
- focus_data_toolkit/generators/engine/context.py +12 -0
- focus_data_toolkit/generators/engine/determinism.py +117 -0
- focus_data_toolkit/generators/engine/json_focus.py +63 -0
- focus_data_toolkit/generators/engine/ladder.py +71 -0
- focus_data_toolkit/generators/engine/scenarios_core.py +380 -0
- focus_data_toolkit/generators/engine/serialize.py +151 -0
- focus_data_toolkit/generators/generate_aws_focus_1_2.py +19 -0
- focus_data_toolkit/generators/generate_aws_focus_1_3.py +20 -0
- focus_data_toolkit/generators/generate_azure_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_azure_focus_1_3.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_3.py +17 -0
- focus_data_toolkit/generators/providers/__init__.py +29 -0
- focus_data_toolkit/generators/providers/aws.py +186 -0
- focus_data_toolkit/generators/providers/azure.py +191 -0
- focus_data_toolkit/generators/providers/gcp.py +194 -0
- focus_data_toolkit/generators/providers/profile.py +123 -0
- focus_data_toolkit/generators/scenarios.py +178 -0
- focus_data_toolkit/generators/versions/__init__.py +17 -0
- focus_data_toolkit/generators/versions/adapter.py +41 -0
- focus_data_toolkit/generators/versions/v1_2.py +111 -0
- focus_data_toolkit/generators/versions/v1_3.py +154 -0
- focus_data_toolkit/io/__init__.py +1 -0
- focus_data_toolkit/io/atomic_writer.py +462 -0
- focus_data_toolkit/io/csv_io.py +128 -0
- focus_data_toolkit/io/parquet_io.py +528 -0
- focus_data_toolkit/io/records.py +92 -0
- focus_data_toolkit/io/row_source.py +117 -0
- focus_data_toolkit/lifecycle.py +342 -0
- focus_data_toolkit/manifest.py +114 -0
- focus_data_toolkit/model/__init__.py +43 -0
- focus_data_toolkit/model/capabilities.py +66 -0
- focus_data_toolkit/model/focus_1_4_decimal_scale.json +10 -0
- focus_data_toolkit/model/focus_1_4_model.json +1913 -0
- focus_data_toolkit/model/focus_1_4_servicesubcategory.json +84 -0
- focus_data_toolkit/model/focus_json_keys.py +112 -0
- focus_data_toolkit/model/iso_4217_currencies.json +23 -0
- focus_data_toolkit/model/json_schema_check.py +205 -0
- focus_data_toolkit/model/json_schemas/allocatedmethoddetailsobjectschema.json +82 -0
- focus_data_toolkit/model/json_schemas/commitmentprogrameligibilitydetailsobjectschema.json +41 -0
- focus_data_toolkit/model/json_schemas/contractappliedobjectschema.json +104 -0
- focus_data_toolkit/model/json_schemas/contractcommitmentapplicabilityobjectschema.json +290 -0
- focus_data_toolkit/model/json_schemas/json_schemas_provenance.json +38 -0
- focus_data_toolkit/model/model_provenance.json +58 -0
- focus_data_toolkit/model/validator.py +498 -0
- focus_data_toolkit/modes.py +18 -0
- focus_data_toolkit/official_validator.py +61 -0
- focus_data_toolkit/progress.py +89 -0
- focus_data_toolkit/provenance.py +106 -0
- focus_data_toolkit/py.typed +1 -0
- focus_data_toolkit/runtime.py +243 -0
- focus_data_toolkit/schema/__init__.py +17 -0
- focus_data_toolkit/schema/detection.py +274 -0
- focus_data_toolkit/schema/registry.py +127 -0
- focus_data_toolkit/storage/__init__.py +1 -0
- focus_data_toolkit/storage/external_index.py +99 -0
- focus_data_toolkit/storage/spill.py +150 -0
- focus_data_toolkit/studio/__init__.py +19 -0
- focus_data_toolkit/studio/app.py +467 -0
- focus_data_toolkit/studio/config.py +42 -0
- focus_data_toolkit/studio/frontend/app.js +214 -0
- focus_data_toolkit/studio/frontend/index.html +101 -0
- focus_data_toolkit/studio/frontend/style.css +60 -0
- focus_data_toolkit/studio/jobs.py +142 -0
- focus_data_toolkit/studio/preview.py +32 -0
- focus_data_toolkit/studio/security.py +125 -0
- focus_data_toolkit/studio/server.py +71 -0
- focus_data_toolkit/supplement/__init__.py +50 -0
- focus_data_toolkit/supplement/adapters/__init__.py +21 -0
- focus_data_toolkit/supplement/adapters/adapters_provenance.json +39 -0
- focus_data_toolkit/supplement/adapters/aws_invoice_summary.json +24 -0
- focus_data_toolkit/supplement/adapters/aws_savings_plans.json +31 -0
- focus_data_toolkit/supplement/adapters/azure_invoice.json +25 -0
- focus_data_toolkit/supplement/adapters/gcp_compute_commitments.json +28 -0
- focus_data_toolkit/supplement/adapters/registry.py +215 -0
- focus_data_toolkit/supplement/apply.py +318 -0
- focus_data_toolkit/supplement/gaps.py +219 -0
- focus_data_toolkit/supplement/kinds.py +118 -0
- focus_data_toolkit/supplement/loader.py +409 -0
- focus_data_toolkit/supplement/spec.py +74 -0
- focus_data_toolkit/supplement/validate.py +215 -0
- focus_data_toolkit/validate/__init__.py +15 -0
- focus_data_toolkit/validate/allocation.py +333 -0
- focus_data_toolkit/validate/bundle.py +254 -0
- focus_data_toolkit/validate/codes.py +93 -0
- focus_data_toolkit/validate/corrections.py +245 -0
- focus_data_toolkit/validate/reconciliation.py +98 -0
- focus_data_toolkit/validate/referential.py +289 -0
- focus_data_toolkit-0.11.0.dist-info/METADATA +519 -0
- focus_data_toolkit-0.11.0.dist-info/RECORD +116 -0
- focus_data_toolkit-0.11.0.dist-info/WHEEL +5 -0
- focus_data_toolkit-0.11.0.dist-info/entry_points.txt +2 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSE +21 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSES/CC-BY-4.0.txt +156 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/NOTICE +60 -0
- focus_data_toolkit-0.11.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Dataclasses describing a cloud provider's synthetic-data profile.
|
|
2
|
+
|
|
3
|
+
These hold only what is genuinely provider-specific. The engine reads the tables and calls
|
|
4
|
+
the callables in a fixed order, so each callable owns its own RNG draw count (which is how
|
|
5
|
+
per-provider byte output is reproduced despite differing id widths/alphabets and — for
|
|
6
|
+
commitments — differing draw *order*).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import random
|
|
12
|
+
from collections.abc import Callable, Mapping
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from decimal import Decimal
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class ServiceSpec:
|
|
19
|
+
"""One synthetic service. ``id_fields`` carries provider-specific resource-id parts
|
|
20
|
+
(AWS ``arn_kind``; Azure ``arm_type``; GCP ``api``/``collection``) opaquely — only the
|
|
21
|
+
provider's ``resource_id`` callable reads them."""
|
|
22
|
+
|
|
23
|
+
name: str
|
|
24
|
+
category: str
|
|
25
|
+
subcategory: str
|
|
26
|
+
resource_type: str
|
|
27
|
+
sku_meter: str # the SKU "Function" (Compute / Storage / Database / ...)
|
|
28
|
+
pricing_unit: str # FOCUS UnitFormat
|
|
29
|
+
description: str
|
|
30
|
+
unit_price_usd: Decimal
|
|
31
|
+
qty_low: Decimal
|
|
32
|
+
qty_high: Decimal
|
|
33
|
+
name_prefix: str
|
|
34
|
+
granularity: str # "hourly" | "daily" | "monthly"
|
|
35
|
+
zonal: bool
|
|
36
|
+
commitment_eligible: bool
|
|
37
|
+
id_fields: Mapping[str, object] = field(default_factory=dict)
|
|
38
|
+
sku_details: Mapping[str, object] = field(default_factory=dict)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class RowContext:
|
|
43
|
+
"""Identity carried from ``base_row`` into a scenario (the group's billing account)."""
|
|
44
|
+
|
|
45
|
+
billing_id: str
|
|
46
|
+
sub_id: str
|
|
47
|
+
sub_name: str
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class ResourceRef:
|
|
52
|
+
"""Everything a provider's ``resource_id`` callable might need.
|
|
53
|
+
|
|
54
|
+
Each provider reads only the fields it uses (AWS: region + billing account; Azure:
|
|
55
|
+
subscription id + name; GCP: project id), so the callable signature stays uniform and
|
|
56
|
+
the callables never draw from the RNG.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
spec: ServiceSpec
|
|
60
|
+
region_id: str
|
|
61
|
+
region_name: str
|
|
62
|
+
billing_id: str
|
|
63
|
+
sub_id: str
|
|
64
|
+
sub_name: str
|
|
65
|
+
resource_name: str
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class CommitmentModel:
|
|
70
|
+
"""Provider-specific commitment terms and id/sku formats.
|
|
71
|
+
|
|
72
|
+
``spend_based`` is drawn once by the engine (``rng.random() < 0.6``) and passed in, so
|
|
73
|
+
the callables never re-draw the spend/usage split. ``commit_id_before_base_row`` captures
|
|
74
|
+
the one real cross-provider *ordering* difference: AWS draws the commitment id (from the
|
|
75
|
+
region) before the purchase ``base_row``; Azure/GCP draw it after, from the subscription /
|
|
76
|
+
project id. The two RNG-drawing callables own their own hex widths.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
commit_id_before_base_row: bool
|
|
80
|
+
commit_id: Callable[[random.Random, str, str, bool], str] # (rng, region_id, sub_id, spend_based)
|
|
81
|
+
commit_resource_name: Callable[[random.Random, bool], str] # (rng, spend_based)
|
|
82
|
+
purchase_sku_id: Callable[[random.Random], str] # (rng) -> commitment purchase SkuId
|
|
83
|
+
purchase_sku_details: Callable[[bool], str] # (spend_based) -> SkuPriceDetails JSON
|
|
84
|
+
purchase_description: Callable[[str], str] # (commit_type) -> Purchase ChargeDescription
|
|
85
|
+
commit_name: Callable[[bool], str]
|
|
86
|
+
commit_type: Callable[[bool], str] # ResourceType + CommitmentDiscountType
|
|
87
|
+
commit_category: Callable[[bool], str] # "Spend" | "Usage"
|
|
88
|
+
commit_unit: Callable[[bool], str] # "USD" | "Hours"
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class ProviderProfile:
|
|
93
|
+
key: str # "aws" | "azure" | "gcp"
|
|
94
|
+
provider_name: str
|
|
95
|
+
publisher_name: str
|
|
96
|
+
service_provider_name: str # FOCUS 1.3 identity (used only by the 1.3 adapter)
|
|
97
|
+
host_provider_name: str
|
|
98
|
+
invoice_issuer_name: str
|
|
99
|
+
billing_account_type: str
|
|
100
|
+
sub_account_type: str
|
|
101
|
+
split_allocation_description: str # "Shared <service> host cost allocated to workload"
|
|
102
|
+
tag_keys: tuple[str, str, str] # (environment, cost-center, owner) tag key names
|
|
103
|
+
invoice_id: Callable[[str], str] # (billing_id) -> InvoiceId
|
|
104
|
+
services: tuple[ServiceSpec, ...]
|
|
105
|
+
regions: tuple[tuple[str, str, tuple[str, ...]], ...]
|
|
106
|
+
billing_accounts: tuple[tuple[str, str], ...]
|
|
107
|
+
sub_accounts: tuple[tuple[str, str], ...]
|
|
108
|
+
resource_id: Callable[[ResourceRef], str] # pure, never draws
|
|
109
|
+
resource_name: Callable[[random.Random, ServiceSpec], str] # usage/purchase resource name
|
|
110
|
+
committed_resource_name: Callable[[random.Random, ServiceSpec, int], str] # committed-usage (index k)
|
|
111
|
+
sku_id: Callable[[random.Random, ServiceSpec], str]
|
|
112
|
+
sku_price_id: Callable[[random.Random], str]
|
|
113
|
+
allocated_resource_id: Callable[[random.Random, str, RowContext, str], str] # (rng, region_id, ctx, workload)
|
|
114
|
+
commitment: CommitmentModel
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def commitment_service(self) -> ServiceSpec:
|
|
118
|
+
"""The service used for commitment and split-allocation rows (services[0])."""
|
|
119
|
+
return self.services[0]
|
|
120
|
+
|
|
121
|
+
@property
|
|
122
|
+
def allowed_subcategories(self) -> frozenset[str]:
|
|
123
|
+
return frozenset(spec.subcategory for spec in self.services)
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Coherent, provider-agnostic FOCUS scenario generators (P1.8 / P1.9 / P1.10).
|
|
2
|
+
|
|
3
|
+
Unlike the large per-provider generators (which emit whole Cost and Usage files), these build
|
|
4
|
+
small, *self-consistent* scenarios that exercise the cross-dataset validators on realistic —
|
|
5
|
+
not random — data:
|
|
6
|
+
|
|
7
|
+
* :func:`split_allocation_group` — a Split Cost Allocation group whose ratios sum to exactly 1
|
|
8
|
+
and whose allocated costs sum to exactly the origin charge (the last consumer absorbs the
|
|
9
|
+
rounding residue), so it passes :func:`~focus_data_toolkit.validate.allocation.validate_split_allocation`.
|
|
10
|
+
* :func:`correction_set` — an original charge plus a signed correction line
|
|
11
|
+
(``ChargeClass="Correction"``) that references it and records the auditable net charge.
|
|
12
|
+
* :func:`billing_lifecycle_instances` — a T0→T6 sequence of
|
|
13
|
+
:class:`~focus_data_toolkit.lifecycle.DatasetInstance` snapshots (period opens, fills, closes;
|
|
14
|
+
invoice is issued, voided, replaced) with only *allowed* status transitions.
|
|
15
|
+
|
|
16
|
+
Everything is deterministic (no RNG, no clock): the same arguments always yield the same rows,
|
|
17
|
+
so the scenarios are byte-reproducible and safe to snapshot in tests.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from collections.abc import Sequence
|
|
23
|
+
from decimal import Decimal
|
|
24
|
+
|
|
25
|
+
from focus_data_toolkit.generators.engine.json_focus import allocated_method_details
|
|
26
|
+
from focus_data_toolkit.lifecycle import DatasetInstance
|
|
27
|
+
|
|
28
|
+
_RATIO_QUANTUM = Decimal("0.000001")
|
|
29
|
+
_COST_QUANTUM = Decimal("0.000001")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def split_allocation_group(
|
|
33
|
+
origin_id: str,
|
|
34
|
+
origin_cost: str | Decimal,
|
|
35
|
+
weights: Sequence[float | int | str | Decimal],
|
|
36
|
+
*,
|
|
37
|
+
method: str = "split-proportional",
|
|
38
|
+
unit: str = "Hours",
|
|
39
|
+
resource_prefix: str = "res",
|
|
40
|
+
) -> list[dict[str, str]]:
|
|
41
|
+
"""Build a valid Split Cost Allocation group distributing ``origin_cost`` over ``weights``.
|
|
42
|
+
|
|
43
|
+
Ratios sum to exactly 1 and allocated ``BilledCost`` sums to exactly ``origin_cost``: each
|
|
44
|
+
consumer gets ``weight/total`` (quantised), and the **last** consumer takes the residue so
|
|
45
|
+
no cents are created or lost. ``weights`` may encode equal (all 1), proportional, or
|
|
46
|
+
weighted allocation, and may be negative for a correction that redistributes a credit.
|
|
47
|
+
"""
|
|
48
|
+
if len(weights) < 1:
|
|
49
|
+
raise ValueError("an allocation group needs at least one consumer")
|
|
50
|
+
origin = Decimal(str(origin_cost))
|
|
51
|
+
decimal_weights = [Decimal(str(w)) for w in weights]
|
|
52
|
+
total = sum(decimal_weights, Decimal(0))
|
|
53
|
+
if total == 0:
|
|
54
|
+
raise ValueError("allocation weights sum to zero; cannot form ratios")
|
|
55
|
+
|
|
56
|
+
ratios: list[Decimal] = []
|
|
57
|
+
costs: list[Decimal] = []
|
|
58
|
+
acc_ratio = Decimal(0)
|
|
59
|
+
acc_cost = Decimal(0)
|
|
60
|
+
last = len(decimal_weights) - 1
|
|
61
|
+
for i, w in enumerate(decimal_weights):
|
|
62
|
+
if i < last:
|
|
63
|
+
ratio = (w / total).quantize(_RATIO_QUANTUM)
|
|
64
|
+
cost = (origin * w / total).quantize(_COST_QUANTUM)
|
|
65
|
+
acc_ratio += ratio
|
|
66
|
+
acc_cost += cost
|
|
67
|
+
else: # last consumer absorbs the residue so both sums stay exact
|
|
68
|
+
ratio = Decimal(1) - acc_ratio
|
|
69
|
+
cost = origin - acc_cost
|
|
70
|
+
ratios.append(ratio)
|
|
71
|
+
costs.append(cost)
|
|
72
|
+
|
|
73
|
+
return [
|
|
74
|
+
{
|
|
75
|
+
"x_SplitOriginId": origin_id,
|
|
76
|
+
"x_SplitOriginCost": str(origin),
|
|
77
|
+
"AllocatedMethodId": method,
|
|
78
|
+
"AllocatedResourceId": f"{resource_prefix}-{i}",
|
|
79
|
+
"AllocatedMethodDetails": allocated_method_details(
|
|
80
|
+
[{"AllocatedRatio": str(ratios[i]), "UsageUnit": unit, "UsageQuantity": "1"}]
|
|
81
|
+
),
|
|
82
|
+
"BilledCost": str(costs[i]),
|
|
83
|
+
"ChargeCategory": "Usage",
|
|
84
|
+
}
|
|
85
|
+
for i in range(len(decimal_weights))
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def correction_set(
|
|
90
|
+
charge_key: str,
|
|
91
|
+
original_cost: str | Decimal,
|
|
92
|
+
corrections: Sequence[str | Decimal],
|
|
93
|
+
*,
|
|
94
|
+
invoice_id: str = "INV-1",
|
|
95
|
+
currency: str = "USD",
|
|
96
|
+
charge_category: str = "Usage",
|
|
97
|
+
) -> list[dict[str, str]]:
|
|
98
|
+
"""An original charge plus one signed correction line per entry in ``corrections``.
|
|
99
|
+
|
|
100
|
+
The original keeps its own ``x_ChargeKey`` (never overwritten); each correction is a
|
|
101
|
+
``ChargeClass="Correction"`` line that references the original via ``x_CorrectionOf`` and
|
|
102
|
+
records the running auditable net (``x_NetCharge`` = original + corrections so far). The net
|
|
103
|
+
of the whole set is therefore verifiable and no history is destroyed.
|
|
104
|
+
"""
|
|
105
|
+
original = Decimal(str(original_cost))
|
|
106
|
+
rows = [
|
|
107
|
+
{
|
|
108
|
+
"x_ChargeKey": charge_key,
|
|
109
|
+
"ChargeClass": "",
|
|
110
|
+
"ChargeCategory": charge_category,
|
|
111
|
+
"BilledCost": str(original),
|
|
112
|
+
"InvoiceId": invoice_id,
|
|
113
|
+
"BillingCurrency": currency,
|
|
114
|
+
}
|
|
115
|
+
]
|
|
116
|
+
running = original
|
|
117
|
+
for n, amount in enumerate(corrections, start=1):
|
|
118
|
+
delta = Decimal(str(amount))
|
|
119
|
+
running += delta
|
|
120
|
+
rows.append(
|
|
121
|
+
{
|
|
122
|
+
"x_ChargeKey": f"{charge_key}-c{n}",
|
|
123
|
+
"x_CorrectionOf": charge_key,
|
|
124
|
+
"ChargeClass": "Correction",
|
|
125
|
+
"ChargeCategory": "Credit" if delta < 0 else charge_category,
|
|
126
|
+
"BilledCost": str(delta),
|
|
127
|
+
"x_NetCharge": str(running),
|
|
128
|
+
"InvoiceId": invoice_id,
|
|
129
|
+
"BillingCurrency": currency,
|
|
130
|
+
}
|
|
131
|
+
)
|
|
132
|
+
return rows
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def billing_lifecycle_instances(
|
|
136
|
+
*,
|
|
137
|
+
invoice_id: str = "INV-1",
|
|
138
|
+
period_start: str = "2026-01-01T00:00:00Z",
|
|
139
|
+
period_end: str = "2026-02-01T00:00:00Z",
|
|
140
|
+
replacement_invoice_id: str = "INV-1-R1",
|
|
141
|
+
) -> list[DatasetInstance]:
|
|
142
|
+
"""A canonical T0→T6 billing lifecycle as a sequence of dataset-instance snapshots.
|
|
143
|
+
|
|
144
|
+
T0 period opens (incomplete) → T1 more Cost and Usage arrives → T2 period closes → T3
|
|
145
|
+
invoice issued → T4 late correction recorded → T5 invoice voided → T6 replacement issued.
|
|
146
|
+
Only allowed status transitions occur; each snapshot links to the previous one.
|
|
147
|
+
"""
|
|
148
|
+
steps = [
|
|
149
|
+
("t0", "Billing Period", False, "Open", None),
|
|
150
|
+
("t1", "Billing Period", False, "Open", "t0"),
|
|
151
|
+
("t2", "Billing Period", True, "Closed", "t1"),
|
|
152
|
+
("t3", "Invoice Detail", True, "Issued", "t2"),
|
|
153
|
+
("t4", "Invoice Detail", True, "Issued", "t3"), # correction recorded, still issued
|
|
154
|
+
("t5", "Invoice Detail", True, "Voided", "t4"),
|
|
155
|
+
("t6", "Invoice Detail", True, "Issued", "t5"), # replacement invoice issued
|
|
156
|
+
]
|
|
157
|
+
instances: list[DatasetInstance] = []
|
|
158
|
+
for order, (instance_id, dataset, complete, status, prev) in enumerate(steps):
|
|
159
|
+
inv = replacement_invoice_id if instance_id == "t6" else invoice_id
|
|
160
|
+
instances.append(
|
|
161
|
+
DatasetInstance(
|
|
162
|
+
instance_id=instance_id,
|
|
163
|
+
dataset=dataset,
|
|
164
|
+
subject_id=inv,
|
|
165
|
+
order=order,
|
|
166
|
+
created=period_start,
|
|
167
|
+
last_updated=period_end,
|
|
168
|
+
complete=complete,
|
|
169
|
+
period_start=period_start,
|
|
170
|
+
period_end=period_end,
|
|
171
|
+
status=status,
|
|
172
|
+
previous_instance_id=prev,
|
|
173
|
+
)
|
|
174
|
+
)
|
|
175
|
+
return instances
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
__all__ = ["billing_lifecycle_instances", "correction_set", "split_allocation_group"]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""FOCUS version adapters (1.2, 1.3).
|
|
2
|
+
|
|
3
|
+
An adapter owns everything that differs between FOCUS versions: the column set, the optional
|
|
4
|
+
Contract Commitment dataset, the scenario ladder, and the small per-version field hooks
|
|
5
|
+
(the 1.3 provider-identity split, ``PricingCurrencyEffectiveCost`` on Tax/Credit, and the
|
|
6
|
+
``ContractApplied`` link). Providers are version-agnostic; adapters are provider-agnostic.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from focus_data_toolkit.generators.versions.adapter import LadderBranch, VersionAdapter
|
|
12
|
+
from focus_data_toolkit.generators.versions.v1_2 import V12
|
|
13
|
+
from focus_data_toolkit.generators.versions.v1_3 import V13
|
|
14
|
+
|
|
15
|
+
ADAPTERS: dict[str, VersionAdapter] = {V12.version: V12, V13.version: V13}
|
|
16
|
+
|
|
17
|
+
__all__ = ["ADAPTERS", "LadderBranch", "V12", "V13", "VersionAdapter"]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""The ``VersionAdapter`` strategy object and the scenario-ladder description."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
from focus_data_toolkit.generators.providers.profile import ProviderProfile
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass(frozen=True)
|
|
12
|
+
class LadderBranch:
|
|
13
|
+
"""One branch of the scenario-selection ladder, evaluated in order.
|
|
14
|
+
|
|
15
|
+
Mirrors the historical ``if/elif`` chain exactly: the first branch whose threshold the
|
|
16
|
+
roll falls under is chosen (``requires_credits`` branches are skipped unless credits are
|
|
17
|
+
enabled); if that branch's ``min_remaining`` guard fails the row falls through to Usage.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
kind: str # "credit" | "tax" | "purchase" | "split" | "commitment"
|
|
21
|
+
threshold: float
|
|
22
|
+
requires_credits: bool = False
|
|
23
|
+
min_remaining: int | None = None
|
|
24
|
+
group: bool = False # True if the builder returns multiple rows (commitment)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True)
|
|
28
|
+
class VersionAdapter:
|
|
29
|
+
version: str # "1.2" | "1.3"
|
|
30
|
+
default_seed: int
|
|
31
|
+
columns: tuple[str, ...]
|
|
32
|
+
contract_commitment_columns: tuple[str, ...] | None
|
|
33
|
+
ladder: tuple[LadderBranch, ...]
|
|
34
|
+
# Keys copied verbatim from a commitment Purchase row onto every linked usage row.
|
|
35
|
+
commitment_identity_keys: tuple[str, ...]
|
|
36
|
+
emits_split_allocation: bool
|
|
37
|
+
# Per-version field hooks (no-ops in 1.2):
|
|
38
|
+
fill_version_identity: Callable[[dict, ProviderProfile], None] # (row, profile)
|
|
39
|
+
on_tax_row: Callable[[dict, str], None] # (row, amount_str)
|
|
40
|
+
on_credit_row: Callable[[dict, str], None] # (row, negative_str)
|
|
41
|
+
on_commit_usage: Callable[[dict, str, str, str], None] # (usage, commit_id, contract_id, effective_str)
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""FOCUS 1.2 adapter: 57-column Cost and Usage, no Contract Commitment dataset."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from focus_data_toolkit.generators.versions.adapter import LadderBranch, VersionAdapter
|
|
6
|
+
|
|
7
|
+
COLUMNS: tuple[str, ...] = (
|
|
8
|
+
"ProviderName",
|
|
9
|
+
"PublisherName",
|
|
10
|
+
"InvoiceIssuerName",
|
|
11
|
+
"InvoiceId",
|
|
12
|
+
"BillingAccountId",
|
|
13
|
+
"BillingAccountName",
|
|
14
|
+
"BillingAccountType",
|
|
15
|
+
"SubAccountId",
|
|
16
|
+
"SubAccountName",
|
|
17
|
+
"SubAccountType",
|
|
18
|
+
"BillingPeriodStart",
|
|
19
|
+
"BillingPeriodEnd",
|
|
20
|
+
"ChargePeriodStart",
|
|
21
|
+
"ChargePeriodEnd",
|
|
22
|
+
"ChargeCategory",
|
|
23
|
+
"ChargeClass",
|
|
24
|
+
"ChargeDescription",
|
|
25
|
+
"ChargeFrequency",
|
|
26
|
+
"BilledCost",
|
|
27
|
+
"EffectiveCost",
|
|
28
|
+
"ListCost",
|
|
29
|
+
"ContractedCost",
|
|
30
|
+
"ListUnitPrice",
|
|
31
|
+
"ContractedUnitPrice",
|
|
32
|
+
"PricingCategory",
|
|
33
|
+
"PricingQuantity",
|
|
34
|
+
"PricingUnit",
|
|
35
|
+
"PricingCurrency",
|
|
36
|
+
"PricingCurrencyContractedUnitPrice",
|
|
37
|
+
"PricingCurrencyEffectiveCost",
|
|
38
|
+
"PricingCurrencyListUnitPrice",
|
|
39
|
+
"BillingCurrency",
|
|
40
|
+
"ConsumedQuantity",
|
|
41
|
+
"ConsumedUnit",
|
|
42
|
+
"ServiceName",
|
|
43
|
+
"ServiceCategory",
|
|
44
|
+
"ServiceSubcategory",
|
|
45
|
+
"SkuId",
|
|
46
|
+
"SkuMeter",
|
|
47
|
+
"SkuPriceId",
|
|
48
|
+
"SkuPriceDetails",
|
|
49
|
+
"ResourceId",
|
|
50
|
+
"ResourceName",
|
|
51
|
+
"ResourceType",
|
|
52
|
+
"RegionId",
|
|
53
|
+
"RegionName",
|
|
54
|
+
"AvailabilityZone",
|
|
55
|
+
"CommitmentDiscountId",
|
|
56
|
+
"CommitmentDiscountName",
|
|
57
|
+
"CommitmentDiscountCategory",
|
|
58
|
+
"CommitmentDiscountType",
|
|
59
|
+
"CommitmentDiscountStatus",
|
|
60
|
+
"CommitmentDiscountQuantity",
|
|
61
|
+
"CommitmentDiscountUnit",
|
|
62
|
+
"CapacityReservationId",
|
|
63
|
+
"CapacityReservationStatus",
|
|
64
|
+
"Tags",
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
assert len(COLUMNS) == 57, f"FOCUS 1.2 must have 57 columns, got {len(COLUMNS)}"
|
|
68
|
+
assert len(set(COLUMNS)) == 57, "FOCUS 1.2 column names must be unique"
|
|
69
|
+
|
|
70
|
+
# In 1.2 the commitment usage rows copied only the account ids from the purchase.
|
|
71
|
+
_COMMITMENT_IDENTITY_KEYS: tuple[str, ...] = (
|
|
72
|
+
"BillingAccountId",
|
|
73
|
+
"SubAccountId",
|
|
74
|
+
"SubAccountName",
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _noop_identity(row: dict, profile: object) -> None:
|
|
79
|
+
"""1.2 has no ServiceProviderName/HostProviderName and does not seed PricingCurrency."""
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _noop_tax(row: dict, amount_str: str) -> None:
|
|
83
|
+
"""1.2 leaves PricingCurrencyEffectiveCost null on Tax."""
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _noop_credit(row: dict, negative_str: str) -> None:
|
|
87
|
+
"""1.2 leaves PricingCurrencyEffectiveCost null on Credit."""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _noop_commit_usage(usage: dict, commit_id: str, contract_id: str, effective_str: str) -> None:
|
|
91
|
+
"""1.2 has no ContractApplied column."""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
V12 = VersionAdapter(
|
|
95
|
+
version="1.2",
|
|
96
|
+
default_seed=1202,
|
|
97
|
+
columns=COLUMNS,
|
|
98
|
+
contract_commitment_columns=None,
|
|
99
|
+
ladder=(
|
|
100
|
+
LadderBranch("credit", 0.05, requires_credits=True),
|
|
101
|
+
LadderBranch("tax", 0.12),
|
|
102
|
+
LadderBranch("purchase", 0.20),
|
|
103
|
+
LadderBranch("commitment", 0.45, min_remaining=6, group=True),
|
|
104
|
+
),
|
|
105
|
+
commitment_identity_keys=_COMMITMENT_IDENTITY_KEYS,
|
|
106
|
+
emits_split_allocation=False,
|
|
107
|
+
fill_version_identity=_noop_identity,
|
|
108
|
+
on_tax_row=_noop_tax,
|
|
109
|
+
on_credit_row=_noop_credit,
|
|
110
|
+
on_commit_usage=_noop_commit_usage,
|
|
111
|
+
)
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""FOCUS 1.3 adapter: 65-column Cost and Usage + 13-column Contract Commitment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from focus_data_toolkit.generators.engine.json_focus import contract_applied
|
|
6
|
+
from focus_data_toolkit.generators.providers.profile import ProviderProfile
|
|
7
|
+
from focus_data_toolkit.generators.versions.adapter import LadderBranch, VersionAdapter
|
|
8
|
+
|
|
9
|
+
COLUMNS: tuple[str, ...] = (
|
|
10
|
+
"ProviderName",
|
|
11
|
+
"PublisherName",
|
|
12
|
+
"ServiceProviderName",
|
|
13
|
+
"HostProviderName",
|
|
14
|
+
"InvoiceIssuerName",
|
|
15
|
+
"InvoiceId",
|
|
16
|
+
"BillingAccountId",
|
|
17
|
+
"BillingAccountName",
|
|
18
|
+
"BillingAccountType",
|
|
19
|
+
"SubAccountId",
|
|
20
|
+
"SubAccountName",
|
|
21
|
+
"SubAccountType",
|
|
22
|
+
"BillingPeriodStart",
|
|
23
|
+
"BillingPeriodEnd",
|
|
24
|
+
"ChargePeriodStart",
|
|
25
|
+
"ChargePeriodEnd",
|
|
26
|
+
"ChargeCategory",
|
|
27
|
+
"ChargeClass",
|
|
28
|
+
"ChargeDescription",
|
|
29
|
+
"ChargeFrequency",
|
|
30
|
+
"BilledCost",
|
|
31
|
+
"EffectiveCost",
|
|
32
|
+
"ListCost",
|
|
33
|
+
"ContractedCost",
|
|
34
|
+
"ListUnitPrice",
|
|
35
|
+
"ContractedUnitPrice",
|
|
36
|
+
"PricingCategory",
|
|
37
|
+
"PricingQuantity",
|
|
38
|
+
"PricingUnit",
|
|
39
|
+
"PricingCurrency",
|
|
40
|
+
"PricingCurrencyContractedUnitPrice",
|
|
41
|
+
"PricingCurrencyEffectiveCost",
|
|
42
|
+
"PricingCurrencyListUnitPrice",
|
|
43
|
+
"BillingCurrency",
|
|
44
|
+
"ConsumedQuantity",
|
|
45
|
+
"ConsumedUnit",
|
|
46
|
+
"ServiceName",
|
|
47
|
+
"ServiceCategory",
|
|
48
|
+
"ServiceSubcategory",
|
|
49
|
+
"SkuId",
|
|
50
|
+
"SkuMeter",
|
|
51
|
+
"SkuPriceId",
|
|
52
|
+
"SkuPriceDetails",
|
|
53
|
+
"ResourceId",
|
|
54
|
+
"ResourceName",
|
|
55
|
+
"ResourceType",
|
|
56
|
+
"RegionId",
|
|
57
|
+
"RegionName",
|
|
58
|
+
"AvailabilityZone",
|
|
59
|
+
"CommitmentDiscountId",
|
|
60
|
+
"CommitmentDiscountName",
|
|
61
|
+
"CommitmentDiscountCategory",
|
|
62
|
+
"CommitmentDiscountType",
|
|
63
|
+
"CommitmentDiscountStatus",
|
|
64
|
+
"CommitmentDiscountQuantity",
|
|
65
|
+
"CommitmentDiscountUnit",
|
|
66
|
+
"ContractApplied",
|
|
67
|
+
"CapacityReservationId",
|
|
68
|
+
"CapacityReservationStatus",
|
|
69
|
+
"AllocatedMethodId",
|
|
70
|
+
"AllocatedMethodDetails",
|
|
71
|
+
"AllocatedResourceId",
|
|
72
|
+
"AllocatedResourceName",
|
|
73
|
+
"AllocatedTags",
|
|
74
|
+
"Tags",
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
assert len(COLUMNS) == 65, f"FOCUS 1.3 Cost and Usage must have 65 columns, got {len(COLUMNS)}"
|
|
78
|
+
assert len(set(COLUMNS)) == 65, "FOCUS 1.3 Cost and Usage column names must be unique"
|
|
79
|
+
|
|
80
|
+
CONTRACT_COMMITMENT_COLUMNS: tuple[str, ...] = (
|
|
81
|
+
"ContractCommitmentId",
|
|
82
|
+
"ContractCommitmentType",
|
|
83
|
+
"ContractCommitmentCategory",
|
|
84
|
+
"ContractCommitmentCost",
|
|
85
|
+
"ContractCommitmentQuantity",
|
|
86
|
+
"ContractCommitmentUnit",
|
|
87
|
+
"ContractCommitmentDescription",
|
|
88
|
+
"ContractCommitmentPeriodStart",
|
|
89
|
+
"ContractCommitmentPeriodEnd",
|
|
90
|
+
"ContractId",
|
|
91
|
+
"ContractPeriodStart",
|
|
92
|
+
"ContractPeriodEnd",
|
|
93
|
+
"BillingCurrency",
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
assert len(CONTRACT_COMMITMENT_COLUMNS) == 13, (
|
|
97
|
+
f"FOCUS 1.3 Contract Commitment must have 13 columns, got {len(CONTRACT_COMMITMENT_COLUMNS)}"
|
|
98
|
+
)
|
|
99
|
+
assert len(set(CONTRACT_COMMITMENT_COLUMNS)) == 13, (
|
|
100
|
+
"FOCUS 1.3 Contract Commitment column names must be unique"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# In 1.3 every linked usage row copies the full billing identity of the commitment.
|
|
104
|
+
_COMMITMENT_IDENTITY_KEYS: tuple[str, ...] = (
|
|
105
|
+
"BillingAccountId",
|
|
106
|
+
"BillingAccountName",
|
|
107
|
+
"BillingAccountType",
|
|
108
|
+
"SubAccountId",
|
|
109
|
+
"SubAccountName",
|
|
110
|
+
"SubAccountType",
|
|
111
|
+
"InvoiceId",
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _fill_identity(row: dict, profile: ProviderProfile) -> None:
|
|
116
|
+
row["ServiceProviderName"] = profile.service_provider_name
|
|
117
|
+
row["HostProviderName"] = profile.host_provider_name
|
|
118
|
+
# Multi-currency generator: PricingCurrency is never null (_set_currency overrides it for
|
|
119
|
+
# priced rows; Tax/Credit keep this USD default).
|
|
120
|
+
row["PricingCurrency"] = "USD"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _on_tax(row: dict, amount_str: str) -> None:
|
|
124
|
+
row["PricingCurrencyEffectiveCost"] = amount_str
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _on_credit(row: dict, negative_str: str) -> None:
|
|
128
|
+
row["PricingCurrencyEffectiveCost"] = negative_str
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _on_commit_usage(usage: dict, commit_id: str, contract_id: str, effective_str: str) -> None:
|
|
132
|
+
# FOCUS 1.3 ContractApplied: the JSON link to the Contract Commitment dataset.
|
|
133
|
+
usage["ContractApplied"] = contract_applied(commit_id, contract_id, effective_str, "1.0000", "Hours")
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
V13 = VersionAdapter(
|
|
137
|
+
version="1.3",
|
|
138
|
+
default_seed=1302,
|
|
139
|
+
columns=COLUMNS,
|
|
140
|
+
contract_commitment_columns=CONTRACT_COMMITMENT_COLUMNS,
|
|
141
|
+
ladder=(
|
|
142
|
+
LadderBranch("credit", 0.05, requires_credits=True),
|
|
143
|
+
LadderBranch("tax", 0.12),
|
|
144
|
+
LadderBranch("purchase", 0.20),
|
|
145
|
+
LadderBranch("split", 0.40),
|
|
146
|
+
LadderBranch("commitment", 0.58, min_remaining=6, group=True),
|
|
147
|
+
),
|
|
148
|
+
commitment_identity_keys=_COMMITMENT_IDENTITY_KEYS,
|
|
149
|
+
emits_split_allocation=True,
|
|
150
|
+
fill_version_identity=_fill_identity,
|
|
151
|
+
on_tax_row=_on_tax,
|
|
152
|
+
on_credit_row=_on_credit,
|
|
153
|
+
on_commit_usage=_on_commit_usage,
|
|
154
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Streaming/atomic I/O layer (P1.5/P1.6/P1.7)."""
|