focus-data-toolkit 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- focus_data_toolkit/__init__.py +69 -0
- focus_data_toolkit/__main__.py +6 -0
- focus_data_toolkit/_version.py +8 -0
- focus_data_toolkit/cli.py +968 -0
- focus_data_toolkit/context/__init__.py +88 -0
- focus_data_toolkit/context/billing.py +54 -0
- focus_data_toolkit/context/provider.py +90 -0
- focus_data_toolkit/convert/__init__.py +708 -0
- focus_data_toolkit/convert/billing_period.py +65 -0
- focus_data_toolkit/convert/contract_applied.py +235 -0
- focus_data_toolkit/convert/contract_commitment.py +182 -0
- focus_data_toolkit/convert/cost_and_usage.py +179 -0
- focus_data_toolkit/convert/detect.py +39 -0
- focus_data_toolkit/convert/invoice_detail.py +199 -0
- focus_data_toolkit/convert/streaming.py +1030 -0
- focus_data_toolkit/errors.py +145 -0
- focus_data_toolkit/focus_json.py +68 -0
- focus_data_toolkit/generators/__init__.py +61 -0
- focus_data_toolkit/generators/_shim.py +43 -0
- focus_data_toolkit/generators/engine/__init__.py +14 -0
- focus_data_toolkit/generators/engine/context.py +12 -0
- focus_data_toolkit/generators/engine/determinism.py +117 -0
- focus_data_toolkit/generators/engine/json_focus.py +63 -0
- focus_data_toolkit/generators/engine/ladder.py +71 -0
- focus_data_toolkit/generators/engine/scenarios_core.py +380 -0
- focus_data_toolkit/generators/engine/serialize.py +151 -0
- focus_data_toolkit/generators/generate_aws_focus_1_2.py +19 -0
- focus_data_toolkit/generators/generate_aws_focus_1_3.py +20 -0
- focus_data_toolkit/generators/generate_azure_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_azure_focus_1_3.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_2.py +17 -0
- focus_data_toolkit/generators/generate_gcp_focus_1_3.py +17 -0
- focus_data_toolkit/generators/providers/__init__.py +29 -0
- focus_data_toolkit/generators/providers/aws.py +186 -0
- focus_data_toolkit/generators/providers/azure.py +191 -0
- focus_data_toolkit/generators/providers/gcp.py +194 -0
- focus_data_toolkit/generators/providers/profile.py +123 -0
- focus_data_toolkit/generators/scenarios.py +178 -0
- focus_data_toolkit/generators/versions/__init__.py +17 -0
- focus_data_toolkit/generators/versions/adapter.py +41 -0
- focus_data_toolkit/generators/versions/v1_2.py +111 -0
- focus_data_toolkit/generators/versions/v1_3.py +154 -0
- focus_data_toolkit/io/__init__.py +1 -0
- focus_data_toolkit/io/atomic_writer.py +462 -0
- focus_data_toolkit/io/csv_io.py +128 -0
- focus_data_toolkit/io/parquet_io.py +528 -0
- focus_data_toolkit/io/records.py +92 -0
- focus_data_toolkit/io/row_source.py +117 -0
- focus_data_toolkit/lifecycle.py +342 -0
- focus_data_toolkit/manifest.py +114 -0
- focus_data_toolkit/model/__init__.py +43 -0
- focus_data_toolkit/model/capabilities.py +66 -0
- focus_data_toolkit/model/focus_1_4_decimal_scale.json +10 -0
- focus_data_toolkit/model/focus_1_4_model.json +1913 -0
- focus_data_toolkit/model/focus_1_4_servicesubcategory.json +84 -0
- focus_data_toolkit/model/focus_json_keys.py +112 -0
- focus_data_toolkit/model/iso_4217_currencies.json +23 -0
- focus_data_toolkit/model/json_schema_check.py +205 -0
- focus_data_toolkit/model/json_schemas/allocatedmethoddetailsobjectschema.json +82 -0
- focus_data_toolkit/model/json_schemas/commitmentprogrameligibilitydetailsobjectschema.json +41 -0
- focus_data_toolkit/model/json_schemas/contractappliedobjectschema.json +104 -0
- focus_data_toolkit/model/json_schemas/contractcommitmentapplicabilityobjectschema.json +290 -0
- focus_data_toolkit/model/json_schemas/json_schemas_provenance.json +38 -0
- focus_data_toolkit/model/model_provenance.json +58 -0
- focus_data_toolkit/model/validator.py +498 -0
- focus_data_toolkit/modes.py +18 -0
- focus_data_toolkit/official_validator.py +61 -0
- focus_data_toolkit/progress.py +89 -0
- focus_data_toolkit/provenance.py +106 -0
- focus_data_toolkit/py.typed +1 -0
- focus_data_toolkit/runtime.py +243 -0
- focus_data_toolkit/schema/__init__.py +17 -0
- focus_data_toolkit/schema/detection.py +274 -0
- focus_data_toolkit/schema/registry.py +127 -0
- focus_data_toolkit/storage/__init__.py +1 -0
- focus_data_toolkit/storage/external_index.py +99 -0
- focus_data_toolkit/storage/spill.py +150 -0
- focus_data_toolkit/studio/__init__.py +19 -0
- focus_data_toolkit/studio/app.py +467 -0
- focus_data_toolkit/studio/config.py +42 -0
- focus_data_toolkit/studio/frontend/app.js +214 -0
- focus_data_toolkit/studio/frontend/index.html +101 -0
- focus_data_toolkit/studio/frontend/style.css +60 -0
- focus_data_toolkit/studio/jobs.py +142 -0
- focus_data_toolkit/studio/preview.py +32 -0
- focus_data_toolkit/studio/security.py +125 -0
- focus_data_toolkit/studio/server.py +71 -0
- focus_data_toolkit/supplement/__init__.py +50 -0
- focus_data_toolkit/supplement/adapters/__init__.py +21 -0
- focus_data_toolkit/supplement/adapters/adapters_provenance.json +39 -0
- focus_data_toolkit/supplement/adapters/aws_invoice_summary.json +24 -0
- focus_data_toolkit/supplement/adapters/aws_savings_plans.json +31 -0
- focus_data_toolkit/supplement/adapters/azure_invoice.json +25 -0
- focus_data_toolkit/supplement/adapters/gcp_compute_commitments.json +28 -0
- focus_data_toolkit/supplement/adapters/registry.py +215 -0
- focus_data_toolkit/supplement/apply.py +318 -0
- focus_data_toolkit/supplement/gaps.py +219 -0
- focus_data_toolkit/supplement/kinds.py +118 -0
- focus_data_toolkit/supplement/loader.py +409 -0
- focus_data_toolkit/supplement/spec.py +74 -0
- focus_data_toolkit/supplement/validate.py +215 -0
- focus_data_toolkit/validate/__init__.py +15 -0
- focus_data_toolkit/validate/allocation.py +333 -0
- focus_data_toolkit/validate/bundle.py +254 -0
- focus_data_toolkit/validate/codes.py +93 -0
- focus_data_toolkit/validate/corrections.py +245 -0
- focus_data_toolkit/validate/reconciliation.py +98 -0
- focus_data_toolkit/validate/referential.py +289 -0
- focus_data_toolkit-0.11.0.dist-info/METADATA +519 -0
- focus_data_toolkit-0.11.0.dist-info/RECORD +116 -0
- focus_data_toolkit-0.11.0.dist-info/WHEEL +5 -0
- focus_data_toolkit-0.11.0.dist-info/entry_points.txt +2 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSE +21 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/LICENSES/CC-BY-4.0.txt +156 -0
- focus_data_toolkit-0.11.0.dist-info/licenses/NOTICE +60 -0
- focus_data_toolkit-0.11.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
{
|
|
2
|
+
"AI Platforms": "AI and Machine Learning",
|
|
3
|
+
"API Management": "Integration",
|
|
4
|
+
"Analytics Platforms": "Analytics",
|
|
5
|
+
"Application Networking": "Networking",
|
|
6
|
+
"Application Platforms": "Web",
|
|
7
|
+
"Architecture": "Management and Governance",
|
|
8
|
+
"Backup Storage": "Storage",
|
|
9
|
+
"Block Storage": "Storage",
|
|
10
|
+
"Bots": "AI and Machine Learning",
|
|
11
|
+
"Business Intelligence": "Analytics",
|
|
12
|
+
"Caching": "Databases",
|
|
13
|
+
"Compliance": "Management and Governance",
|
|
14
|
+
"Containers": "Compute",
|
|
15
|
+
"Content Creation": "Media",
|
|
16
|
+
"Content Delivery": "Networking",
|
|
17
|
+
"Continuous Integration and Deployment": "Developer Tools",
|
|
18
|
+
"Cost Management": "Management and Governance",
|
|
19
|
+
"Data Governance": "Management and Governance",
|
|
20
|
+
"Data Migration": "Migration",
|
|
21
|
+
"Data Processing": "Analytics",
|
|
22
|
+
"Data Warehouses": "Databases",
|
|
23
|
+
"Developer Platforms": "Developer Tools",
|
|
24
|
+
"Development Environments": "Developer Tools",
|
|
25
|
+
"Disaster Recovery": "Management and Governance",
|
|
26
|
+
"End User Computing": "Compute",
|
|
27
|
+
"Endpoint Management": "Management and Governance",
|
|
28
|
+
"File Storage": "Storage",
|
|
29
|
+
"Gaming": "Media",
|
|
30
|
+
"Generative AI": "AI and Machine Learning",
|
|
31
|
+
"Identity and Access Management": "Identity",
|
|
32
|
+
"IoT Analytics": "Internet of Things",
|
|
33
|
+
"IoT Platforms": "Internet of Things",
|
|
34
|
+
"Ledger Databases": "Databases",
|
|
35
|
+
"Machine Learning": "AI and Machine Learning",
|
|
36
|
+
"Media Streaming": "Media",
|
|
37
|
+
"Messaging": "Integration",
|
|
38
|
+
"Mixed Reality": "Media",
|
|
39
|
+
"Multicloud Integration": "Multicloud",
|
|
40
|
+
"Natural Language Processing": "AI and Machine Learning",
|
|
41
|
+
"Network Connectivity": "Networking",
|
|
42
|
+
"Network Infrastructure": "Networking",
|
|
43
|
+
"Network Routing": "Networking",
|
|
44
|
+
"Network Security": "Networking",
|
|
45
|
+
"NoSQL Databases": "Databases",
|
|
46
|
+
"Object Storage": "Storage",
|
|
47
|
+
"Observability": "Management and Governance",
|
|
48
|
+
"Other (AI and Machine Learning)": "AI and Machine Learning",
|
|
49
|
+
"Other (Analytics)": "Analytics",
|
|
50
|
+
"Other (Business Applications)": "Business Applications",
|
|
51
|
+
"Other (Compute)": "Compute",
|
|
52
|
+
"Other (Databases)": "Databases",
|
|
53
|
+
"Other (Developer Tools)": "Developer Tools",
|
|
54
|
+
"Other (Identity)": "Identity",
|
|
55
|
+
"Other (Integration)": "Integration",
|
|
56
|
+
"Other (Internet of Things)": "Internet of Things",
|
|
57
|
+
"Other (Management and Governance)": "Management and Governance",
|
|
58
|
+
"Other (Media)": "Media",
|
|
59
|
+
"Other (Migration)": "Migration",
|
|
60
|
+
"Other (Mobile)": "Mobile",
|
|
61
|
+
"Other (Multicloud)": "Multicloud",
|
|
62
|
+
"Other (Networking)": "Networking",
|
|
63
|
+
"Other (Other)": "Other",
|
|
64
|
+
"Other (Security)": "Security",
|
|
65
|
+
"Other (Storage)": "Storage",
|
|
66
|
+
"Other (Web)": "Web",
|
|
67
|
+
"Productivity and Collaboration": "Business Applications",
|
|
68
|
+
"Quality Assurance": "Developer Tools",
|
|
69
|
+
"Quantum Compute": "Compute",
|
|
70
|
+
"Relational Databases": "Databases",
|
|
71
|
+
"Resource Migration": "Migration",
|
|
72
|
+
"Search": "Analytics",
|
|
73
|
+
"Secret Management": "Security",
|
|
74
|
+
"Security Posture Management": "Security",
|
|
75
|
+
"Serverless Compute": "Compute",
|
|
76
|
+
"Source Code Management": "Developer Tools",
|
|
77
|
+
"Storage Platforms": "Storage",
|
|
78
|
+
"Streaming Analytics": "Analytics",
|
|
79
|
+
"Support": "Management and Governance",
|
|
80
|
+
"Threat Detection and Response": "Security",
|
|
81
|
+
"Time Series Databases": "Databases",
|
|
82
|
+
"Virtual Machines": "Compute",
|
|
83
|
+
"Workflow Orchestration": "Integration"
|
|
84
|
+
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""FOCUS-defined property keys for JSON / Key-Value typed columns.
|
|
2
|
+
|
|
3
|
+
Sourced verbatim from the frozen FOCUS specification tags
|
|
4
|
+
(``FinOps-Open-Cost-and-Usage-Spec/FOCUS_Spec`` at ``v1.3`` / ``v1.4``). These
|
|
5
|
+
registries drive the ``x_`` custom-key rule: inside a FOCUS JSON column, any key
|
|
6
|
+
that is **not** a FOCUS-defined property MUST be prefixed with ``x_`` (FOCUS
|
|
7
|
+
attributes ``CustomColumnHandling`` / per-column ``KeyValueFormat`` /
|
|
8
|
+
``JsonObjectFormat``).
|
|
9
|
+
|
|
10
|
+
Important exceptions: ``Tags`` and ``AllocatedTags`` are Key-Value columns whose
|
|
11
|
+
keys are arbitrary user tag names — the ``x_`` rule does **not** apply to them
|
|
12
|
+
(FOCUS ``tags.md``: a single user-defined tag scheme carries no prefix). They are
|
|
13
|
+
deliberately absent from :data:`XPREFIX_ENFORCED_COLUMNS`.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
# --- Key-Value columns with a FOCUS-defined property set ------------------- #
|
|
19
|
+
|
|
20
|
+
# SkuPriceDetails FOCUS-defined properties (identical in v1.3 and v1.4):
|
|
21
|
+
# skupricedetails.md "FOCUS-Defined Properties" table — 13 keys.
|
|
22
|
+
SKU_PRICE_DETAILS_KEYS: frozenset[str] = frozenset(
|
|
23
|
+
{
|
|
24
|
+
"CoreCount",
|
|
25
|
+
"DiskMaxIops",
|
|
26
|
+
"DiskSpace",
|
|
27
|
+
"DiskType",
|
|
28
|
+
"GpuCount",
|
|
29
|
+
"InstanceType",
|
|
30
|
+
"InstanceSeries",
|
|
31
|
+
"MemorySize",
|
|
32
|
+
"NetworkMaxIops",
|
|
33
|
+
"NetworkMaxThroughput",
|
|
34
|
+
"OperatingSystem",
|
|
35
|
+
"Redundancy",
|
|
36
|
+
"StorageClass",
|
|
37
|
+
}
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
# InvoiceDetailGrain FOCUS-defined properties (v1.4 invoicedetailgrain.md).
|
|
41
|
+
# Each links to a Cost and Usage column; the JSON key is that column's Column ID.
|
|
42
|
+
INVOICE_DETAIL_GRAIN_KEYS: frozenset[str] = frozenset(
|
|
43
|
+
{
|
|
44
|
+
"ContractId",
|
|
45
|
+
"RegionId",
|
|
46
|
+
"ResourceId",
|
|
47
|
+
"ResourceType",
|
|
48
|
+
"ServiceName",
|
|
49
|
+
"SkuId",
|
|
50
|
+
"SkuMeter",
|
|
51
|
+
"SkuPriceId",
|
|
52
|
+
"SubAccountId",
|
|
53
|
+
}
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
# --- JSON-Object columns with a top-level ``Elements`` array --------------- #
|
|
57
|
+
|
|
58
|
+
# ContractApplied element keys. The identifier keys changed casing between
|
|
59
|
+
# versions (contractapplied.md @ v1.3 vs v1.4); the three metric keys are stable.
|
|
60
|
+
CONTRACT_APPLIED_ELEMENT_KEYS_1_3: frozenset[str] = frozenset(
|
|
61
|
+
{
|
|
62
|
+
"ContractID",
|
|
63
|
+
"ContractCommitmentID",
|
|
64
|
+
"ContractCommitmentAppliedCost",
|
|
65
|
+
"ContractCommitmentAppliedQuantity",
|
|
66
|
+
"ContractCommitmentAppliedUnit",
|
|
67
|
+
}
|
|
68
|
+
)
|
|
69
|
+
CONTRACT_APPLIED_ELEMENT_KEYS_1_4: frozenset[str] = frozenset(
|
|
70
|
+
{
|
|
71
|
+
"ContractId",
|
|
72
|
+
"ContractCommitmentId",
|
|
73
|
+
"ContractCommitmentAppliedCost",
|
|
74
|
+
"ContractCommitmentAppliedQuantity",
|
|
75
|
+
"ContractCommitmentAppliedUnit",
|
|
76
|
+
}
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# AllocatedMethodDetails element keys (allocatedmethoddetails.md, v1.3 & v1.4).
|
|
80
|
+
ALLOCATED_METHOD_DETAILS_ELEMENT_KEYS: frozenset[str] = frozenset(
|
|
81
|
+
{
|
|
82
|
+
"AllocatedRatio",
|
|
83
|
+
"UsageUnit",
|
|
84
|
+
"UsageQuantity",
|
|
85
|
+
}
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# CommitmentProgramEligibilityDetails element keys
|
|
89
|
+
# (commitmentprogrameligibilitydetails.md @ v1.4; top-level array key is
|
|
90
|
+
# ``CommitmentPrograms``, not ``Elements``).
|
|
91
|
+
COMMITMENT_PROGRAM_ELIGIBILITY_ELEMENT_KEYS: frozenset[str] = frozenset({"ProgramType"})
|
|
92
|
+
|
|
93
|
+
# Columns whose custom keys MUST be ``x_``-prefixed, mapped to their FOCUS-defined
|
|
94
|
+
# key set. Key-Value columns match keys at the top level. Array-of-objects columns
|
|
95
|
+
# map to ``(top_level_array_key, element_focus_keys)``: top-level keys must be the
|
|
96
|
+
# array key or ``x_``-prefixed, and element keys must be FOCUS-defined or
|
|
97
|
+
# ``x_``-prefixed (custom properties are allowed at both levels per the v1.4 column
|
|
98
|
+
# docs). ``ContractApplied`` is validated in depth by
|
|
99
|
+
# ``focus_data_toolkit.convert.contract_applied.parse`` and is therefore absent
|
|
100
|
+
# here. Tags / AllocatedTags are intentionally excluded (arbitrary tag keys are
|
|
101
|
+
# allowed).
|
|
102
|
+
XPREFIX_ENFORCED_KEYVALUE_COLUMNS: dict[str, frozenset[str]] = {
|
|
103
|
+
"SkuPriceDetails": SKU_PRICE_DETAILS_KEYS,
|
|
104
|
+
"InvoiceDetailGrain": INVOICE_DETAIL_GRAIN_KEYS,
|
|
105
|
+
}
|
|
106
|
+
XPREFIX_ENFORCED_ELEMENTS_COLUMNS: dict[str, tuple[str, frozenset[str]]] = {
|
|
107
|
+
"AllocatedMethodDetails": ("Elements", ALLOCATED_METHOD_DETAILS_ELEMENT_KEYS),
|
|
108
|
+
"CommitmentProgramEligibilityDetails": (
|
|
109
|
+
"CommitmentPrograms",
|
|
110
|
+
COMMITMENT_PROGRAM_ELIGIBILITY_ELEMENT_KEYS,
|
|
111
|
+
),
|
|
112
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_source": "ISO 4217 active alphabetic currency codes (incl. fund/precious-metal/test codes X*).",
|
|
3
|
+
"codes": [
|
|
4
|
+
"AED", "AFN", "ALL", "AMD", "ANG", "AOA", "ARS", "AUD", "AWG", "AZN",
|
|
5
|
+
"BAM", "BBD", "BDT", "BGN", "BHD", "BIF", "BMD", "BND", "BOB", "BOV",
|
|
6
|
+
"BRL", "BSD", "BTN", "BWP", "BYN", "BZD", "CAD", "CDF", "CHE", "CHF",
|
|
7
|
+
"CHW", "CLF", "CLP", "CNY", "COP", "COU", "CRC", "CUP", "CVE", "CZK",
|
|
8
|
+
"DJF", "DKK", "DOP", "DZD", "EGP", "ERN", "ETB", "EUR", "FJD", "FKP",
|
|
9
|
+
"GBP", "GEL", "GHS", "GIP", "GMD", "GNF", "GTQ", "GYD", "HKD", "HNL",
|
|
10
|
+
"HTG", "HUF", "IDR", "ILS", "INR", "IQD", "IRR", "ISK", "JMD", "JOD",
|
|
11
|
+
"JPY", "KES", "KGS", "KHR", "KMF", "KPW", "KRW", "KWD", "KYD", "KZT",
|
|
12
|
+
"LAK", "LBP", "LKR", "LRD", "LSL", "LYD", "MAD", "MDL", "MGA", "MKD",
|
|
13
|
+
"MMK", "MNT", "MOP", "MRU", "MUR", "MVR", "MWK", "MXN", "MXV", "MYR",
|
|
14
|
+
"MZN", "NAD", "NGN", "NIO", "NOK", "NPR", "NZD", "OMR", "PAB", "PEN",
|
|
15
|
+
"PGK", "PHP", "PKR", "PLN", "PYG", "QAR", "RON", "RSD", "RUB", "RWF",
|
|
16
|
+
"SAR", "SBD", "SCR", "SDG", "SEK", "SGD", "SHP", "SLE", "SOS", "SRD",
|
|
17
|
+
"SSP", "STN", "SVC", "SYP", "SZL", "THB", "TJS", "TMT", "TND", "TOP",
|
|
18
|
+
"TRY", "TTD", "TWD", "TZS", "UAH", "UGX", "USD", "USN", "UYI", "UYU",
|
|
19
|
+
"UYW", "UZS", "VED", "VES", "VND", "VUV", "WST", "XAF", "XAG", "XAU",
|
|
20
|
+
"XBA", "XBB", "XBC", "XBD", "XCD", "XCG", "XDR", "XOF", "XPD", "XPF",
|
|
21
|
+
"XPT", "XSU", "XTS", "XUA", "XXX", "YER", "ZAR", "ZMW", "ZWG"
|
|
22
|
+
]
|
|
23
|
+
}
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""Validate FOCUS JSON-object column values against the vendored official schemas.
|
|
2
|
+
|
|
3
|
+
The FOCUS specification publishes normative JSON Schemas (draft 2020-12) for its
|
|
4
|
+
JSON-object columns under ``specification/schemas/datasets/``. Those files are
|
|
5
|
+
vendored verbatim in ``model/json_schemas/`` (see ``json_schemas_provenance.json``)
|
|
6
|
+
and evaluated here with a small interpreter covering exactly the keyword subset the
|
|
7
|
+
official schemas use — no runtime dependency, and the vendored files (not hand-coded
|
|
8
|
+
rules) remain the single source of truth.
|
|
9
|
+
|
|
10
|
+
Supported keywords: ``$ref`` (``#/$defs/...``), ``type`` (incl. type lists),
|
|
11
|
+
``properties``, ``patternProperties``, ``additionalProperties`` (boolean),
|
|
12
|
+
``required``, ``items``, ``minItems`` / ``maxItems``, ``contains``, ``enum``,
|
|
13
|
+
``const``, ``minimum`` / ``maximum``, ``allOf`` / ``anyOf`` / ``oneOf`` / ``not``,
|
|
14
|
+
``if`` / ``then``, and boolean schemas. An unsupported keyword raises — a vendored
|
|
15
|
+
schema update that outgrows the subset must fail loudly, never silently pass.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import math
|
|
22
|
+
import re
|
|
23
|
+
from functools import cache
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
SCHEMA_DIR = Path(__file__).resolve().parent / "json_schemas"
|
|
27
|
+
PROVENANCE_PATH = SCHEMA_DIR / "json_schemas_provenance.json"
|
|
28
|
+
|
|
29
|
+
# FOCUS column -> vendored official schema file.
|
|
30
|
+
OFFICIAL_SCHEMA_COLUMNS: dict[str, str] = {
|
|
31
|
+
"ContractApplied": "contractappliedobjectschema.json",
|
|
32
|
+
"AllocatedMethodDetails": "allocatedmethoddetailsobjectschema.json",
|
|
33
|
+
"CommitmentProgramEligibilityDetails": "commitmentprogrameligibilitydetailsobjectschema.json",
|
|
34
|
+
"ContractCommitmentApplicability": "contractcommitmentapplicabilityobjectschema.json",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
# Keywords the interpreter evaluates, plus annotations it may safely ignore.
|
|
38
|
+
_HANDLED = frozenset({
|
|
39
|
+
"$ref", "type", "properties", "patternProperties", "additionalProperties",
|
|
40
|
+
"required", "items", "minItems", "maxItems", "contains", "enum", "const",
|
|
41
|
+
"minimum", "maximum", "allOf", "anyOf", "oneOf", "not", "if", "then",
|
|
42
|
+
})
|
|
43
|
+
_ANNOTATIONS = frozenset({"$schema", "$id", "$defs", "title", "description", "default"})
|
|
44
|
+
|
|
45
|
+
def _is_number(v: object) -> bool:
|
|
46
|
+
# Standard JSON has no NaN/Infinity; Python's parser may still produce them from
|
|
47
|
+
# the non-standard constants, so a non-finite float is never a schema "number".
|
|
48
|
+
if isinstance(v, bool) or not isinstance(v, int | float):
|
|
49
|
+
return False
|
|
50
|
+
return math.isfinite(v)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_TYPE_CHECKS = {
|
|
54
|
+
"object": lambda v: isinstance(v, dict),
|
|
55
|
+
"array": lambda v: isinstance(v, list),
|
|
56
|
+
"string": lambda v: isinstance(v, str),
|
|
57
|
+
"number": _is_number,
|
|
58
|
+
"integer": lambda v: _is_number(v) and (isinstance(v, int) or v.is_integer()),
|
|
59
|
+
"boolean": lambda v: isinstance(v, bool),
|
|
60
|
+
"null": lambda v: v is None,
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class UnsupportedSchemaKeyword(ValueError):
|
|
65
|
+
"""A vendored schema uses a keyword outside the interpreter's subset."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@cache
|
|
69
|
+
def load_official_schema(filename: str) -> dict:
|
|
70
|
+
schema = json.loads((SCHEMA_DIR / filename).read_text(encoding="utf-8"))
|
|
71
|
+
_assert_supported(schema)
|
|
72
|
+
return schema
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _assert_supported(schema: object) -> None:
|
|
76
|
+
if isinstance(schema, bool):
|
|
77
|
+
return
|
|
78
|
+
if isinstance(schema, dict):
|
|
79
|
+
for key, sub in schema.items():
|
|
80
|
+
if key in _ANNOTATIONS:
|
|
81
|
+
continue
|
|
82
|
+
if key not in _HANDLED:
|
|
83
|
+
raise UnsupportedSchemaKeyword(f"unsupported JSON Schema keyword {key!r}")
|
|
84
|
+
if key in ("properties", "patternProperties"):
|
|
85
|
+
for nested in sub.values():
|
|
86
|
+
_assert_supported(nested)
|
|
87
|
+
elif key in ("allOf", "anyOf", "oneOf"):
|
|
88
|
+
for nested in sub:
|
|
89
|
+
_assert_supported(nested)
|
|
90
|
+
elif key in ("items", "contains", "not", "if", "then", "additionalProperties"):
|
|
91
|
+
_assert_supported(sub)
|
|
92
|
+
for nested in schema.get("$defs", {}).values():
|
|
93
|
+
_assert_supported(nested)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def check_against_official_schema(column: str, obj: object) -> list[str]:
|
|
97
|
+
"""Return the official-schema violations for ``obj`` (empty list = conformant)."""
|
|
98
|
+
schema = load_official_schema(OFFICIAL_SCHEMA_COLUMNS[column])
|
|
99
|
+
return _validate(obj, schema, schema, "$")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _resolve_ref(root: dict, ref: str) -> dict:
|
|
103
|
+
if not ref.startswith("#/"): # the official schemas only use local refs
|
|
104
|
+
raise UnsupportedSchemaKeyword(f"unsupported $ref target {ref!r}")
|
|
105
|
+
node: object = root
|
|
106
|
+
for part in ref[2:].split("/"):
|
|
107
|
+
node = node[part] # type: ignore[index]
|
|
108
|
+
return node # type: ignore[return-value]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _type_ok(value: object, spec: str | list[str]) -> bool:
|
|
112
|
+
names = [spec] if isinstance(spec, str) else spec
|
|
113
|
+
return any(_TYPE_CHECKS[n](value) for n in names)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _json_equal(a: object, b: object) -> bool:
|
|
117
|
+
# JSON equality: booleans are not numbers (Python's bool == int would conflate them).
|
|
118
|
+
if isinstance(a, bool) is not isinstance(b, bool):
|
|
119
|
+
return False
|
|
120
|
+
return a == b
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _validate(value: object, schema: object, root: dict, path: str) -> list[str]:
|
|
124
|
+
if schema is True:
|
|
125
|
+
return []
|
|
126
|
+
if schema is False:
|
|
127
|
+
return [f"{path}: not permitted here"]
|
|
128
|
+
assert isinstance(schema, dict)
|
|
129
|
+
|
|
130
|
+
if "$ref" in schema:
|
|
131
|
+
ref_errors = _validate(value, _resolve_ref(root, schema["$ref"]), root, path)
|
|
132
|
+
if ref_errors:
|
|
133
|
+
return ref_errors
|
|
134
|
+
|
|
135
|
+
spec_type = schema.get("type")
|
|
136
|
+
if spec_type is not None and not _type_ok(value, spec_type):
|
|
137
|
+
return [f"{path}: expected type {spec_type}"]
|
|
138
|
+
|
|
139
|
+
errors: list[str] = []
|
|
140
|
+
if "enum" in schema and not any(_json_equal(value, v) for v in schema["enum"]):
|
|
141
|
+
errors.append(f"{path}: value not in {schema['enum']}")
|
|
142
|
+
if "const" in schema and not _json_equal(value, schema["const"]):
|
|
143
|
+
errors.append(f"{path}: expected constant {schema['const']!r}")
|
|
144
|
+
|
|
145
|
+
for sub in schema.get("allOf", ()):
|
|
146
|
+
errors.extend(_validate(value, sub, root, path))
|
|
147
|
+
if "anyOf" in schema and all(_validate(value, sub, root, path) for sub in schema["anyOf"]):
|
|
148
|
+
errors.append(f"{path}: matches none of the allowed forms (anyOf)")
|
|
149
|
+
if "oneOf" in schema:
|
|
150
|
+
matched = sum(1 for sub in schema["oneOf"] if not _validate(value, sub, root, path))
|
|
151
|
+
if matched != 1:
|
|
152
|
+
errors.append(f"{path}: must match exactly one allowed form (matched {matched})")
|
|
153
|
+
if "not" in schema and not _validate(value, schema["not"], root, path):
|
|
154
|
+
errors.append(f"{path}: matches a prohibited form")
|
|
155
|
+
if "if" in schema and not _validate(value, schema["if"], root, path):
|
|
156
|
+
errors.extend(_validate(value, schema.get("then", True), root, path))
|
|
157
|
+
|
|
158
|
+
if isinstance(value, dict):
|
|
159
|
+
errors.extend(_validate_object(value, schema, root, path))
|
|
160
|
+
elif isinstance(value, list):
|
|
161
|
+
errors.extend(_validate_array(value, schema, root, path))
|
|
162
|
+
elif isinstance(value, int | float) and not isinstance(value, bool):
|
|
163
|
+
if "minimum" in schema and value < schema["minimum"]:
|
|
164
|
+
errors.append(f"{path}: {value} is below minimum {schema['minimum']}")
|
|
165
|
+
if "maximum" in schema and value > schema["maximum"]:
|
|
166
|
+
errors.append(f"{path}: {value} is above maximum {schema['maximum']}")
|
|
167
|
+
return errors
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _validate_object(value: dict, schema: dict, root: dict, path: str) -> list[str]:
|
|
171
|
+
errors: list[str] = []
|
|
172
|
+
for req in schema.get("required", ()):
|
|
173
|
+
if req not in value:
|
|
174
|
+
errors.append(f"{path}: required property {req!r} is missing")
|
|
175
|
+
properties = schema.get("properties", {})
|
|
176
|
+
patterns = {re.compile(p): s for p, s in schema.get("patternProperties", {}).items()}
|
|
177
|
+
additional = schema.get("additionalProperties", True)
|
|
178
|
+
for key, item in value.items():
|
|
179
|
+
matched = False
|
|
180
|
+
if key in properties:
|
|
181
|
+
matched = True
|
|
182
|
+
errors.extend(_validate(item, properties[key], root, f"{path}.{key}"))
|
|
183
|
+
for pattern, sub in patterns.items():
|
|
184
|
+
if pattern.search(key):
|
|
185
|
+
matched = True
|
|
186
|
+
errors.extend(_validate(item, sub, root, f"{path}.{key}"))
|
|
187
|
+
if not matched:
|
|
188
|
+
errors.extend(_validate(item, additional, root, f"{path}.{key}"))
|
|
189
|
+
return errors
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _validate_array(value: list, schema: dict, root: dict, path: str) -> list[str]:
|
|
193
|
+
errors: list[str] = []
|
|
194
|
+
if "minItems" in schema and len(value) < schema["minItems"]:
|
|
195
|
+
errors.append(f"{path}: needs at least {schema['minItems']} item(s)")
|
|
196
|
+
if "maxItems" in schema and len(value) > schema["maxItems"]:
|
|
197
|
+
errors.append(f"{path}: allows at most {schema['maxItems']} item(s)")
|
|
198
|
+
if "items" in schema:
|
|
199
|
+
for i, item in enumerate(value):
|
|
200
|
+
errors.extend(_validate(item, schema["items"], root, f"{path}[{i}]"))
|
|
201
|
+
if "contains" in schema and all(
|
|
202
|
+
_validate(item, schema["contains"], root, path) for item in value
|
|
203
|
+
):
|
|
204
|
+
errors.append(f"{path}: no item matches the required form (contains)")
|
|
205
|
+
return errors
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://focus.finops.org/schemas/allocatedmethoddetailsobject.json",
|
|
4
|
+
"title": "Allocated Method Details Object",
|
|
5
|
+
"description": "Schema for validating the Allocated Method Details column JSON object structure in FOCUS datasets",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": [
|
|
8
|
+
"Elements"
|
|
9
|
+
],
|
|
10
|
+
"properties": {
|
|
11
|
+
"Elements": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"description": "Array containing one or more objects describing allocation properties",
|
|
14
|
+
"minItems": 1,
|
|
15
|
+
"items": {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"required": [
|
|
18
|
+
"AllocatedRatio"
|
|
19
|
+
],
|
|
20
|
+
"properties": {
|
|
21
|
+
"AllocatedRatio": {
|
|
22
|
+
"type": "number",
|
|
23
|
+
"description": "Percentage of overall cost derived from corresponding method and metric",
|
|
24
|
+
"minimum": 0,
|
|
25
|
+
"maximum": 1
|
|
26
|
+
},
|
|
27
|
+
"UsageUnit": {
|
|
28
|
+
"type": [
|
|
29
|
+
"string",
|
|
30
|
+
"null"
|
|
31
|
+
],
|
|
32
|
+
"description": "Unit being measured used to calculate allocation"
|
|
33
|
+
},
|
|
34
|
+
"UsageQuantity": {
|
|
35
|
+
"type": [
|
|
36
|
+
"number",
|
|
37
|
+
"null"
|
|
38
|
+
],
|
|
39
|
+
"description": "Volume of UsageUnit consumed or used"
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
"anyOf": [
|
|
43
|
+
{
|
|
44
|
+
"description": "Scenario 1: UsageQuantity is absent or explicitly null. UsageUnit can be anything (or absent).",
|
|
45
|
+
"properties": {
|
|
46
|
+
"UsageQuantity": {
|
|
47
|
+
"type": "null"
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"description": "Scenario 2: UsageQuantity is provided and is a number. UsageUnit MUST be provided and MUST be a string.",
|
|
53
|
+
"required": [
|
|
54
|
+
"UsageQuantity",
|
|
55
|
+
"UsageUnit"
|
|
56
|
+
],
|
|
57
|
+
"properties": {
|
|
58
|
+
"UsageQuantity": {
|
|
59
|
+
"type": "number"
|
|
60
|
+
},
|
|
61
|
+
"UsageUnit": {
|
|
62
|
+
"type": "string"
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
],
|
|
67
|
+
"patternProperties": {
|
|
68
|
+
"^x_[A-Z][a-zA-Z0-9]*$": {
|
|
69
|
+
"description": "Custom data generator-defined properties prefixed with x_ in PascalCase format"
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
"additionalProperties": false
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
},
|
|
76
|
+
"patternProperties": {
|
|
77
|
+
"^x_[A-Z][a-zA-Z0-9]*$": {
|
|
78
|
+
"description": "Custom data generator-defined top-level properties prefixed with x_ in PascalCase format"
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
"additionalProperties": false
|
|
82
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://focus.finops.org/schemas/commitmentprogrameligibilitydetailsobject.json",
|
|
4
|
+
"title": "Commitment Program Eligibility Details Object",
|
|
5
|
+
"description": "Schema for validating the Commitment Program Eligibility Details column JSON object structure in FOCUS datasets",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": [
|
|
8
|
+
"CommitmentPrograms"
|
|
9
|
+
],
|
|
10
|
+
"properties": {
|
|
11
|
+
"CommitmentPrograms": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"description": "Array of objects identifying commitment programs for which the usage is eligible.",
|
|
14
|
+
"minItems": 1,
|
|
15
|
+
"items": {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"required": [
|
|
18
|
+
"ProgramType"
|
|
19
|
+
],
|
|
20
|
+
"properties": {
|
|
21
|
+
"ProgramType": {
|
|
22
|
+
"type": "string",
|
|
23
|
+
"description": "The specific type of commitment program (e.g., discount or capacity reservation) available for this usage."
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"patternProperties": {
|
|
27
|
+
"^x_[A-Z][a-zA-Z0-9]*$": {
|
|
28
|
+
"description": "Custom data generator-defined properties prefixed with x_ in PascalCase format"
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"additionalProperties": false
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"patternProperties": {
|
|
36
|
+
"^x_[A-Z][a-zA-Z0-9]*$": {
|
|
37
|
+
"description": "Custom data generator-defined top-level properties prefixed with x_ in PascalCase format"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"additionalProperties": false
|
|
41
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://focus.finops.org/schemas/contractappliedobject.json",
|
|
4
|
+
"title": "Contract Applied Object",
|
|
5
|
+
"description": "Schema for validating the Contract Applied column JSON object structure in FOCUS datasets",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": [
|
|
8
|
+
"Elements"
|
|
9
|
+
],
|
|
10
|
+
"properties": {
|
|
11
|
+
"Elements": {
|
|
12
|
+
"type": "array",
|
|
13
|
+
"description": "Array containing one or more objects describing contract commitments applied to the charge",
|
|
14
|
+
"minItems": 1,
|
|
15
|
+
"items": {
|
|
16
|
+
"type": "object",
|
|
17
|
+
"required": [
|
|
18
|
+
"ContractId",
|
|
19
|
+
"ContractCommitmentId"
|
|
20
|
+
],
|
|
21
|
+
"properties": {
|
|
22
|
+
"ContractId": {
|
|
23
|
+
"type": "string",
|
|
24
|
+
"description": "Unique identifier for the contract"
|
|
25
|
+
},
|
|
26
|
+
"ContractCommitmentId": {
|
|
27
|
+
"type": "string",
|
|
28
|
+
"description": "Unique identifier for the contract commitment term"
|
|
29
|
+
},
|
|
30
|
+
"ContractCommitmentAppliedCost": {
|
|
31
|
+
"type": [
|
|
32
|
+
"number",
|
|
33
|
+
"null"
|
|
34
|
+
],
|
|
35
|
+
"description": "Cost value of the charge applied to the contract commitment"
|
|
36
|
+
},
|
|
37
|
+
"ContractCommitmentAppliedQuantity": {
|
|
38
|
+
"type": [
|
|
39
|
+
"number",
|
|
40
|
+
"null"
|
|
41
|
+
],
|
|
42
|
+
"description": "Quantity of usage applied to the contract commitment"
|
|
43
|
+
},
|
|
44
|
+
"ContractCommitmentAppliedUnit": {
|
|
45
|
+
"type": [
|
|
46
|
+
"string",
|
|
47
|
+
"null"
|
|
48
|
+
],
|
|
49
|
+
"description": "Unit of measure for the applied quantity"
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
"oneOf": [
|
|
53
|
+
{
|
|
54
|
+
"description": "Scenario 1: Cost has a value. Quantity and Unit MUST be absent or explicitly null.",
|
|
55
|
+
"required": [
|
|
56
|
+
"ContractCommitmentAppliedCost"
|
|
57
|
+
],
|
|
58
|
+
"properties": {
|
|
59
|
+
"ContractCommitmentAppliedCost": {
|
|
60
|
+
"type": "number"
|
|
61
|
+
},
|
|
62
|
+
"ContractCommitmentAppliedQuantity": {
|
|
63
|
+
"type": "null"
|
|
64
|
+
},
|
|
65
|
+
"ContractCommitmentAppliedUnit": {
|
|
66
|
+
"type": "null"
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"description": "Scenario 2: Quantity and Unit have values. Cost MUST be absent or explicitly null.",
|
|
72
|
+
"required": [
|
|
73
|
+
"ContractCommitmentAppliedQuantity",
|
|
74
|
+
"ContractCommitmentAppliedUnit"
|
|
75
|
+
],
|
|
76
|
+
"properties": {
|
|
77
|
+
"ContractCommitmentAppliedCost": {
|
|
78
|
+
"type": "null"
|
|
79
|
+
},
|
|
80
|
+
"ContractCommitmentAppliedQuantity": {
|
|
81
|
+
"type": "number"
|
|
82
|
+
},
|
|
83
|
+
"ContractCommitmentAppliedUnit": {
|
|
84
|
+
"type": "string"
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
"patternProperties": {
|
|
90
|
+
"^x_[A-Z][a-zA-Z0-9]*$": {
|
|
91
|
+
"description": "Custom data generator-defined properties prefixed with x_ in PascalCase format"
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
"additionalProperties": false
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
"patternProperties": {
|
|
99
|
+
"^x_[A-Z][a-zA-Z0-9]*$": {
|
|
100
|
+
"description": "Custom data generator-defined top-level properties prefixed with x_ in PascalCase format"
|
|
101
|
+
}
|
|
102
|
+
},
|
|
103
|
+
"additionalProperties": false
|
|
104
|
+
}
|