intentumdiff-python 0.0.1__py3-none-win_arm64.whl → 0.0.3b1__py3-none-win_arm64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- intentumdiff/__init__.py +9 -0
- intentumdiff/__main__.py +12 -0
- intentumdiff/_differ_gate.py +0 -136
- intentumdiff/_differ_presentation.py +17 -735
- intentumdiff/_differ_runtime.py +13 -64
- intentumdiff/analysis/compile_commands.py +26 -141
- intentumdiff/analysis/cross_file.py +23 -25
- intentumdiff/analysis/diagnostics.py +1 -164
- intentumdiff/analysis/guardrail_reports.py +31 -33
- intentumdiff/analysis/guardrails.py +111 -188
- intentumdiff/analysis/schema_resolver.py +72 -360
- intentumdiff/analysis/user_schemas.py +57 -254
- intentumdiff/api.py +21 -0
- intentumdiff/cache/sqlite_store.py +5 -23
- intentumdiff/cli/__init__.py +59 -0
- intentumdiff/cli/_commands.py +243 -137
- intentumdiff/cli/_parser.py +51 -56
- intentumdiff/cli/_shared.py +151 -116
- intentumdiff/content_type.py +13 -30
- intentumdiff/core/commit_differ.py +137 -286
- intentumdiff/core/diffignore.py +28 -34
- intentumdiff/core/index.py +16 -18
- intentumdiff/core/indexer.py +148 -169
- intentumdiff/core/models.py +279 -83
- intentumdiff/differ.py +366 -1000
- intentumdiff/github_app.py +40 -27
- intentumdiff/intentumdiff_rust_core/intentumdiff_rust_core.dll +0 -0
- intentumdiff/live_server.py +197 -34
- intentumdiff/lsp/client.py +49 -43
- intentumdiff/lsp/enricher.py +19 -119
- intentumdiff/lsp/exceptions.py +6 -7
- intentumdiff/lsp/servers.py +10 -11
- intentumdiff/lsp_server/_codelens.py +24 -57
- intentumdiff/lsp_server/_diagnostics.py +23 -51
- intentumdiff/lsp_server/_handlers.py +67 -9
- intentumdiff/lsp_server/server.py +28 -8
- intentumdiff/plugins/adapter.py +5 -0
- intentumdiff/plugins/builtins.py +7 -14
- intentumdiff/plugins/exceptions.py +28 -0
- intentumdiff/plugins/hub.py +39 -43
- intentumdiff/plugins/loader.py +229 -215
- intentumdiff/plugins/registry.py +259 -207
- intentumdiff/plugins/registry.schema.json +156 -15
- intentumdiff/plugins/registry_schema.py +47 -140
- intentumdiff/rust_core.py +623 -1326
- intentumdiff/serve/_app.py +26 -35
- intentumdiff/sources/base.py +24 -0
- intentumdiff/sources/file_source.py +15 -2
- intentumdiff/sources/git_source.py +64 -185
- intentumdiff/sources/live_buffer_source.py +45 -5
- intentumdiff/sources/patch_source.py +36 -109
- intentumdiff/vcs/_safe_path.py +7 -0
- intentumdiff/vcs/base.py +6 -0
- intentumdiff/vcs/git_backend.py +1 -1
- intentumdiff/vcs/git_cli.py +0 -27
- intentumdiff/vcs/hg_backend.py +1 -1
- intentumdiff/vcs/perforce_backend.py +7 -1
- intentumdiff/vcs/svn_backend.py +1 -1
- intentumdiff/wasm/abap_parser.wasm +0 -0
- intentumdiff/wasm/adf_parser.wasm +0 -0
- intentumdiff/wasm/asciidoc_parser.wasm +0 -0
- intentumdiff/wasm/asm_parser.wasm +0 -0
- intentumdiff/wasm/assemblyscript_parser.wasm +0 -0
- intentumdiff/wasm/astro_parser.wasm +0 -0
- intentumdiff/wasm/bash_parser.wasm +0 -0
- intentumdiff/wasm/clojure_parser.wasm +0 -0
- intentumdiff/wasm/cmake_parser.wasm +0 -0
- intentumdiff/wasm/cpp_parser.wasm +0 -0
- intentumdiff/wasm/csharp_parser.wasm +0 -0
- intentumdiff/wasm/css_parser.wasm +0 -0
- intentumdiff/wasm/dart_parser.wasm +0 -0
- intentumdiff/wasm/databricks_parser.wasm +0 -0
- intentumdiff/wasm/dax_parser.wasm +0 -0
- intentumdiff/wasm/delphi_parser.wasm +0 -0
- intentumdiff/wasm/dockerfile_parser.wasm +0 -0
- intentumdiff/wasm/elixir_parser.wasm +0 -0
- intentumdiff/wasm/freebasic_parser.wasm +0 -0
- intentumdiff/wasm/generic_parser.wasm +0 -0
- intentumdiff/wasm/gitignore_parser.wasm +0 -0
- intentumdiff/wasm/go_parser.wasm +0 -0
- intentumdiff/wasm/gomod_parser.wasm +0 -0
- intentumdiff/wasm/graphql_parser.wasm +0 -0
- intentumdiff/wasm/groovy_parser.wasm +0 -0
- intentumdiff/wasm/haskell_parser.wasm +0 -0
- intentumdiff/wasm/html_parser.wasm +0 -0
- intentumdiff/wasm/html_renderer.wasm +0 -0
- intentumdiff/wasm/ini_parser.wasm +0 -0
- intentumdiff/wasm/java_parser.wasm +0 -0
- intentumdiff/wasm/js_ts_parser.wasm +0 -0
- intentumdiff/wasm/json_parser.wasm +0 -0
- intentumdiff/wasm/kotlin_parser.wasm +0 -0
- intentumdiff/wasm/latex_parser.wasm +0 -0
- intentumdiff/wasm/llm_renderer.wasm +0 -0
- intentumdiff/wasm/lua_parser.wasm +0 -0
- intentumdiff/wasm/make_parser.wasm +0 -0
- intentumdiff/wasm/markdown_parser.wasm +0 -0
- intentumdiff/wasm/mdx_parser.wasm +0 -0
- intentumdiff/wasm/ocaml_parser.wasm +0 -0
- intentumdiff/wasm/odin_parser.wasm +0 -0
- intentumdiff/wasm/parser_manifest.json +81 -2
- intentumdiff/wasm/patch_renderer.wasm +0 -0
- intentumdiff/wasm/perl_parser.wasm +0 -0
- intentumdiff/wasm/php_parser.wasm +0 -0
- intentumdiff/wasm/plsql_parser.wasm +0 -0
- intentumdiff/wasm/po_parser.wasm +0 -0
- intentumdiff/wasm/postscript_parser.wasm +0 -0
- intentumdiff/wasm/powershell_parser.wasm +0 -0
- intentumdiff/wasm/proto_parser.wasm +0 -0
- intentumdiff/wasm/puppet_parser.wasm +0 -0
- intentumdiff/wasm/python_parser.wasm +0 -0
- intentumdiff/wasm/qsharp_parser.wasm +0 -0
- intentumdiff/wasm/r_parser.wasm +0 -0
- intentumdiff/wasm/reasonml_parser.wasm +0 -0
- intentumdiff/wasm/ruby_parser.wasm +0 -0
- intentumdiff/wasm/rust_parser.wasm +0 -0
- intentumdiff/wasm/sas_parser.wasm +0 -0
- intentumdiff/wasm/scala_parser.wasm +0 -0
- intentumdiff/wasm/scss_parser.wasm +0 -0
- intentumdiff/wasm/sql_parser.wasm +0 -0
- intentumdiff/wasm/squirrel_parser.wasm +0 -0
- intentumdiff/wasm/svelte_parser.wasm +0 -0
- intentumdiff/wasm/swift_parser.wasm +0 -0
- intentumdiff/wasm/terminal_renderer.wasm +0 -0
- intentumdiff/wasm/terraform_parser.wasm +0 -0
- intentumdiff/wasm/toml_parser.wasm +0 -0
- intentumdiff/wasm/tsql_parser.wasm +0 -0
- intentumdiff/wasm/vbnet_parser.wasm +0 -0
- intentumdiff/wasm/vue_parser.wasm +0 -0
- intentumdiff/wasm/wasm_provenance.json +884 -0
- intentumdiff/wasm/wat_parser.wasm +0 -0
- intentumdiff/wasm/xml_parser.wasm +0 -0
- intentumdiff/wasm/yaml_parser.wasm +0 -0
- intentumdiff/wasm/zig_parser.wasm +0 -0
- intentumdiff/watcher.py +15 -17
- intentumdiff_python-0.0.3b1.dist-info/METADATA +188 -0
- intentumdiff_python-0.0.3b1.dist-info/RECORD +164 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/WHEEL +1 -1
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/sboms/intentumdiff-rust-core.cyclonedx.json +905 -379
- intentumdiff/analysis/invariances.py +0 -1207
- intentumdiff/analysis/keyed_profiles.py +0 -25
- intentumdiff/analysis/resource_profiles.py +0 -14
- intentumdiff/analysis/text_review.py +0 -46
- intentumdiff/core/cst_serializer.py +0 -101
- intentumdiff/invariances/__init__.py +0 -2
- intentumdiff/invariances/rules.schema.json +0 -283
- intentumdiff/invariances/rules.yaml +0 -349
- intentumdiff_python-0.0.1.dist-info/METADATA +0 -112
- intentumdiff_python-0.0.1.dist-info/RECORD +0 -165
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/entry_points.txt +0 -0
- {intentumdiff_python-0.0.1.dist-info → intentumdiff_python-0.0.3b1.dist-info}/licenses/LICENSE +0 -0
|
@@ -6,8 +6,8 @@ import hashlib
|
|
|
6
6
|
import ipaddress
|
|
7
7
|
import json
|
|
8
8
|
import logging
|
|
9
|
+
import math
|
|
9
10
|
import os
|
|
10
|
-
import re
|
|
11
11
|
import socket
|
|
12
12
|
import ssl
|
|
13
13
|
import subprocess
|
|
@@ -17,12 +17,15 @@ import urllib.parse
|
|
|
17
17
|
import urllib.request
|
|
18
18
|
from collections.abc import Callable, Mapping
|
|
19
19
|
from dataclasses import dataclass
|
|
20
|
+
from functools import partial
|
|
20
21
|
from pathlib import Path
|
|
21
22
|
from typing import Any
|
|
22
23
|
|
|
24
|
+
from intentumdiff.rust_core import _c_abi_call
|
|
25
|
+
|
|
23
26
|
from intentumdiff.analysis.user_schemas import (
|
|
24
27
|
load_user_schema_profiles,
|
|
25
|
-
|
|
28
|
+
_profile_payload,
|
|
26
29
|
)
|
|
27
30
|
|
|
28
31
|
logger = logging.getLogger(__name__)
|
|
@@ -30,66 +33,8 @@ logger = logging.getLogger(__name__)
|
|
|
30
33
|
DEFAULT_SCHEMA_TTL_SECONDS = 86_400
|
|
31
34
|
DEFAULT_TIMEOUT_SECONDS = 5.0
|
|
32
35
|
DEFAULT_MAX_BYTES = 5 * 1024 * 1024
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
"git",
|
|
36
|
-
"id",
|
|
37
|
-
"job_cluster_key",
|
|
38
|
-
"key",
|
|
39
|
-
"local",
|
|
40
|
-
"name",
|
|
41
|
-
"operationid",
|
|
42
|
-
"package",
|
|
43
|
-
"task",
|
|
44
|
-
"task_key",
|
|
45
|
-
}
|
|
46
|
-
)
|
|
47
|
-
|
|
48
|
-
DBT_SCHEMA_BASE = "https://raw.githubusercontent.com/dbt-labs/dbt-jsonschema/main/schemas/latest"
|
|
49
|
-
DBT_SCHEMA_URLS = {
|
|
50
|
-
"dbt_cloud": f"{DBT_SCHEMA_BASE}/dbt_cloud-latest.json",
|
|
51
|
-
"dbt_project": f"{DBT_SCHEMA_BASE}/dbt_project-latest.json",
|
|
52
|
-
"dbt_yml_files": f"{DBT_SCHEMA_BASE}/dbt_yml_files-latest.json",
|
|
53
|
-
"dependencies": f"{DBT_SCHEMA_BASE}/dependencies-latest.json",
|
|
54
|
-
"packages": f"{DBT_SCHEMA_BASE}/packages-latest.json",
|
|
55
|
-
"selectors": f"{DBT_SCHEMA_BASE}/selectors-latest.json",
|
|
56
|
-
}
|
|
57
|
-
ADVISORY_ADF_NO_RAW_SCHEMA_URL = (
|
|
58
|
-
"https://learn.microsoft.com/en-sg/answers/questions/2125264/"
|
|
59
|
-
"are-there-json-schemas-for-adf-source-files"
|
|
60
|
-
)
|
|
61
|
-
DATABRICKS_BUNDLE_SCHEMA_URL = (
|
|
62
|
-
"https://raw.githubusercontent.com/databricks/cli/main/bundle/schema/jsonschema.json"
|
|
63
|
-
)
|
|
64
|
-
OPENAPI_31_SCHEMA_URL = "https://spec.openapis.org/oas/3.1/schema/2022-10-07"
|
|
65
|
-
OPENAPI_30_SCHEMA_URL = "https://spec.openapis.org/oas/3.0/schema/2021-09-28"
|
|
66
|
-
KUBERNETES_SCHEMA_URL = "https://json.schemastore.org/kubernetes.json"
|
|
67
|
-
GITHUB_WORKFLOW_SCHEMA_URL = "https://json.schemastore.org/github-workflow.json"
|
|
68
|
-
AZURE_PIPELINES_SCHEMA_URL = "https://json.schemastore.org/azure-pipelines.json"
|
|
69
|
-
DBT_IDENTITY_FIELDS = frozenset(
|
|
70
|
-
{
|
|
71
|
-
"exposure",
|
|
72
|
-
"macro",
|
|
73
|
-
"metric",
|
|
74
|
-
"model",
|
|
75
|
-
"models",
|
|
76
|
-
"name",
|
|
77
|
-
"package_name",
|
|
78
|
-
"packages",
|
|
79
|
-
"project_name",
|
|
80
|
-
"resource_type",
|
|
81
|
-
"selector",
|
|
82
|
-
"selectors",
|
|
83
|
-
"seed",
|
|
84
|
-
"snapshot",
|
|
85
|
-
"snapshots",
|
|
86
|
-
"source",
|
|
87
|
-
"sources",
|
|
88
|
-
"test",
|
|
89
|
-
"tests",
|
|
90
|
-
"version",
|
|
91
|
-
}
|
|
92
|
-
)
|
|
36
|
+
# Schema identity, provider selection and the built-in provider URL catalogue are owned
|
|
37
|
+
# by core (``schema_profiles``); this module only performs the host fetch/cache I/O.
|
|
93
38
|
|
|
94
39
|
|
|
95
40
|
@dataclass(frozen=True)
|
|
@@ -152,33 +97,19 @@ def resolve_schema(
|
|
|
152
97
|
) -> SchemaResolution:
|
|
153
98
|
values = env or os.environ
|
|
154
99
|
mode = _schema_fetch_mode(values)
|
|
155
|
-
declared = discover_declared_schema(content, language)
|
|
156
|
-
# User-registered profiles (issue #63) outrank the embedded fetch and the
|
|
157
|
-
# built-in provider registry: a claimed $schema URL or a filename/marker
|
|
158
|
-
# match resolves OFFLINE from the local descriptor — no fetch, no logging
|
|
159
|
-
# of proprietary schema content.
|
|
160
100
|
user_profiles, _user_errors = load_user_schema_profiles(values)
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
filename=filename,
|
|
165
|
-
content=content,
|
|
166
|
-
declared_url=declared,
|
|
167
|
-
)
|
|
168
|
-
if profile is not None:
|
|
169
|
-
return SchemaResolution(
|
|
170
|
-
provider_id=profile.provider_id,
|
|
171
|
-
status="user-profile",
|
|
172
|
-
source_url=profile.schema_path,
|
|
173
|
-
schema=profile.schema,
|
|
174
|
-
identity_fields=profile.identity_fields,
|
|
175
|
-
cache_fingerprint=profile.fingerprint,
|
|
176
|
-
)
|
|
177
|
-
candidate = (
|
|
178
|
-
SchemaCandidate("embedded", declared)
|
|
179
|
-
if declared
|
|
180
|
-
else provider_schema_candidate(filename, language, content)
|
|
101
|
+
decision = _schema_operation(
|
|
102
|
+
"resolve", profiles=[_profile_payload(p) for p in user_profiles],
|
|
103
|
+
filename=filename, language=language, content=content,
|
|
181
104
|
)
|
|
105
|
+
if decision["profile_index"] is not None:
|
|
106
|
+
profile = user_profiles[decision["profile_index"]]
|
|
107
|
+
return SchemaResolution(
|
|
108
|
+
provider_id=profile.provider_id, status="user-profile",
|
|
109
|
+
source_url=profile.schema_path, schema=profile.schema,
|
|
110
|
+
identity_fields=profile.identity_fields, cache_fingerprint=profile.fingerprint,
|
|
111
|
+
)
|
|
112
|
+
candidate = _candidate_from_payload(decision["candidate"])
|
|
182
113
|
if candidate is None:
|
|
183
114
|
return SchemaResolution(provider_id="none", status="not_found")
|
|
184
115
|
if candidate.status:
|
|
@@ -263,7 +194,7 @@ def resolve_schema(
|
|
|
263
194
|
ttl,
|
|
264
195
|
current_time,
|
|
265
196
|
values,
|
|
266
|
-
fetcher or _fetch_https,
|
|
197
|
+
fetcher or partial(_fetch_https, env=values),
|
|
267
198
|
)
|
|
268
199
|
except Exception as exc:
|
|
269
200
|
logger.debug("Schema fetch failed for %s", candidate.url, exc_info=True)
|
|
@@ -278,135 +209,31 @@ def resolve_schema(
|
|
|
278
209
|
)
|
|
279
210
|
|
|
280
211
|
|
|
212
|
+
def _schema_operation(operation: str, **payload: Any) -> Any:
|
|
213
|
+
return _c_abi_call("schema_profiles", json.dumps({"operation": operation, **payload}))
|
|
214
|
+
|
|
215
|
+
|
|
281
216
|
def discover_declared_schema(content: str, language: str | None = None) -> str | None:
|
|
282
|
-
|
|
283
|
-
if language_id in {"json", "adf"} or _looks_like_json(content):
|
|
284
|
-
try:
|
|
285
|
-
parsed = json.loads(content)
|
|
286
|
-
except json.JSONDecodeError:
|
|
287
|
-
parsed = None
|
|
288
|
-
if isinstance(parsed, dict):
|
|
289
|
-
value = parsed.get("$schema")
|
|
290
|
-
if isinstance(value, str) and value.strip():
|
|
291
|
-
return value.strip()
|
|
292
|
-
|
|
293
|
-
for line in content.splitlines()[:5]:
|
|
294
|
-
match = re.match(r"\s*#\s*yaml-language-server:\s*\$schema=(\S+)\s*$", line)
|
|
295
|
-
if match:
|
|
296
|
-
return match.group(1).strip()
|
|
297
|
-
return None
|
|
217
|
+
return _schema_operation("discover", content=content, language=language)
|
|
298
218
|
|
|
299
219
|
|
|
300
220
|
def provider_schema_candidate(
|
|
301
|
-
filename: str,
|
|
302
|
-
language: str | None,
|
|
303
|
-
content: str = "",
|
|
221
|
+
filename: str, language: str | None, content: str = "",
|
|
304
222
|
) -> SchemaCandidate | None:
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
language_id = (language or "").lower()
|
|
308
|
-
|
|
309
|
-
if url := _openapi_schema_url(basename, content):
|
|
310
|
-
version = "3.1" if url == OPENAPI_31_SCHEMA_URL else "3.0"
|
|
311
|
-
return SchemaCandidate(
|
|
312
|
-
f"openapi:{version}",
|
|
313
|
-
url,
|
|
314
|
-
identity_fields=frozenset({"operationId", "name"}),
|
|
315
|
-
)
|
|
316
|
-
if _looks_like_kubernetes_manifest(basename, content):
|
|
317
|
-
return SchemaCandidate(
|
|
318
|
-
"kubernetes:manifest",
|
|
319
|
-
KUBERNETES_SCHEMA_URL,
|
|
320
|
-
identity_fields=frozenset({"name"}),
|
|
321
|
-
)
|
|
322
|
-
if _looks_like_github_workflow(normalized_path, content):
|
|
323
|
-
return SchemaCandidate(
|
|
324
|
-
"github-actions:workflow",
|
|
325
|
-
GITHUB_WORKFLOW_SCHEMA_URL,
|
|
326
|
-
identity_fields=frozenset({"id", "name", "run", "runs-on", "uses"}),
|
|
327
|
-
)
|
|
328
|
-
if _looks_like_azure_pipeline(normalized_path, content):
|
|
329
|
-
return SchemaCandidate(
|
|
330
|
-
"azure-pipelines:pipeline",
|
|
331
|
-
AZURE_PIPELINES_SCHEMA_URL,
|
|
332
|
-
identity_fields=frozenset({"job", "stage", "task", "script", "displayName"}),
|
|
333
|
-
)
|
|
223
|
+
value = _schema_operation("provider", filename=filename, language=language, content=content)
|
|
224
|
+
return _candidate_from_payload(value)
|
|
334
225
|
|
|
335
|
-
dbt_url = _dbt_schema_url(basename, language_id, content)
|
|
336
|
-
if dbt_url:
|
|
337
|
-
key = dbt_url.rsplit("/", 1)[-1].replace("-latest.json", "")
|
|
338
|
-
return SchemaCandidate(
|
|
339
|
-
f"dbt:{key}",
|
|
340
|
-
dbt_url,
|
|
341
|
-
identity_fields=DBT_IDENTITY_FIELDS,
|
|
342
|
-
)
|
|
343
|
-
|
|
344
|
-
if language_id in {"databricks", "databricks-workflow"} or basename in {
|
|
345
|
-
"databricks.yml",
|
|
346
|
-
"databricks.yaml",
|
|
347
|
-
}:
|
|
348
|
-
return SchemaCandidate(
|
|
349
|
-
"databricks:bundle",
|
|
350
|
-
DATABRICKS_BUNDLE_SCHEMA_URL,
|
|
351
|
-
command=("databricks", "bundle", "schema"),
|
|
352
|
-
identity_fields=frozenset(
|
|
353
|
-
{
|
|
354
|
-
"depends_on",
|
|
355
|
-
"job_cluster_key",
|
|
356
|
-
"library",
|
|
357
|
-
"libraries",
|
|
358
|
-
"task_key",
|
|
359
|
-
}
|
|
360
|
-
),
|
|
361
|
-
)
|
|
362
226
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
"adf:no_raw_schema",
|
|
366
|
-
None,
|
|
367
|
-
status="no_raw_schema",
|
|
368
|
-
advisory_url=ADVISORY_ADF_NO_RAW_SCHEMA_URL,
|
|
369
|
-
)
|
|
370
|
-
if language_id in {"hcl", "terraform"} or basename.endswith((".tf", ".hcl")):
|
|
227
|
+
def _candidate_from_payload(value: dict[str, Any] | None) -> SchemaCandidate | None:
|
|
228
|
+
if value is None:
|
|
371
229
|
return None
|
|
372
|
-
|
|
230
|
+
value["command"] = tuple(value["command"])
|
|
231
|
+
value["identity_fields"] = frozenset(value["identity_fields"])
|
|
232
|
+
return SchemaCandidate(**value)
|
|
373
233
|
|
|
374
234
|
|
|
375
235
|
def derive_identity_fields(schema: dict[str, Any] | None) -> frozenset[str]:
|
|
376
|
-
|
|
377
|
-
return frozenset()
|
|
378
|
-
found: set[str] = set()
|
|
379
|
-
stack: list[Any] = [schema]
|
|
380
|
-
seen: set[int] = set()
|
|
381
|
-
while stack:
|
|
382
|
-
current = stack.pop()
|
|
383
|
-
current_id = id(current)
|
|
384
|
-
if current_id in seen:
|
|
385
|
-
continue
|
|
386
|
-
seen.add(current_id)
|
|
387
|
-
if isinstance(current, dict):
|
|
388
|
-
properties = current.get("properties")
|
|
389
|
-
if isinstance(properties, dict):
|
|
390
|
-
for key, value in properties.items():
|
|
391
|
-
normalized = _normalize_identity_field(key)
|
|
392
|
-
if normalized in IDENTITY_FIELD_CANDIDATES:
|
|
393
|
-
found.add(normalized)
|
|
394
|
-
stack.append(value)
|
|
395
|
-
for key in (
|
|
396
|
-
"items",
|
|
397
|
-
"anyOf",
|
|
398
|
-
"oneOf",
|
|
399
|
-
"allOf",
|
|
400
|
-
"$defs",
|
|
401
|
-
"definitions",
|
|
402
|
-
"patternProperties",
|
|
403
|
-
):
|
|
404
|
-
value = current.get(key)
|
|
405
|
-
if value is not None:
|
|
406
|
-
stack.append(value)
|
|
407
|
-
elif isinstance(current, list):
|
|
408
|
-
stack.extend(current)
|
|
409
|
-
return frozenset(found)
|
|
236
|
+
return frozenset(_schema_operation("derive", schema=schema))
|
|
410
237
|
|
|
411
238
|
|
|
412
239
|
def _identity_fields_for(
|
|
@@ -416,12 +243,6 @@ def _identity_fields_for(
|
|
|
416
243
|
return frozenset({*candidate.identity_fields, *derive_identity_fields(schema)})
|
|
417
244
|
|
|
418
245
|
|
|
419
|
-
def schema_cache_fingerprint(resolution: SchemaResolution | None) -> str:
|
|
420
|
-
if resolution is None:
|
|
421
|
-
return "schema:none"
|
|
422
|
-
return resolution.cache_fingerprint
|
|
423
|
-
|
|
424
|
-
|
|
425
246
|
def schema_resolution_metadata(
|
|
426
247
|
resolution: SchemaResolution | None,
|
|
427
248
|
) -> dict[str, Any] | None:
|
|
@@ -453,7 +274,7 @@ def _fetch_or_cache(
|
|
|
453
274
|
url = candidate.url
|
|
454
275
|
if url is None:
|
|
455
276
|
raise ValueError("schema candidate has no URL to fetch")
|
|
456
|
-
cached = _read_cache(candidate, root)
|
|
277
|
+
cached = _read_cache(candidate, root, now)
|
|
457
278
|
if cached and now - cached["fetched_at"] < ttl:
|
|
458
279
|
return _resolution_from_cache(candidate, cached, "cache_hit")
|
|
459
280
|
|
|
@@ -505,19 +326,11 @@ def _resolve_from_cache(
|
|
|
505
326
|
*,
|
|
506
327
|
allow_stale: bool,
|
|
507
328
|
) -> SchemaResolution:
|
|
508
|
-
cached = _read_cache(candidate, root)
|
|
509
|
-
if not cached:
|
|
329
|
+
cached = _read_cache(candidate, root, now)
|
|
330
|
+
if not cached or (not allow_stale and now - cached["fetched_at"] >= ttl):
|
|
510
331
|
return SchemaResolution(
|
|
511
|
-
provider_id=candidate.provider_id,
|
|
512
|
-
|
|
513
|
-
status="cache_miss",
|
|
514
|
-
identity_fields=candidate.identity_fields,
|
|
515
|
-
)
|
|
516
|
-
if not allow_stale and now - cached["fetched_at"] >= ttl:
|
|
517
|
-
return SchemaResolution(
|
|
518
|
-
provider_id=candidate.provider_id,
|
|
519
|
-
source_url=candidate.url,
|
|
520
|
-
status="cache_stale",
|
|
332
|
+
provider_id=candidate.provider_id, source_url=candidate.url,
|
|
333
|
+
status="cache_miss" if not cached else "cache_stale",
|
|
521
334
|
identity_fields=candidate.identity_fields,
|
|
522
335
|
)
|
|
523
336
|
status = "cache_hit" if now - cached["fetched_at"] < ttl else "stale_cache"
|
|
@@ -548,21 +361,28 @@ def _resolution_from_cache(
|
|
|
548
361
|
)
|
|
549
362
|
|
|
550
363
|
|
|
551
|
-
def _read_cache(candidate: SchemaCandidate, root: Path) -> dict[str, Any] | None:
|
|
364
|
+
def _read_cache(candidate: SchemaCandidate, root: Path, now: float) -> dict[str, Any] | None:
|
|
365
|
+
"""A complete cache record, or None (a cache miss) for a missing or malformed one (#307)."""
|
|
552
366
|
path = _cache_file(candidate, root)
|
|
553
367
|
try:
|
|
554
368
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
369
|
+
except (OSError, ValueError, RecursionError):
|
|
370
|
+
data = None
|
|
371
|
+
stamp = data.get("fetched_at") if isinstance(data, dict) else None
|
|
372
|
+
if (type(stamp) in (int, float) and math.isfinite(stamp) and stamp <= now
|
|
373
|
+
and isinstance(data.get("schema"), dict)):
|
|
374
|
+
return data
|
|
375
|
+
if path.exists():
|
|
376
|
+
logger.warning("Ignoring malformed schema cache record %s", path)
|
|
559
377
|
return None
|
|
560
378
|
|
|
561
379
|
|
|
562
380
|
def _write_cache(candidate: SchemaCandidate, root: Path, data: Mapping[str, Any]) -> None:
|
|
563
381
|
path = _cache_file(candidate, root)
|
|
564
382
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
565
|
-
path.
|
|
383
|
+
temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") # readers never see a partial
|
|
384
|
+
temporary.write_text(json.dumps(data, sort_keys=True, indent=2), encoding="utf-8")
|
|
385
|
+
os.replace(temporary, path)
|
|
566
386
|
|
|
567
387
|
|
|
568
388
|
def _cache_file(candidate: SchemaCandidate, root: Path) -> Path:
|
|
@@ -571,11 +391,13 @@ def _cache_file(candidate: SchemaCandidate, root: Path) -> Path:
|
|
|
571
391
|
return root / f"{digest}.json"
|
|
572
392
|
|
|
573
393
|
|
|
574
|
-
def _fetch_https(
|
|
394
|
+
def _fetch_https(
|
|
395
|
+
url: str, headers: Mapping[str, str], *, env: Mapping[str, str] | None = None
|
|
396
|
+
) -> FetchResponse:
|
|
575
397
|
request = urllib.request.Request(url, headers=dict(headers)) # noqa: S310
|
|
576
398
|
context = ssl.create_default_context()
|
|
577
399
|
opener = urllib.request.build_opener(
|
|
578
|
-
_HttpsOnlyRedirectHandler,
|
|
400
|
+
_HttpsOnlyRedirectHandler(env),
|
|
579
401
|
urllib.request.HTTPSHandler(context=context),
|
|
580
402
|
)
|
|
581
403
|
with opener.open( # noqa: S310 - URL and redirects are validated before fetch.
|
|
@@ -593,6 +415,10 @@ def _fetch_https(url: str, headers: Mapping[str, str]) -> FetchResponse:
|
|
|
593
415
|
|
|
594
416
|
|
|
595
417
|
class _HttpsOnlyRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
418
|
+
def __init__(self, env: Mapping[str, str] | None = None) -> None:
|
|
419
|
+
super().__init__()
|
|
420
|
+
self._schema_env = env or {}
|
|
421
|
+
|
|
596
422
|
def redirect_request(
|
|
597
423
|
self,
|
|
598
424
|
req: urllib.request.Request,
|
|
@@ -610,6 +436,11 @@ class _HttpsOnlyRedirectHandler(urllib.request.HTTPRedirectHandler):
|
|
|
610
436
|
dict(headers),
|
|
611
437
|
fp,
|
|
612
438
|
)
|
|
439
|
+
# Every HTTPS redirect is a new network target. Check it before
|
|
440
|
+
# urllib opens the redirected request, not only after receiving bytes
|
|
441
|
+
# from its final_url (#256). Honour the same explicit private-host
|
|
442
|
+
# override as the initial request.
|
|
443
|
+
_validate_schema_url(newurl, self._schema_env)
|
|
613
444
|
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
614
445
|
|
|
615
446
|
|
|
@@ -641,9 +472,9 @@ def _schema_commands_allowed(env: Mapping[str, str]) -> bool:
|
|
|
641
472
|
|
|
642
473
|
def _schema_ttl_seconds(env: Mapping[str, str]) -> int:
|
|
643
474
|
raw = env.get("INTENTUMDIFF_SCHEMA_CACHE_TTL_SECONDS", "")
|
|
644
|
-
try:
|
|
475
|
+
try: # nan / inf / exponent overflow fall back to the default (#307)
|
|
645
476
|
return max(0, int(float(raw)))
|
|
646
|
-
except ValueError:
|
|
477
|
+
except (ValueError, OverflowError):
|
|
647
478
|
return DEFAULT_SCHEMA_TTL_SECONDS
|
|
648
479
|
|
|
649
480
|
|
|
@@ -674,8 +505,12 @@ def _validate_schema_url(url: str, env: Mapping[str, str]) -> None:
|
|
|
674
505
|
def _is_private_host(host: str) -> bool:
|
|
675
506
|
try:
|
|
676
507
|
addresses = {info[4][0] for info in socket.getaddrinfo(host, None)}
|
|
677
|
-
except OSError:
|
|
678
|
-
|
|
508
|
+
except OSError as exc:
|
|
509
|
+
# DNS failure must not certify a host as public: urllib would perform
|
|
510
|
+
# its own later lookup, which could resolve to a private address (#255).
|
|
511
|
+
raise ValueError(f"schema URL host could not be resolved: {host}") from exc
|
|
512
|
+
if not addresses:
|
|
513
|
+
raise ValueError(f"schema URL host returned no addresses: {host}")
|
|
679
514
|
for address in addresses:
|
|
680
515
|
try:
|
|
681
516
|
ip = ipaddress.ip_address(address)
|
|
@@ -691,126 +526,3 @@ def _is_private_host(host: str) -> bool:
|
|
|
691
526
|
):
|
|
692
527
|
return True
|
|
693
528
|
return False
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
def _dbt_schema_url(basename: str, language_id: str, content: str) -> str | None:
|
|
697
|
-
if basename in {"dbt_project.yml", "dbt_project.yaml"}:
|
|
698
|
-
return DBT_SCHEMA_URLS["dbt_project"]
|
|
699
|
-
if basename in {"packages.yml", "packages.yaml"}:
|
|
700
|
-
return DBT_SCHEMA_URLS["packages"]
|
|
701
|
-
if basename in {"dependencies.yml", "dependencies.yaml"}:
|
|
702
|
-
return DBT_SCHEMA_URLS["dependencies"]
|
|
703
|
-
if basename in {"selectors.yml", "selectors.yaml"}:
|
|
704
|
-
return DBT_SCHEMA_URLS["selectors"]
|
|
705
|
-
if basename in {"dbt_cloud.yml", "dbt_cloud.yaml"}:
|
|
706
|
-
return DBT_SCHEMA_URLS["dbt_cloud"]
|
|
707
|
-
if language_id in {"dbt-yaml", "dbt-config", "dbt-packages"}:
|
|
708
|
-
if language_id == "dbt-config":
|
|
709
|
-
return DBT_SCHEMA_URLS["dbt_project"]
|
|
710
|
-
if language_id == "dbt-packages":
|
|
711
|
-
return DBT_SCHEMA_URLS["packages"]
|
|
712
|
-
return DBT_SCHEMA_URLS["dbt_yml_files"]
|
|
713
|
-
markers = ("version", "models", "sources", "exposures", "seeds", "snapshots")
|
|
714
|
-
if basename.endswith((".yml", ".yaml")) and any(
|
|
715
|
-
re.search(rf"(?m)^\s*{re.escape(marker)}\s*:", content) for marker in markers
|
|
716
|
-
):
|
|
717
|
-
return DBT_SCHEMA_URLS["dbt_yml_files"]
|
|
718
|
-
return None
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
def _openapi_schema_url(basename: str, content: str) -> str | None:
|
|
722
|
-
if not basename.endswith((".json", ".yaml", ".yml")):
|
|
723
|
-
return None
|
|
724
|
-
lowered = basename.lower()
|
|
725
|
-
named_like_openapi = lowered in {
|
|
726
|
-
"openapi.json",
|
|
727
|
-
"openapi.yaml",
|
|
728
|
-
"openapi.yml",
|
|
729
|
-
"swagger.json",
|
|
730
|
-
"swagger.yaml",
|
|
731
|
-
"swagger.yml",
|
|
732
|
-
}
|
|
733
|
-
version = _top_level_scalar(content, "openapi")
|
|
734
|
-
if version:
|
|
735
|
-
if version.startswith("3.1"):
|
|
736
|
-
return OPENAPI_31_SCHEMA_URL
|
|
737
|
-
return OPENAPI_30_SCHEMA_URL
|
|
738
|
-
if _top_level_scalar(content, "swagger") or named_like_openapi:
|
|
739
|
-
return OPENAPI_30_SCHEMA_URL
|
|
740
|
-
return None
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
def _looks_like_kubernetes_manifest(basename: str, content: str) -> bool:
|
|
744
|
-
if not basename.endswith((".json", ".yaml", ".yml")):
|
|
745
|
-
return False
|
|
746
|
-
return bool(
|
|
747
|
-
_top_level_scalar(content, "apiVersion")
|
|
748
|
-
and _top_level_scalar(content, "kind")
|
|
749
|
-
and re.search(r"(?m)^\s*metadata\s*:", content)
|
|
750
|
-
)
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
def _looks_like_github_workflow(path: str, content: str) -> bool:
|
|
754
|
-
if not path.endswith((".yaml", ".yml")):
|
|
755
|
-
return False
|
|
756
|
-
if "/.github/workflows/" in f"/{path}":
|
|
757
|
-
return True
|
|
758
|
-
return bool(
|
|
759
|
-
re.search(r"(?m)^\s*on\s*:", content)
|
|
760
|
-
and re.search(r"(?m)^\s*jobs\s*:", content)
|
|
761
|
-
and (
|
|
762
|
-
re.search(r"(?m)^\s*uses\s*:", content)
|
|
763
|
-
or re.search(r"(?m)^\s*runs-on\s*:", content)
|
|
764
|
-
)
|
|
765
|
-
)
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
def _looks_like_azure_pipeline(path: str, content: str) -> bool:
|
|
769
|
-
basename = path.rsplit("/", 1)[-1]
|
|
770
|
-
if not path.endswith((".yaml", ".yml")):
|
|
771
|
-
return False
|
|
772
|
-
if basename in {"azure-pipelines.yml", "azure-pipelines.yaml"}:
|
|
773
|
-
return True
|
|
774
|
-
return bool(
|
|
775
|
-
re.search(r"(?m)^\s*(trigger|pr)\s*:", content)
|
|
776
|
-
and re.search(r"(?m)^\s*(stages|jobs|steps)\s*:", content)
|
|
777
|
-
and (
|
|
778
|
-
re.search(r"(?m)^\s*pool\s*:", content)
|
|
779
|
-
or re.search(r"(?m)^\s*vmImage\s*:", content)
|
|
780
|
-
or re.search(r"(?m)^\s*task\s*:", content)
|
|
781
|
-
)
|
|
782
|
-
)
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
def _top_level_scalar(content: str, key: str) -> str:
|
|
786
|
-
try:
|
|
787
|
-
parsed = json.loads(content)
|
|
788
|
-
except json.JSONDecodeError:
|
|
789
|
-
parsed = None
|
|
790
|
-
if isinstance(parsed, dict):
|
|
791
|
-
value = parsed.get(key)
|
|
792
|
-
return str(value).strip() if value is not None else ""
|
|
793
|
-
match = re.search(rf"(?m)^{re.escape(key)}\s*:\s*['\"]?([^'\"\n#]+)", content)
|
|
794
|
-
return match.group(1).strip() if match else ""
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
def _looks_like_adf_path(basename: str) -> bool:
|
|
798
|
-
direct_names = {"pipeline.json", "dataset.json", "linkedservice.json", "factory.json"}
|
|
799
|
-
return basename in direct_names or any(
|
|
800
|
-
basename.endswith(suffix)
|
|
801
|
-
for suffix in (
|
|
802
|
-
".pipeline.json",
|
|
803
|
-
".dataset.json",
|
|
804
|
-
".linkedservice.json",
|
|
805
|
-
".trigger.json",
|
|
806
|
-
)
|
|
807
|
-
)
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
def _normalize_identity_field(name: str) -> str:
|
|
811
|
-
return name.strip().lower().replace("-", "_")
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
def _looks_like_json(content: str) -> bool:
|
|
815
|
-
stripped = content.lstrip()
|
|
816
|
-
return stripped.startswith("{")
|