xp-libs-analytics 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xp_libs_analytics/__init__.py +18 -0
- xp_libs_analytics/engine/__init__.py +1 -0
- xp_libs_analytics/engine/backend.py +379 -0
- xp_libs_analytics/engine/planner.py +450 -0
- xp_libs_analytics/engine/worker.py +121 -0
- xp_libs_analytics/errors/__init__.py +43 -0
- xp_libs_analytics/errors/catalog.json +75 -0
- xp_libs_analytics/errors/codes.py +81 -0
- xp_libs_analytics/ingest/__init__.py +309 -0
- xp_libs_analytics/interfaces.py +70 -0
- xp_libs_analytics/matching/__init__.py +1 -0
- xp_libs_analytics/matching/compiler.py +344 -0
- xp_libs_analytics/matching/diagnostics.py +115 -0
- xp_libs_analytics/matching/models.py +98 -0
- xp_libs_analytics/matching/normalize.py +16 -0
- xp_libs_analytics/matching/patterns.py +71 -0
- xp_libs_analytics/mcp/__init__.py +1 -0
- xp_libs_analytics/mcp/server.py +248 -0
- xp_libs_analytics/models/__init__.py +654 -0
- xp_libs_analytics/parser/__init__.py +142 -0
- xp_libs_analytics/policy/__init__.py +69 -0
- xp_libs_analytics/py.typed +0 -0
- xp_libs_analytics/registry.py +204 -0
- xp_libs_analytics/sdk/__init__.py +259 -0
- xp_libs_analytics/skills/generic-analytics/SKILL.md +236 -0
- xp_libs_analytics/skills/generic-analytics/examples/README.md +69 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/01_count_all.yaml +8 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/02_filter_and_count.yaml +15 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/03_group_stats_and_topk.yaml +21 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/04_nested_or_predicates.yaml +17 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/05_is_missing.yaml +11 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/06_missing_vs_null.yaml +17 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/07_conditional_aggregation.yaml +19 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/08_ratio_if.yaml +16 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/09_tumbling_window.yaml +23 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/10_sliding_window.yaml +32 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/11_explicit_cast.yaml +16 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/12_text_range_distinct.yaml +23 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/13_summary_statistics.yaml +15 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/14_nested_and_not.yaml +20 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/15_conditional_sum.yaml +13 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/16_mixed_types_cast.yaml +17 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/17_relative_last_hour.yaml +15 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/18_string_exact.yaml +11 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/19_string_normalization.yaml +15 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/20_prefix_suffix.yaml +16 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/21_whole_word.yaml +13 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/22_ilike.yaml +11 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/23_rust_regex.yaml +11 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/24_fuzzy.yaml +17 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/25_multi_field_matching.yaml +24 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/26_conditional_fuzzy.yaml +20 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/27_temporal_fuzzy.yaml +38 -0
- xp_libs_analytics/skills/generic-analytics/examples/queries/28_safe_matching_diagnostics.yaml +22 -0
- xp_libs_analytics/skills/generic-analytics/references/errors.md +55 -0
- xp_libs_analytics/skills/generic-analytics/references/query-language-full.md +306 -0
- xp_libs_analytics/skills/generic-analytics/references/query-language.md +39 -0
- xp_libs_analytics/skills/generic-analytics/references/string-matching-full.md +122 -0
- xp_libs_analytics/skills/generic-analytics/references/string-matching.md +11 -0
- xp_libs_analytics-0.4.0.dist-info/METADATA +199 -0
- xp_libs_analytics-0.4.0.dist-info/RECORD +65 -0
- xp_libs_analytics-0.4.0.dist-info/WHEEL +4 -0
- xp_libs_analytics-0.4.0.dist-info/entry_points.txt +2 -0
- xp_libs_analytics-0.4.0.dist-info/licenses/LICENSE +22 -0
- xp_libs_analytics-0.4.0.dist-info/licenses/THIRD_PARTY_NOTICES.md +175 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Restricted YAML boundary; no global PyYAML resolver mutation."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
import re
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import yaml
|
|
8
|
+
from yaml import events, nodes
|
|
9
|
+
|
|
10
|
+
from xp_libs_analytics.errors import AnalyticsError
|
|
11
|
+
from xp_libs_analytics.policy import Limits
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def fault(code: str, mark: Any) -> AnalyticsError:
|
|
15
|
+
return AnalyticsError(code, line=mark.line + 1, column=mark.column + 1)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class RestrictedLoader(yaml.SafeLoader):
|
|
19
|
+
yaml_implicit_resolvers: dict[str, list[tuple[str, re.Pattern[str]]]] = {}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
for tag, pattern, initial in [
|
|
23
|
+
("bool", r"^(?:true|false)$", "tf"),
|
|
24
|
+
("null", r"^null$", "n"),
|
|
25
|
+
("int", r"^-?(?:0|[1-9][0-9]*)$", "-0123456789"),
|
|
26
|
+
(
|
|
27
|
+
"float",
|
|
28
|
+
r"^-?(?:0|[1-9][0-9]*)(?:\.[0-9]+(?:[eE][+-]?[0-9]+)?|[eE][+-]?[0-9]+)$",
|
|
29
|
+
"-0123456789",
|
|
30
|
+
),
|
|
31
|
+
]:
|
|
32
|
+
RestrictedLoader.add_implicit_resolver("tag:yaml.org,2002:" + tag, re.compile(pattern), initial)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
DEFAULT_LIMITS = Limits()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def parse_yaml(
|
|
39
|
+
text: str,
|
|
40
|
+
limits: Limits = DEFAULT_LIMITS,
|
|
41
|
+
*,
|
|
42
|
+
source_marks: dict[str, tuple[int, int]] | None = None,
|
|
43
|
+
) -> dict[str, Any]:
|
|
44
|
+
if not isinstance(text, str):
|
|
45
|
+
raise AnalyticsError("QUERY_MALFORMED")
|
|
46
|
+
try:
|
|
47
|
+
encoded = text.encode("utf-8", "strict")
|
|
48
|
+
except UnicodeError:
|
|
49
|
+
raise AnalyticsError("QUERY_YAML_SCALAR_INVALID") from None
|
|
50
|
+
if len(encoded) > limits.max_query_bytes:
|
|
51
|
+
raise AnalyticsError("QUERY_YAML_COMPLEXITY_LIMIT")
|
|
52
|
+
depth = docs = count = 0
|
|
53
|
+
widths: list[int] = []
|
|
54
|
+
try:
|
|
55
|
+
for event in yaml.parse(text, Loader=RestrictedLoader):
|
|
56
|
+
count += 1
|
|
57
|
+
if count > limits.max_yaml_nodes:
|
|
58
|
+
raise fault("QUERY_YAML_COMPLEXITY_LIMIT", event.start_mark)
|
|
59
|
+
if isinstance(event, events.DocumentStartEvent):
|
|
60
|
+
docs += 1
|
|
61
|
+
if docs > 1:
|
|
62
|
+
raise fault("QUERY_YAML_MULTI_DOCUMENT", event.start_mark)
|
|
63
|
+
if event.tags or event.version:
|
|
64
|
+
raise fault("QUERY_YAML_TAG_FORBIDDEN", event.start_mark)
|
|
65
|
+
if isinstance(event, events.AliasEvent):
|
|
66
|
+
raise fault("QUERY_YAML_ALIAS_FORBIDDEN", event.start_mark)
|
|
67
|
+
if getattr(event, "anchor", None):
|
|
68
|
+
raise fault("QUERY_YAML_ANCHOR_FORBIDDEN", event.start_mark)
|
|
69
|
+
if getattr(event, "tag", None):
|
|
70
|
+
raise fault("QUERY_YAML_TAG_FORBIDDEN", event.start_mark)
|
|
71
|
+
if isinstance(event, (events.ScalarEvent, events.CollectionStartEvent)) and widths:
|
|
72
|
+
widths[-1] += 1
|
|
73
|
+
if widths[-1] > limits.max_collection_items:
|
|
74
|
+
raise fault("QUERY_YAML_COMPLEXITY_LIMIT", event.start_mark)
|
|
75
|
+
if isinstance(event, events.CollectionStartEvent):
|
|
76
|
+
depth += 1
|
|
77
|
+
widths.append(0)
|
|
78
|
+
if depth > limits.max_yaml_depth:
|
|
79
|
+
raise fault("QUERY_YAML_COMPLEXITY_LIMIT", event.start_mark)
|
|
80
|
+
elif isinstance(event, events.CollectionEndEvent):
|
|
81
|
+
depth -= 1
|
|
82
|
+
widths.pop()
|
|
83
|
+
elif isinstance(event, events.ScalarEvent):
|
|
84
|
+
if len(event.value) > limits.max_scalar_chars:
|
|
85
|
+
raise fault("QUERY_YAML_COMPLEXITY_LIMIT", event.start_mark)
|
|
86
|
+
if any(
|
|
87
|
+
ord(c) < 32 and c not in "\n\r\t" or 0xD800 <= ord(c) <= 0xDFFF
|
|
88
|
+
for c in event.value
|
|
89
|
+
):
|
|
90
|
+
raise fault("QUERY_YAML_SCALAR_INVALID", event.start_mark)
|
|
91
|
+
loader = RestrictedLoader(text)
|
|
92
|
+
marks: dict[str, tuple[int, int]] = {}
|
|
93
|
+
try:
|
|
94
|
+
root = loader.get_single_node()
|
|
95
|
+
|
|
96
|
+
def convert(node: Any, path: str) -> Any:
|
|
97
|
+
if node is not None:
|
|
98
|
+
marks[path] = (node.start_mark.line + 1, node.start_mark.column + 1)
|
|
99
|
+
if isinstance(node, nodes.MappingNode):
|
|
100
|
+
result = {}
|
|
101
|
+
for key, value in node.value:
|
|
102
|
+
k = convert(key, path)
|
|
103
|
+
if not isinstance(k, str):
|
|
104
|
+
raise fault("QUERY_YAML_ROOT_INVALID", key.start_mark)
|
|
105
|
+
if k == "<<":
|
|
106
|
+
raise fault("QUERY_YAML_ALIAS_FORBIDDEN", key.start_mark)
|
|
107
|
+
if k in result:
|
|
108
|
+
raise fault("QUERY_YAML_DUPLICATE_KEY", key.start_mark)
|
|
109
|
+
result[k] = convert(value, path + "." + k)
|
|
110
|
+
return result
|
|
111
|
+
if isinstance(node, nodes.SequenceNode):
|
|
112
|
+
return [convert(v, path + f"[{i}]") for i, v in enumerate(node.value)]
|
|
113
|
+
if isinstance(node, nodes.ScalarNode):
|
|
114
|
+
value = loader.construct_object(node)
|
|
115
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
116
|
+
raise fault("QUERY_YAML_SCALAR_INVALID", node.start_mark)
|
|
117
|
+
if (
|
|
118
|
+
isinstance(value, int)
|
|
119
|
+
and not isinstance(value, bool)
|
|
120
|
+
and not -(2**63) <= value < 2**63
|
|
121
|
+
):
|
|
122
|
+
raise fault("QUERY_YAML_SCALAR_INVALID", node.start_mark)
|
|
123
|
+
if node.style is None and re.match(
|
|
124
|
+
r"^(?:[+-]?\.(?:nan|inf)|[+-]?0[xob][0-9a-f]+|0[0-9]+)$", node.value, re.I
|
|
125
|
+
):
|
|
126
|
+
raise fault("QUERY_YAML_SCALAR_INVALID", node.start_mark)
|
|
127
|
+
return value
|
|
128
|
+
raise AnalyticsError("QUERY_YAML_ROOT_INVALID")
|
|
129
|
+
|
|
130
|
+
result = convert(root, "$")
|
|
131
|
+
finally:
|
|
132
|
+
loader.dispose()
|
|
133
|
+
if not isinstance(result, dict) or not result:
|
|
134
|
+
raise AnalyticsError("QUERY_YAML_ROOT_INVALID")
|
|
135
|
+
if source_marks is not None:
|
|
136
|
+
source_marks.update(marks)
|
|
137
|
+
return result
|
|
138
|
+
except yaml.YAMLError as exc:
|
|
139
|
+
mark = getattr(exc, "problem_mark", None)
|
|
140
|
+
if mark:
|
|
141
|
+
raise fault("QUERY_YAML_INVALID", mark) from None
|
|
142
|
+
raise AnalyticsError("QUERY_YAML_INVALID") from None
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Host-owned policy. Request documents cannot override these limits."""
|
|
2
|
+
|
|
3
|
+
import threading
|
|
4
|
+
from collections import deque
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass(frozen=True)
|
|
9
|
+
class Limits:
|
|
10
|
+
max_bytes: int = 8 * 1024 * 1024
|
|
11
|
+
max_rows: int = 100_000
|
|
12
|
+
max_columns: int = 256
|
|
13
|
+
max_depth: int = 12
|
|
14
|
+
max_record_bytes: int = 128 * 1024
|
|
15
|
+
max_datasets: int = 8
|
|
16
|
+
max_total_bytes: int = 64 * 1024 * 1024
|
|
17
|
+
ttl_seconds: int = 900
|
|
18
|
+
max_result_bytes: int = 1024 * 1024
|
|
19
|
+
max_groups: int = 10_000
|
|
20
|
+
max_windows: int = 10_000
|
|
21
|
+
max_expanded_rows: int = 1_200_000
|
|
22
|
+
max_distinct: int = 10_000
|
|
23
|
+
max_query_bytes: int = 65_536
|
|
24
|
+
max_yaml_depth: int = 16
|
|
25
|
+
max_yaml_nodes: int = 4096
|
|
26
|
+
max_scalar_chars: int = 1024
|
|
27
|
+
max_collection_items: int = 1024
|
|
28
|
+
max_concurrency: int = 2
|
|
29
|
+
max_queries_per_minute: int = 120
|
|
30
|
+
timeout_seconds: float = 15.0
|
|
31
|
+
memory_bytes: int = 1024 * 1024 * 1024
|
|
32
|
+
min_samples: int = 1
|
|
33
|
+
allow_null_cast: bool = True
|
|
34
|
+
max_match_fields: int = 10
|
|
35
|
+
max_match_patterns: int = 10
|
|
36
|
+
max_match_work: int = 40
|
|
37
|
+
max_match_literal_chars: int = 1024
|
|
38
|
+
max_regex_bytes: int = 512
|
|
39
|
+
max_match_value_chars: int = 1024
|
|
40
|
+
max_fuzzy_candidates: int = 10_000
|
|
41
|
+
max_match_normalized_bytes: int = 8 * 1024 * 1024
|
|
42
|
+
max_fuzzy_comparisons: int = 100_000
|
|
43
|
+
min_fuzzy_query_chars: int = 3
|
|
44
|
+
min_fuzzy_score: float = 70.0
|
|
45
|
+
max_match_diagnostic_values: int = 20
|
|
46
|
+
max_match_diagnostic_candidates: int = 10_000
|
|
47
|
+
allow_match_raw_values: bool = False
|
|
48
|
+
allow_match_catch_all: bool = False
|
|
49
|
+
match_pushdown: bool = True
|
|
50
|
+
force_matching_isolation: bool = True
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class Principal:
|
|
55
|
+
id: str = "local"
|
|
56
|
+
fields: frozenset[str] | None = None
|
|
57
|
+
scopes: frozenset[str] = frozenset({"read", "upload", "release"})
|
|
58
|
+
|
|
59
|
+
match_raw_fields: frozenset[str] = frozenset()
|
|
60
|
+
|
|
61
|
+
def permits(self, field: str) -> bool:
|
|
62
|
+
return self.fields is None or field in self.fields
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class RequestBudget:
|
|
66
|
+
def __init__(self, limits: Limits) -> None:
|
|
67
|
+
self.slots = threading.BoundedSemaphore(limits.max_concurrency)
|
|
68
|
+
self.calls: deque[float] = deque()
|
|
69
|
+
self.lock = threading.Lock()
|
|
File without changes
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Context-local logical sources, atomic publication and pinned immutable snapshots."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import threading
|
|
7
|
+
from collections.abc import Callable, Iterator
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
from typing import Any, Literal, Protocol
|
|
12
|
+
|
|
13
|
+
from xp_libs_analytics.errors import AnalyticsError
|
|
14
|
+
from xp_libs_analytics.ingest import DatasetHandle, DatasetStore, Snapshot
|
|
15
|
+
from xp_libs_analytics.policy import Principal
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SnapshotSource(Protocol):
|
|
19
|
+
"""Trusted source binding; execution consumes snapshots, never DataFrames or user code."""
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def handle(self) -> DatasetHandle: ...
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def expires(self) -> datetime: ...
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def mode(self) -> Literal["snapshot"]: ...
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class RecordsSnapshotSource:
|
|
33
|
+
handle: DatasetHandle
|
|
34
|
+
expires: datetime
|
|
35
|
+
mode: Literal["snapshot"] = "snapshot"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class DatasetInfo:
|
|
40
|
+
"""Public binding metadata; opaque IDs remain internal."""
|
|
41
|
+
|
|
42
|
+
name: str
|
|
43
|
+
revision: int
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def validate_name(name: str) -> None:
|
|
47
|
+
if type(name) is not str or not re.fullmatch(r"[A-Za-z][A-Za-z0-9_-]{0,63}", name):
|
|
48
|
+
raise AnalyticsError("DATASET_NAME_INVALID")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _Namespace:
|
|
52
|
+
def __init__(self, owner: str) -> None:
|
|
53
|
+
self.bindings: dict[str, SnapshotSource] = {}
|
|
54
|
+
self.lock = threading.RLock()
|
|
55
|
+
self.closed = False
|
|
56
|
+
self.owner = owner
|
|
57
|
+
self.generation = 0
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class DatasetRegistry:
|
|
61
|
+
"""A bounded namespace owned by one execution context, never a global name table."""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
store: DatasetStore,
|
|
66
|
+
principal: Callable[[], Principal],
|
|
67
|
+
clock: Callable[[], datetime],
|
|
68
|
+
*,
|
|
69
|
+
namespace: _Namespace | None = None,
|
|
70
|
+
) -> None:
|
|
71
|
+
self._store, self._principal, self._clock = store, principal, clock
|
|
72
|
+
self._namespace = namespace or _Namespace(principal().id)
|
|
73
|
+
self._bindings = self._namespace.bindings
|
|
74
|
+
self._lock = self._namespace.lock
|
|
75
|
+
|
|
76
|
+
def _check(self, scope: str) -> Principal:
|
|
77
|
+
principal = self._principal()
|
|
78
|
+
if (
|
|
79
|
+
self._namespace.closed
|
|
80
|
+
or scope not in principal.scopes
|
|
81
|
+
or principal.id != self._namespace.owner
|
|
82
|
+
):
|
|
83
|
+
raise AnalyticsError("ACCESS_DENIED")
|
|
84
|
+
return principal
|
|
85
|
+
|
|
86
|
+
def _source(self, name: str, revision: int | None = None) -> SnapshotSource:
|
|
87
|
+
validate_name(name)
|
|
88
|
+
source = self._bindings.get(name)
|
|
89
|
+
if source is None:
|
|
90
|
+
# A foreign context's name is indistinguishable from an unknown name.
|
|
91
|
+
raise AnalyticsError("DATASET_NAME_UNKNOWN")
|
|
92
|
+
if revision is not None and (
|
|
93
|
+
type(revision) is not int or revision != source.handle.revision
|
|
94
|
+
):
|
|
95
|
+
raise AnalyticsError("DATASET_SNAPSHOT_STALE")
|
|
96
|
+
return source
|
|
97
|
+
|
|
98
|
+
def _publish(
|
|
99
|
+
self,
|
|
100
|
+
name: str,
|
|
101
|
+
records: Any,
|
|
102
|
+
*,
|
|
103
|
+
replace: bool,
|
|
104
|
+
ndjson: bool,
|
|
105
|
+
expected_revision: int | None = None,
|
|
106
|
+
) -> DatasetInfo:
|
|
107
|
+
validate_name(name)
|
|
108
|
+
with self._lock:
|
|
109
|
+
principal = self._check("upload")
|
|
110
|
+
old = self._source(name, expected_revision) if replace else None
|
|
111
|
+
if not replace and name in self._bindings:
|
|
112
|
+
raise AnalyticsError("DATASET_NAME_DUPLICATE")
|
|
113
|
+
if not replace and len(self._bindings) >= self._store.limits.max_datasets:
|
|
114
|
+
raise AnalyticsError("RESOURCE_DATASETS_LIMIT")
|
|
115
|
+
now = self._clock()
|
|
116
|
+
revision = self._namespace.generation + 1
|
|
117
|
+
handle = self._store.register(
|
|
118
|
+
records,
|
|
119
|
+
principal,
|
|
120
|
+
now,
|
|
121
|
+
ndjson=ndjson,
|
|
122
|
+
replaces=old.handle if old else None,
|
|
123
|
+
revision=revision,
|
|
124
|
+
)
|
|
125
|
+
snapshot = self._store.get_snapshot(handle, principal, now)
|
|
126
|
+
self._namespace.generation = revision
|
|
127
|
+
self._bindings[name] = RecordsSnapshotSource(handle, snapshot.expires)
|
|
128
|
+
return DatasetInfo(name, revision)
|
|
129
|
+
|
|
130
|
+
def register_records(self, name: str, records: Any, *, ndjson: bool = False) -> DatasetInfo:
|
|
131
|
+
return self._publish(name, records, replace=False, ndjson=ndjson)
|
|
132
|
+
|
|
133
|
+
def replace_records(
|
|
134
|
+
self,
|
|
135
|
+
name: str,
|
|
136
|
+
records: Any,
|
|
137
|
+
*,
|
|
138
|
+
ndjson: bool = False,
|
|
139
|
+
expected_revision: int | None = None,
|
|
140
|
+
) -> DatasetInfo:
|
|
141
|
+
return self._publish(
|
|
142
|
+
name, records, replace=True, ndjson=ndjson, expected_revision=expected_revision
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
@contextmanager
|
|
146
|
+
def snapshot(self, name: str, *, revision: int | None = None) -> Iterator[Snapshot]:
|
|
147
|
+
with self._lock:
|
|
148
|
+
principal = self._check("read")
|
|
149
|
+
source = self._source(name, revision)
|
|
150
|
+
now = self._clock()
|
|
151
|
+
if source.expires <= now:
|
|
152
|
+
raise AnalyticsError("DATASET_SNAPSHOT_STALE")
|
|
153
|
+
snapshot = self._store.acquire(source.handle, principal, now)
|
|
154
|
+
try:
|
|
155
|
+
yield snapshot
|
|
156
|
+
finally:
|
|
157
|
+
self._store.unpin(source.handle)
|
|
158
|
+
|
|
159
|
+
def names(self) -> list[str]:
|
|
160
|
+
with self._lock:
|
|
161
|
+
self._check("read")
|
|
162
|
+
return sorted(self._bindings)
|
|
163
|
+
|
|
164
|
+
def describe(self, name: str) -> dict[str, Any]:
|
|
165
|
+
with self.snapshot(name) as snapshot:
|
|
166
|
+
principal = self._check("read")
|
|
167
|
+
return {
|
|
168
|
+
"name": name,
|
|
169
|
+
"revision": snapshot.handle.revision,
|
|
170
|
+
"records": snapshot.row_count,
|
|
171
|
+
"source_kind": snapshot.source_kind,
|
|
172
|
+
"mode": "snapshot",
|
|
173
|
+
"expires": snapshot.expires.isoformat(),
|
|
174
|
+
"fields": {
|
|
175
|
+
p: info for p, info in snapshot.schema().items() if principal.permits(p)
|
|
176
|
+
},
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
def list_datasets(self) -> list[dict[str, Any]]:
|
|
180
|
+
datasets = []
|
|
181
|
+
for name in self.names():
|
|
182
|
+
try:
|
|
183
|
+
datasets.append(self.describe(name))
|
|
184
|
+
except AnalyticsError as exc:
|
|
185
|
+
if exc.code not in {"DATASET_NAME_UNKNOWN", "DATASET_SNAPSHOT_STALE"}:
|
|
186
|
+
raise
|
|
187
|
+
return datasets
|
|
188
|
+
|
|
189
|
+
def unregister_dataset(self, name: str, *, expected_revision: int | None = None) -> None:
|
|
190
|
+
with self._lock:
|
|
191
|
+
self._check("release")
|
|
192
|
+
source = self._source(name, expected_revision)
|
|
193
|
+
# Expired bindings can still be removed. No query can resolve them again.
|
|
194
|
+
with self._store._lock:
|
|
195
|
+
self._store._retire(source.handle.id)
|
|
196
|
+
del self._bindings[name]
|
|
197
|
+
|
|
198
|
+
def close(self) -> None:
|
|
199
|
+
with self._lock:
|
|
200
|
+
for source in self._bindings.values():
|
|
201
|
+
with self._store._lock:
|
|
202
|
+
self._store._retire(source.handle.id)
|
|
203
|
+
self._bindings.clear()
|
|
204
|
+
self._namespace.closed = True
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
"""Headless Python API; identity is supplied by the trusted application."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Callable, Iterator
|
|
8
|
+
from contextlib import contextmanager
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from time import monotonic
|
|
12
|
+
from typing import Any
|
|
13
|
+
from uuid import uuid4
|
|
14
|
+
|
|
15
|
+
from xp_libs_analytics.engine.planner import LogicalPlan, plan_query
|
|
16
|
+
from xp_libs_analytics.errors import CATALOG, AnalyticsError
|
|
17
|
+
from xp_libs_analytics.ingest import DatasetStore, Snapshot, utc_now
|
|
18
|
+
from xp_libs_analytics.models import AGG_OPS, FILTER_OPS, AnalyticsQuery, parse_query
|
|
19
|
+
from xp_libs_analytics.policy import Limits, Principal
|
|
20
|
+
from xp_libs_analytics.registry import DatasetInfo, DatasetRegistry
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class AnalyticsEngine:
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
*,
|
|
27
|
+
limits: Limits | None = None,
|
|
28
|
+
principal: Principal | None = None,
|
|
29
|
+
store: DatasetStore | None = None,
|
|
30
|
+
clock: Callable[[], datetime] = utc_now,
|
|
31
|
+
isolated: bool = False,
|
|
32
|
+
telemetry: Callable[[dict[str, Any]], None] | None = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
self.limits = limits or Limits()
|
|
35
|
+
self.principal = principal or Principal()
|
|
36
|
+
self.store = store or DatasetStore(self.limits)
|
|
37
|
+
self.registry = DatasetRegistry(self.store, lambda: self.principal, lambda: self.clock())
|
|
38
|
+
self.clock = clock
|
|
39
|
+
self.isolated = isolated
|
|
40
|
+
self.telemetry = telemetry
|
|
41
|
+
request_budget = self.store.request_budget(self.principal)
|
|
42
|
+
self._slots = request_budget.slots
|
|
43
|
+
self._calls = request_budget.calls
|
|
44
|
+
self._quota_lock = request_budget.lock
|
|
45
|
+
|
|
46
|
+
def register_records(self, name: str, records: Any, *, ndjson: bool = False) -> DatasetInfo:
|
|
47
|
+
return self.registry.register_records(name, records, ndjson=ndjson)
|
|
48
|
+
|
|
49
|
+
def replace_records(
|
|
50
|
+
self,
|
|
51
|
+
name: str,
|
|
52
|
+
records: Any,
|
|
53
|
+
*,
|
|
54
|
+
ndjson: bool = False,
|
|
55
|
+
expected_revision: int | None = None,
|
|
56
|
+
) -> DatasetInfo:
|
|
57
|
+
return self.registry.replace_records(
|
|
58
|
+
name, records, ndjson=ndjson, expected_revision=expected_revision
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def describe(self, name: str) -> dict[str, Any]:
|
|
62
|
+
return self.registry.describe(name)
|
|
63
|
+
|
|
64
|
+
def list_datasets(self) -> list[dict[str, Any]]:
|
|
65
|
+
return self.registry.list_datasets()
|
|
66
|
+
|
|
67
|
+
def unregister_dataset(self, name: str, *, expected_revision: int | None = None) -> None:
|
|
68
|
+
self.registry.unregister_dataset(name, expected_revision=expected_revision)
|
|
69
|
+
|
|
70
|
+
def register_stream(self, name: str, source: Any) -> None:
|
|
71
|
+
"""Reserved seam; V1 cannot admit or execute continuous sources."""
|
|
72
|
+
raise AnalyticsError("QUERY_UNSUPPORTED_FEATURE")
|
|
73
|
+
|
|
74
|
+
def create_context(self) -> AnalyticsEngine:
|
|
75
|
+
"""Create an independent namespace with shared host resource/owner budgets."""
|
|
76
|
+
return AnalyticsEngine(
|
|
77
|
+
limits=self.limits,
|
|
78
|
+
principal=self.principal,
|
|
79
|
+
store=self.store,
|
|
80
|
+
clock=self.clock,
|
|
81
|
+
isolated=self.isolated,
|
|
82
|
+
telemetry=self.telemetry,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
def _authorized_view(self, principal: Principal) -> AnalyticsEngine:
|
|
86
|
+
if principal.id != self.principal.id:
|
|
87
|
+
raise AnalyticsError("ACCESS_DENIED")
|
|
88
|
+
view = AnalyticsEngine(
|
|
89
|
+
limits=self.limits,
|
|
90
|
+
principal=principal,
|
|
91
|
+
store=self.store,
|
|
92
|
+
clock=self.clock,
|
|
93
|
+
isolated=self.isolated,
|
|
94
|
+
telemetry=self.telemetry,
|
|
95
|
+
)
|
|
96
|
+
view.registry = DatasetRegistry(
|
|
97
|
+
self.store,
|
|
98
|
+
lambda: view.principal,
|
|
99
|
+
lambda: view.clock(),
|
|
100
|
+
namespace=self.registry._namespace,
|
|
101
|
+
)
|
|
102
|
+
return view
|
|
103
|
+
|
|
104
|
+
def close(self) -> None:
|
|
105
|
+
self.registry.close()
|
|
106
|
+
|
|
107
|
+
def __enter__(self) -> AnalyticsEngine:
|
|
108
|
+
return self
|
|
109
|
+
|
|
110
|
+
def __exit__(self, *args: Any) -> None:
|
|
111
|
+
self.close()
|
|
112
|
+
|
|
113
|
+
def capabilities(self) -> dict[str, Any]:
|
|
114
|
+
from dataclasses import asdict
|
|
115
|
+
|
|
116
|
+
return {
|
|
117
|
+
"version": "analytics.query/v1",
|
|
118
|
+
"authoring_format": "yaml",
|
|
119
|
+
"mode": "snapshot",
|
|
120
|
+
"filters": FILTER_OPS,
|
|
121
|
+
"matching": {
|
|
122
|
+
"algorithms": ["ratio", "wratio", "token_sort"],
|
|
123
|
+
"normalization": {
|
|
124
|
+
"case_sensitive": True,
|
|
125
|
+
"unicode": ["none", "nfc", "nfkc"],
|
|
126
|
+
"trim": False,
|
|
127
|
+
},
|
|
128
|
+
"diagnostics": "options.matching_details",
|
|
129
|
+
"reference": "analytics://string-matching/v1",
|
|
130
|
+
"entity_inference": False,
|
|
131
|
+
},
|
|
132
|
+
"aggregations": AGG_OPS,
|
|
133
|
+
"limits": asdict(self.limits),
|
|
134
|
+
"schema": AnalyticsQuery.model_json_schema(),
|
|
135
|
+
"execution_isolated": self.isolated,
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
@contextmanager
|
|
139
|
+
def _prepare(self, text: str) -> Iterator[tuple[LogicalPlan, Snapshot, datetime]]:
|
|
140
|
+
query = parse_query(text, self.limits)
|
|
141
|
+
with self.registry.snapshot(query.dataset.name) as snapshot:
|
|
142
|
+
now = self.clock()
|
|
143
|
+
yield plan_query(query, snapshot, self.principal, self.limits, now), snapshot, now
|
|
144
|
+
|
|
145
|
+
def validate_query(self, query_yaml: str) -> dict[str, Any]:
|
|
146
|
+
return self.explain_query(query_yaml)
|
|
147
|
+
|
|
148
|
+
def explain_query(self, query_yaml: str) -> dict[str, Any]:
|
|
149
|
+
try:
|
|
150
|
+
self._take_quota()
|
|
151
|
+
with self._prepare(query_yaml) as (plan, snapshot, _):
|
|
152
|
+
return {
|
|
153
|
+
"status": "OK",
|
|
154
|
+
"plan": plan.explain(),
|
|
155
|
+
"dataset": {
|
|
156
|
+
"name": json.loads(plan.query_json)["dataset"]["name"],
|
|
157
|
+
"revision": snapshot.handle.revision,
|
|
158
|
+
},
|
|
159
|
+
}
|
|
160
|
+
except AnalyticsError as exc:
|
|
161
|
+
return exc.envelope()
|
|
162
|
+
except Exception:
|
|
163
|
+
return AnalyticsError("INTERNAL_ERROR").envelope()
|
|
164
|
+
|
|
165
|
+
def _take_quota(self) -> None:
|
|
166
|
+
with self._quota_lock:
|
|
167
|
+
now = monotonic()
|
|
168
|
+
while self._calls and self._calls[0] < now - 60:
|
|
169
|
+
self._calls.popleft()
|
|
170
|
+
if len(self._calls) >= self.limits.max_queries_per_minute:
|
|
171
|
+
raise AnalyticsError("QUERY_TOO_EXPENSIVE")
|
|
172
|
+
self._calls.append(now)
|
|
173
|
+
|
|
174
|
+
def query_yaml(self, query_yaml: str) -> dict[str, Any]:
|
|
175
|
+
query_id = uuid4().hex
|
|
176
|
+
started = monotonic()
|
|
177
|
+
event: dict[str, Any] = {
|
|
178
|
+
"query_id": query_id,
|
|
179
|
+
"event": "query",
|
|
180
|
+
"principal_hash": hashlib.sha256(self.principal.id.encode()).hexdigest(),
|
|
181
|
+
"outcome": "ERROR",
|
|
182
|
+
}
|
|
183
|
+
acquired = False
|
|
184
|
+
try:
|
|
185
|
+
self._take_quota()
|
|
186
|
+
acquired = self._slots.acquire(blocking=False)
|
|
187
|
+
if not acquired:
|
|
188
|
+
raise AnalyticsError("RESOURCE_CONCURRENCY_LIMIT")
|
|
189
|
+
with self._prepare(query_yaml) as (plan, snapshot, now):
|
|
190
|
+
event["plan_fingerprint"] = plan.fingerprint
|
|
191
|
+
event["snapshot_id"] = snapshot.handle.id
|
|
192
|
+
event["snapshot_revision"] = snapshot.handle.revision
|
|
193
|
+
if (
|
|
194
|
+
self.isolated
|
|
195
|
+
or self.limits.force_matching_isolation
|
|
196
|
+
and json.loads(plan.matching_cost_json).get("requires_isolation")
|
|
197
|
+
):
|
|
198
|
+
from xp_libs_analytics.engine.worker import execute_isolated
|
|
199
|
+
|
|
200
|
+
result = execute_isolated(plan, self.limits)
|
|
201
|
+
else:
|
|
202
|
+
from xp_libs_analytics.engine.backend import execute_plan
|
|
203
|
+
|
|
204
|
+
result = execute_plan(plan, self.limits)
|
|
205
|
+
result.update(
|
|
206
|
+
version="analytics.result/v1",
|
|
207
|
+
query_id=query_id,
|
|
208
|
+
dataset={
|
|
209
|
+
"name": json.loads(plan.query_json)["dataset"]["name"],
|
|
210
|
+
"revision": snapshot.handle.revision,
|
|
211
|
+
},
|
|
212
|
+
)
|
|
213
|
+
if result["status"] != "ERROR":
|
|
214
|
+
result["evidence"].update(
|
|
215
|
+
schema_revision=snapshot.handle.revision,
|
|
216
|
+
source_kind=snapshot.source_kind,
|
|
217
|
+
computed_at=now.isoformat().replace("+00:00", "Z"),
|
|
218
|
+
)
|
|
219
|
+
if (
|
|
220
|
+
len(json.dumps(result, allow_nan=False).encode("utf-8"))
|
|
221
|
+
> self.limits.max_result_bytes
|
|
222
|
+
):
|
|
223
|
+
raise AnalyticsError("RESOURCE_RESULT_LIMIT")
|
|
224
|
+
event["outcome"] = result["status"]
|
|
225
|
+
if result["status"] == "ERROR":
|
|
226
|
+
event["error_code"] = result["error"]["code"]
|
|
227
|
+
return result
|
|
228
|
+
except AnalyticsError as exc:
|
|
229
|
+
event["error_code"] = exc.code.value
|
|
230
|
+
return exc.envelope(query_id)
|
|
231
|
+
except Exception:
|
|
232
|
+
event["error_code"] = "EXECUTION_BACKEND_ERROR"
|
|
233
|
+
return AnalyticsError("EXECUTION_BACKEND_ERROR").envelope(query_id)
|
|
234
|
+
finally:
|
|
235
|
+
if acquired:
|
|
236
|
+
self._slots.release()
|
|
237
|
+
if self.telemetry is not None:
|
|
238
|
+
event["duration_seconds"] = monotonic() - started
|
|
239
|
+
try:
|
|
240
|
+
self.telemetry(event)
|
|
241
|
+
except Exception: # noqa: S110 -- do not log sensitive host hook exceptions
|
|
242
|
+
# A host observability outage cannot alter analytical results.
|
|
243
|
+
pass
|
|
244
|
+
|
|
245
|
+
def query_file(self, path: Path) -> dict[str, Any]:
|
|
246
|
+
if path.suffix.lower() not in (".yaml", ".yml"):
|
|
247
|
+
return AnalyticsError("QUERY_MALFORMED").envelope()
|
|
248
|
+
try:
|
|
249
|
+
with path.open("rb") as stream:
|
|
250
|
+
text = stream.read(self.limits.max_query_bytes + 1).decode("utf-8")
|
|
251
|
+
return self.query_yaml(text)
|
|
252
|
+
except (OSError, UnicodeError):
|
|
253
|
+
return AnalyticsError("QUERY_YAML_INVALID").envelope()
|
|
254
|
+
|
|
255
|
+
@staticmethod
|
|
256
|
+
def error_info(code: str) -> dict[str, Any]:
|
|
257
|
+
if code not in CATALOG:
|
|
258
|
+
return AnalyticsError("MCP_INVALID_TOOL_CALL").envelope()
|
|
259
|
+
return {"code": code, "message": CATALOG[code]}
|