prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
"""The engine a request runs on.
|
|
2
|
+
|
|
3
|
+
- `one(engine)`: every request on one engine, queued on its lock.
|
|
4
|
+
- `DatabricksUsers`: on a Databricks App, each request on an engine over a
|
|
5
|
+
SQL warehouse connection opened with the signed-in user's token, which
|
|
6
|
+
the app forwards as `X-Forwarded-Access-Token`; Unity Catalog's grants
|
|
7
|
+
are that person's. An engine is kept per user and token for `idle`
|
|
8
|
+
seconds after its last request, then closed. The app's service principal
|
|
9
|
+
is used only when `service_principal=True` and no user token came.
|
|
10
|
+
With `from_bearer` (`prograph serve --per-user`, off an App) the user's
|
|
11
|
+
credential is the request's `Authorization: Bearer` instead.
|
|
12
|
+
|
|
13
|
+
The schema is read once (`schema_from`) and shared: a GraphSchema is not
|
|
14
|
+
written to by a query.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import hashlib
|
|
20
|
+
import math
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from typing import Any, Callable, Dict, Optional, Tuple
|
|
24
|
+
|
|
25
|
+
USER_TOKEN = "x-forwarded-access-token"
|
|
26
|
+
USER_NAME = "x-forwarded-email"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def schema_from(config: Optional[str] = None, r2rml: Optional[str] = None) -> Any:
|
|
30
|
+
"""A GraphSchema from a graph YAML (`prograph.config`) or an R2RML
|
|
31
|
+
mapping (`prograph.r2rml`), whichever is named."""
|
|
32
|
+
if (config is None) == (r2rml is None):
|
|
33
|
+
raise ValueError("name one of a graph config (--config) or an R2RML mapping (--r2rml)")
|
|
34
|
+
if r2rml is not None:
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from prograph.r2rml.schema import load
|
|
37
|
+
return load(Path(r2rml).read_text())
|
|
38
|
+
from prograph.config import load_config
|
|
39
|
+
return load_config(config).to_schema()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def one(engine: Any) -> Callable[[Any], Any]:
|
|
43
|
+
def source(request: Any) -> Any:
|
|
44
|
+
return engine
|
|
45
|
+
# Every request is on this engine: its mapping is the server's (the Service Description reads it).
|
|
46
|
+
source.single = True # type: ignore[attr-defined]
|
|
47
|
+
return source
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def local_engine(config: Optional[str] = None, r2rml: Optional[str] = None,
|
|
51
|
+
duckdb_path: Optional[str] = None) -> Any:
|
|
52
|
+
"""An engine on this machine: over a DuckDB database file (opened read
|
|
53
|
+
only), or over the files a graph YAML names (`ProGraph.from_config`)."""
|
|
54
|
+
from prograph import ProGraph
|
|
55
|
+
if duckdb_path is not None:
|
|
56
|
+
import duckdb
|
|
57
|
+
return ProGraph(schema=schema_from(config, r2rml), backend="duckdb",
|
|
58
|
+
connection=duckdb.connect(duckdb_path, read_only=True))
|
|
59
|
+
if config is None:
|
|
60
|
+
raise ValueError("--r2rml needs the database it maps (--duckdb or a warehouse)")
|
|
61
|
+
return ProGraph.from_config(config)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class _Held:
|
|
65
|
+
"""A user's engine, when it was last asked for or let go, whose it is,
|
|
66
|
+
when its credential stops (monotonic; None: only idleness closes it),
|
|
67
|
+
and how many requests are running on it."""
|
|
68
|
+
__slots__ = ("engine", "last", "user", "until", "busy", "key")
|
|
69
|
+
|
|
70
|
+
def __init__(self, engine: Any, last: float, user: str, until: Optional[float]):
|
|
71
|
+
self.engine, self.last, self.user, self.until = engine, last, user, until
|
|
72
|
+
self.busy, self.key = 0, ""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _Build:
|
|
76
|
+
"""One build per credential at a time, and how many requests wait on it,
|
|
77
|
+
so the lock is dropped only when nobody holds or waits for it."""
|
|
78
|
+
__slots__ = ("lock", "waiting")
|
|
79
|
+
|
|
80
|
+
def __init__(self) -> None:
|
|
81
|
+
self.lock, self.waiting = threading.Lock(), 0
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
REFUSED_FOR = 30.0 # seconds a refused credential is answered from memory
|
|
85
|
+
REFUSED_KEPT = 10000 # the most refused credentials remembered
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class DatabricksUsers:
|
|
89
|
+
"""An engine per signed-in user over one SQL warehouse.
|
|
90
|
+
|
|
91
|
+
Where the user's credential comes from is the server's mode, chosen at
|
|
92
|
+
start (`prograph serve`): on a Databricks App the proxy forwards it as
|
|
93
|
+
`X-Forwarded-Access-Token` (the default); with `from_bearer` (`--per-user`,
|
|
94
|
+
off an App) the caller sends it as `Authorization: Bearer`, a Databricks
|
|
95
|
+
token used as it is or an identity provider's JWT exchanged for one
|
|
96
|
+
(`prograph.endpoint.credentials`).
|
|
97
|
+
|
|
98
|
+
Engines are kept by the SHA-256 of the credential the request carried.
|
|
99
|
+
A request holds its engine until `release` (the endpoint calls it when
|
|
100
|
+
the answer is sent): an engine is closed `idle` seconds after its last
|
|
101
|
+
request ended and none is running, by a timer
|
|
102
|
+
every `sweep_every` seconds as well as on the next request. One whose
|
|
103
|
+
credential expires (an exchanged token's `expires_in`, a JWT's `exp`)
|
|
104
|
+
takes no new request and is closed when its last one ends. At most
|
|
105
|
+
`max_engines` are open: a request that would open one more is a 503
|
|
106
|
+
with `Retry-After` until one closes; open engines are never closed to
|
|
107
|
+
make room. With `from_bearer`, a credential refused is answered 401
|
|
108
|
+
from memory for `REFUSED_FOR` seconds, with no call to the workspace.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
def __init__(self, schema: Any, server_hostname: str, http_path: str,
|
|
112
|
+
catalog: Optional[str] = None, db_schema: Optional[str] = None,
|
|
113
|
+
idle: float = 900.0, service_principal: bool = False,
|
|
114
|
+
connect: Optional[Callable[..., Any]] = None,
|
|
115
|
+
from_bearer: bool = False, max_engines: int = 64,
|
|
116
|
+
sweep_every: Optional[float] = None, workspace: Any = None,
|
|
117
|
+
refused_for: float = REFUSED_FOR,
|
|
118
|
+
statement_timeout: Optional[float] = None):
|
|
119
|
+
if from_bearer and service_principal:
|
|
120
|
+
raise ValueError("a per-user server has no service principal: every request "
|
|
121
|
+
"carries its user's credential")
|
|
122
|
+
if max_engines < 1:
|
|
123
|
+
raise ValueError("max_engines is at least 1")
|
|
124
|
+
if idle < 0:
|
|
125
|
+
raise ValueError("idle is not negative")
|
|
126
|
+
self.schema, self.hostname, self.http_path = schema, server_hostname, http_path
|
|
127
|
+
self.catalog, self.db_schema = catalog, db_schema
|
|
128
|
+
self.idle, self.service_principal = idle, service_principal
|
|
129
|
+
self.from_bearer, self.max_engines = from_bearer, max_engines
|
|
130
|
+
self.sweep_every = sweep_every if sweep_every is not None else max(0.05, min(60.0, idle / 4))
|
|
131
|
+
self.refused_for = refused_for
|
|
132
|
+
# The warehouse cancels a statement past this many seconds (Databricks SQL's
|
|
133
|
+
# STATEMENT_TIMEOUT, set on each connection's session): the server's --timeout.
|
|
134
|
+
self.statement_timeout = statement_timeout
|
|
135
|
+
self._connect = connect or self._sql_connect
|
|
136
|
+
if workspace is None:
|
|
137
|
+
from prograph.endpoint.credentials import Workspace
|
|
138
|
+
workspace = Workspace(server_hostname)
|
|
139
|
+
self.workspace = workspace
|
|
140
|
+
self._engines: Dict[str, _Held] = {}
|
|
141
|
+
# Every open engine by id(engine): those in `_engines` and those
|
|
142
|
+
# retired (credential ended) while a request still runs on them.
|
|
143
|
+
self._open_by_id: Dict[int, _Held] = {}
|
|
144
|
+
self._refused: Dict[str, Tuple[float, str]] = {}
|
|
145
|
+
self._opening = 0
|
|
146
|
+
self._lock = threading.Lock()
|
|
147
|
+
# One build per user at a time: a second request of the same user
|
|
148
|
+
# waits for the first's engine instead of opening another.
|
|
149
|
+
self._building: Dict[str, _Build] = {}
|
|
150
|
+
self._stop = threading.Event()
|
|
151
|
+
self._sweeper: Optional[threading.Thread] = None
|
|
152
|
+
|
|
153
|
+
def _session(self) -> Dict[str, Any]:
|
|
154
|
+
"""The session configuration every connection opens with: STATEMENT_TIMEOUT, whole seconds
|
|
155
|
+
(at least 1), where the server has a timeout; the warehouse halts a statement past it."""
|
|
156
|
+
if not self.statement_timeout:
|
|
157
|
+
return {}
|
|
158
|
+
return {"session_configuration": {"STATEMENT_TIMEOUT": str(max(1, math.ceil(self.statement_timeout)))}}
|
|
159
|
+
|
|
160
|
+
def _sql_connect(self, **kwargs: Any) -> Any:
|
|
161
|
+
import databricks.sql as dbsql
|
|
162
|
+
return dbsql.connect(server_hostname=self.hostname, http_path=self.http_path,
|
|
163
|
+
catalog=self.catalog, schema=self.db_schema, **kwargs)
|
|
164
|
+
|
|
165
|
+
def _service_principal_kwargs(self) -> Dict[str, Any]:
|
|
166
|
+
from databricks.sdk.core import Config, oauth_service_principal
|
|
167
|
+
config = Config()
|
|
168
|
+
return {"credentials_provider": lambda: oauth_service_principal(config)}
|
|
169
|
+
|
|
170
|
+
def __call__(self, request: Any) -> Any:
|
|
171
|
+
from prograph.endpoint.app import ProtocolError
|
|
172
|
+
if self.from_bearer:
|
|
173
|
+
from prograph.endpoint.credentials import bearer
|
|
174
|
+
token = bearer(request.headers)
|
|
175
|
+
if not token:
|
|
176
|
+
raise ProtocolError(401, "send your own credential as Authorization: Bearer <token>: "
|
|
177
|
+
"a Databricks OAuth token (databricks auth token) or your "
|
|
178
|
+
"identity provider's token")
|
|
179
|
+
key: str = hashlib.sha256(token.encode()).hexdigest()
|
|
180
|
+
kwargs: Optional[Dict[str, Any]] = {"access_token": token}
|
|
181
|
+
self._refuse_if_refused(key)
|
|
182
|
+
else:
|
|
183
|
+
token = request.headers.get(USER_TOKEN)
|
|
184
|
+
if token:
|
|
185
|
+
key = hashlib.sha256(token.encode()).hexdigest()
|
|
186
|
+
kwargs = {"access_token": token}
|
|
187
|
+
elif self.service_principal:
|
|
188
|
+
key, kwargs = "service-principal", None
|
|
189
|
+
else:
|
|
190
|
+
raise ProtocolError(401, "no signed-in user's token: enable user authorization "
|
|
191
|
+
"(the sql scope) for the app")
|
|
192
|
+
with self._lock:
|
|
193
|
+
build = self._building.setdefault(key, _Build())
|
|
194
|
+
build.waiting += 1
|
|
195
|
+
try:
|
|
196
|
+
with build.lock:
|
|
197
|
+
if self.from_bearer:
|
|
198
|
+
# A request that waited on one refused is refused alike.
|
|
199
|
+
self._refuse_if_refused(key)
|
|
200
|
+
return self._engine_for(key, kwargs, token, request)
|
|
201
|
+
finally:
|
|
202
|
+
with self._lock:
|
|
203
|
+
build.waiting -= 1
|
|
204
|
+
# Dropped only when no request holds or waits on it, and the
|
|
205
|
+
# credential has no engine (one refused, or closed since):
|
|
206
|
+
# dropped earlier, a later request would build beside a waiter.
|
|
207
|
+
if build.waiting == 0 and key not in self._engines \
|
|
208
|
+
and self._building.get(key) is build:
|
|
209
|
+
del self._building[key]
|
|
210
|
+
|
|
211
|
+
def _refuse_if_refused(self, key: str) -> None:
|
|
212
|
+
from prograph.endpoint.app import ProtocolError
|
|
213
|
+
now = time.monotonic()
|
|
214
|
+
with self._lock:
|
|
215
|
+
held = self._refused.get(key)
|
|
216
|
+
if held is not None and held[0] <= now:
|
|
217
|
+
del self._refused[key]
|
|
218
|
+
held = None
|
|
219
|
+
if held is not None:
|
|
220
|
+
raise ProtocolError(401, held[1], challenge='error="invalid_token"')
|
|
221
|
+
|
|
222
|
+
def _remember_refusal(self, key: str, reason: str) -> None:
|
|
223
|
+
now = time.monotonic()
|
|
224
|
+
with self._lock:
|
|
225
|
+
if len(self._refused) >= REFUSED_KEPT:
|
|
226
|
+
for old in [k for k, (until, _) in self._refused.items() if until <= now]:
|
|
227
|
+
del self._refused[old]
|
|
228
|
+
while len(self._refused) >= REFUSED_KEPT:
|
|
229
|
+
del self._refused[next(iter(self._refused))]
|
|
230
|
+
self._refused[key] = (now + self.refused_for, reason)
|
|
231
|
+
|
|
232
|
+
def _engine_for(self, key: str, kwargs: Optional[Dict[str, Any]], token: Optional[str],
|
|
233
|
+
request: Any) -> Any:
|
|
234
|
+
from prograph.endpoint.app import ProtocolError
|
|
235
|
+
now = time.monotonic()
|
|
236
|
+
with self._lock:
|
|
237
|
+
closing = self._take_stale(now)
|
|
238
|
+
held = self._engines.get(key)
|
|
239
|
+
if held is not None:
|
|
240
|
+
held.last, held.busy = now, held.busy + 1
|
|
241
|
+
elif len(self._open_by_id) + self._opening >= self.max_engines:
|
|
242
|
+
retry = max(1, int(min((h.last + self.idle - now for h in self._engines.values()
|
|
243
|
+
if not h.busy), default=self.idle)) + 1)
|
|
244
|
+
full = True
|
|
245
|
+
else:
|
|
246
|
+
self._opening += 1
|
|
247
|
+
full = False
|
|
248
|
+
self._close(closing, "idle")
|
|
249
|
+
if held is not None:
|
|
250
|
+
return held.engine
|
|
251
|
+
if full:
|
|
252
|
+
self._log().warning("sparql_endpoint_engine_bound", max_engines=self.max_engines)
|
|
253
|
+
raise ProtocolError(503, f"{self.max_engines} users' engines are open, the server's "
|
|
254
|
+
f"bound (--max-user-engines): try again in {retry} s",
|
|
255
|
+
headers={"Retry-After": str(retry)})
|
|
256
|
+
try:
|
|
257
|
+
held = self._open(key, kwargs, token, request, now)
|
|
258
|
+
finally:
|
|
259
|
+
with self._lock:
|
|
260
|
+
self._opening -= 1
|
|
261
|
+
held.busy, held.key = 1, key
|
|
262
|
+
with self._lock:
|
|
263
|
+
replaced = self._engines.get(key)
|
|
264
|
+
self._engines[key] = held
|
|
265
|
+
self._open_by_id[id(held.engine)] = held
|
|
266
|
+
if replaced is not None: # not reached while one build per key holds
|
|
267
|
+
self._retire([replaced])
|
|
268
|
+
self._start_sweeper()
|
|
269
|
+
return held.engine
|
|
270
|
+
|
|
271
|
+
def release(self, engine: Any) -> None:
|
|
272
|
+
"""A request on `engine` has ended (its answer sent, or failed)."""
|
|
273
|
+
now = time.monotonic()
|
|
274
|
+
with self._lock:
|
|
275
|
+
held = self._open_by_id.get(id(engine))
|
|
276
|
+
if held is None or held.engine is not engine:
|
|
277
|
+
return
|
|
278
|
+
held.busy = max(0, held.busy - 1)
|
|
279
|
+
held.last = now
|
|
280
|
+
retired = held.busy == 0 and self._engines.get(held.key) is not held
|
|
281
|
+
if retired:
|
|
282
|
+
del self._open_by_id[id(engine)]
|
|
283
|
+
if retired:
|
|
284
|
+
self._close([held], "expired")
|
|
285
|
+
|
|
286
|
+
def _open(self, key: str, kwargs: Optional[Dict[str, Any]], token: Optional[str], request: Any,
|
|
287
|
+
now: float) -> _Held:
|
|
288
|
+
"""A new engine on a new connection: the user's, or the service
|
|
289
|
+
principal's when `kwargs` is None."""
|
|
290
|
+
from prograph import ProGraph
|
|
291
|
+
from prograph.endpoint import credentials as C
|
|
292
|
+
from prograph.endpoint.app import ProtocolError
|
|
293
|
+
log = self._log()
|
|
294
|
+
until: Optional[float] = None
|
|
295
|
+
secrets = [token]
|
|
296
|
+
if self.from_bearer:
|
|
297
|
+
assert token is not None
|
|
298
|
+
if C.unreadable_expiry(token):
|
|
299
|
+
said = "the token's exp is not a time: send a valid token"
|
|
300
|
+
self._remember_refusal(key, said)
|
|
301
|
+
raise ProtocolError(401, said, challenge='error="invalid_token"')
|
|
302
|
+
until = C.wall_to_monotonic(C.expires_at(token))
|
|
303
|
+
if C.needs_exchange(token, self.hostname):
|
|
304
|
+
started = time.monotonic()
|
|
305
|
+
try:
|
|
306
|
+
exchanged, lives = self.workspace.exchange(token)
|
|
307
|
+
except C.CredentialRefused as refused:
|
|
308
|
+
log.warning("sparql_endpoint_exchange_refused", reason=str(refused)[:300])
|
|
309
|
+
self._remember_refusal(key, str(refused))
|
|
310
|
+
raise ProtocolError(401, str(refused), challenge='error="invalid_token"') from None
|
|
311
|
+
except C.CheckUnavailable as failed:
|
|
312
|
+
log.warning("sparql_endpoint_exchange_failed", reason=str(failed)[:300])
|
|
313
|
+
raise ProtocolError(failed.status, str(failed), headers=(
|
|
314
|
+
{"Retry-After": failed.retry_after} if failed.retry_after else None)) from None
|
|
315
|
+
secrets.append(exchanged)
|
|
316
|
+
log.info("sparql_endpoint_exchange", ms=round((time.monotonic() - started) * 1000))
|
|
317
|
+
kwargs = {"access_token": exchanged}
|
|
318
|
+
if lives is not None:
|
|
319
|
+
# A minute's margin: no query starts on a token about to stop.
|
|
320
|
+
ends = now + max(0.0, lives - 60.0)
|
|
321
|
+
until = ends if until is None else min(until, ends)
|
|
322
|
+
if kwargs is None:
|
|
323
|
+
kwargs = self._service_principal_kwargs()
|
|
324
|
+
who = request.headers.get(USER_NAME) if not self.from_bearer else None
|
|
325
|
+
who = who or ("a user" if token else "the service principal")
|
|
326
|
+
started = time.monotonic()
|
|
327
|
+
log.info("sparql_endpoint_connect", user=who, http_path=self.http_path)
|
|
328
|
+
try:
|
|
329
|
+
connection = self._connect(**kwargs, **self._session())
|
|
330
|
+
connected = time.monotonic()
|
|
331
|
+
engine = ProGraph(schema=self.schema, backend="databricks_sql", connection=connection)
|
|
332
|
+
except Exception as error:
|
|
333
|
+
if not self.from_bearer:
|
|
334
|
+
raise
|
|
335
|
+
reason = C.scrubbed(f"{type(error).__name__}: {error}", secrets)[:300]
|
|
336
|
+
if C.refused_by_warehouse(error):
|
|
337
|
+
log.warning("sparql_endpoint_connect_refused", reason=reason)
|
|
338
|
+
said = f"the SQL warehouse did not accept the credential ({reason})"
|
|
339
|
+
self._remember_refusal(key, said)
|
|
340
|
+
raise ProtocolError(401, said, challenge='error="invalid_token"') from None
|
|
341
|
+
log.warning("sparql_endpoint_connect_failed", reason=reason)
|
|
342
|
+
raise ProtocolError(502, f"the SQL warehouse could not be reached or failed ({reason}): "
|
|
343
|
+
"try again") from None
|
|
344
|
+
if self.from_bearer:
|
|
345
|
+
who = self.workspace.user_name(kwargs["access_token"]) or "a user (name not given)"
|
|
346
|
+
log.info("sparql_endpoint_engine", user=who, connect_ms=round((connected - started) * 1000),
|
|
347
|
+
engine_ms=round((time.monotonic() - connected) * 1000))
|
|
348
|
+
return _Held(engine, now, who, until)
|
|
349
|
+
|
|
350
|
+
def _take_stale(self, now: float) -> list:
|
|
351
|
+
"""Under the lock: remove from use the engines idle past `idle`
|
|
352
|
+
with no request running, and those past their credential's end;
|
|
353
|
+
return the ones to close now (an ended one still running a request
|
|
354
|
+
is closed by `release` when it ends)."""
|
|
355
|
+
stale = []
|
|
356
|
+
for key, held in list(self._engines.items()):
|
|
357
|
+
idle = not held.busy and now - held.last > self.idle
|
|
358
|
+
ended = held.until is not None and now >= held.until
|
|
359
|
+
if idle or ended:
|
|
360
|
+
del self._engines[key]
|
|
361
|
+
build = self._building.get(key)
|
|
362
|
+
if build is not None and build.waiting == 0:
|
|
363
|
+
del self._building[key]
|
|
364
|
+
if not held.busy:
|
|
365
|
+
self._open_by_id.pop(id(held.engine), None)
|
|
366
|
+
stale.append(held)
|
|
367
|
+
return stale
|
|
368
|
+
|
|
369
|
+
def _retire(self, held: list) -> None:
|
|
370
|
+
with self._lock:
|
|
371
|
+
closing = [h for h in held if not h.busy]
|
|
372
|
+
for h in closing:
|
|
373
|
+
self._open_by_id.pop(id(h.engine), None)
|
|
374
|
+
self._close(closing, "replaced")
|
|
375
|
+
|
|
376
|
+
def _close(self, stale: list, why: str) -> None:
|
|
377
|
+
for held in stale:
|
|
378
|
+
connection = getattr(held.engine, "_connection", None)
|
|
379
|
+
try:
|
|
380
|
+
if connection is not None:
|
|
381
|
+
connection.close()
|
|
382
|
+
except Exception as error: # noqa: BLE001 - a closed or broken connection
|
|
383
|
+
self._log().warning("sparql_endpoint_close_failed", error=str(error)[:200])
|
|
384
|
+
self._log().info("sparql_endpoint_engine_closed", user=held.user, why=why)
|
|
385
|
+
|
|
386
|
+
def sweep(self) -> None:
|
|
387
|
+
"""Close every engine idle past `idle` or past its credential."""
|
|
388
|
+
with self._lock:
|
|
389
|
+
stale = self._take_stale(time.monotonic())
|
|
390
|
+
self._close(stale, "idle")
|
|
391
|
+
|
|
392
|
+
def _start_sweeper(self) -> None:
|
|
393
|
+
with self._lock:
|
|
394
|
+
if self._sweeper is not None or self._stop.is_set():
|
|
395
|
+
return
|
|
396
|
+
self._sweeper = threading.Thread(target=self._sweep_loop, name="prograph-engine-sweeper",
|
|
397
|
+
daemon=True)
|
|
398
|
+
self._sweeper.start()
|
|
399
|
+
|
|
400
|
+
def _sweep_loop(self) -> None:
|
|
401
|
+
while not self._stop.wait(self.sweep_every):
|
|
402
|
+
try:
|
|
403
|
+
self.sweep()
|
|
404
|
+
except Exception as error: # noqa: BLE001 - the timer outlives one failure
|
|
405
|
+
self._log().warning("sparql_endpoint_sweep_failed", error=str(error)[:200])
|
|
406
|
+
|
|
407
|
+
def close(self) -> None:
|
|
408
|
+
"""Stop the timer and close every engine (the server's shutdown)."""
|
|
409
|
+
self._stop.set()
|
|
410
|
+
with self._lock:
|
|
411
|
+
held = list(self._open_by_id.values())
|
|
412
|
+
self._engines, self._open_by_id = {}, {}
|
|
413
|
+
self._close(held, "shutdown")
|
|
414
|
+
|
|
415
|
+
@staticmethod
|
|
416
|
+
def _log() -> Any:
|
|
417
|
+
from prograph.logging import get_logger
|
|
418
|
+
return get_logger("prograph.endpoint")
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Which result format answers a request: the `format=` parameter if given,
|
|
2
|
+
else the `Accept` header (RFC 9110 12.5.1: q-values, `type/*`, `*/*`)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from typing import List, Optional, Sequence, Union
|
|
7
|
+
|
|
8
|
+
# `format=` (or `output=`) as many endpoints read it: a media type, or a
|
|
9
|
+
# short name read by the query's form. SPARQLWrapper sends its return
|
|
10
|
+
# format so (`format=xml` for RDF/XML of a CONSTRUCT, `n3`, `json-ld`), as
|
|
11
|
+
# Fuseki reads it: `xml` is results XML for a SELECT or ASK, RDF/XML for a
|
|
12
|
+
# graph.
|
|
13
|
+
SOLUTION_NAMES = {
|
|
14
|
+
"json": "application/sparql-results+json",
|
|
15
|
+
"xml": "application/sparql-results+xml",
|
|
16
|
+
"csv": "text/csv",
|
|
17
|
+
"tsv": "text/tab-separated-values",
|
|
18
|
+
}
|
|
19
|
+
GRAPH_NAMES = {
|
|
20
|
+
"turtle": "text/turtle",
|
|
21
|
+
"ttl": "text/turtle",
|
|
22
|
+
"n3": "text/turtle",
|
|
23
|
+
"nt": "application/n-triples",
|
|
24
|
+
"ntriples": "application/n-triples",
|
|
25
|
+
"n-triples": "application/n-triples",
|
|
26
|
+
"xml": "application/rdf+xml",
|
|
27
|
+
"rdf": "application/rdf+xml",
|
|
28
|
+
"rdfxml": "application/rdf+xml",
|
|
29
|
+
"rdf+xml": "application/rdf+xml",
|
|
30
|
+
"json": "application/ld+json",
|
|
31
|
+
"jsonld": "application/ld+json",
|
|
32
|
+
"json-ld": "application/ld+json",
|
|
33
|
+
}
|
|
34
|
+
SHORT_NAMES = {**GRAPH_NAMES, **SOLUTION_NAMES}
|
|
35
|
+
|
|
36
|
+
# Media types a client may name for one the server writes.
|
|
37
|
+
ALIASES = {
|
|
38
|
+
"application/json": "application/sparql-results+json",
|
|
39
|
+
"application/xml": "application/sparql-results+xml",
|
|
40
|
+
"text/xml": "application/sparql-results+xml",
|
|
41
|
+
"application/x-turtle": "text/turtle",
|
|
42
|
+
"text/plain": "application/n-triples",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _accepted(header: str) -> List[tuple]:
|
|
47
|
+
"""(media range, q) in the header's order, q above 0."""
|
|
48
|
+
out = []
|
|
49
|
+
for part in header.split(","):
|
|
50
|
+
fields = [f.strip() for f in part.split(";")]
|
|
51
|
+
if not fields[0]:
|
|
52
|
+
continue
|
|
53
|
+
q = 1.0
|
|
54
|
+
for f in fields[1:]:
|
|
55
|
+
if f.lower().startswith("q="):
|
|
56
|
+
try:
|
|
57
|
+
q = float(f[2:])
|
|
58
|
+
except ValueError:
|
|
59
|
+
q = 0.0
|
|
60
|
+
if q > 0:
|
|
61
|
+
out.append((fields[0].lower(), q))
|
|
62
|
+
return out
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _named(value: str, graph: bool) -> str:
|
|
66
|
+
"""A `format=` value as a media type: a short name for the form, or the
|
|
67
|
+
media type itself (a `+` a query string decoded as a space put back)."""
|
|
68
|
+
name = value.strip().lower().replace(" ", "+")
|
|
69
|
+
names = GRAPH_NAMES if graph else SOLUTION_NAMES
|
|
70
|
+
return names.get(name, ALIASES.get(name, name))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def choose(accept: Optional[str], formats: Union[None, str, Sequence[str]], offered: List[str],
|
|
74
|
+
default: str) -> Optional[str]:
|
|
75
|
+
"""The media type to answer in, of `offered`; None when nothing the
|
|
76
|
+
client asks for is offered (406). `formats` are the request's `format=`
|
|
77
|
+
and `output=` values: the first one on offer wins; when none is, the
|
|
78
|
+
`Accept` header decides. With no Accept, or `*/*`, `default`."""
|
|
79
|
+
graph = any(m.startswith(("text/turtle", "application/n-triples", "application/rdf+xml",
|
|
80
|
+
"application/ld+json")) for m in offered)
|
|
81
|
+
values = [formats] if isinstance(formats, str) else list(formats or [])
|
|
82
|
+
for value in values:
|
|
83
|
+
media = _named(value, graph)
|
|
84
|
+
if media in offered:
|
|
85
|
+
return media
|
|
86
|
+
if not accept or not accept.strip():
|
|
87
|
+
return default
|
|
88
|
+
best, best_q, best_rank = None, 0.0, -1
|
|
89
|
+
for media_range, q in _accepted(accept):
|
|
90
|
+
media_range = ALIASES.get(media_range, media_range)
|
|
91
|
+
if media_range == "*/*":
|
|
92
|
+
candidates, rank = [default], 0
|
|
93
|
+
elif media_range.endswith("/*"):
|
|
94
|
+
prefix = media_range[:-1]
|
|
95
|
+
candidates = [m for m in ([default] + offered) if m.startswith(prefix)]
|
|
96
|
+
rank = 1
|
|
97
|
+
else:
|
|
98
|
+
candidates, rank = [m for m in offered if m == media_range], 2
|
|
99
|
+
if candidates and (q > best_q or (q == best_q and rank > best_rank)):
|
|
100
|
+
best, best_q, best_rank = candidates[0], q, rank
|
|
101
|
+
return best
|