prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,418 @@
1
+ """The engine a request runs on.
2
+
3
+ - `one(engine)`: every request on one engine, queued on its lock.
4
+ - `DatabricksUsers`: on a Databricks App, each request on an engine over a
5
+ SQL warehouse connection opened with the signed-in user's token, which
6
+ the app forwards as `X-Forwarded-Access-Token`; Unity Catalog's grants
7
+ are that person's. An engine is kept per user and token for `idle`
8
+ seconds after its last request, then closed. The app's service principal
9
+ is used only when `service_principal=True` and no user token came.
10
+ With `from_bearer` (`prograph serve --per-user`, off an App) the user's
11
+ credential is the request's `Authorization: Bearer` instead.
12
+
13
+ The schema is read once (`schema_from`) and shared: a GraphSchema is not
14
+ written to by a query.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import hashlib
20
+ import math
21
+ import threading
22
+ import time
23
+ from typing import Any, Callable, Dict, Optional, Tuple
24
+
25
+ USER_TOKEN = "x-forwarded-access-token"
26
+ USER_NAME = "x-forwarded-email"
27
+
28
+
29
+ def schema_from(config: Optional[str] = None, r2rml: Optional[str] = None) -> Any:
30
+ """A GraphSchema from a graph YAML (`prograph.config`) or an R2RML
31
+ mapping (`prograph.r2rml`), whichever is named."""
32
+ if (config is None) == (r2rml is None):
33
+ raise ValueError("name one of a graph config (--config) or an R2RML mapping (--r2rml)")
34
+ if r2rml is not None:
35
+ from pathlib import Path
36
+ from prograph.r2rml.schema import load
37
+ return load(Path(r2rml).read_text())
38
+ from prograph.config import load_config
39
+ return load_config(config).to_schema()
40
+
41
+
42
+ def one(engine: Any) -> Callable[[Any], Any]:
43
+ def source(request: Any) -> Any:
44
+ return engine
45
+ # Every request is on this engine: its mapping is the server's (the Service Description reads it).
46
+ source.single = True # type: ignore[attr-defined]
47
+ return source
48
+
49
+
50
+ def local_engine(config: Optional[str] = None, r2rml: Optional[str] = None,
51
+ duckdb_path: Optional[str] = None) -> Any:
52
+ """An engine on this machine: over a DuckDB database file (opened read
53
+ only), or over the files a graph YAML names (`ProGraph.from_config`)."""
54
+ from prograph import ProGraph
55
+ if duckdb_path is not None:
56
+ import duckdb
57
+ return ProGraph(schema=schema_from(config, r2rml), backend="duckdb",
58
+ connection=duckdb.connect(duckdb_path, read_only=True))
59
+ if config is None:
60
+ raise ValueError("--r2rml needs the database it maps (--duckdb or a warehouse)")
61
+ return ProGraph.from_config(config)
62
+
63
+
64
+ class _Held:
65
+ """A user's engine, when it was last asked for or let go, whose it is,
66
+ when its credential stops (monotonic; None: only idleness closes it),
67
+ and how many requests are running on it."""
68
+ __slots__ = ("engine", "last", "user", "until", "busy", "key")
69
+
70
+ def __init__(self, engine: Any, last: float, user: str, until: Optional[float]):
71
+ self.engine, self.last, self.user, self.until = engine, last, user, until
72
+ self.busy, self.key = 0, ""
73
+
74
+
75
+ class _Build:
76
+ """One build per credential at a time, and how many requests wait on it,
77
+ so the lock is dropped only when nobody holds or waits for it."""
78
+ __slots__ = ("lock", "waiting")
79
+
80
+ def __init__(self) -> None:
81
+ self.lock, self.waiting = threading.Lock(), 0
82
+
83
+
84
+ REFUSED_FOR = 30.0 # seconds a refused credential is answered from memory
85
+ REFUSED_KEPT = 10000 # the most refused credentials remembered
86
+
87
+
88
+ class DatabricksUsers:
89
+ """An engine per signed-in user over one SQL warehouse.
90
+
91
+ Where the user's credential comes from is the server's mode, chosen at
92
+ start (`prograph serve`): on a Databricks App the proxy forwards it as
93
+ `X-Forwarded-Access-Token` (the default); with `from_bearer` (`--per-user`,
94
+ off an App) the caller sends it as `Authorization: Bearer`, a Databricks
95
+ token used as it is or an identity provider's JWT exchanged for one
96
+ (`prograph.endpoint.credentials`).
97
+
98
+ Engines are kept by the SHA-256 of the credential the request carried.
99
+ A request holds its engine until `release` (the endpoint calls it when
100
+ the answer is sent): an engine is closed `idle` seconds after its last
101
+ request ended and none is running, by a timer
102
+ every `sweep_every` seconds as well as on the next request. One whose
103
+ credential expires (an exchanged token's `expires_in`, a JWT's `exp`)
104
+ takes no new request and is closed when its last one ends. At most
105
+ `max_engines` are open: a request that would open one more is a 503
106
+ with `Retry-After` until one closes; open engines are never closed to
107
+ make room. With `from_bearer`, a credential refused is answered 401
108
+ from memory for `REFUSED_FOR` seconds, with no call to the workspace.
109
+ """
110
+
111
+ def __init__(self, schema: Any, server_hostname: str, http_path: str,
112
+ catalog: Optional[str] = None, db_schema: Optional[str] = None,
113
+ idle: float = 900.0, service_principal: bool = False,
114
+ connect: Optional[Callable[..., Any]] = None,
115
+ from_bearer: bool = False, max_engines: int = 64,
116
+ sweep_every: Optional[float] = None, workspace: Any = None,
117
+ refused_for: float = REFUSED_FOR,
118
+ statement_timeout: Optional[float] = None):
119
+ if from_bearer and service_principal:
120
+ raise ValueError("a per-user server has no service principal: every request "
121
+ "carries its user's credential")
122
+ if max_engines < 1:
123
+ raise ValueError("max_engines is at least 1")
124
+ if idle < 0:
125
+ raise ValueError("idle is not negative")
126
+ self.schema, self.hostname, self.http_path = schema, server_hostname, http_path
127
+ self.catalog, self.db_schema = catalog, db_schema
128
+ self.idle, self.service_principal = idle, service_principal
129
+ self.from_bearer, self.max_engines = from_bearer, max_engines
130
+ self.sweep_every = sweep_every if sweep_every is not None else max(0.05, min(60.0, idle / 4))
131
+ self.refused_for = refused_for
132
+ # The warehouse cancels a statement past this many seconds (Databricks SQL's
133
+ # STATEMENT_TIMEOUT, set on each connection's session): the server's --timeout.
134
+ self.statement_timeout = statement_timeout
135
+ self._connect = connect or self._sql_connect
136
+ if workspace is None:
137
+ from prograph.endpoint.credentials import Workspace
138
+ workspace = Workspace(server_hostname)
139
+ self.workspace = workspace
140
+ self._engines: Dict[str, _Held] = {}
141
+ # Every open engine by id(engine): those in `_engines` and those
142
+ # retired (credential ended) while a request still runs on them.
143
+ self._open_by_id: Dict[int, _Held] = {}
144
+ self._refused: Dict[str, Tuple[float, str]] = {}
145
+ self._opening = 0
146
+ self._lock = threading.Lock()
147
+ # One build per user at a time: a second request of the same user
148
+ # waits for the first's engine instead of opening another.
149
+ self._building: Dict[str, _Build] = {}
150
+ self._stop = threading.Event()
151
+ self._sweeper: Optional[threading.Thread] = None
152
+
153
+ def _session(self) -> Dict[str, Any]:
154
+ """The session configuration every connection opens with: STATEMENT_TIMEOUT, whole seconds
155
+ (at least 1), where the server has a timeout; the warehouse halts a statement past it."""
156
+ if not self.statement_timeout:
157
+ return {}
158
+ return {"session_configuration": {"STATEMENT_TIMEOUT": str(max(1, math.ceil(self.statement_timeout)))}}
159
+
160
+ def _sql_connect(self, **kwargs: Any) -> Any:
161
+ import databricks.sql as dbsql
162
+ return dbsql.connect(server_hostname=self.hostname, http_path=self.http_path,
163
+ catalog=self.catalog, schema=self.db_schema, **kwargs)
164
+
165
+ def _service_principal_kwargs(self) -> Dict[str, Any]:
166
+ from databricks.sdk.core import Config, oauth_service_principal
167
+ config = Config()
168
+ return {"credentials_provider": lambda: oauth_service_principal(config)}
169
+
170
+ def __call__(self, request: Any) -> Any:
171
+ from prograph.endpoint.app import ProtocolError
172
+ if self.from_bearer:
173
+ from prograph.endpoint.credentials import bearer
174
+ token = bearer(request.headers)
175
+ if not token:
176
+ raise ProtocolError(401, "send your own credential as Authorization: Bearer <token>: "
177
+ "a Databricks OAuth token (databricks auth token) or your "
178
+ "identity provider's token")
179
+ key: str = hashlib.sha256(token.encode()).hexdigest()
180
+ kwargs: Optional[Dict[str, Any]] = {"access_token": token}
181
+ self._refuse_if_refused(key)
182
+ else:
183
+ token = request.headers.get(USER_TOKEN)
184
+ if token:
185
+ key = hashlib.sha256(token.encode()).hexdigest()
186
+ kwargs = {"access_token": token}
187
+ elif self.service_principal:
188
+ key, kwargs = "service-principal", None
189
+ else:
190
+ raise ProtocolError(401, "no signed-in user's token: enable user authorization "
191
+ "(the sql scope) for the app")
192
+ with self._lock:
193
+ build = self._building.setdefault(key, _Build())
194
+ build.waiting += 1
195
+ try:
196
+ with build.lock:
197
+ if self.from_bearer:
198
+ # A request that waited on one refused is refused alike.
199
+ self._refuse_if_refused(key)
200
+ return self._engine_for(key, kwargs, token, request)
201
+ finally:
202
+ with self._lock:
203
+ build.waiting -= 1
204
+ # Dropped only when no request holds or waits on it, and the
205
+ # credential has no engine (one refused, or closed since):
206
+ # dropped earlier, a later request would build beside a waiter.
207
+ if build.waiting == 0 and key not in self._engines \
208
+ and self._building.get(key) is build:
209
+ del self._building[key]
210
+
211
+ def _refuse_if_refused(self, key: str) -> None:
212
+ from prograph.endpoint.app import ProtocolError
213
+ now = time.monotonic()
214
+ with self._lock:
215
+ held = self._refused.get(key)
216
+ if held is not None and held[0] <= now:
217
+ del self._refused[key]
218
+ held = None
219
+ if held is not None:
220
+ raise ProtocolError(401, held[1], challenge='error="invalid_token"')
221
+
222
+ def _remember_refusal(self, key: str, reason: str) -> None:
223
+ now = time.monotonic()
224
+ with self._lock:
225
+ if len(self._refused) >= REFUSED_KEPT:
226
+ for old in [k for k, (until, _) in self._refused.items() if until <= now]:
227
+ del self._refused[old]
228
+ while len(self._refused) >= REFUSED_KEPT:
229
+ del self._refused[next(iter(self._refused))]
230
+ self._refused[key] = (now + self.refused_for, reason)
231
+
232
+ def _engine_for(self, key: str, kwargs: Optional[Dict[str, Any]], token: Optional[str],
233
+ request: Any) -> Any:
234
+ from prograph.endpoint.app import ProtocolError
235
+ now = time.monotonic()
236
+ with self._lock:
237
+ closing = self._take_stale(now)
238
+ held = self._engines.get(key)
239
+ if held is not None:
240
+ held.last, held.busy = now, held.busy + 1
241
+ elif len(self._open_by_id) + self._opening >= self.max_engines:
242
+ retry = max(1, int(min((h.last + self.idle - now for h in self._engines.values()
243
+ if not h.busy), default=self.idle)) + 1)
244
+ full = True
245
+ else:
246
+ self._opening += 1
247
+ full = False
248
+ self._close(closing, "idle")
249
+ if held is not None:
250
+ return held.engine
251
+ if full:
252
+ self._log().warning("sparql_endpoint_engine_bound", max_engines=self.max_engines)
253
+ raise ProtocolError(503, f"{self.max_engines} users' engines are open, the server's "
254
+ f"bound (--max-user-engines): try again in {retry} s",
255
+ headers={"Retry-After": str(retry)})
256
+ try:
257
+ held = self._open(key, kwargs, token, request, now)
258
+ finally:
259
+ with self._lock:
260
+ self._opening -= 1
261
+ held.busy, held.key = 1, key
262
+ with self._lock:
263
+ replaced = self._engines.get(key)
264
+ self._engines[key] = held
265
+ self._open_by_id[id(held.engine)] = held
266
+ if replaced is not None: # not reached while one build per key holds
267
+ self._retire([replaced])
268
+ self._start_sweeper()
269
+ return held.engine
270
+
271
+ def release(self, engine: Any) -> None:
272
+ """A request on `engine` has ended (its answer sent, or failed)."""
273
+ now = time.monotonic()
274
+ with self._lock:
275
+ held = self._open_by_id.get(id(engine))
276
+ if held is None or held.engine is not engine:
277
+ return
278
+ held.busy = max(0, held.busy - 1)
279
+ held.last = now
280
+ retired = held.busy == 0 and self._engines.get(held.key) is not held
281
+ if retired:
282
+ del self._open_by_id[id(engine)]
283
+ if retired:
284
+ self._close([held], "expired")
285
+
286
+ def _open(self, key: str, kwargs: Optional[Dict[str, Any]], token: Optional[str], request: Any,
287
+ now: float) -> _Held:
288
+ """A new engine on a new connection: the user's, or the service
289
+ principal's when `kwargs` is None."""
290
+ from prograph import ProGraph
291
+ from prograph.endpoint import credentials as C
292
+ from prograph.endpoint.app import ProtocolError
293
+ log = self._log()
294
+ until: Optional[float] = None
295
+ secrets = [token]
296
+ if self.from_bearer:
297
+ assert token is not None
298
+ if C.unreadable_expiry(token):
299
+ said = "the token's exp is not a time: send a valid token"
300
+ self._remember_refusal(key, said)
301
+ raise ProtocolError(401, said, challenge='error="invalid_token"')
302
+ until = C.wall_to_monotonic(C.expires_at(token))
303
+ if C.needs_exchange(token, self.hostname):
304
+ started = time.monotonic()
305
+ try:
306
+ exchanged, lives = self.workspace.exchange(token)
307
+ except C.CredentialRefused as refused:
308
+ log.warning("sparql_endpoint_exchange_refused", reason=str(refused)[:300])
309
+ self._remember_refusal(key, str(refused))
310
+ raise ProtocolError(401, str(refused), challenge='error="invalid_token"') from None
311
+ except C.CheckUnavailable as failed:
312
+ log.warning("sparql_endpoint_exchange_failed", reason=str(failed)[:300])
313
+ raise ProtocolError(failed.status, str(failed), headers=(
314
+ {"Retry-After": failed.retry_after} if failed.retry_after else None)) from None
315
+ secrets.append(exchanged)
316
+ log.info("sparql_endpoint_exchange", ms=round((time.monotonic() - started) * 1000))
317
+ kwargs = {"access_token": exchanged}
318
+ if lives is not None:
319
+ # A minute's margin: no query starts on a token about to stop.
320
+ ends = now + max(0.0, lives - 60.0)
321
+ until = ends if until is None else min(until, ends)
322
+ if kwargs is None:
323
+ kwargs = self._service_principal_kwargs()
324
+ who = request.headers.get(USER_NAME) if not self.from_bearer else None
325
+ who = who or ("a user" if token else "the service principal")
326
+ started = time.monotonic()
327
+ log.info("sparql_endpoint_connect", user=who, http_path=self.http_path)
328
+ try:
329
+ connection = self._connect(**kwargs, **self._session())
330
+ connected = time.monotonic()
331
+ engine = ProGraph(schema=self.schema, backend="databricks_sql", connection=connection)
332
+ except Exception as error:
333
+ if not self.from_bearer:
334
+ raise
335
+ reason = C.scrubbed(f"{type(error).__name__}: {error}", secrets)[:300]
336
+ if C.refused_by_warehouse(error):
337
+ log.warning("sparql_endpoint_connect_refused", reason=reason)
338
+ said = f"the SQL warehouse did not accept the credential ({reason})"
339
+ self._remember_refusal(key, said)
340
+ raise ProtocolError(401, said, challenge='error="invalid_token"') from None
341
+ log.warning("sparql_endpoint_connect_failed", reason=reason)
342
+ raise ProtocolError(502, f"the SQL warehouse could not be reached or failed ({reason}): "
343
+ "try again") from None
344
+ if self.from_bearer:
345
+ who = self.workspace.user_name(kwargs["access_token"]) or "a user (name not given)"
346
+ log.info("sparql_endpoint_engine", user=who, connect_ms=round((connected - started) * 1000),
347
+ engine_ms=round((time.monotonic() - connected) * 1000))
348
+ return _Held(engine, now, who, until)
349
+
350
+ def _take_stale(self, now: float) -> list:
351
+ """Under the lock: remove from use the engines idle past `idle`
352
+ with no request running, and those past their credential's end;
353
+ return the ones to close now (an ended one still running a request
354
+ is closed by `release` when it ends)."""
355
+ stale = []
356
+ for key, held in list(self._engines.items()):
357
+ idle = not held.busy and now - held.last > self.idle
358
+ ended = held.until is not None and now >= held.until
359
+ if idle or ended:
360
+ del self._engines[key]
361
+ build = self._building.get(key)
362
+ if build is not None and build.waiting == 0:
363
+ del self._building[key]
364
+ if not held.busy:
365
+ self._open_by_id.pop(id(held.engine), None)
366
+ stale.append(held)
367
+ return stale
368
+
369
+ def _retire(self, held: list) -> None:
370
+ with self._lock:
371
+ closing = [h for h in held if not h.busy]
372
+ for h in closing:
373
+ self._open_by_id.pop(id(h.engine), None)
374
+ self._close(closing, "replaced")
375
+
376
+ def _close(self, stale: list, why: str) -> None:
377
+ for held in stale:
378
+ connection = getattr(held.engine, "_connection", None)
379
+ try:
380
+ if connection is not None:
381
+ connection.close()
382
+ except Exception as error: # noqa: BLE001 - a closed or broken connection
383
+ self._log().warning("sparql_endpoint_close_failed", error=str(error)[:200])
384
+ self._log().info("sparql_endpoint_engine_closed", user=held.user, why=why)
385
+
386
+ def sweep(self) -> None:
387
+ """Close every engine idle past `idle` or past its credential."""
388
+ with self._lock:
389
+ stale = self._take_stale(time.monotonic())
390
+ self._close(stale, "idle")
391
+
392
+ def _start_sweeper(self) -> None:
393
+ with self._lock:
394
+ if self._sweeper is not None or self._stop.is_set():
395
+ return
396
+ self._sweeper = threading.Thread(target=self._sweep_loop, name="prograph-engine-sweeper",
397
+ daemon=True)
398
+ self._sweeper.start()
399
+
400
+ def _sweep_loop(self) -> None:
401
+ while not self._stop.wait(self.sweep_every):
402
+ try:
403
+ self.sweep()
404
+ except Exception as error: # noqa: BLE001 - the timer outlives one failure
405
+ self._log().warning("sparql_endpoint_sweep_failed", error=str(error)[:200])
406
+
407
+ def close(self) -> None:
408
+ """Stop the timer and close every engine (the server's shutdown)."""
409
+ self._stop.set()
410
+ with self._lock:
411
+ held = list(self._open_by_id.values())
412
+ self._engines, self._open_by_id = {}, {}
413
+ self._close(held, "shutdown")
414
+
415
+ @staticmethod
416
+ def _log() -> Any:
417
+ from prograph.logging import get_logger
418
+ return get_logger("prograph.endpoint")
@@ -0,0 +1,101 @@
1
+ """Which result format answers a request: the `format=` parameter if given,
2
+ else the `Accept` header (RFC 9110 12.5.1: q-values, `type/*`, `*/*`)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from typing import List, Optional, Sequence, Union
7
+
8
+ # `format=` (or `output=`) as many endpoints read it: a media type, or a
9
+ # short name read by the query's form. SPARQLWrapper sends its return
10
+ # format so (`format=xml` for RDF/XML of a CONSTRUCT, `n3`, `json-ld`), as
11
+ # Fuseki reads it: `xml` is results XML for a SELECT or ASK, RDF/XML for a
12
+ # graph.
13
+ SOLUTION_NAMES = {
14
+ "json": "application/sparql-results+json",
15
+ "xml": "application/sparql-results+xml",
16
+ "csv": "text/csv",
17
+ "tsv": "text/tab-separated-values",
18
+ }
19
+ GRAPH_NAMES = {
20
+ "turtle": "text/turtle",
21
+ "ttl": "text/turtle",
22
+ "n3": "text/turtle",
23
+ "nt": "application/n-triples",
24
+ "ntriples": "application/n-triples",
25
+ "n-triples": "application/n-triples",
26
+ "xml": "application/rdf+xml",
27
+ "rdf": "application/rdf+xml",
28
+ "rdfxml": "application/rdf+xml",
29
+ "rdf+xml": "application/rdf+xml",
30
+ "json": "application/ld+json",
31
+ "jsonld": "application/ld+json",
32
+ "json-ld": "application/ld+json",
33
+ }
34
+ SHORT_NAMES = {**GRAPH_NAMES, **SOLUTION_NAMES}
35
+
36
+ # Media types a client may name for one the server writes.
37
+ ALIASES = {
38
+ "application/json": "application/sparql-results+json",
39
+ "application/xml": "application/sparql-results+xml",
40
+ "text/xml": "application/sparql-results+xml",
41
+ "application/x-turtle": "text/turtle",
42
+ "text/plain": "application/n-triples",
43
+ }
44
+
45
+
46
+ def _accepted(header: str) -> List[tuple]:
47
+ """(media range, q) in the header's order, q above 0."""
48
+ out = []
49
+ for part in header.split(","):
50
+ fields = [f.strip() for f in part.split(";")]
51
+ if not fields[0]:
52
+ continue
53
+ q = 1.0
54
+ for f in fields[1:]:
55
+ if f.lower().startswith("q="):
56
+ try:
57
+ q = float(f[2:])
58
+ except ValueError:
59
+ q = 0.0
60
+ if q > 0:
61
+ out.append((fields[0].lower(), q))
62
+ return out
63
+
64
+
65
+ def _named(value: str, graph: bool) -> str:
66
+ """A `format=` value as a media type: a short name for the form, or the
67
+ media type itself (a `+` a query string decoded as a space put back)."""
68
+ name = value.strip().lower().replace(" ", "+")
69
+ names = GRAPH_NAMES if graph else SOLUTION_NAMES
70
+ return names.get(name, ALIASES.get(name, name))
71
+
72
+
73
+ def choose(accept: Optional[str], formats: Union[None, str, Sequence[str]], offered: List[str],
74
+ default: str) -> Optional[str]:
75
+ """The media type to answer in, of `offered`; None when nothing the
76
+ client asks for is offered (406). `formats` are the request's `format=`
77
+ and `output=` values: the first one on offer wins; when none is, the
78
+ `Accept` header decides. With no Accept, or `*/*`, `default`."""
79
+ graph = any(m.startswith(("text/turtle", "application/n-triples", "application/rdf+xml",
80
+ "application/ld+json")) for m in offered)
81
+ values = [formats] if isinstance(formats, str) else list(formats or [])
82
+ for value in values:
83
+ media = _named(value, graph)
84
+ if media in offered:
85
+ return media
86
+ if not accept or not accept.strip():
87
+ return default
88
+ best, best_q, best_rank = None, 0.0, -1
89
+ for media_range, q in _accepted(accept):
90
+ media_range = ALIASES.get(media_range, media_range)
91
+ if media_range == "*/*":
92
+ candidates, rank = [default], 0
93
+ elif media_range.endswith("/*"):
94
+ prefix = media_range[:-1]
95
+ candidates = [m for m in ([default] + offered) if m.startswith(prefix)]
96
+ rank = 1
97
+ else:
98
+ candidates, rank = [m for m in offered if m == media_range], 2
99
+ if candidates and (q > best_q or (q == best_q and rank > best_rank)):
100
+ best, best_q, best_rank = candidates[0], q, rank
101
+ return best