prograph 1.3.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (237) hide show
  1. prograph/__init__.py +36 -0
  2. prograph/__main__.py +5 -0
  3. prograph/_version.py +375 -0
  4. prograph/cli/__init__.py +1 -0
  5. prograph/cli/_iri_utils.py +124 -0
  6. prograph/cli/emit_genie_space.py +107 -0
  7. prograph/cli/generate_views.py +140 -0
  8. prograph/cli/main.py +420 -0
  9. prograph/cli/schema_inferrer.py +746 -0
  10. prograph/cli/ttl_loader.py +289 -0
  11. prograph/cli/upload_genie_space.py +84 -0
  12. prograph/config/__init__.py +27 -0
  13. prograph/config/yaml_config.py +848 -0
  14. prograph/demos/__init__.py +14 -0
  15. prograph/demos/turbofan/README.md +74 -0
  16. prograph/demos/turbofan/SCENARIO.md +209 -0
  17. prograph/demos/turbofan/__init__.py +75 -0
  18. prograph/demos/turbofan/create_tables.sql +206 -0
  19. prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
  20. prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
  21. prograph/demos/turbofan/demo.py +162 -0
  22. prograph/demos/turbofan/genie_qa_notebook.py +434 -0
  23. prograph/demos/turbofan/graph.yaml +1098 -0
  24. prograph/demos/turbofan/instance_graph_demo.py +32 -0
  25. prograph/demos/turbofan/load.py +223 -0
  26. prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
  27. prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
  28. prograph/demos/turbofan/payload-shapes.ttl +33 -0
  29. prograph/demos/turbofan/queries/example-queries.sparql +215 -0
  30. prograph/demos/turbofan/shapes.ttl +75 -0
  31. prograph/demos/turbofan/tables.py +106 -0
  32. prograph/demos/turbofan/turbofan_schema.py +393 -0
  33. prograph/demos/turbofan/validation_demo.py +153 -0
  34. prograph/describe.py +231 -0
  35. prograph/endpoint/__init__.py +12 -0
  36. prograph/endpoint/app.py +722 -0
  37. prograph/endpoint/cli.py +280 -0
  38. prograph/endpoint/credentials.py +240 -0
  39. prograph/endpoint/engines.py +418 -0
  40. prograph/endpoint/negotiate.py +101 -0
  41. prograph/endpoint/results.py +181 -0
  42. prograph/endpoint/service.py +31 -0
  43. prograph/engine.py +2718 -0
  44. prograph/execution.py +455 -0
  45. prograph/executors/__init__.py +6 -0
  46. prograph/executors/base.py +11 -0
  47. prograph/executors/databricks_sql_executor.py +51 -0
  48. prograph/executors/duckdb_executor.py +24 -0
  49. prograph/executors/mysql_executor.py +20 -0
  50. prograph/executors/oracle_executor.py +24 -0
  51. prograph/executors/pandas_executor.py +72 -0
  52. prograph/executors/postgresql_executor.py +35 -0
  53. prograph/executors/spark_executor.py +88 -0
  54. prograph/executors/sqlserver_executor.py +35 -0
  55. prograph/extras/__init__.py +1 -0
  56. prograph/extras/graphql/__init__.py +33 -0
  57. prograph/extras/graphql/execute.py +25 -0
  58. prograph/extras/graphql/schema.py +314 -0
  59. prograph/extras/instance_graph.py +559 -0
  60. prograph/genie/__init__.py +83 -0
  61. prograph/genie/databricks_uploader.py +306 -0
  62. prograph/genie/deploy.py +393 -0
  63. prograph/genie/formats.py +414 -0
  64. prograph/genie/instruction_generator.py +375 -0
  65. prograph/genie/sample_query_generator.py +256 -0
  66. prograph/genie/shacl_views.py +318 -0
  67. prograph/genie/space_emitter.py +329 -0
  68. prograph/genie/view_generator.py +628 -0
  69. prograph/genie/view_writer.py +172 -0
  70. prograph/graph/__init__.py +1 -0
  71. prograph/graph/ansi_session.py +348 -0
  72. prograph/graph/databricks_sql_graph.py +300 -0
  73. prograph/graph/drivers.py +143 -0
  74. prograph/graph/duckdb_graph.py +68 -0
  75. prograph/graph/mysql_graph.py +38 -0
  76. prograph/graph/oracle_graph.py +285 -0
  77. prograph/graph/pandas_graph.py +343 -0
  78. prograph/graph/postgresql_graph.py +644 -0
  79. prograph/graph/spark_graph.py +556 -0
  80. prograph/graph/sql_graph.py +83 -0
  81. prograph/graph/sqlserver_graph.py +72 -0
  82. prograph/hints.py +95 -0
  83. prograph/logging.py +550 -0
  84. prograph/mcp/__init__.py +29 -0
  85. prograph/mcp/__main__.py +180 -0
  86. prograph/mcp/bootstrap.py +98 -0
  87. prograph/mcp/server.py +601 -0
  88. prograph/mcp/sparql.py +397 -0
  89. prograph/mcp/tools.py +504 -0
  90. prograph/nlp/__init__.py +79 -0
  91. prograph/nlp/client.py +305 -0
  92. prograph/nlp/databricks_tools.py +369 -0
  93. prograph/nlp/graph_context.py +313 -0
  94. prograph/nlp/interface.py +652 -0
  95. prograph/nlp/prompts/__init__.py +19 -0
  96. prograph/nlp/prompts/examples.py +258 -0
  97. prograph/nlp/prompts/system.py +239 -0
  98. prograph/nlp/providers/__init__.py +14 -0
  99. prograph/nlp/providers/anthropic.py +274 -0
  100. prograph/nlp/providers/databricks.py +312 -0
  101. prograph/nlp/providers/openai.py +203 -0
  102. prograph/nlp/schema_context.py +78 -0
  103. prograph/ontology/__init__.py +53 -0
  104. prograph/ontology/alignment.py +542 -0
  105. prograph/ontology/hierarchy.py +310 -0
  106. prograph/ontology/imports.py +587 -0
  107. prograph/ontology/layer.py +991 -0
  108. prograph/ontology/skos.py +61 -0
  109. prograph/owl/__init__.py +27 -0
  110. prograph/owl/defined_classes.py +323 -0
  111. prograph/parameters.py +242 -0
  112. prograph/parser/__init__.py +1 -0
  113. prograph/parser/cypher/CypherLexer.py +836 -0
  114. prograph/parser/cypher/CypherListener.py +966 -0
  115. prograph/parser/cypher/CypherParser.py +10854 -0
  116. prograph/parser/cypher/CypherVisitor.py +543 -0
  117. prograph/parser/cypher/__init__.py +1 -0
  118. prograph/parser/sparql/SparqlLexer.py +801 -0
  119. prograph/parser/sparql/SparqlParser.py +10146 -0
  120. prograph/parser/sparql/SparqlParserListener.py +1056 -0
  121. prograph/parser/sparql/SparqlParserVisitor.py +593 -0
  122. prograph/parser/sparql/__init__.py +8 -0
  123. prograph/plan/__init__.py +3 -0
  124. prograph/plan/ir/__init__.py +1 -0
  125. prograph/plan/ir/casts.py +338 -0
  126. prograph/plan/ir/consistency.py +397 -0
  127. prograph/plan/ir/construct.py +90 -0
  128. prograph/plan/ir/cost.py +234 -0
  129. prograph/plan/ir/cypher_answer.py +1081 -0
  130. prograph/plan/ir/cypher_apoc.py +717 -0
  131. prograph/plan/ir/cypher_apoc_convert.py +505 -0
  132. prograph/plan/ir/cypher_apoc_date.py +845 -0
  133. prograph/plan/ir/cypher_apoc_map.py +496 -0
  134. prograph/plan/ir/cypher_apoc_meta.py +384 -0
  135. prograph/plan/ir/cypher_apoc_path.py +597 -0
  136. prograph/plan/ir/cypher_apoc_procedures.py +163 -0
  137. prograph/plan/ir/cypher_apoc_text.py +785 -0
  138. prograph/plan/ir/cypher_front.py +5474 -0
  139. prograph/plan/ir/cypher_functions.py +565 -0
  140. prograph/plan/ir/cypher_lists.py +935 -0
  141. prograph/plan/ir/cypher_maps.py +370 -0
  142. prograph/plan/ir/cypher_mixed.py +604 -0
  143. prograph/plan/ir/cypher_ontology.py +226 -0
  144. prograph/plan/ir/cypher_procedures.py +191 -0
  145. prograph/plan/ir/cypher_runtime_errors.py +107 -0
  146. prograph/plan/ir/cypher_temporal.py +1357 -0
  147. prograph/plan/ir/cypher_temporal_arith.py +339 -0
  148. prograph/plan/ir/cypher_values.py +546 -0
  149. prograph/plan/ir/cypher_write.py +612 -0
  150. prograph/plan/ir/dataset.py +643 -0
  151. prograph/plan/ir/dates.py +102 -0
  152. prograph/plan/ir/entry.py +992 -0
  153. prograph/plan/ir/expr.py +260 -0
  154. prograph/plan/ir/facts.py +1171 -0
  155. prograph/plan/ir/fresh.py +291 -0
  156. prograph/plan/ir/freshness.py +625 -0
  157. prograph/plan/ir/identity.py +797 -0
  158. prograph/plan/ir/java_datetime.py +265 -0
  159. prograph/plan/ir/java_regex.py +391 -0
  160. prograph/plan/ir/join_order.py +39 -0
  161. prograph/plan/ir/key_checks.py +433 -0
  162. prograph/plan/ir/keys.py +454 -0
  163. prograph/plan/ir/lower.py +5323 -0
  164. prograph/plan/ir/ops.py +619 -0
  165. prograph/plan/ir/owl_rl.py +1119 -0
  166. prograph/plan/ir/owl_rl_rules.py +725 -0
  167. prograph/plan/ir/profiles.py +694 -0
  168. prograph/plan/ir/ql.py +987 -0
  169. prograph/plan/ir/rdfs.py +342 -0
  170. prograph/plan/ir/rewrite.py +1782 -0
  171. prograph/plan/ir/saturate.py +339 -0
  172. prograph/plan/ir/semantics.py +103 -0
  173. prograph/plan/ir/slot.py +509 -0
  174. prograph/plan/ir/sparql_front.py +3418 -0
  175. prograph/plan/ir/strings.py +534 -0
  176. prograph/plan/ir/term_functions.py +274 -0
  177. prograph/plan/ir/terms.py +84 -0
  178. prograph/plan/ir/xpath_regex.py +264 -0
  179. prograph/plan/run_record.py +65 -0
  180. prograph/plan/sql/__init__.py +21 -0
  181. prograph/plan/sql/dialect.py +3726 -0
  182. prograph/plan/sql/mysql_serializer.py +103 -0
  183. prograph/plan/sql/names.py +65 -0
  184. prograph/plan/sql/oracle_serializer.py +306 -0
  185. prograph/plan/sql/pg_serializer.py +501 -0
  186. prograph/plan/sql/rel_algebra.py +781 -0
  187. prograph/plan/sql/sql_serializer.py +1419 -0
  188. prograph/plan/sql/sqlserver_serializer.py +603 -0
  189. prograph/provenance.py +222 -0
  190. prograph/r2rml/__init__.py +48 -0
  191. prograph/r2rml/catalog.py +259 -0
  192. prograph/r2rml/compile.py +372 -0
  193. prograph/r2rml/direct.py +140 -0
  194. prograph/r2rml/export.py +495 -0
  195. prograph/r2rml/model.py +180 -0
  196. prograph/r2rml/parser.py +546 -0
  197. prograph/r2rml/schema.py +120 -0
  198. prograph/read_only.py +122 -0
  199. prograph/results.py +157 -0
  200. prograph/rules/__init__.py +168 -0
  201. prograph/rules/__main__.py +32 -0
  202. prograph/schema/__init__.py +60 -0
  203. prograph/schema/change_set.py +50 -0
  204. prograph/schema/collations.py +198 -0
  205. prograph/schema/compile_schema.py +388 -0
  206. prograph/schema/context.py +845 -0
  207. prograph/schema/examples.py +234 -0
  208. prograph/schema/iri.py +817 -0
  209. prograph/schema/literals.py +94 -0
  210. prograph/schema/lpg_view.py +617 -0
  211. prograph/schema/mapping.py +1803 -0
  212. prograph/schema/maps_schema.py +40 -0
  213. prograph/schema/naming.py +77 -0
  214. prograph/schema/query_sources.py +59 -0
  215. prograph/schema/rows.py +215 -0
  216. prograph/schema/source_map.py +1103 -0
  217. prograph/schema/store_view.py +459 -0
  218. prograph/schema/types.py +569 -0
  219. prograph/schema/xsd.py +569 -0
  220. prograph/shacl/__init__.py +30 -0
  221. prograph/shacl/dqv.py +58 -0
  222. prograph/shacl/generate.py +255 -0
  223. prograph/shacl/report.py +138 -0
  224. prograph/shacl/validator.py +164 -0
  225. prograph/store.py +1737 -0
  226. prograph/utils/__init__.py +6 -0
  227. prograph/utils/exceptions.py +308 -0
  228. prograph/utils/helpers.py +126 -0
  229. prograph/warnings.py +46 -0
  230. prograph/writes.py +369 -0
  231. prograph-1.3.3.dist-info/METADATA +281 -0
  232. prograph-1.3.3.dist-info/RECORD +237 -0
  233. prograph-1.3.3.dist-info/WHEEL +5 -0
  234. prograph-1.3.3.dist-info/entry_points.txt +3 -0
  235. prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
  236. prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
  237. prograph-1.3.3.dist-info/top_level.txt +1 -0
@@ -0,0 +1,280 @@
1
+ """`prograph serve`: the SPARQL endpoint on a host and port."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ from typing import Any
7
+ from prograph.rules import module_home
8
+
9
+ module_home("server-authentication", "A server's authentication and each user's credential",
10
+ terms=("token", "authentication", "bearer", "--no-auth", "credential", "per-user"))
11
+
12
+ LOCAL_HOSTS = ("127.0.0.1", "localhost", "::1")
13
+ TOKEN_ENV = "PROGRAPH_ENDPOINT_TOKEN"
14
+ # A Databricks App does not expand variables in app.yaml's command, so the
15
+ # warehouse id comes from the environment, which app.yaml's env sets.
16
+ WAREHOUSE_ENV = "PROGRAPH_WAREHOUSE_ID"
17
+ MODEL_ENV = "PROGRAPH_MCP_MODEL"
18
+ # A server answers queries others write (an MCP client's model, a SPARQL client): a query past this
19
+ # many seconds is an error, and a Databricks SQL warehouse cancels it. --timeout 0 for none.
20
+ DEFAULT_TIMEOUT = 300.0
21
+ # The model `ask` writes SPARQL with when none is named, on the databricks provider: the
22
+ # library client's default, one home.
23
+ from prograph.nlp.providers.databricks import DEFAULT_MODEL # noqa: E402
24
+
25
+
26
+ def add_parser(subparsers: Any) -> None:
27
+ p = subparsers.add_parser(
28
+ "serve", help="Serve a mapping as a SPARQL 1.1 Protocol endpoint (pip install 'prograph[endpoint]')")
29
+ source = p.add_argument_group("the mapping")
30
+ source.add_argument("--config", help="a ProGraph graph YAML (its sources, or the tables it names)")
31
+ source.add_argument("--r2rml", help="an R2RML mapping (Turtle) over --duckdb or a warehouse")
32
+ data = p.add_argument_group("the data")
33
+ data.add_argument("--duckdb", dest="duckdb_path", help="a DuckDB database file, opened read only")
34
+ data.add_argument("--warehouse-http-path",
35
+ help="a Databricks SQL warehouse's HTTP path; each query runs as the signed-in "
36
+ "user of a Databricks App (DATABRICKS_HOST names the workspace)")
37
+ data.add_argument("--warehouse-id", default=os.environ.get(WAREHOUSE_ENV) or None,
38
+ help=f"a SQL warehouse's id, for its HTTP path (default ${WAREHOUSE_ENV}, which a "
39
+ "Databricks App's app.yaml sets from its sql-warehouse resource)")
40
+ data.add_argument("--catalog", help="the warehouse's default catalog")
41
+ data.add_argument("--schema", dest="db_schema", help="the warehouse's default schema")
42
+ data.add_argument("--service-principal", action="store_true",
43
+ help="with no user token, query as the app's service principal")
44
+ p.add_argument("--host", default="127.0.0.1", help="address to bind (default 127.0.0.1)")
45
+ p.add_argument("--port", type=int, default=int(os.environ.get("DATABRICKS_APP_PORT", "8000")),
46
+ help="port (default $DATABRICKS_APP_PORT or 8000)")
47
+ p.add_argument("--token-file", help=f"a file holding the bearer token (else ${TOKEN_ENV})")
48
+ p.add_argument("--no-auth", action="store_true",
49
+ help="serve off localhost with no token (behind a proxy that signs users in)")
50
+ users = p.add_argument_group(
51
+ "each user's own credential, off a Databricks App (--per-user)")
52
+ users.add_argument("--per-user", action="store_true",
53
+ help="over a warehouse: each request's Authorization: Bearer is its user's own "
54
+ "credential, a Databricks OAuth token used as it is or an identity "
55
+ "provider's JWT exchanged for one (OAuth token federation); the query runs "
56
+ "as that user. Not with a server token, --no-auth or --service-principal")
57
+ users.add_argument("--idp-issuer", metavar="URL",
58
+ help="with --per-user --mcp: the identity provider's issuer URL, named as the "
59
+ "authorization server in /.well-known/oauth-protected-resource so MCP "
60
+ "clients can sign users in (set --endpoint-iri to the public URL)")
61
+ users.add_argument("--max-user-engines", type=int, default=64,
62
+ help="over a warehouse: the most users' engines open at once (default 64); a "
63
+ "request that would open another is 503 with Retry-After until one "
64
+ "idles out")
65
+ users.add_argument("--user-idle", type=float, default=900.0,
66
+ help="over a warehouse: seconds after a user's last request their engine and "
67
+ "connection are closed, on a timer (default 900)")
68
+ p.add_argument("--row-cap", type=int, help="rows past which a SELECT is an error")
69
+ p.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT,
70
+ help=f"seconds past which a query is an error (default {DEFAULT_TIMEOUT:g}; 0 for none). "
71
+ "On a Databricks SQL warehouse the warehouse cancels the statement "
72
+ "(STATEMENT_TIMEOUT); on other engines the query runs to its end and its "
73
+ "answer is the error")
74
+ p.add_argument("--endpoint-iri", help="the endpoint's public URL, for the service description")
75
+ mcp = p.add_argument_group("MCP: the SPARQL tools at /mcp, under the same authentication")
76
+ mcp.add_argument("--mcp", action="store_true",
77
+ help="serve the MCP tools get_schema, run_sparql, ask and explain_sparql at /mcp "
78
+ "(streamable HTTP; pip install 'prograph[mcp]')")
79
+ mcp.add_argument("--mcp-model", default=os.environ.get(MODEL_ENV) or None,
80
+ help=f"the model ask writes SPARQL with: a Databricks Model Serving endpoint's name "
81
+ f"(default ${MODEL_ENV}, else {DEFAULT_MODEL} on the databricks provider, the "
82
+ f"model measured; another provider needs one named)")
83
+ mcp.add_argument("--mcp-provider", choices=["databricks", "anthropic", "openai"], default="databricks",
84
+ help="where the model is served (default databricks)")
85
+ mcp.add_argument("--mcp-extra-tools", action="store_true",
86
+ help="also serve the ontology, SHACL and views tools")
87
+
88
+
89
+ def server_token(token_file: Any) -> Any:
90
+ """The server's own bearer token, from `--token-file` or
91
+ $PROGRAPH_ENDPOINT_TOKEN, or None."""
92
+ token = None
93
+ if token_file:
94
+ with open(token_file) as f:
95
+ token = f.read().strip()
96
+ return token or os.environ.get(TOKEN_ENV) or None
97
+
98
+
99
+ def require_authentication(host: Any, token: Any, no_auth: bool, server: str = "serving") -> None:
100
+ """Off localhost a server needs a token unless --no-auth says a proxy
101
+ signs users in. `prograph serve` and
102
+ `python -m prograph.mcp` both ask here."""
103
+ if token is None and (host or "127.0.0.1") not in LOCAL_HOSTS and not no_auth:
104
+ raise SystemExit(f"{server} on {host} needs a token (--token-file or ${TOKEN_ENV}) or --no-auth")
105
+
106
+
107
+ def token_for(args: Any) -> Any:
108
+ """The bearer token, or None. Off localhost a server needs a token
109
+ unless --no-auth says a proxy signs users in.
110
+
111
+ The modes, chosen at start: a server token (every request carries it);
112
+ over a warehouse on a Databricks App, each user's token from the proxy
113
+ (`X-Forwarded-Access-Token`); with --per-user, each user's own
114
+ credential in `Authorization`, so the server has no token of its own
115
+ and a combination that would give `Authorization` two meanings, or
116
+ answer a request with no credential, stops here."""
117
+ per_user = getattr(args, "per_user", False)
118
+ if getattr(args, "max_user_engines", 1) < 1:
119
+ raise SystemExit("--max-user-engines is at least 1")
120
+ if getattr(args, "user_idle", 0.0) < 0:
121
+ raise SystemExit("--user-idle is seconds, not negative")
122
+ token = server_token(args.token_file)
123
+ if per_user:
124
+ if not (args.warehouse_http_path or getattr(args, "warehouse_id", None)):
125
+ raise SystemExit("--per-user needs a warehouse (--warehouse-id or --warehouse-http-path): "
126
+ "it opens each user's own connection")
127
+ if token is not None:
128
+ raise SystemExit("--per-user reads each user's credential from Authorization, where a "
129
+ f"server token (--token-file or ${TOKEN_ENV}) would be compared: "
130
+ "use one or the other" + ("" if args.token_file else
131
+ f" (unset ${TOKEN_ENV})"))
132
+ if args.no_auth:
133
+ raise SystemExit("--per-user and --no-auth: every request of a per-user server carries "
134
+ "its user's credential; drop --no-auth")
135
+ if args.service_principal:
136
+ raise SystemExit("--per-user and --service-principal: a per-user server answers no "
137
+ "request without its user's credential; drop --service-principal")
138
+ if getattr(args, "idp_issuer", None) and not getattr(args, "mcp", False):
139
+ raise SystemExit("--idp-issuer is for MCP clients' sign-in: it needs --mcp")
140
+ if getattr(args, "idp_issuer", None) and not getattr(args, "endpoint_iri", None):
141
+ raise SystemExit("--idp-issuer needs --endpoint-iri, the server's public URL: the "
142
+ "resource metadata names it, and the bind address is not it")
143
+ return None
144
+ if getattr(args, "idp_issuer", None):
145
+ raise SystemExit("--idp-issuer names where per-user clients sign in: it needs --per-user")
146
+ # As each signed-in user (a warehouse, no service principal) a request
147
+ # without a user token is refused, and one with a token is the
148
+ # warehouse's to check; the service principal answers without one, so
149
+ # it needs the server's token, or --no-auth behind a proxy that signs
150
+ # users in.
151
+ as_each_user = bool(args.warehouse_http_path or getattr(args, "warehouse_id", None)) \
152
+ and not args.service_principal
153
+ if not as_each_user:
154
+ try:
155
+ require_authentication(args.host, token, args.no_auth)
156
+ except SystemExit as stop:
157
+ raise SystemExit(str(stop) + (" (--service-principal answers requests that carry no "
158
+ "user token)" if args.service_principal else "")) from None
159
+ return token
160
+
161
+
162
+ def workspace_host(value: str) -> str:
163
+ """The workspace's host name from DATABRICKS_HOST, given with or
164
+ without `https://` or `http://` and a trailing path: every call to it
165
+ is made over https."""
166
+ from urllib.parse import urlparse
167
+ value = value.strip()
168
+ if not value:
169
+ return ""
170
+ parsed = urlparse(value if "://" in value else f"https://{value}")
171
+ return parsed.netloc
172
+
173
+
174
+ def run(args: Any) -> None:
175
+ import uvicorn
176
+
177
+ from prograph.endpoint import engines
178
+ from prograph.endpoint.app import Options, create_app
179
+ from prograph.logging import get_logger
180
+
181
+ for name in ("warehouse_id", "warehouse_http_path"):
182
+ value = getattr(args, name, None)
183
+ if value and value.startswith("$"):
184
+ raise SystemExit(f"--{name.replace('_', '-')} is {value!r}, an unexpanded variable: "
185
+ f"set ${WAREHOUSE_ENV} in app.yaml's env instead (a Databricks App does "
186
+ "not expand variables in its command)")
187
+ if getattr(args, "warehouse_id", None) and not args.warehouse_http_path:
188
+ args.warehouse_http_path = f"/sql/1.0/warehouses/{args.warehouse_id}"
189
+ token = token_for(args)
190
+ args.timeout = timeout_of(args)
191
+ if args.warehouse_http_path:
192
+ host = workspace_host(os.environ.get("DATABRICKS_HOST", ""))
193
+ if not host:
194
+ raise SystemExit("--warehouse-http-path needs DATABRICKS_HOST")
195
+ schema = engines.schema_from(args.config, args.r2rml)
196
+ per_user = getattr(args, "per_user", False)
197
+ source = engines.DatabricksUsers(schema, host,
198
+ args.warehouse_http_path, catalog=args.catalog,
199
+ db_schema=args.db_schema,
200
+ service_principal=args.service_principal,
201
+ from_bearer=per_user,
202
+ max_engines=getattr(args, "max_user_engines", 64),
203
+ idle=getattr(args, "user_idle", 900.0),
204
+ statement_timeout=args.timeout)
205
+ how = ("databricks_sql, as each user's own credential (Authorization)" if per_user
206
+ else "databricks_sql, as the signed-in user")
207
+ else:
208
+ engine = engines.local_engine(args.config, args.r2rml, args.duckdb_path)
209
+ source = engines.one(engine)
210
+ how = "duckdb file" if args.duckdb_path else "graph config"
211
+ iri = args.endpoint_iri or f"http://{args.host}:{args.port}/sparql"
212
+ options = Options(row_cap=args.row_cap, timeout=args.timeout, endpoint_iri=iri, token=token,
213
+ per_user=getattr(args, "per_user", False),
214
+ authorization_server=getattr(args, "idp_issuer", None))
215
+ auth = ("per-user" if options.per_user else "token" if token
216
+ else "app user" if args.warehouse_http_path else "none")
217
+ get_logger("prograph.endpoint").info(
218
+ "sparql_endpoint_start", endpoint=iri, engine=how, auth=auth,
219
+ row_cap=args.row_cap, timeout=args.timeout)
220
+ # The running queries by id, one table for /sparql and the MCP tools: either cancels either's.
221
+ from prograph.execution import RunningQueries
222
+ queries = RunningQueries()
223
+ mcp = mcp_server(args, source, options, queries) if getattr(args, "mcp", False) else None
224
+ uvicorn.run(create_app(source, options, mcp=mcp, queries=queries), host=args.host, port=args.port,
225
+ log_level="info")
226
+
227
+
228
+ def mcp_server(args: Any, source: Any, options: Options, queries: Any = None) -> Any:
229
+ """The MCP server `prograph serve --mcp` serves at /mcp: stateless, so
230
+ each POST stands alone, with no `initialize` handshake before it and no
231
+ `mcp-session-id` after. Databricks Genie Code adds a Databricks App as a
232
+ custom MCP server only when it is stateless ("Your app must be stateless",
233
+ https://docs.databricks.com/aws/en/assistant/mcp), and behind the Apps
234
+ proxy a session held in one process's memory cannot be relied on to meet
235
+ its next request. Each POST still carries its own user token, so a call
236
+ runs on that user's engine as before. Responses stay streamable HTTP's
237
+ event stream, as in that page's example (`stateless_http=True` alone);
238
+ stdio (`prograph-mcp --sparql`) is unchanged."""
239
+ from prograph.mcp.server import build_sparql_server
240
+ from prograph.logging import get_logger
241
+ mcp = build_sparql_server(source, llm=model_client(args),
242
+ extra_tools=args.mcp_extra_tools, host=args.host,
243
+ row_cap=options.row_cap, timeout=options.timeout,
244
+ stateless_http=True, queries=queries)
245
+ get_logger("prograph.endpoint").info("sparql_endpoint_mcp", path="/mcp", model=model_name(args),
246
+ provider=args.mcp_provider, stateless=True)
247
+ return mcp
248
+
249
+
250
+ def timeout_of(args: Any) -> Any:
251
+ """The server's timeout in seconds: `--timeout`, `DEFAULT_TIMEOUT` when not given, None for 0."""
252
+ value = getattr(args, "timeout", DEFAULT_TIMEOUT)
253
+ if value is not None and value < 0:
254
+ raise SystemExit("--timeout is a number of seconds, 0 for none")
255
+ return value or None
256
+
257
+
258
+ def model_name(args: Any) -> Any:
259
+ """The model `ask` writes queries with: the one named, else `DEFAULT_MODEL` on the databricks
260
+ provider; None for another provider with no model named."""
261
+ return getattr(args, "mcp_model", None) or (
262
+ DEFAULT_MODEL if getattr(args, "mcp_provider", "databricks") == "databricks" else None)
263
+
264
+
265
+ def model_client(args: Any) -> Any:
266
+ """A function returning the client `ask` writes queries with: the model
267
+ named, else `DEFAULT_MODEL` on the databricks provider; None for another
268
+ provider with no model named. Called at the first `ask`, so a server starts
269
+ without reaching the model."""
270
+ name = model_name(args)
271
+ if not name:
272
+ return None
273
+
274
+ def build() -> Any:
275
+ from prograph.nlp import providers
276
+ cls = {"databricks": "DatabricksLLMClient", "anthropic": "AnthropicLLMClient",
277
+ "openai": "OpenAILLMClient"}[args.mcp_provider]
278
+ return getattr(providers, cls)(model=name)
279
+
280
+ return build
@@ -0,0 +1,240 @@
1
+ """A caller's own credential, off a Databricks App (`prograph serve --per-user`).
2
+
3
+ The caller sends `Authorization: Bearer <token>`, one of:
4
+
5
+ - a Databricks token (an OAuth access token from `databricks auth token`,
6
+ or a personal access token): the warehouse connection is opened with it
7
+ as it is;
8
+ - an identity provider's JWT: exchanged for a Databricks token through
9
+ OAuth token federation (`Workspace.exchange`, RFC 8693 at the
10
+ workspace's `/oidc/v1/token`), and the connection opened with that.
11
+
12
+ The two are told apart by shape (`needs_exchange`): a token that is not a
13
+ JWT, or a JWT the workspace itself issued (its `iss` is the workspace's
14
+ `/oidc`), is a Databricks token; any other JWT is exchanged. The JWT's
15
+ claims are read unverified, and only to choose that route: ProGraph checks
16
+ no signature and takes no identity from a claim. Databricks' exchange (its
17
+ federation policy: issuer, audience, subject) and the warehouse's
18
+ connection are the checks; a refusal by either is a 401.
19
+
20
+ The user's name, for the log events only, is the workspace's SCIM `Me`
21
+ (`Workspace.user_name`) under the Databricks token. No token is logged or
22
+ put in an error message (`scrubbed`).
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import base64
28
+ import json
29
+ import time
30
+ import urllib.error
31
+ import urllib.parse
32
+ import urllib.request
33
+ from typing import Any, Callable, Dict, Iterable, Optional, Tuple
34
+
35
+ EXCHANGE_GRANT = "urn:ietf:params:oauth:grant-type:token-exchange"
36
+ JWT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"
37
+ EXCHANGE_PATH = "/oidc/v1/token"
38
+ ME_PATH = "/api/2.0/preview/scim/v2/Me"
39
+
40
+ # (method, url, headers, body) -> (status, body bytes)
41
+ Http = Callable[[str, str, Dict[str, str], Optional[bytes]], Tuple[int, bytes]]
42
+
43
+
44
+ class CredentialRefused(Exception):
45
+ """The credential was refused; the message holds no token."""
46
+
47
+
48
+ class CheckUnavailable(Exception):
49
+ """The credential could not be checked (the workspace or warehouse
50
+ unreachable, failing or rate-limiting): not the credential's fault, so
51
+ not a 401 that would make a client throw a good token away. `status`
52
+ is 503 for a rate limit, else 502; the message holds no token."""
53
+
54
+ def __init__(self, message: str, status: int = 502, retry_after: Optional[str] = None):
55
+ super().__init__(message)
56
+ self.status, self.retry_after = status, retry_after
57
+
58
+
59
+ # OAuth error codes (RFC 6749 5.2, RFC 8693 2.2.2) that refuse the token
60
+ # or the client; any other failure of the exchange is not the token's.
61
+ REFUSING_CODES = ("invalid_grant", "invalid_request", "invalid_client", "unauthorized_client",
62
+ "invalid_scope", "invalid_target", "access_denied")
63
+
64
+
65
+ def refused_by_warehouse(error: BaseException) -> bool:
66
+ """True when the warehouse refused the credential at connection (the
67
+ connector's `RequestError` context says HTTP 401 or 403, or 400 at the
68
+ session's open), False when the connection failed for another reason
69
+ (unreachable, 5xx, timeout, a malformed HTTP path)."""
70
+ context = getattr(error, "context", None)
71
+ code = context.get("http-code") if isinstance(context, dict) else None
72
+ if isinstance(code, int):
73
+ if code in (401, 403):
74
+ return True
75
+ # Opening the session, the warehouse answers a forged or malformed
76
+ # token, or one the connector cannot read (an `exp` of 1e400), with
77
+ # HTTP 400 and no body. A wrong catalog or schema fails later, as a
78
+ # Thrift error; a malformed HTTP path says MALFORMED_REQUEST.
79
+ return code == 400 and "MALFORMED_REQUEST" not in str(error)
80
+ return "invalid access token" in str(error).lower()
81
+
82
+
83
+ def bearer(headers: Any) -> Optional[str]:
84
+ """The token of an `Authorization: Bearer <token>` header, or None."""
85
+ value = headers.get("authorization") or ""
86
+ scheme, _, token = value.strip().partition(" ")
87
+ if scheme.lower() != "bearer":
88
+ return None
89
+ return token.strip() or None
90
+
91
+
92
+ def unverified_claims(token: str) -> Optional[Dict[str, Any]]:
93
+ """A JWT's payload, unverified, or None when `token` is not a JWT
94
+ (three base64url parts, the second a JSON object). For choosing a
95
+ route and bounding a cache's life only: never for identity."""
96
+ parts = token.split(".")
97
+ if len(parts) != 3 or not all(parts):
98
+ return None
99
+ try:
100
+ payload = base64.urlsafe_b64decode(parts[1] + "=" * (-len(parts[1]) % 4))
101
+ claims = json.loads(payload)
102
+ except (ValueError, UnicodeDecodeError):
103
+ return None
104
+ return claims if isinstance(claims, dict) else None
105
+
106
+
107
+ def needs_exchange(token: str, workspace_host: str) -> bool:
108
+ """True for an identity provider's JWT, to be exchanged; False for a
109
+ Databricks token: not a JWT (a personal access token), or a JWT whose
110
+ issuer is this workspace (an OAuth access token it issued)."""
111
+ claims = unverified_claims(token)
112
+ if claims is None:
113
+ return False
114
+ issuer = claims.get("iss")
115
+ if not isinstance(issuer, str):
116
+ return False
117
+ host = urllib.parse.urlparse(issuer).netloc.lower()
118
+ return host != workspace_host.lower()
119
+
120
+
121
+ def expires_at(token: str) -> Optional[float]:
122
+ """The wall-clock time a JWT says it expires (`exp`), or None. Read to
123
+ keep no engine past its caller's credential, never to extend one."""
124
+ import math
125
+ claims = unverified_claims(token) or {}
126
+ exp = claims.get("exp")
127
+ if not isinstance(exp, (int, float)) or isinstance(exp, bool):
128
+ return None
129
+ try:
130
+ when = float(exp)
131
+ except OverflowError: # an integer past a float's range
132
+ return None
133
+ # A value no clock reaches (inf, nan, past year 9999) is no expiry: the
134
+ # exchange and the warehouse still check the token.
135
+ return when if math.isfinite(when) and abs(when) < 253402300800 else None
136
+
137
+
138
+ def unreadable_expiry(token: str) -> bool:
139
+ """True for a JWT whose `exp` is present but no time a clock reaches
140
+ (`1e400`, a string): the connector cannot read it either (it fails
141
+ with "timestamp out of range"), and no valid token carries one."""
142
+ claims = unverified_claims(token)
143
+ return claims is not None and "exp" in claims and expires_at(token) is None
144
+
145
+
146
+ def scrubbed(text: str, secrets: Iterable[Optional[str]]) -> str:
147
+ """`text` with each secret in it replaced by `<token>`."""
148
+ for secret in secrets:
149
+ if secret:
150
+ text = text.replace(secret, "<token>")
151
+ return text
152
+
153
+
154
+ def _urllib(method: str, url: str, headers: Dict[str, str], body: Optional[bytes]) -> Tuple[int, bytes]:
155
+ request = urllib.request.Request(url, data=body, headers=headers, method=method)
156
+ try:
157
+ with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 - https, a named host
158
+ return response.status, response.read()
159
+ except urllib.error.HTTPError as error:
160
+ return error.code, error.read() or b""
161
+
162
+
163
+ class Workspace:
164
+ """The two workspace calls a per-user server makes besides the
165
+ warehouse's: the token exchange and the user's name. `http` is the
166
+ HTTP client (`urllib` by default; a stub in tests)."""
167
+
168
+ def __init__(self, host: str, http: Optional[Http] = None):
169
+ self.host = host
170
+ self._http = http or _urllib
171
+
172
+ def exchange(self, subject_token: str) -> Tuple[str, Optional[float]]:
173
+ """A Databricks access token for an identity provider's JWT, and the
174
+ seconds it lives (`expires_in`), by OAuth token federation: RFC 8693
175
+ token exchange, `scope=all-apis`, no client id (an account-wide
176
+ federation policy). A refusal is `CredentialRefused`."""
177
+ body = urllib.parse.urlencode({
178
+ "grant_type": EXCHANGE_GRANT,
179
+ "subject_token": subject_token,
180
+ "subject_token_type": JWT_TOKEN_TYPE,
181
+ "scope": "all-apis",
182
+ }).encode()
183
+ try:
184
+ status, raw = self._http("POST", f"https://{self.host}{EXCHANGE_PATH}",
185
+ {"Content-Type": "application/x-www-form-urlencoded",
186
+ "Accept": "application/json"}, body)
187
+ except Exception as error: # noqa: BLE001 - the workspace unreachable
188
+ raise CheckUnavailable(f"the workspace's token exchange could not be reached: "
189
+ f"{type(error).__name__}") from None
190
+ try:
191
+ answer = json.loads(raw or b"{}")
192
+ except ValueError:
193
+ answer = {}
194
+ if not isinstance(answer, dict):
195
+ answer = {}
196
+ token = answer.get("access_token")
197
+ if status != 200 or not isinstance(token, str) or not token:
198
+ code = answer.get("error") or f"HTTP {status}"
199
+ said = answer.get("error_description") or ""
200
+ said = scrubbed(str(said), [subject_token])[:200]
201
+ refusing = status in (401, 403) or (status == 400 and code in REFUSING_CODES)
202
+ if not refusing:
203
+ raise CheckUnavailable(
204
+ f"the workspace's token exchange failed ({code}): try again",
205
+ status=503 if status == 429 else 502,
206
+ retry_after="30" if status == 429 else None)
207
+ raise CredentialRefused(
208
+ f"the workspace refused to exchange the identity provider's token ({code}"
209
+ + (f": {said}" if said else "") + "): check the account's federation policy "
210
+ "(issuer, audience, subject claim)")
211
+ lives = answer.get("expires_in")
212
+ return token, float(lives) if isinstance(lives, (int, float)) else None
213
+
214
+ def user_name(self, token: str) -> Optional[str]:
215
+ """The user's name (`userName`, else the primary e-mail) from SCIM
216
+ `Me`, or None when the workspace does not say. For logs only."""
217
+ try:
218
+ status, raw = self._http("GET", f"https://{self.host}{ME_PATH}",
219
+ {"Authorization": f"Bearer {token}", "Accept": "application/json"},
220
+ None)
221
+ if status != 200:
222
+ return None
223
+ me = json.loads(raw)
224
+ except Exception: # noqa: BLE001 - a name is not needed to answer
225
+ return None
226
+ if not isinstance(me, dict):
227
+ return None
228
+ name = me.get("userName")
229
+ if not name:
230
+ emails = [e for e in me.get("emails") or [] if isinstance(e, dict)]
231
+ primary = [e for e in emails if e.get("primary")] or emails
232
+ name = primary[0].get("value") if primary else None
233
+ return str(name) if name else None
234
+
235
+
236
+ def wall_to_monotonic(wall: Optional[float]) -> Optional[float]:
237
+ """A wall-clock time as `time.monotonic()` would read it then."""
238
+ if wall is None:
239
+ return None
240
+ return time.monotonic() + (wall - time.time())