prograph 1.3.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prograph/__init__.py +36 -0
- prograph/__main__.py +5 -0
- prograph/_version.py +375 -0
- prograph/cli/__init__.py +1 -0
- prograph/cli/_iri_utils.py +124 -0
- prograph/cli/emit_genie_space.py +107 -0
- prograph/cli/generate_views.py +140 -0
- prograph/cli/main.py +420 -0
- prograph/cli/schema_inferrer.py +746 -0
- prograph/cli/ttl_loader.py +289 -0
- prograph/cli/upload_genie_space.py +84 -0
- prograph/config/__init__.py +27 -0
- prograph/config/yaml_config.py +848 -0
- prograph/demos/__init__.py +14 -0
- prograph/demos/turbofan/README.md +74 -0
- prograph/demos/turbofan/SCENARIO.md +209 -0
- prograph/demos/turbofan/__init__.py +75 -0
- prograph/demos/turbofan/create_tables.sql +206 -0
- prograph/demos/turbofan/data/engine-instances.ttl +1123 -0
- prograph/demos/turbofan/data/sensor-timeseries.csv +90 -0
- prograph/demos/turbofan/demo.py +162 -0
- prograph/demos/turbofan/genie_qa_notebook.py +434 -0
- prograph/demos/turbofan/graph.yaml +1098 -0
- prograph/demos/turbofan/instance_graph_demo.py +32 -0
- prograph/demos/turbofan/load.py +223 -0
- prograph/demos/turbofan/mapping.r2rml.ttl +991 -0
- prograph/demos/turbofan/ontology/engine-maintenance.ttl +606 -0
- prograph/demos/turbofan/payload-shapes.ttl +33 -0
- prograph/demos/turbofan/queries/example-queries.sparql +215 -0
- prograph/demos/turbofan/shapes.ttl +75 -0
- prograph/demos/turbofan/tables.py +106 -0
- prograph/demos/turbofan/turbofan_schema.py +393 -0
- prograph/demos/turbofan/validation_demo.py +153 -0
- prograph/describe.py +231 -0
- prograph/endpoint/__init__.py +12 -0
- prograph/endpoint/app.py +722 -0
- prograph/endpoint/cli.py +280 -0
- prograph/endpoint/credentials.py +240 -0
- prograph/endpoint/engines.py +418 -0
- prograph/endpoint/negotiate.py +101 -0
- prograph/endpoint/results.py +181 -0
- prograph/endpoint/service.py +31 -0
- prograph/engine.py +2718 -0
- prograph/execution.py +455 -0
- prograph/executors/__init__.py +6 -0
- prograph/executors/base.py +11 -0
- prograph/executors/databricks_sql_executor.py +51 -0
- prograph/executors/duckdb_executor.py +24 -0
- prograph/executors/mysql_executor.py +20 -0
- prograph/executors/oracle_executor.py +24 -0
- prograph/executors/pandas_executor.py +72 -0
- prograph/executors/postgresql_executor.py +35 -0
- prograph/executors/spark_executor.py +88 -0
- prograph/executors/sqlserver_executor.py +35 -0
- prograph/extras/__init__.py +1 -0
- prograph/extras/graphql/__init__.py +33 -0
- prograph/extras/graphql/execute.py +25 -0
- prograph/extras/graphql/schema.py +314 -0
- prograph/extras/instance_graph.py +559 -0
- prograph/genie/__init__.py +83 -0
- prograph/genie/databricks_uploader.py +306 -0
- prograph/genie/deploy.py +393 -0
- prograph/genie/formats.py +414 -0
- prograph/genie/instruction_generator.py +375 -0
- prograph/genie/sample_query_generator.py +256 -0
- prograph/genie/shacl_views.py +318 -0
- prograph/genie/space_emitter.py +329 -0
- prograph/genie/view_generator.py +628 -0
- prograph/genie/view_writer.py +172 -0
- prograph/graph/__init__.py +1 -0
- prograph/graph/ansi_session.py +348 -0
- prograph/graph/databricks_sql_graph.py +300 -0
- prograph/graph/drivers.py +143 -0
- prograph/graph/duckdb_graph.py +68 -0
- prograph/graph/mysql_graph.py +38 -0
- prograph/graph/oracle_graph.py +285 -0
- prograph/graph/pandas_graph.py +343 -0
- prograph/graph/postgresql_graph.py +644 -0
- prograph/graph/spark_graph.py +556 -0
- prograph/graph/sql_graph.py +83 -0
- prograph/graph/sqlserver_graph.py +72 -0
- prograph/hints.py +95 -0
- prograph/logging.py +550 -0
- prograph/mcp/__init__.py +29 -0
- prograph/mcp/__main__.py +180 -0
- prograph/mcp/bootstrap.py +98 -0
- prograph/mcp/server.py +601 -0
- prograph/mcp/sparql.py +397 -0
- prograph/mcp/tools.py +504 -0
- prograph/nlp/__init__.py +79 -0
- prograph/nlp/client.py +305 -0
- prograph/nlp/databricks_tools.py +369 -0
- prograph/nlp/graph_context.py +313 -0
- prograph/nlp/interface.py +652 -0
- prograph/nlp/prompts/__init__.py +19 -0
- prograph/nlp/prompts/examples.py +258 -0
- prograph/nlp/prompts/system.py +239 -0
- prograph/nlp/providers/__init__.py +14 -0
- prograph/nlp/providers/anthropic.py +274 -0
- prograph/nlp/providers/databricks.py +312 -0
- prograph/nlp/providers/openai.py +203 -0
- prograph/nlp/schema_context.py +78 -0
- prograph/ontology/__init__.py +53 -0
- prograph/ontology/alignment.py +542 -0
- prograph/ontology/hierarchy.py +310 -0
- prograph/ontology/imports.py +587 -0
- prograph/ontology/layer.py +991 -0
- prograph/ontology/skos.py +61 -0
- prograph/owl/__init__.py +27 -0
- prograph/owl/defined_classes.py +323 -0
- prograph/parameters.py +242 -0
- prograph/parser/__init__.py +1 -0
- prograph/parser/cypher/CypherLexer.py +836 -0
- prograph/parser/cypher/CypherListener.py +966 -0
- prograph/parser/cypher/CypherParser.py +10854 -0
- prograph/parser/cypher/CypherVisitor.py +543 -0
- prograph/parser/cypher/__init__.py +1 -0
- prograph/parser/sparql/SparqlLexer.py +801 -0
- prograph/parser/sparql/SparqlParser.py +10146 -0
- prograph/parser/sparql/SparqlParserListener.py +1056 -0
- prograph/parser/sparql/SparqlParserVisitor.py +593 -0
- prograph/parser/sparql/__init__.py +8 -0
- prograph/plan/__init__.py +3 -0
- prograph/plan/ir/__init__.py +1 -0
- prograph/plan/ir/casts.py +338 -0
- prograph/plan/ir/consistency.py +397 -0
- prograph/plan/ir/construct.py +90 -0
- prograph/plan/ir/cost.py +234 -0
- prograph/plan/ir/cypher_answer.py +1081 -0
- prograph/plan/ir/cypher_apoc.py +717 -0
- prograph/plan/ir/cypher_apoc_convert.py +505 -0
- prograph/plan/ir/cypher_apoc_date.py +845 -0
- prograph/plan/ir/cypher_apoc_map.py +496 -0
- prograph/plan/ir/cypher_apoc_meta.py +384 -0
- prograph/plan/ir/cypher_apoc_path.py +597 -0
- prograph/plan/ir/cypher_apoc_procedures.py +163 -0
- prograph/plan/ir/cypher_apoc_text.py +785 -0
- prograph/plan/ir/cypher_front.py +5474 -0
- prograph/plan/ir/cypher_functions.py +565 -0
- prograph/plan/ir/cypher_lists.py +935 -0
- prograph/plan/ir/cypher_maps.py +370 -0
- prograph/plan/ir/cypher_mixed.py +604 -0
- prograph/plan/ir/cypher_ontology.py +226 -0
- prograph/plan/ir/cypher_procedures.py +191 -0
- prograph/plan/ir/cypher_runtime_errors.py +107 -0
- prograph/plan/ir/cypher_temporal.py +1357 -0
- prograph/plan/ir/cypher_temporal_arith.py +339 -0
- prograph/plan/ir/cypher_values.py +546 -0
- prograph/plan/ir/cypher_write.py +612 -0
- prograph/plan/ir/dataset.py +643 -0
- prograph/plan/ir/dates.py +102 -0
- prograph/plan/ir/entry.py +992 -0
- prograph/plan/ir/expr.py +260 -0
- prograph/plan/ir/facts.py +1171 -0
- prograph/plan/ir/fresh.py +291 -0
- prograph/plan/ir/freshness.py +625 -0
- prograph/plan/ir/identity.py +797 -0
- prograph/plan/ir/java_datetime.py +265 -0
- prograph/plan/ir/java_regex.py +391 -0
- prograph/plan/ir/join_order.py +39 -0
- prograph/plan/ir/key_checks.py +433 -0
- prograph/plan/ir/keys.py +454 -0
- prograph/plan/ir/lower.py +5323 -0
- prograph/plan/ir/ops.py +619 -0
- prograph/plan/ir/owl_rl.py +1119 -0
- prograph/plan/ir/owl_rl_rules.py +725 -0
- prograph/plan/ir/profiles.py +694 -0
- prograph/plan/ir/ql.py +987 -0
- prograph/plan/ir/rdfs.py +342 -0
- prograph/plan/ir/rewrite.py +1782 -0
- prograph/plan/ir/saturate.py +339 -0
- prograph/plan/ir/semantics.py +103 -0
- prograph/plan/ir/slot.py +509 -0
- prograph/plan/ir/sparql_front.py +3418 -0
- prograph/plan/ir/strings.py +534 -0
- prograph/plan/ir/term_functions.py +274 -0
- prograph/plan/ir/terms.py +84 -0
- prograph/plan/ir/xpath_regex.py +264 -0
- prograph/plan/run_record.py +65 -0
- prograph/plan/sql/__init__.py +21 -0
- prograph/plan/sql/dialect.py +3726 -0
- prograph/plan/sql/mysql_serializer.py +103 -0
- prograph/plan/sql/names.py +65 -0
- prograph/plan/sql/oracle_serializer.py +306 -0
- prograph/plan/sql/pg_serializer.py +501 -0
- prograph/plan/sql/rel_algebra.py +781 -0
- prograph/plan/sql/sql_serializer.py +1419 -0
- prograph/plan/sql/sqlserver_serializer.py +603 -0
- prograph/provenance.py +222 -0
- prograph/r2rml/__init__.py +48 -0
- prograph/r2rml/catalog.py +259 -0
- prograph/r2rml/compile.py +372 -0
- prograph/r2rml/direct.py +140 -0
- prograph/r2rml/export.py +495 -0
- prograph/r2rml/model.py +180 -0
- prograph/r2rml/parser.py +546 -0
- prograph/r2rml/schema.py +120 -0
- prograph/read_only.py +122 -0
- prograph/results.py +157 -0
- prograph/rules/__init__.py +168 -0
- prograph/rules/__main__.py +32 -0
- prograph/schema/__init__.py +60 -0
- prograph/schema/change_set.py +50 -0
- prograph/schema/collations.py +198 -0
- prograph/schema/compile_schema.py +388 -0
- prograph/schema/context.py +845 -0
- prograph/schema/examples.py +234 -0
- prograph/schema/iri.py +817 -0
- prograph/schema/literals.py +94 -0
- prograph/schema/lpg_view.py +617 -0
- prograph/schema/mapping.py +1803 -0
- prograph/schema/maps_schema.py +40 -0
- prograph/schema/naming.py +77 -0
- prograph/schema/query_sources.py +59 -0
- prograph/schema/rows.py +215 -0
- prograph/schema/source_map.py +1103 -0
- prograph/schema/store_view.py +459 -0
- prograph/schema/types.py +569 -0
- prograph/schema/xsd.py +569 -0
- prograph/shacl/__init__.py +30 -0
- prograph/shacl/dqv.py +58 -0
- prograph/shacl/generate.py +255 -0
- prograph/shacl/report.py +138 -0
- prograph/shacl/validator.py +164 -0
- prograph/store.py +1737 -0
- prograph/utils/__init__.py +6 -0
- prograph/utils/exceptions.py +308 -0
- prograph/utils/helpers.py +126 -0
- prograph/warnings.py +46 -0
- prograph/writes.py +369 -0
- prograph-1.3.3.dist-info/METADATA +281 -0
- prograph-1.3.3.dist-info/RECORD +237 -0
- prograph-1.3.3.dist-info/WHEEL +5 -0
- prograph-1.3.3.dist-info/entry_points.txt +3 -0
- prograph-1.3.3.dist-info/licenses/LICENSE +201 -0
- prograph-1.3.3.dist-info/licenses/NOTICE +10 -0
- prograph-1.3.3.dist-info/top_level.txt +1 -0
prograph/endpoint/cli.py
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""`prograph serve`: the SPARQL endpoint on a host and port."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import Any
|
|
7
|
+
from prograph.rules import module_home
|
|
8
|
+
|
|
9
|
+
module_home("server-authentication", "A server's authentication and each user's credential",
|
|
10
|
+
terms=("token", "authentication", "bearer", "--no-auth", "credential", "per-user"))
|
|
11
|
+
|
|
12
|
+
LOCAL_HOSTS = ("127.0.0.1", "localhost", "::1")
|
|
13
|
+
TOKEN_ENV = "PROGRAPH_ENDPOINT_TOKEN"
|
|
14
|
+
# A Databricks App does not expand variables in app.yaml's command, so the
|
|
15
|
+
# warehouse id comes from the environment, which app.yaml's env sets.
|
|
16
|
+
WAREHOUSE_ENV = "PROGRAPH_WAREHOUSE_ID"
|
|
17
|
+
MODEL_ENV = "PROGRAPH_MCP_MODEL"
|
|
18
|
+
# A server answers queries others write (an MCP client's model, a SPARQL client): a query past this
|
|
19
|
+
# many seconds is an error, and a Databricks SQL warehouse cancels it. --timeout 0 for none.
|
|
20
|
+
DEFAULT_TIMEOUT = 300.0
|
|
21
|
+
# The model `ask` writes SPARQL with when none is named, on the databricks provider: the
|
|
22
|
+
# library client's default, one home.
|
|
23
|
+
from prograph.nlp.providers.databricks import DEFAULT_MODEL # noqa: E402
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def add_parser(subparsers: Any) -> None:
|
|
27
|
+
p = subparsers.add_parser(
|
|
28
|
+
"serve", help="Serve a mapping as a SPARQL 1.1 Protocol endpoint (pip install 'prograph[endpoint]')")
|
|
29
|
+
source = p.add_argument_group("the mapping")
|
|
30
|
+
source.add_argument("--config", help="a ProGraph graph YAML (its sources, or the tables it names)")
|
|
31
|
+
source.add_argument("--r2rml", help="an R2RML mapping (Turtle) over --duckdb or a warehouse")
|
|
32
|
+
data = p.add_argument_group("the data")
|
|
33
|
+
data.add_argument("--duckdb", dest="duckdb_path", help="a DuckDB database file, opened read only")
|
|
34
|
+
data.add_argument("--warehouse-http-path",
|
|
35
|
+
help="a Databricks SQL warehouse's HTTP path; each query runs as the signed-in "
|
|
36
|
+
"user of a Databricks App (DATABRICKS_HOST names the workspace)")
|
|
37
|
+
data.add_argument("--warehouse-id", default=os.environ.get(WAREHOUSE_ENV) or None,
|
|
38
|
+
help=f"a SQL warehouse's id, for its HTTP path (default ${WAREHOUSE_ENV}, which a "
|
|
39
|
+
"Databricks App's app.yaml sets from its sql-warehouse resource)")
|
|
40
|
+
data.add_argument("--catalog", help="the warehouse's default catalog")
|
|
41
|
+
data.add_argument("--schema", dest="db_schema", help="the warehouse's default schema")
|
|
42
|
+
data.add_argument("--service-principal", action="store_true",
|
|
43
|
+
help="with no user token, query as the app's service principal")
|
|
44
|
+
p.add_argument("--host", default="127.0.0.1", help="address to bind (default 127.0.0.1)")
|
|
45
|
+
p.add_argument("--port", type=int, default=int(os.environ.get("DATABRICKS_APP_PORT", "8000")),
|
|
46
|
+
help="port (default $DATABRICKS_APP_PORT or 8000)")
|
|
47
|
+
p.add_argument("--token-file", help=f"a file holding the bearer token (else ${TOKEN_ENV})")
|
|
48
|
+
p.add_argument("--no-auth", action="store_true",
|
|
49
|
+
help="serve off localhost with no token (behind a proxy that signs users in)")
|
|
50
|
+
users = p.add_argument_group(
|
|
51
|
+
"each user's own credential, off a Databricks App (--per-user)")
|
|
52
|
+
users.add_argument("--per-user", action="store_true",
|
|
53
|
+
help="over a warehouse: each request's Authorization: Bearer is its user's own "
|
|
54
|
+
"credential, a Databricks OAuth token used as it is or an identity "
|
|
55
|
+
"provider's JWT exchanged for one (OAuth token federation); the query runs "
|
|
56
|
+
"as that user. Not with a server token, --no-auth or --service-principal")
|
|
57
|
+
users.add_argument("--idp-issuer", metavar="URL",
|
|
58
|
+
help="with --per-user --mcp: the identity provider's issuer URL, named as the "
|
|
59
|
+
"authorization server in /.well-known/oauth-protected-resource so MCP "
|
|
60
|
+
"clients can sign users in (set --endpoint-iri to the public URL)")
|
|
61
|
+
users.add_argument("--max-user-engines", type=int, default=64,
|
|
62
|
+
help="over a warehouse: the most users' engines open at once (default 64); a "
|
|
63
|
+
"request that would open another is 503 with Retry-After until one "
|
|
64
|
+
"idles out")
|
|
65
|
+
users.add_argument("--user-idle", type=float, default=900.0,
|
|
66
|
+
help="over a warehouse: seconds after a user's last request their engine and "
|
|
67
|
+
"connection are closed, on a timer (default 900)")
|
|
68
|
+
p.add_argument("--row-cap", type=int, help="rows past which a SELECT is an error")
|
|
69
|
+
p.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT,
|
|
70
|
+
help=f"seconds past which a query is an error (default {DEFAULT_TIMEOUT:g}; 0 for none). "
|
|
71
|
+
"On a Databricks SQL warehouse the warehouse cancels the statement "
|
|
72
|
+
"(STATEMENT_TIMEOUT); on other engines the query runs to its end and its "
|
|
73
|
+
"answer is the error")
|
|
74
|
+
p.add_argument("--endpoint-iri", help="the endpoint's public URL, for the service description")
|
|
75
|
+
mcp = p.add_argument_group("MCP: the SPARQL tools at /mcp, under the same authentication")
|
|
76
|
+
mcp.add_argument("--mcp", action="store_true",
|
|
77
|
+
help="serve the MCP tools get_schema, run_sparql, ask and explain_sparql at /mcp "
|
|
78
|
+
"(streamable HTTP; pip install 'prograph[mcp]')")
|
|
79
|
+
mcp.add_argument("--mcp-model", default=os.environ.get(MODEL_ENV) or None,
|
|
80
|
+
help=f"the model ask writes SPARQL with: a Databricks Model Serving endpoint's name "
|
|
81
|
+
f"(default ${MODEL_ENV}, else {DEFAULT_MODEL} on the databricks provider, the "
|
|
82
|
+
f"model measured; another provider needs one named)")
|
|
83
|
+
mcp.add_argument("--mcp-provider", choices=["databricks", "anthropic", "openai"], default="databricks",
|
|
84
|
+
help="where the model is served (default databricks)")
|
|
85
|
+
mcp.add_argument("--mcp-extra-tools", action="store_true",
|
|
86
|
+
help="also serve the ontology, SHACL and views tools")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def server_token(token_file: Any) -> Any:
|
|
90
|
+
"""The server's own bearer token, from `--token-file` or
|
|
91
|
+
$PROGRAPH_ENDPOINT_TOKEN, or None."""
|
|
92
|
+
token = None
|
|
93
|
+
if token_file:
|
|
94
|
+
with open(token_file) as f:
|
|
95
|
+
token = f.read().strip()
|
|
96
|
+
return token or os.environ.get(TOKEN_ENV) or None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def require_authentication(host: Any, token: Any, no_auth: bool, server: str = "serving") -> None:
|
|
100
|
+
"""Off localhost a server needs a token unless --no-auth says a proxy
|
|
101
|
+
signs users in. `prograph serve` and
|
|
102
|
+
`python -m prograph.mcp` both ask here."""
|
|
103
|
+
if token is None and (host or "127.0.0.1") not in LOCAL_HOSTS and not no_auth:
|
|
104
|
+
raise SystemExit(f"{server} on {host} needs a token (--token-file or ${TOKEN_ENV}) or --no-auth")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def token_for(args: Any) -> Any:
|
|
108
|
+
"""The bearer token, or None. Off localhost a server needs a token
|
|
109
|
+
unless --no-auth says a proxy signs users in.
|
|
110
|
+
|
|
111
|
+
The modes, chosen at start: a server token (every request carries it);
|
|
112
|
+
over a warehouse on a Databricks App, each user's token from the proxy
|
|
113
|
+
(`X-Forwarded-Access-Token`); with --per-user, each user's own
|
|
114
|
+
credential in `Authorization`, so the server has no token of its own
|
|
115
|
+
and a combination that would give `Authorization` two meanings, or
|
|
116
|
+
answer a request with no credential, stops here."""
|
|
117
|
+
per_user = getattr(args, "per_user", False)
|
|
118
|
+
if getattr(args, "max_user_engines", 1) < 1:
|
|
119
|
+
raise SystemExit("--max-user-engines is at least 1")
|
|
120
|
+
if getattr(args, "user_idle", 0.0) < 0:
|
|
121
|
+
raise SystemExit("--user-idle is seconds, not negative")
|
|
122
|
+
token = server_token(args.token_file)
|
|
123
|
+
if per_user:
|
|
124
|
+
if not (args.warehouse_http_path or getattr(args, "warehouse_id", None)):
|
|
125
|
+
raise SystemExit("--per-user needs a warehouse (--warehouse-id or --warehouse-http-path): "
|
|
126
|
+
"it opens each user's own connection")
|
|
127
|
+
if token is not None:
|
|
128
|
+
raise SystemExit("--per-user reads each user's credential from Authorization, where a "
|
|
129
|
+
f"server token (--token-file or ${TOKEN_ENV}) would be compared: "
|
|
130
|
+
"use one or the other" + ("" if args.token_file else
|
|
131
|
+
f" (unset ${TOKEN_ENV})"))
|
|
132
|
+
if args.no_auth:
|
|
133
|
+
raise SystemExit("--per-user and --no-auth: every request of a per-user server carries "
|
|
134
|
+
"its user's credential; drop --no-auth")
|
|
135
|
+
if args.service_principal:
|
|
136
|
+
raise SystemExit("--per-user and --service-principal: a per-user server answers no "
|
|
137
|
+
"request without its user's credential; drop --service-principal")
|
|
138
|
+
if getattr(args, "idp_issuer", None) and not getattr(args, "mcp", False):
|
|
139
|
+
raise SystemExit("--idp-issuer is for MCP clients' sign-in: it needs --mcp")
|
|
140
|
+
if getattr(args, "idp_issuer", None) and not getattr(args, "endpoint_iri", None):
|
|
141
|
+
raise SystemExit("--idp-issuer needs --endpoint-iri, the server's public URL: the "
|
|
142
|
+
"resource metadata names it, and the bind address is not it")
|
|
143
|
+
return None
|
|
144
|
+
if getattr(args, "idp_issuer", None):
|
|
145
|
+
raise SystemExit("--idp-issuer names where per-user clients sign in: it needs --per-user")
|
|
146
|
+
# As each signed-in user (a warehouse, no service principal) a request
|
|
147
|
+
# without a user token is refused, and one with a token is the
|
|
148
|
+
# warehouse's to check; the service principal answers without one, so
|
|
149
|
+
# it needs the server's token, or --no-auth behind a proxy that signs
|
|
150
|
+
# users in.
|
|
151
|
+
as_each_user = bool(args.warehouse_http_path or getattr(args, "warehouse_id", None)) \
|
|
152
|
+
and not args.service_principal
|
|
153
|
+
if not as_each_user:
|
|
154
|
+
try:
|
|
155
|
+
require_authentication(args.host, token, args.no_auth)
|
|
156
|
+
except SystemExit as stop:
|
|
157
|
+
raise SystemExit(str(stop) + (" (--service-principal answers requests that carry no "
|
|
158
|
+
"user token)" if args.service_principal else "")) from None
|
|
159
|
+
return token
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def workspace_host(value: str) -> str:
|
|
163
|
+
"""The workspace's host name from DATABRICKS_HOST, given with or
|
|
164
|
+
without `https://` or `http://` and a trailing path: every call to it
|
|
165
|
+
is made over https."""
|
|
166
|
+
from urllib.parse import urlparse
|
|
167
|
+
value = value.strip()
|
|
168
|
+
if not value:
|
|
169
|
+
return ""
|
|
170
|
+
parsed = urlparse(value if "://" in value else f"https://{value}")
|
|
171
|
+
return parsed.netloc
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def run(args: Any) -> None:
|
|
175
|
+
import uvicorn
|
|
176
|
+
|
|
177
|
+
from prograph.endpoint import engines
|
|
178
|
+
from prograph.endpoint.app import Options, create_app
|
|
179
|
+
from prograph.logging import get_logger
|
|
180
|
+
|
|
181
|
+
for name in ("warehouse_id", "warehouse_http_path"):
|
|
182
|
+
value = getattr(args, name, None)
|
|
183
|
+
if value and value.startswith("$"):
|
|
184
|
+
raise SystemExit(f"--{name.replace('_', '-')} is {value!r}, an unexpanded variable: "
|
|
185
|
+
f"set ${WAREHOUSE_ENV} in app.yaml's env instead (a Databricks App does "
|
|
186
|
+
"not expand variables in its command)")
|
|
187
|
+
if getattr(args, "warehouse_id", None) and not args.warehouse_http_path:
|
|
188
|
+
args.warehouse_http_path = f"/sql/1.0/warehouses/{args.warehouse_id}"
|
|
189
|
+
token = token_for(args)
|
|
190
|
+
args.timeout = timeout_of(args)
|
|
191
|
+
if args.warehouse_http_path:
|
|
192
|
+
host = workspace_host(os.environ.get("DATABRICKS_HOST", ""))
|
|
193
|
+
if not host:
|
|
194
|
+
raise SystemExit("--warehouse-http-path needs DATABRICKS_HOST")
|
|
195
|
+
schema = engines.schema_from(args.config, args.r2rml)
|
|
196
|
+
per_user = getattr(args, "per_user", False)
|
|
197
|
+
source = engines.DatabricksUsers(schema, host,
|
|
198
|
+
args.warehouse_http_path, catalog=args.catalog,
|
|
199
|
+
db_schema=args.db_schema,
|
|
200
|
+
service_principal=args.service_principal,
|
|
201
|
+
from_bearer=per_user,
|
|
202
|
+
max_engines=getattr(args, "max_user_engines", 64),
|
|
203
|
+
idle=getattr(args, "user_idle", 900.0),
|
|
204
|
+
statement_timeout=args.timeout)
|
|
205
|
+
how = ("databricks_sql, as each user's own credential (Authorization)" if per_user
|
|
206
|
+
else "databricks_sql, as the signed-in user")
|
|
207
|
+
else:
|
|
208
|
+
engine = engines.local_engine(args.config, args.r2rml, args.duckdb_path)
|
|
209
|
+
source = engines.one(engine)
|
|
210
|
+
how = "duckdb file" if args.duckdb_path else "graph config"
|
|
211
|
+
iri = args.endpoint_iri or f"http://{args.host}:{args.port}/sparql"
|
|
212
|
+
options = Options(row_cap=args.row_cap, timeout=args.timeout, endpoint_iri=iri, token=token,
|
|
213
|
+
per_user=getattr(args, "per_user", False),
|
|
214
|
+
authorization_server=getattr(args, "idp_issuer", None))
|
|
215
|
+
auth = ("per-user" if options.per_user else "token" if token
|
|
216
|
+
else "app user" if args.warehouse_http_path else "none")
|
|
217
|
+
get_logger("prograph.endpoint").info(
|
|
218
|
+
"sparql_endpoint_start", endpoint=iri, engine=how, auth=auth,
|
|
219
|
+
row_cap=args.row_cap, timeout=args.timeout)
|
|
220
|
+
# The running queries by id, one table for /sparql and the MCP tools: either cancels either's.
|
|
221
|
+
from prograph.execution import RunningQueries
|
|
222
|
+
queries = RunningQueries()
|
|
223
|
+
mcp = mcp_server(args, source, options, queries) if getattr(args, "mcp", False) else None
|
|
224
|
+
uvicorn.run(create_app(source, options, mcp=mcp, queries=queries), host=args.host, port=args.port,
|
|
225
|
+
log_level="info")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def mcp_server(args: Any, source: Any, options: Options, queries: Any = None) -> Any:
|
|
229
|
+
"""The MCP server `prograph serve --mcp` serves at /mcp: stateless, so
|
|
230
|
+
each POST stands alone, with no `initialize` handshake before it and no
|
|
231
|
+
`mcp-session-id` after. Databricks Genie Code adds a Databricks App as a
|
|
232
|
+
custom MCP server only when it is stateless ("Your app must be stateless",
|
|
233
|
+
https://docs.databricks.com/aws/en/assistant/mcp), and behind the Apps
|
|
234
|
+
proxy a session held in one process's memory cannot be relied on to meet
|
|
235
|
+
its next request. Each POST still carries its own user token, so a call
|
|
236
|
+
runs on that user's engine as before. Responses stay streamable HTTP's
|
|
237
|
+
event stream, as in that page's example (`stateless_http=True` alone);
|
|
238
|
+
stdio (`prograph-mcp --sparql`) is unchanged."""
|
|
239
|
+
from prograph.mcp.server import build_sparql_server
|
|
240
|
+
from prograph.logging import get_logger
|
|
241
|
+
mcp = build_sparql_server(source, llm=model_client(args),
|
|
242
|
+
extra_tools=args.mcp_extra_tools, host=args.host,
|
|
243
|
+
row_cap=options.row_cap, timeout=options.timeout,
|
|
244
|
+
stateless_http=True, queries=queries)
|
|
245
|
+
get_logger("prograph.endpoint").info("sparql_endpoint_mcp", path="/mcp", model=model_name(args),
|
|
246
|
+
provider=args.mcp_provider, stateless=True)
|
|
247
|
+
return mcp
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def timeout_of(args: Any) -> Any:
|
|
251
|
+
"""The server's timeout in seconds: `--timeout`, `DEFAULT_TIMEOUT` when not given, None for 0."""
|
|
252
|
+
value = getattr(args, "timeout", DEFAULT_TIMEOUT)
|
|
253
|
+
if value is not None and value < 0:
|
|
254
|
+
raise SystemExit("--timeout is a number of seconds, 0 for none")
|
|
255
|
+
return value or None
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def model_name(args: Any) -> Any:
|
|
259
|
+
"""The model `ask` writes queries with: the one named, else `DEFAULT_MODEL` on the databricks
|
|
260
|
+
provider; None for another provider with no model named."""
|
|
261
|
+
return getattr(args, "mcp_model", None) or (
|
|
262
|
+
DEFAULT_MODEL if getattr(args, "mcp_provider", "databricks") == "databricks" else None)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def model_client(args: Any) -> Any:
|
|
266
|
+
"""A function returning the client `ask` writes queries with: the model
|
|
267
|
+
named, else `DEFAULT_MODEL` on the databricks provider; None for another
|
|
268
|
+
provider with no model named. Called at the first `ask`, so a server starts
|
|
269
|
+
without reaching the model."""
|
|
270
|
+
name = model_name(args)
|
|
271
|
+
if not name:
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
def build() -> Any:
|
|
275
|
+
from prograph.nlp import providers
|
|
276
|
+
cls = {"databricks": "DatabricksLLMClient", "anthropic": "AnthropicLLMClient",
|
|
277
|
+
"openai": "OpenAILLMClient"}[args.mcp_provider]
|
|
278
|
+
return getattr(providers, cls)(model=name)
|
|
279
|
+
|
|
280
|
+
return build
|
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
"""A caller's own credential, off a Databricks App (`prograph serve --per-user`).
|
|
2
|
+
|
|
3
|
+
The caller sends `Authorization: Bearer <token>`, one of:
|
|
4
|
+
|
|
5
|
+
- a Databricks token (an OAuth access token from `databricks auth token`,
|
|
6
|
+
or a personal access token): the warehouse connection is opened with it
|
|
7
|
+
as it is;
|
|
8
|
+
- an identity provider's JWT: exchanged for a Databricks token through
|
|
9
|
+
OAuth token federation (`Workspace.exchange`, RFC 8693 at the
|
|
10
|
+
workspace's `/oidc/v1/token`), and the connection opened with that.
|
|
11
|
+
|
|
12
|
+
The two are told apart by shape (`needs_exchange`): a token that is not a
|
|
13
|
+
JWT, or a JWT the workspace itself issued (its `iss` is the workspace's
|
|
14
|
+
`/oidc`), is a Databricks token; any other JWT is exchanged. The JWT's
|
|
15
|
+
claims are read unverified, and only to choose that route: ProGraph checks
|
|
16
|
+
no signature and takes no identity from a claim. Databricks' exchange (its
|
|
17
|
+
federation policy: issuer, audience, subject) and the warehouse's
|
|
18
|
+
connection are the checks; a refusal by either is a 401.
|
|
19
|
+
|
|
20
|
+
The user's name, for the log events only, is the workspace's SCIM `Me`
|
|
21
|
+
(`Workspace.user_name`) under the Databricks token. No token is logged or
|
|
22
|
+
put in an error message (`scrubbed`).
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import base64
|
|
28
|
+
import json
|
|
29
|
+
import time
|
|
30
|
+
import urllib.error
|
|
31
|
+
import urllib.parse
|
|
32
|
+
import urllib.request
|
|
33
|
+
from typing import Any, Callable, Dict, Iterable, Optional, Tuple
|
|
34
|
+
|
|
35
|
+
EXCHANGE_GRANT = "urn:ietf:params:oauth:grant-type:token-exchange"
|
|
36
|
+
JWT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt"
|
|
37
|
+
EXCHANGE_PATH = "/oidc/v1/token"
|
|
38
|
+
ME_PATH = "/api/2.0/preview/scim/v2/Me"
|
|
39
|
+
|
|
40
|
+
# (method, url, headers, body) -> (status, body bytes)
|
|
41
|
+
Http = Callable[[str, str, Dict[str, str], Optional[bytes]], Tuple[int, bytes]]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class CredentialRefused(Exception):
|
|
45
|
+
"""The credential was refused; the message holds no token."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class CheckUnavailable(Exception):
|
|
49
|
+
"""The credential could not be checked (the workspace or warehouse
|
|
50
|
+
unreachable, failing or rate-limiting): not the credential's fault, so
|
|
51
|
+
not a 401 that would make a client throw a good token away. `status`
|
|
52
|
+
is 503 for a rate limit, else 502; the message holds no token."""
|
|
53
|
+
|
|
54
|
+
def __init__(self, message: str, status: int = 502, retry_after: Optional[str] = None):
|
|
55
|
+
super().__init__(message)
|
|
56
|
+
self.status, self.retry_after = status, retry_after
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# OAuth error codes (RFC 6749 5.2, RFC 8693 2.2.2) that refuse the token
|
|
60
|
+
# or the client; any other failure of the exchange is not the token's.
|
|
61
|
+
REFUSING_CODES = ("invalid_grant", "invalid_request", "invalid_client", "unauthorized_client",
|
|
62
|
+
"invalid_scope", "invalid_target", "access_denied")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def refused_by_warehouse(error: BaseException) -> bool:
|
|
66
|
+
"""True when the warehouse refused the credential at connection (the
|
|
67
|
+
connector's `RequestError` context says HTTP 401 or 403, or 400 at the
|
|
68
|
+
session's open), False when the connection failed for another reason
|
|
69
|
+
(unreachable, 5xx, timeout, a malformed HTTP path)."""
|
|
70
|
+
context = getattr(error, "context", None)
|
|
71
|
+
code = context.get("http-code") if isinstance(context, dict) else None
|
|
72
|
+
if isinstance(code, int):
|
|
73
|
+
if code in (401, 403):
|
|
74
|
+
return True
|
|
75
|
+
# Opening the session, the warehouse answers a forged or malformed
|
|
76
|
+
# token, or one the connector cannot read (an `exp` of 1e400), with
|
|
77
|
+
# HTTP 400 and no body. A wrong catalog or schema fails later, as a
|
|
78
|
+
# Thrift error; a malformed HTTP path says MALFORMED_REQUEST.
|
|
79
|
+
return code == 400 and "MALFORMED_REQUEST" not in str(error)
|
|
80
|
+
return "invalid access token" in str(error).lower()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def bearer(headers: Any) -> Optional[str]:
|
|
84
|
+
"""The token of an `Authorization: Bearer <token>` header, or None."""
|
|
85
|
+
value = headers.get("authorization") or ""
|
|
86
|
+
scheme, _, token = value.strip().partition(" ")
|
|
87
|
+
if scheme.lower() != "bearer":
|
|
88
|
+
return None
|
|
89
|
+
return token.strip() or None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def unverified_claims(token: str) -> Optional[Dict[str, Any]]:
|
|
93
|
+
"""A JWT's payload, unverified, or None when `token` is not a JWT
|
|
94
|
+
(three base64url parts, the second a JSON object). For choosing a
|
|
95
|
+
route and bounding a cache's life only: never for identity."""
|
|
96
|
+
parts = token.split(".")
|
|
97
|
+
if len(parts) != 3 or not all(parts):
|
|
98
|
+
return None
|
|
99
|
+
try:
|
|
100
|
+
payload = base64.urlsafe_b64decode(parts[1] + "=" * (-len(parts[1]) % 4))
|
|
101
|
+
claims = json.loads(payload)
|
|
102
|
+
except (ValueError, UnicodeDecodeError):
|
|
103
|
+
return None
|
|
104
|
+
return claims if isinstance(claims, dict) else None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def needs_exchange(token: str, workspace_host: str) -> bool:
|
|
108
|
+
"""True for an identity provider's JWT, to be exchanged; False for a
|
|
109
|
+
Databricks token: not a JWT (a personal access token), or a JWT whose
|
|
110
|
+
issuer is this workspace (an OAuth access token it issued)."""
|
|
111
|
+
claims = unverified_claims(token)
|
|
112
|
+
if claims is None:
|
|
113
|
+
return False
|
|
114
|
+
issuer = claims.get("iss")
|
|
115
|
+
if not isinstance(issuer, str):
|
|
116
|
+
return False
|
|
117
|
+
host = urllib.parse.urlparse(issuer).netloc.lower()
|
|
118
|
+
return host != workspace_host.lower()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def expires_at(token: str) -> Optional[float]:
|
|
122
|
+
"""The wall-clock time a JWT says it expires (`exp`), or None. Read to
|
|
123
|
+
keep no engine past its caller's credential, never to extend one."""
|
|
124
|
+
import math
|
|
125
|
+
claims = unverified_claims(token) or {}
|
|
126
|
+
exp = claims.get("exp")
|
|
127
|
+
if not isinstance(exp, (int, float)) or isinstance(exp, bool):
|
|
128
|
+
return None
|
|
129
|
+
try:
|
|
130
|
+
when = float(exp)
|
|
131
|
+
except OverflowError: # an integer past a float's range
|
|
132
|
+
return None
|
|
133
|
+
# A value no clock reaches (inf, nan, past year 9999) is no expiry: the
|
|
134
|
+
# exchange and the warehouse still check the token.
|
|
135
|
+
return when if math.isfinite(when) and abs(when) < 253402300800 else None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def unreadable_expiry(token: str) -> bool:
|
|
139
|
+
"""True for a JWT whose `exp` is present but no time a clock reaches
|
|
140
|
+
(`1e400`, a string): the connector cannot read it either (it fails
|
|
141
|
+
with "timestamp out of range"), and no valid token carries one."""
|
|
142
|
+
claims = unverified_claims(token)
|
|
143
|
+
return claims is not None and "exp" in claims and expires_at(token) is None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def scrubbed(text: str, secrets: Iterable[Optional[str]]) -> str:
|
|
147
|
+
"""`text` with each secret in it replaced by `<token>`."""
|
|
148
|
+
for secret in secrets:
|
|
149
|
+
if secret:
|
|
150
|
+
text = text.replace(secret, "<token>")
|
|
151
|
+
return text
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _urllib(method: str, url: str, headers: Dict[str, str], body: Optional[bytes]) -> Tuple[int, bytes]:
|
|
155
|
+
request = urllib.request.Request(url, data=body, headers=headers, method=method)
|
|
156
|
+
try:
|
|
157
|
+
with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 - https, a named host
|
|
158
|
+
return response.status, response.read()
|
|
159
|
+
except urllib.error.HTTPError as error:
|
|
160
|
+
return error.code, error.read() or b""
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class Workspace:
|
|
164
|
+
"""The two workspace calls a per-user server makes besides the
|
|
165
|
+
warehouse's: the token exchange and the user's name. `http` is the
|
|
166
|
+
HTTP client (`urllib` by default; a stub in tests)."""
|
|
167
|
+
|
|
168
|
+
def __init__(self, host: str, http: Optional[Http] = None):
|
|
169
|
+
self.host = host
|
|
170
|
+
self._http = http or _urllib
|
|
171
|
+
|
|
172
|
+
def exchange(self, subject_token: str) -> Tuple[str, Optional[float]]:
|
|
173
|
+
"""A Databricks access token for an identity provider's JWT, and the
|
|
174
|
+
seconds it lives (`expires_in`), by OAuth token federation: RFC 8693
|
|
175
|
+
token exchange, `scope=all-apis`, no client id (an account-wide
|
|
176
|
+
federation policy). A refusal is `CredentialRefused`."""
|
|
177
|
+
body = urllib.parse.urlencode({
|
|
178
|
+
"grant_type": EXCHANGE_GRANT,
|
|
179
|
+
"subject_token": subject_token,
|
|
180
|
+
"subject_token_type": JWT_TOKEN_TYPE,
|
|
181
|
+
"scope": "all-apis",
|
|
182
|
+
}).encode()
|
|
183
|
+
try:
|
|
184
|
+
status, raw = self._http("POST", f"https://{self.host}{EXCHANGE_PATH}",
|
|
185
|
+
{"Content-Type": "application/x-www-form-urlencoded",
|
|
186
|
+
"Accept": "application/json"}, body)
|
|
187
|
+
except Exception as error: # noqa: BLE001 - the workspace unreachable
|
|
188
|
+
raise CheckUnavailable(f"the workspace's token exchange could not be reached: "
|
|
189
|
+
f"{type(error).__name__}") from None
|
|
190
|
+
try:
|
|
191
|
+
answer = json.loads(raw or b"{}")
|
|
192
|
+
except ValueError:
|
|
193
|
+
answer = {}
|
|
194
|
+
if not isinstance(answer, dict):
|
|
195
|
+
answer = {}
|
|
196
|
+
token = answer.get("access_token")
|
|
197
|
+
if status != 200 or not isinstance(token, str) or not token:
|
|
198
|
+
code = answer.get("error") or f"HTTP {status}"
|
|
199
|
+
said = answer.get("error_description") or ""
|
|
200
|
+
said = scrubbed(str(said), [subject_token])[:200]
|
|
201
|
+
refusing = status in (401, 403) or (status == 400 and code in REFUSING_CODES)
|
|
202
|
+
if not refusing:
|
|
203
|
+
raise CheckUnavailable(
|
|
204
|
+
f"the workspace's token exchange failed ({code}): try again",
|
|
205
|
+
status=503 if status == 429 else 502,
|
|
206
|
+
retry_after="30" if status == 429 else None)
|
|
207
|
+
raise CredentialRefused(
|
|
208
|
+
f"the workspace refused to exchange the identity provider's token ({code}"
|
|
209
|
+
+ (f": {said}" if said else "") + "): check the account's federation policy "
|
|
210
|
+
"(issuer, audience, subject claim)")
|
|
211
|
+
lives = answer.get("expires_in")
|
|
212
|
+
return token, float(lives) if isinstance(lives, (int, float)) else None
|
|
213
|
+
|
|
214
|
+
def user_name(self, token: str) -> Optional[str]:
|
|
215
|
+
"""The user's name (`userName`, else the primary e-mail) from SCIM
|
|
216
|
+
`Me`, or None when the workspace does not say. For logs only."""
|
|
217
|
+
try:
|
|
218
|
+
status, raw = self._http("GET", f"https://{self.host}{ME_PATH}",
|
|
219
|
+
{"Authorization": f"Bearer {token}", "Accept": "application/json"},
|
|
220
|
+
None)
|
|
221
|
+
if status != 200:
|
|
222
|
+
return None
|
|
223
|
+
me = json.loads(raw)
|
|
224
|
+
except Exception: # noqa: BLE001 - a name is not needed to answer
|
|
225
|
+
return None
|
|
226
|
+
if not isinstance(me, dict):
|
|
227
|
+
return None
|
|
228
|
+
name = me.get("userName")
|
|
229
|
+
if not name:
|
|
230
|
+
emails = [e for e in me.get("emails") or [] if isinstance(e, dict)]
|
|
231
|
+
primary = [e for e in emails if e.get("primary")] or emails
|
|
232
|
+
name = primary[0].get("value") if primary else None
|
|
233
|
+
return str(name) if name else None
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def wall_to_monotonic(wall: Optional[float]) -> Optional[float]:
|
|
237
|
+
"""A wall-clock time as `time.monotonic()` would read it then."""
|
|
238
|
+
if wall is None:
|
|
239
|
+
return None
|
|
240
|
+
return time.monotonic() + (wall - time.time())
|