point-topic-access 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- point_topic_access-0.1.0/.gitignore +10 -0
- point_topic_access-0.1.0/HANDOFF.md +140 -0
- point_topic_access-0.1.0/PKG-INFO +143 -0
- point_topic_access-0.1.0/README.md +125 -0
- point_topic_access-0.1.0/pyproject.toml +35 -0
- point_topic_access-0.1.0/src/pt_access/__init__.py +15 -0
- point_topic_access-0.1.0/src/pt_access/claims.py +97 -0
- point_topic_access-0.1.0/src/pt_access/composer.py +258 -0
- point_topic_access-0.1.0/src/pt_access/contract.py +70 -0
- point_topic_access-0.1.0/src/pt_access/jwt.py +153 -0
- point_topic_access-0.1.0/src/pt_access/ontology-permission-contract.json +23 -0
- point_topic_access-0.1.0/src/pt_access/provisioning.py +381 -0
- point_topic_access-0.1.0/tests/test_claims.py +98 -0
- point_topic_access-0.1.0/tests/test_composer.py +395 -0
- point_topic_access-0.1.0/tests/test_contract.py +44 -0
- point_topic_access-0.1.0/tests/test_jwt.py +185 -0
- point_topic_access-0.1.0/tests/test_provisioning.py +336 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# HANDOFF — full context for the agent continuing this repo
|
|
2
|
+
|
|
3
|
+
**Read this before touching anything.** This scaffold was built inside
|
|
4
|
+
`point-topic-mcp/pt-access` (2026-08-07) and is intended to be moved to
|
|
5
|
+
`~/Projects/point-topic-access` and pushed as a new private repo
|
|
6
|
+
(`Point-Topic/point-topic-access`). It is the shared permission logic for Point
|
|
7
|
+
Topic's data apps. All 93 tests pass; no git history exists yet.
|
|
8
|
+
|
|
9
|
+
## Mission
|
|
10
|
+
|
|
11
|
+
Give every Point Topic data app the same identity + data-permission system the MCP
|
|
12
|
+
server runs in production:
|
|
13
|
+
|
|
14
|
+
1. **Identity**: sub-site Auth0 tenant (`point-topic.eu.auth0.com`) — same users/orgs
|
|
15
|
+
everywhere, no per-app logins.
|
|
16
|
+
2. **Data access**: per-org `productPermissions.<data_source> = {field: string[]}` from
|
|
17
|
+
the sub-site (products + geo filters), enforced as ClickHouse RESTRICTIVE row
|
|
18
|
+
policies, provisioned fresh on every login.
|
|
19
|
+
3. **One codebase for the logic** (this package) so MCP / ontology app / query agent
|
|
20
|
+
cannot drift.
|
|
21
|
+
|
|
22
|
+
Tracked by **onto-app-new#25** (the plan: access model, repo-by-repo work, rollout).
|
|
23
|
+
The ontology app's own ticket #18 (permissions built into the Snowflake ontology) is
|
|
24
|
+
**superseded — not being built** (commented on 2026-08-07).
|
|
25
|
+
|
|
26
|
+
## What was verified against live systems (2026-08-07) — trust, don't re-derive
|
|
27
|
+
|
|
28
|
+
- **Geo filters stay out of the JWT** — the Auth0 Action
|
|
29
|
+
(`sub-site/terraform/modules/auth0/actions.tf`) stamps only scalar permission values;
|
|
30
|
+
its own comment documents why: lists bloat tokens to hundreds of KB, hitting proxy
|
|
31
|
+
header limits. Scope is read fresh from sub-site Mongo (this package does that).
|
|
32
|
+
- **Claim shape** (freeze, additive-only): `pt_org_id`, `pt_org_name`, `products[]`
|
|
33
|
+
as `[{name, permissions:{scalar}}]`. `local-pricing-dashboard` and
|
|
34
|
+
`european-fttp-forecasts` SPAs read `products[].permissions.role` — a shape change
|
|
35
|
+
broke the local-pricing dashboard once already (2026-08-05, reverted).
|
|
36
|
+
- **Action stamping is generic**: new products/roles are stamped automatically; no
|
|
37
|
+
Action change needed for the `ontology_app` product.
|
|
38
|
+
- **Per-query role activation** (not SET ROLE on a shared connection): the ontology
|
|
39
|
+
app's backend uses thread-local ClickHouse clients shared across users;
|
|
40
|
+
scoping must be `client.query(sql, settings={"role": "org_<id>"})` — the pattern in
|
|
41
|
+
MCP `tools/database_tools.py::_org_role_settings`.
|
|
42
|
+
- **PREWHERE fix**: profile attached to the **service user**
|
|
43
|
+
(`ALTER USER <u> ADD PROFILES 'mcp_scoped_org_prewhere'`), `optimize_move_to_prewhere=0`.
|
|
44
|
+
Live-proven failures: per-query SETTINGS under readonly → error 164;
|
|
45
|
+
`GRANT SETTINGS PROFILE ... TO <role>` is a syntax error on CH 26.3;
|
|
46
|
+
role-attached profiles don't apply under readonly.
|
|
47
|
+
- **MCP ClickHouse** (`db.point-topic.com:443`) and the **ontology app's ClickHouse**
|
|
48
|
+
(docker container on 109.169.53.90, localhost:8123, user `default`) are DIFFERENT
|
|
49
|
+
instances — the app needs its own provisioning with its own service user (planned:
|
|
50
|
+
`onto_app`, readonly, DEFAULT ROLE NONE; `default` kept only for the business-data
|
|
51
|
+
refresh writes).
|
|
52
|
+
- **Product schema pattern** for the new `ontology_app` product (sub-site):
|
|
53
|
+
`permissionField: 'role'`, `permissionFieldType: 'select'`,
|
|
54
|
+
`permissionFieldOptions: ['admin', 'viewer']` — same as `local_pricing`.
|
|
55
|
+
- **Sub-site contract copy**: `sub-site/apps/ui/src/organisations/config/
|
|
56
|
+
ontology-permission-contract.json` (filterFields.ts is the UI registry).
|
|
57
|
+
- The ontology app's edit endpoints (`POST/PUT/DELETE /api/{add,edit,delete}-entry`)
|
|
58
|
+
have **NO server-side auth today** — closing that gap is part of the integration.
|
|
59
|
+
|
|
60
|
+
## Decisions locked in
|
|
61
|
+
|
|
62
|
+
- **Mongo-first policy source**: `fetch_org_datasets` reads sub-site Mongo directly
|
|
63
|
+
(proven path). A future sub-site API endpoint (`/permission-policy`) can replace the
|
|
64
|
+
Mongo read behind the same spec boundary — the composer/provisioning never change.
|
|
65
|
+
- **No fallback patterns** (house rule): adapters are explicit choices, never
|
|
66
|
+
try-A-else-B.
|
|
67
|
+
- **Contract-driven evolution**: new data source / filter field = fixture change +
|
|
68
|
+
tests, not composer code (the declarative refactor below).
|
|
69
|
+
- **Viewer/admin** for the ontology app come from `ontology_app` product role
|
|
70
|
+
(`products[].permissions.role`), NOT from a new claim.
|
|
71
|
+
- **SPARQL scoping deferred**: ontop queries Snowflake, not ClickHouse — row policies
|
|
72
|
+
can't reach it; SPARQL stays PT-org-only in v1.
|
|
73
|
+
|
|
74
|
+
## Next steps (in order)
|
|
75
|
+
|
|
76
|
+
1. **Declarative composer refactor**: make `composer.py` interpret the fixture
|
|
77
|
+
(`fields` as data: `kind: in_literal | containment`, column, descendantTypes;
|
|
78
|
+
`compatibility` map in the fixture) instead of hardcoded branches. Golden tests
|
|
79
|
+
already pin every SQL shape — refactor with them green. (The MCP handoff's
|
|
80
|
+
proposed `GEO_DIMENSIONS` refactor — now is the time.)
|
|
81
|
+
2. **`spec.py`**: the normalised policy spec (claims/Mongo → `{ds: {filters:
|
|
82
|
+
[{field, values}]}}`) as the single normalisation point — becomes the seam for the
|
|
83
|
+
sub-site API adapter.
|
|
84
|
+
3. **CI workflows**: pytest + ruff + pyright; **drift guard** (fetch sub-site contract
|
|
85
|
+
copy via `gh api`, fail on mismatch); shape-pin test for the claim contract is
|
|
86
|
+
already covered in `tests/test_claims.py`.
|
|
87
|
+
4. **`point-topic-mcp` swap**: delete local `core/sql_filter_composer.py`,
|
|
88
|
+
`core/org_provisioning.py`, fixture; depend on this package; keep
|
|
89
|
+
`auth/product_mapping.py`, `auth/middleware.py`, `_org_role_settings`. Verify
|
|
90
|
+
nothing else uses `SUB_SITE_MONGODB_URI` before dropping it. 242 MCP tests + E2E
|
|
91
|
+
(`infrastructure/verify_http_mcp.py`) must stay green.
|
|
92
|
+
5. **`onto-app-new` integration** (issue #25): Express `express-openid-connect` v3
|
|
93
|
+
(verified current: `issuerBaseURL`, `baseURL`, `clientID`, `clientSecret`, `secret`,
|
|
94
|
+
`authRequired: false`, `auth0Logout: true`); FastAPI JWT dependency on all `/api/*`
|
|
95
|
+
(except `/health`, `/api/external/*`); `/api/me`; role gates (admin = modify/SPARQL/
|
|
96
|
+
database tools/refresh; viewer = read-only); per-query `settings={"role": ...}` in
|
|
97
|
+
`database.py`; drop `permission_service.py` Snowflake chain + `STG_USER` login +
|
|
98
|
+
`data_restrictions` client param. Verify `cto_measurement_record_staging` exists in
|
|
99
|
+
the app's CH before setting measurement tables.
|
|
100
|
+
6. **Sub-site**: seed `ontology_app` product; assign PT org (admin) + client orgs;
|
|
101
|
+
no Action change. **Auth0**: new Regular Web App client (terraform in
|
|
102
|
+
`sub-site/terraform/modules/auth0/`), callbacks for
|
|
103
|
+
`ontology.point-topic.com` (+dev), secret → `ptserver-secret` AWS secret.
|
|
104
|
+
7. **`upc_query_agent`**: adopt `claims.py` (it already reads the claims in
|
|
105
|
+
`api/auth_handler.py`).
|
|
106
|
+
|
|
107
|
+
## Environment / secrets (never inline, never commit)
|
|
108
|
+
|
|
109
|
+
- Sub-site Mongo: `~/.SUB_SITE_MONGODB_URI_PROD` / `_DEV` (read via
|
|
110
|
+
`$(cat ...)`, chmod 600, never printed).
|
|
111
|
+
- Auth0 M2M: `~/.agents/.env.auth0` (`AUTH0_PROD_DOMAIN`, `AUTH0_PROD_CLIENT_ID`,
|
|
112
|
+
`AUTH0_PROD_CLIENT_SECRET`); Management API for tenant ops; terraform is the source
|
|
113
|
+
of truth for tenant config in the sub-site repo.
|
|
114
|
+
- ClickHouse: MCP = `~/.clickhouse.env` on the dev Mac (host `db.point-topic.com:443`);
|
|
115
|
+
ontology app CH = docker on RapidSwitch box (`root@109.169.53.90`, key
|
|
116
|
+
`~/.ssh/rapidswitch_server`), localhost:8123.
|
|
117
|
+
- Load the `credential-boundary` / `point-topic-auth0` / `sub-site-mongodb` skills for
|
|
118
|
+
any operation touching these.
|
|
119
|
+
|
|
120
|
+
## House rules (user's, non-negotiable)
|
|
121
|
+
|
|
122
|
+
- **Never commit/push without explicit approval.** Present the diff, wait for
|
|
123
|
+
"commit/push". The user reviews in the editor.
|
|
124
|
+
- Terminal tool: pass `cd` as a parameter, never `cd X && ...` in the command.
|
|
125
|
+
- `gh` account must be `peterdonaghey` (never `pt-agent`).
|
|
126
|
+
- No fallback/try-A-else-B patterns in code; deliberate checks only.
|
|
127
|
+
- Secrets never in commands/chat; rotate immediately if leaked.
|
|
128
|
+
- The sub-site repo is read-only for agents (user explicit) — sub-site changes are
|
|
129
|
+
authored by the user or explicitly authorised.
|
|
130
|
+
- This scaffold lives at `pt-access/` inside the MCP repo working tree until the user
|
|
131
|
+
moves it to `~/Projects/point-topic-access`; do not commit it as part of the MCP.
|
|
132
|
+
|
|
133
|
+
## Status when handed over
|
|
134
|
+
|
|
135
|
+
- `src/pt_access/`: claims, jwt, composer, provisioning, contract, fixture — ported
|
|
136
|
+
verbatim from the MCP (imports adapted), plus new claims/jwt/contract modules.
|
|
137
|
+
- `tests/`: 93 tests pass (composer 39, provisioning 34, jwt 11, claims 16, contract 3)
|
|
138
|
+
via `pytest -q` (uses `pythonpath = ["src"]`; no install needed).
|
|
139
|
+
- NOT done yet: declarative composer refactor, spec.py, CI workflows, any consumer
|
|
140
|
+
migration. The MCP still contains its own copies (do not delete until the swap).
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: point-topic-access
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Shared permission -> ClickHouse access-control logic for Point Topic apps (claims, JWT verification, geo filter composer, per-org provisioning)
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.12
|
|
7
|
+
Requires-Dist: clickhouse-connect>=0.8.0
|
|
8
|
+
Requires-Dist: httpx>=0.28.0
|
|
9
|
+
Requires-Dist: pymongo>=4.10.0
|
|
10
|
+
Requires-Dist: python-jose>=3.3.0
|
|
11
|
+
Provides-Extra: dev
|
|
12
|
+
Requires-Dist: cryptography>=44.0.0; extra == 'dev'
|
|
13
|
+
Requires-Dist: pyright>=1.1.0; extra == 'dev'
|
|
14
|
+
Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
|
|
15
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
16
|
+
Requires-Dist: ruff>=0.9.0; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# pt-access — shared permission → ClickHouse access-control logic
|
|
20
|
+
|
|
21
|
+
One place for the permission pipeline every Point Topic data app needs. Ported from the
|
|
22
|
+
production-proven `point-topic-mcp` implementation (issues #100/#104/#105) so the MCP
|
|
23
|
+
server, the ontology web app (`onto-app-new`) and future consumers cannot drift.
|
|
24
|
+
|
|
25
|
+
> Full design context: [`HANDOFF.md`](HANDOFF.md). Tracked by
|
|
26
|
+
> [onto-app-new#25](https://github.com/Point-Topic/onto-app-new/issues/25)
|
|
27
|
+
> (Auth0 login + per-org dataset/geo permissions).
|
|
28
|
+
|
|
29
|
+
## The pipeline
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
JWT claims ──► policy spec ──► SQL predicate ──► ClickHouse DDL
|
|
33
|
+
(what you (normalised (composer: (provisioning:
|
|
34
|
+
have) dataset + escaping, role, grants,
|
|
35
|
+
filter rows) containment, row policies,
|
|
36
|
+
OR-grouping) settings profile)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Identity comes from the shared sub-site Auth0 tenant (`point-topic.eu.auth0.com`). The
|
|
40
|
+
Post-Login Action stamps `pt_org_id` and `products[]` (`[{name, permissions}]`, scalar
|
|
41
|
+
values only). **Geo filters are never in the token** — list values would bloat tokens
|
|
42
|
+
past proxy header limits (documented decision in the Action's own code) — they are read
|
|
43
|
+
**fresh from sub-site MongoDB** (`organisations.productPermissions.<ds> = {field: string[]}`)
|
|
44
|
+
on every login/session, then converted to ClickHouse row policies.
|
|
45
|
+
|
|
46
|
+
## Modules
|
|
47
|
+
|
|
48
|
+
| Module | What | Ported from |
|
|
49
|
+
|---|---|---|
|
|
50
|
+
| `claims.py` | `get_org_id`, `get_held_products`, `is_pt_admin`, `has_data_source_access`, `get_product_role` | MCP `auth/middleware.py` (pure parts) |
|
|
51
|
+
| `jwt.py` | `Auth0TokenVerifier` — JWKS cache, RS256, issuer/audience/exp checks | MCP `auth/auth0_helpers.py` (FastMCP wrapper stays in the MCP) |
|
|
52
|
+
| `composer.py` | `compose_sql_filter` / `compose_geo_predicate` — sub-site permission values → per-dataset SQL disjunction | MCP `core/sql_filter_composer.py` (verbatim, live-verified SQL shapes) |
|
|
53
|
+
| `provisioning.py` | `fetch_org_datasets` (sub-site Mongo read) + `resolve_org_config` + `provision_org_role` / `provision_org_on_login` (ClickHouse DDL, serialisation lock) | MCP `core/org_provisioning.py` (verbatim) |
|
|
54
|
+
| `contract.py` | `load_contract` / `validate_contract` — fixture loader + code↔contract consistency check | MCP `core/ontology-permission-contract.json` + `test_permission_contract.py` |
|
|
55
|
+
|
|
56
|
+
Instance-specific config (CH host/port/creds, `grant_to` service user, measurement
|
|
57
|
+
tables, PREWHERE profile) stays in the consuming app's environment — this package takes
|
|
58
|
+
it as parameters, never hardcodes it. `provision_org_role()` already takes `grant_to`
|
|
59
|
+
per call; the convenience wrapper uses `GRANT_TO_USER` (env `MCP_CLICKHOUSE_GRANT_TO_USER`,
|
|
60
|
+
default `mcp_service`) — rename/re-purpose per instance when wiring a new consumer.
|
|
61
|
+
|
|
62
|
+
## Usage
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from pt_access.jwt import Auth0TokenVerifier
|
|
66
|
+
from pt_access.claims import get_org_id, get_held_products, is_pt_admin
|
|
67
|
+
from pt_access.provisioning import provision_org_on_login
|
|
68
|
+
|
|
69
|
+
# 1. Verify the bearer token (FastAPI dependency, etc.)
|
|
70
|
+
claims = await Auth0TokenVerifier(
|
|
71
|
+
auth0_domain="point-topic.eu.auth0.com",
|
|
72
|
+
audience="<your_client_id>", # ID token audience = client_id
|
|
73
|
+
).verify_token(bearer)
|
|
74
|
+
|
|
75
|
+
# 2. Provision the org's ClickHouse role + row policy (on login / session start)
|
|
76
|
+
if not is_pt_admin(claims):
|
|
77
|
+
summary = provision_org_on_login(get_org_id(claims), get_held_products(claims))
|
|
78
|
+
# summary: {"role": "org_<id>", "granted_to", "using", "tables", "warnings"}
|
|
79
|
+
|
|
80
|
+
# 3. Per-query scoping on the app's READ client (never SET ROLE on a shared
|
|
81
|
+
# connection — thread-local clients are shared across users):
|
|
82
|
+
# client.query(sql, settings={"role": "org_<id>"})
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`provision_org_on_login` is a no-op when `CLICKHOUSE_PROVISIONING_USER`/`_PASSWORD` are
|
|
86
|
+
unset; without a provisioning credential orgs fail closed on the engine (role never
|
|
87
|
+
granted). Requires `SUB_SITE_MONGODB_URI` (read-only sub-site Mongo user) for the fresh
|
|
88
|
+
per-org read.
|
|
89
|
+
|
|
90
|
+
## Behaviour contract (do not "fix" — each rule exists because of a live incident)
|
|
91
|
+
|
|
92
|
+
- **Fail closed**: no held data-source products → deny (`REVOKE` role + row policy
|
|
93
|
+
`USING 0`); unknown/empty filter fields → that dataset excluded, ALL excluded →
|
|
94
|
+
deny; a held product with no stored values → bare `(DATA_SOURCE='<ds>')` (omit-empty
|
|
95
|
+
contract = full access to that product's data).
|
|
96
|
+
- **Per-dataset disjunction**: `(DATA_SOURCE='upc' AND <geo>) OR (DATA_SOURCE='gbs')` —
|
|
97
|
+
never a bare `DATA_SOURCE IN (...) AND <geo>` blob.
|
|
98
|
+
- **OR-grouping**: multiple predicates must be wrapped `(A OR B)` *inside* the
|
|
99
|
+
`DATA_SOURCE` guard — SQL precedence otherwise leaks other datasets' rows at the
|
|
100
|
+
postcode (found live on prod 2026-08-05).
|
|
101
|
+
- **PREWHERE**: row policies on non-sorting-key columns return 0 rows under PREWHERE
|
|
102
|
+
(ClickHouse GH #85222). Provisioning creates a settings profile
|
|
103
|
+
(`optimize_move_to_prewhere = 0`) and attaches it to the **service user** — per-query
|
|
104
|
+
`SETTINGS` is blocked under `readonly=1` (error 164) and role-attached profiles don't
|
|
105
|
+
apply. Trade-off: applies to every query through that user.
|
|
106
|
+
- **Convergence**: full re-provision on every login (no drift check, ~320ms), serialised
|
|
107
|
+
by a global lock (row-policy `DROP+CREATE` races → `ACCESS_ENTITY_ALREADY_EXISTS`,
|
|
108
|
+
code 493). Deny orgs are always re-provisioned too (REVOKE converges).
|
|
109
|
+
- **Escaping**: every literal is single-quote-doubled (`King's Lynn`); the row policy is
|
|
110
|
+
the enforcement boundary, so a quote in an admin-entered value must never escape the
|
|
111
|
+
literal.
|
|
112
|
+
|
|
113
|
+
## The contract fixture — the single evolvable artifact
|
|
114
|
+
|
|
115
|
+
`src/pt_access/ontology-permission-contract.json` mirrors
|
|
116
|
+
`sub-site/apps/ui/src/organisations/config/ontology-permission-contract.json`
|
|
117
|
+
(sub-site UI renders exactly what this declares). Adding a **new data source** = a
|
|
118
|
+
`dataSources` entry; a **new filter field** = a `fields` entry (+ `fieldProducts` /
|
|
119
|
+
`compatibility` where applicable). Both are fixture changes, never composer code
|
|
120
|
+
changes. The package CI drift-guard workflow fetches the sub-site copy via `gh api` and
|
|
121
|
+
fails on mismatch; `tests/test_contract.py` (via `contract.validate_contract()`) checks
|
|
122
|
+
the code constants against the fixture.
|
|
123
|
+
|
|
124
|
+
## Consumers
|
|
125
|
+
|
|
126
|
+
| App | Status | Notes |
|
|
127
|
+
|---|---|---|
|
|
128
|
+
| `point-topic-mcp` | to migrate | swap local modules for this package; keep FastMCP middleware + tool→product map |
|
|
129
|
+
| `onto-app-new` | to integrate | issue #25; adds `ontology_app` product (role admin/viewer), JWT auth on all routes, per-query role settings |
|
|
130
|
+
| `upc_query_agent` | follow-up | already reads `pt_org_id`/`products[]` in `api/auth_handler.py` — adopt `claims.py` |
|
|
131
|
+
| `local-pricing-dashboard`, `european-fttp-forecasts` | don't break | read `products[].permissions.role` — the claim shape is pinned by tests |
|
|
132
|
+
|
|
133
|
+
## Development
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
uv sync # or use any venv with the deps
|
|
137
|
+
uv run pytest -q # 93 tests, no network/DB needed (all mocked)
|
|
138
|
+
uv run ruff check src tests
|
|
139
|
+
uv run pyright # requires pyright install; config in pyproject
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Distribution: private GitHub tag, pinned by consumers
|
|
143
|
+
(`pip install git+https://github.com/Point-Topic/point-topic-access@v0.1.0`).
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# pt-access — shared permission → ClickHouse access-control logic
|
|
2
|
+
|
|
3
|
+
One place for the permission pipeline every Point Topic data app needs. Ported from the
|
|
4
|
+
production-proven `point-topic-mcp` implementation (issues #100/#104/#105) so the MCP
|
|
5
|
+
server, the ontology web app (`onto-app-new`) and future consumers cannot drift.
|
|
6
|
+
|
|
7
|
+
> Full design context: [`HANDOFF.md`](HANDOFF.md). Tracked by
|
|
8
|
+
> [onto-app-new#25](https://github.com/Point-Topic/onto-app-new/issues/25)
|
|
9
|
+
> (Auth0 login + per-org dataset/geo permissions).
|
|
10
|
+
|
|
11
|
+
## The pipeline
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
JWT claims ──► policy spec ──► SQL predicate ──► ClickHouse DDL
|
|
15
|
+
(what you (normalised (composer: (provisioning:
|
|
16
|
+
have) dataset + escaping, role, grants,
|
|
17
|
+
filter rows) containment, row policies,
|
|
18
|
+
OR-grouping) settings profile)
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Identity comes from the shared sub-site Auth0 tenant (`point-topic.eu.auth0.com`). The
|
|
22
|
+
Post-Login Action stamps `pt_org_id` and `products[]` (`[{name, permissions}]`, scalar
|
|
23
|
+
values only). **Geo filters are never in the token** — list values would bloat tokens
|
|
24
|
+
past proxy header limits (documented decision in the Action's own code) — they are read
|
|
25
|
+
**fresh from sub-site MongoDB** (`organisations.productPermissions.<ds> = {field: string[]}`)
|
|
26
|
+
on every login/session, then converted to ClickHouse row policies.
|
|
27
|
+
|
|
28
|
+
## Modules
|
|
29
|
+
|
|
30
|
+
| Module | What | Ported from |
|
|
31
|
+
|---|---|---|
|
|
32
|
+
| `claims.py` | `get_org_id`, `get_held_products`, `is_pt_admin`, `has_data_source_access`, `get_product_role` | MCP `auth/middleware.py` (pure parts) |
|
|
33
|
+
| `jwt.py` | `Auth0TokenVerifier` — JWKS cache, RS256, issuer/audience/exp checks | MCP `auth/auth0_helpers.py` (FastMCP wrapper stays in the MCP) |
|
|
34
|
+
| `composer.py` | `compose_sql_filter` / `compose_geo_predicate` — sub-site permission values → per-dataset SQL disjunction | MCP `core/sql_filter_composer.py` (verbatim, live-verified SQL shapes) |
|
|
35
|
+
| `provisioning.py` | `fetch_org_datasets` (sub-site Mongo read) + `resolve_org_config` + `provision_org_role` / `provision_org_on_login` (ClickHouse DDL, serialisation lock) | MCP `core/org_provisioning.py` (verbatim) |
|
|
36
|
+
| `contract.py` | `load_contract` / `validate_contract` — fixture loader + code↔contract consistency check | MCP `core/ontology-permission-contract.json` + `test_permission_contract.py` |
|
|
37
|
+
|
|
38
|
+
Instance-specific config (CH host/port/creds, `grant_to` service user, measurement
|
|
39
|
+
tables, PREWHERE profile) stays in the consuming app's environment — this package takes
|
|
40
|
+
it as parameters, never hardcodes it. `provision_org_role()` already takes `grant_to`
|
|
41
|
+
per call; the convenience wrapper uses `GRANT_TO_USER` (env `MCP_CLICKHOUSE_GRANT_TO_USER`,
|
|
42
|
+
default `mcp_service`) — rename/re-purpose per instance when wiring a new consumer.
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from pt_access.jwt import Auth0TokenVerifier
|
|
48
|
+
from pt_access.claims import get_org_id, get_held_products, is_pt_admin
|
|
49
|
+
from pt_access.provisioning import provision_org_on_login
|
|
50
|
+
|
|
51
|
+
# 1. Verify the bearer token (FastAPI dependency, etc.)
|
|
52
|
+
claims = await Auth0TokenVerifier(
|
|
53
|
+
auth0_domain="point-topic.eu.auth0.com",
|
|
54
|
+
audience="<your_client_id>", # ID token audience = client_id
|
|
55
|
+
).verify_token(bearer)
|
|
56
|
+
|
|
57
|
+
# 2. Provision the org's ClickHouse role + row policy (on login / session start)
|
|
58
|
+
if not is_pt_admin(claims):
|
|
59
|
+
summary = provision_org_on_login(get_org_id(claims), get_held_products(claims))
|
|
60
|
+
# summary: {"role": "org_<id>", "granted_to", "using", "tables", "warnings"}
|
|
61
|
+
|
|
62
|
+
# 3. Per-query scoping on the app's READ client (never SET ROLE on a shared
|
|
63
|
+
# connection — thread-local clients are shared across users):
|
|
64
|
+
# client.query(sql, settings={"role": "org_<id>"})
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`provision_org_on_login` is a no-op when `CLICKHOUSE_PROVISIONING_USER`/`_PASSWORD` are
|
|
68
|
+
unset; without a provisioning credential orgs fail closed on the engine (role never
|
|
69
|
+
granted). Requires `SUB_SITE_MONGODB_URI` (read-only sub-site Mongo user) for the fresh
|
|
70
|
+
per-org read.
|
|
71
|
+
|
|
72
|
+
## Behaviour contract (do not "fix" — each rule exists because of a live incident)
|
|
73
|
+
|
|
74
|
+
- **Fail closed**: no held data-source products → deny (`REVOKE` role + row policy
|
|
75
|
+
`USING 0`); unknown/empty filter fields → that dataset excluded, ALL excluded →
|
|
76
|
+
deny; a held product with no stored values → bare `(DATA_SOURCE='<ds>')` (omit-empty
|
|
77
|
+
contract = full access to that product's data).
|
|
78
|
+
- **Per-dataset disjunction**: `(DATA_SOURCE='upc' AND <geo>) OR (DATA_SOURCE='gbs')` —
|
|
79
|
+
never a bare `DATA_SOURCE IN (...) AND <geo>` blob.
|
|
80
|
+
- **OR-grouping**: multiple predicates must be wrapped `(A OR B)` *inside* the
|
|
81
|
+
`DATA_SOURCE` guard — SQL precedence otherwise leaks other datasets' rows at the
|
|
82
|
+
postcode (found live on prod 2026-08-05).
|
|
83
|
+
- **PREWHERE**: row policies on non-sorting-key columns return 0 rows under PREWHERE
|
|
84
|
+
(ClickHouse GH #85222). Provisioning creates a settings profile
|
|
85
|
+
(`optimize_move_to_prewhere = 0`) and attaches it to the **service user** — per-query
|
|
86
|
+
`SETTINGS` is blocked under `readonly=1` (error 164) and role-attached profiles don't
|
|
87
|
+
apply. Trade-off: applies to every query through that user.
|
|
88
|
+
- **Convergence**: full re-provision on every login (no drift check, ~320ms), serialised
|
|
89
|
+
by a global lock (row-policy `DROP+CREATE` races → `ACCESS_ENTITY_ALREADY_EXISTS`,
|
|
90
|
+
code 493). Deny orgs are always re-provisioned too (REVOKE converges).
|
|
91
|
+
- **Escaping**: every literal is single-quote-doubled (`King's Lynn`); the row policy is
|
|
92
|
+
the enforcement boundary, so a quote in an admin-entered value must never escape the
|
|
93
|
+
literal.
|
|
94
|
+
|
|
95
|
+
## The contract fixture — the single evolvable artifact
|
|
96
|
+
|
|
97
|
+
`src/pt_access/ontology-permission-contract.json` mirrors
|
|
98
|
+
`sub-site/apps/ui/src/organisations/config/ontology-permission-contract.json`
|
|
99
|
+
(sub-site UI renders exactly what this declares). Adding a **new data source** = a
|
|
100
|
+
`dataSources` entry; a **new filter field** = a `fields` entry (+ `fieldProducts` /
|
|
101
|
+
`compatibility` where applicable). Both are fixture changes, never composer code
|
|
102
|
+
changes. The package CI drift-guard workflow fetches the sub-site copy via `gh api` and
|
|
103
|
+
fails on mismatch; `tests/test_contract.py` (via `contract.validate_contract()`) checks
|
|
104
|
+
the code constants against the fixture.
|
|
105
|
+
|
|
106
|
+
## Consumers
|
|
107
|
+
|
|
108
|
+
| App | Status | Notes |
|
|
109
|
+
|---|---|---|
|
|
110
|
+
| `point-topic-mcp` | to migrate | swap local modules for this package; keep FastMCP middleware + tool→product map |
|
|
111
|
+
| `onto-app-new` | to integrate | issue #25; adds `ontology_app` product (role admin/viewer), JWT auth on all routes, per-query role settings |
|
|
112
|
+
| `upc_query_agent` | follow-up | already reads `pt_org_id`/`products[]` in `api/auth_handler.py` — adopt `claims.py` |
|
|
113
|
+
| `local-pricing-dashboard`, `european-fttp-forecasts` | don't break | read `products[].permissions.role` — the claim shape is pinned by tests |
|
|
114
|
+
|
|
115
|
+
## Development
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
uv sync # or use any venv with the deps
|
|
119
|
+
uv run pytest -q # 93 tests, no network/DB needed (all mocked)
|
|
120
|
+
uv run ruff check src tests
|
|
121
|
+
uv run pyright # requires pyright install; config in pyproject
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Distribution: private GitHub tag, pinned by consumers
|
|
125
|
+
(`pip install git+https://github.com/Point-Topic/point-topic-access@v0.1.0`).
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "point-topic-access"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Shared permission -> ClickHouse access-control logic for Point Topic apps (claims, JWT verification, geo filter composer, per-org provisioning)"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
license = {text = "MIT"}
|
|
8
|
+
dependencies = [
|
|
9
|
+
"clickhouse-connect>=0.8.0",
|
|
10
|
+
"httpx>=0.28.0",
|
|
11
|
+
"python-jose>=3.3.0",
|
|
12
|
+
"pymongo>=4.10.0",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
dev = [
|
|
17
|
+
"pytest>=8.0.0",
|
|
18
|
+
"pytest-asyncio>=0.24.0",
|
|
19
|
+
"ruff>=0.9.0",
|
|
20
|
+
"pyright>=1.1.0",
|
|
21
|
+
"cryptography>=44.0.0",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[build-system]
|
|
25
|
+
requires = ["hatchling"]
|
|
26
|
+
build-backend = "hatchling.build"
|
|
27
|
+
|
|
28
|
+
[tool.hatch.build.targets.wheel]
|
|
29
|
+
packages = ["src/pt_access"]
|
|
30
|
+
|
|
31
|
+
[tool.pytest.ini_options]
|
|
32
|
+
asyncio_mode = "auto"
|
|
33
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
34
|
+
testpaths = ["tests"]
|
|
35
|
+
pythonpath = ["src"]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""pt-access: shared permission → ClickHouse access-control logic.
|
|
2
|
+
|
|
3
|
+
Centralises the permission pipeline used by Point Topic's data apps (MCP
|
|
4
|
+
server, ontology web app, query agent):
|
|
5
|
+
|
|
6
|
+
JWT claims ──► policy spec ──► SQL predicate ──► ClickHouse DDL
|
|
7
|
+
(what you (normalised (composer: (provisioning:
|
|
8
|
+
have) dataset + escaping, role, grants,
|
|
9
|
+
filter rows) containment, row policies,
|
|
10
|
+
OR-grouping) settings profile)
|
|
11
|
+
|
|
12
|
+
See README.md for the architecture and HANDOFF.md for the full context.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""JWT claim helpers — the identity/entitlement surface every app shares.
|
|
2
|
+
|
|
3
|
+
Ported from point-topic-mcp auth/middleware.py (the FastMCP middleware itself
|
|
4
|
+
stays in the MCP; these pure functions are transport-agnostic and take claims
|
|
5
|
+
as an argument — no request context).
|
|
6
|
+
|
|
7
|
+
JWT shape (Auth0 post-login Action, live-verified): `pt_org_id`,
|
|
8
|
+
`pt_org_name`, `products[]` as `[{name, permissions}]` — or legacy flat
|
|
9
|
+
strings, handled defensively.
|
|
10
|
+
|
|
11
|
+
Claim contract (FREEZE — additive-only changes):
|
|
12
|
+
- `pt_org_id`, `pt_org_name` (NOT `org_id`/`org_name`: Auth0 reserved names)
|
|
13
|
+
- `products[]` entries: `{"name": "<product_id>", "permissions": {<scalar fields>}}`
|
|
14
|
+
— `permissions` holds SCALAR values only (e.g. `role`); list-valued geo
|
|
15
|
+
filters are NEVER stamped into tokens (token size / proxy header limits),
|
|
16
|
+
they are read fresh from sub-site MongoDB. Client apps
|
|
17
|
+
(local-pricing-dashboard, european-fttp-forecasts) read
|
|
18
|
+
`products[].permissions.role` — do not change this shape.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from pt_access.provisioning import ALL_DATA_SOURCES, POINT_TOPIC_ORG_ID
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def get_org_id(claims: dict | None) -> str | None:
|
|
29
|
+
"""Org id from claims (`pt_org_id`)."""
|
|
30
|
+
return (claims or {}).get("pt_org_id")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def get_held_products(claims: dict | None) -> list[str]:
|
|
34
|
+
"""Product names from the JWT `products[]` claim.
|
|
35
|
+
|
|
36
|
+
Handles both the current shape (`[{name, permissions}]`) and legacy flat
|
|
37
|
+
strings (`["ukplus", "gbs"]`).
|
|
38
|
+
"""
|
|
39
|
+
held: list[str] = []
|
|
40
|
+
for entry in (claims or {}).get("products") or []:
|
|
41
|
+
if isinstance(entry, dict):
|
|
42
|
+
name = entry.get("name")
|
|
43
|
+
if isinstance(name, str):
|
|
44
|
+
held.append(name)
|
|
45
|
+
elif isinstance(entry, str):
|
|
46
|
+
held.append(entry)
|
|
47
|
+
return held
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_pt_admin(claims: dict | None) -> bool:
|
|
51
|
+
"""True when the request is from the Point Topic org (internal admin)."""
|
|
52
|
+
return get_org_id(claims) == POINT_TOPIC_ORG_ID
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def has_data_source_access(claims: dict | None) -> bool:
|
|
56
|
+
"""True when the org holds any data-source product (or legacy `ontology`).
|
|
57
|
+
|
|
58
|
+
The ontology tool family is unlocked by ANY data-source product (sub-site
|
|
59
|
+
PR #110: one product per DATA_SOURCE). The retired `ontology` product name
|
|
60
|
+
is still accepted during the transition window so pre-cutover tokens keep
|
|
61
|
+
working.
|
|
62
|
+
"""
|
|
63
|
+
held = set(get_held_products(claims))
|
|
64
|
+
return bool(held & set(ALL_DATA_SOURCES)) or "ontology" in held
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def get_product_permission(claims: dict | None, product: str, field: str) -> Any:
|
|
68
|
+
"""A scalar permission value for one product, e.g. the app role.
|
|
69
|
+
|
|
70
|
+
``products[].permissions`` holds scalar values only (the Action stamps
|
|
71
|
+
non-object values; list-valued geo filters never reach the token).
|
|
72
|
+
Returns None when the product or field is absent.
|
|
73
|
+
"""
|
|
74
|
+
for entry in (claims or {}).get("products") or []:
|
|
75
|
+
if isinstance(entry, dict) and entry.get("name") == product:
|
|
76
|
+
permissions = entry.get("permissions") or {}
|
|
77
|
+
if isinstance(permissions, dict):
|
|
78
|
+
return permissions.get(field)
|
|
79
|
+
return None
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def get_product_role(claims: dict | None, product: str) -> str | None:
|
|
84
|
+
"""Convenience: the `role` permission of one product (e.g. admin/viewer)."""
|
|
85
|
+
role = get_product_permission(claims, product, "role")
|
|
86
|
+
return role if isinstance(role, str) else None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
__all__ = [
|
|
90
|
+
"get_org_id",
|
|
91
|
+
"get_held_products",
|
|
92
|
+
"is_pt_admin",
|
|
93
|
+
"has_data_source_access",
|
|
94
|
+
"get_product_permission",
|
|
95
|
+
"get_product_role",
|
|
96
|
+
"POINT_TOPIC_ORG_ID",
|
|
97
|
+
]
|