northstar-sdk 1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- northstar_sdk-1.0/.env.example +4 -0
- northstar_sdk-1.0/.gitignore +79 -0
- northstar_sdk-1.0/.python-version +1 -0
- northstar_sdk-1.0/PKG-INFO +146 -0
- northstar_sdk-1.0/PUBLISHING.md +70 -0
- northstar_sdk-1.0/README.md +124 -0
- northstar_sdk-1.0/docs/document-collections.md +215 -0
- northstar_sdk-1.0/docs/register.md +409 -0
- northstar_sdk-1.0/pyproject.toml +42 -0
- northstar_sdk-1.0/src/northstar_sdk/__init__.py +44 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/__init__.py +46 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/base.py +200 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/dashboard_folders.py +116 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/dashboards.py +187 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/data_sources.py +75 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/datasets.py +73 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/exceptions.py +13 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/figures.py +93 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/kpis.py +82 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/page.py +50 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/search.py +83 -0
- northstar_sdk-1.0/src/northstar_sdk/catalogue/teams.py +69 -0
- northstar_sdk-1.0/src/northstar_sdk/client.py +299 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/__init__.py +72 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/_access.py +74 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/dashboard.py +372 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/dashboard_folder.py +150 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/data_source.py +136 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/data_source_category.py +115 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/dataset.py +483 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/dataset_ref.py +49 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/document_collection.py +153 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/figure.py +179 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/kpi.py +206 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/organisation.py +115 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/storage_system.py +182 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/team.py +99 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/team_ref.py +32 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/user.py +364 -0
- northstar_sdk-1.0/src/northstar_sdk/handles/user_ref.py +44 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/__init__.py +27 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/dashboard_folders.py +67 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/dashboards.py +126 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/data_source_categories.py +57 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/data_sources.py +104 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/datasets.py +137 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/document_collections.py +118 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/figures.py +113 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/kpis.py +140 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/organisations.py +78 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/storage_systems.py +102 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/teams.py +47 -0
- northstar_sdk-1.0/src/northstar_sdk/namespaces/users.py +63 -0
- northstar_sdk-1.0/src/northstar_sdk/orchestration.py +326 -0
- northstar_sdk-1.0/src/northstar_sdk/py.typed +0 -0
- northstar_sdk-1.0/src/northstar_sdk/types/__init__.py +62 -0
- northstar_sdk-1.0/src/northstar_sdk/types/json.py +8 -0
- northstar_sdk-1.0/src/northstar_sdk/types/params.py +134 -0
- northstar_sdk-1.0/src/northstar_sdk/types/responses.py +59 -0
- northstar_sdk-1.0/tests/catalogue/__init__.py +0 -0
- northstar_sdk-1.0/tests/catalogue/test_catalogue.py +127 -0
- northstar_sdk-1.0/tests/handles/__init__.py +0 -0
- northstar_sdk-1.0/tests/handles/test_user.py +122 -0
- northstar_sdk-1.0/tests/namespaces/__init__.py +0 -0
- northstar_sdk-1.0/tests/namespaces/test_dashboards.py +181 -0
- northstar_sdk-1.0/tests/namespaces/test_datasets.py +62 -0
- northstar_sdk-1.0/tests/namespaces/test_document_collections.py +98 -0
- northstar_sdk-1.0/tests/namespaces/test_kpis.py +53 -0
- northstar_sdk-1.0/tests/test_typing_params.py +133 -0
- northstar_sdk-1.0/uv.lock +81 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
|
2
|
+
|
|
3
|
+
# Dependencies
|
|
4
|
+
**/node_modules/
|
|
5
|
+
**/.pnp
|
|
6
|
+
.pnp.*
|
|
7
|
+
.yarn/*
|
|
8
|
+
!.yarn/patches
|
|
9
|
+
!.yarn/plugins
|
|
10
|
+
!.yarn/releases
|
|
11
|
+
!.yarn/versions
|
|
12
|
+
|
|
13
|
+
# Testing
|
|
14
|
+
**/coverage/
|
|
15
|
+
|
|
16
|
+
# Next.js
|
|
17
|
+
**/.next/
|
|
18
|
+
**/out/
|
|
19
|
+
|
|
20
|
+
# Production
|
|
21
|
+
**/build/
|
|
22
|
+
|
|
23
|
+
# Python
|
|
24
|
+
**/__pycache__/
|
|
25
|
+
**/*.py[cod]
|
|
26
|
+
**/.venv/
|
|
27
|
+
**/dist/
|
|
28
|
+
**/wheels/
|
|
29
|
+
**/*.egg-info/
|
|
30
|
+
**/.ruff_cache/
|
|
31
|
+
**/.pytest_cache/
|
|
32
|
+
|
|
33
|
+
# TypeScript
|
|
34
|
+
**/*.tsbuildinfo
|
|
35
|
+
**/next-env.d.ts
|
|
36
|
+
|
|
37
|
+
# Prisma
|
|
38
|
+
**/prisma/generated/
|
|
39
|
+
|
|
40
|
+
# Environment variables
|
|
41
|
+
.env
|
|
42
|
+
.env.*
|
|
43
|
+
!.env.example
|
|
44
|
+
!.env.*.example
|
|
45
|
+
|
|
46
|
+
# Debug logs
|
|
47
|
+
npm-debug.log*
|
|
48
|
+
yarn-debug.log*
|
|
49
|
+
yarn-error.log*
|
|
50
|
+
.pnpm-debug.log*
|
|
51
|
+
|
|
52
|
+
# Deployment
|
|
53
|
+
**/.vercel/
|
|
54
|
+
|
|
55
|
+
# OS
|
|
56
|
+
**/.DS_Store
|
|
57
|
+
**/Thumbs.db
|
|
58
|
+
|
|
59
|
+
# IDE — ignore personal settings, keep shared workspace config
|
|
60
|
+
**/.vscode/*
|
|
61
|
+
!**/.vscode/settings.json
|
|
62
|
+
!**/.vscode/extensions.json
|
|
63
|
+
!**/.vscode/tasks.json
|
|
64
|
+
!**/.vscode/launch.json
|
|
65
|
+
**/.idea/
|
|
66
|
+
|
|
67
|
+
# Cursor
|
|
68
|
+
**/.cursor/
|
|
69
|
+
|
|
70
|
+
# Local themes & branding
|
|
71
|
+
**/src/config/themes/
|
|
72
|
+
**/public/branding/
|
|
73
|
+
|
|
74
|
+
# Notebooks
|
|
75
|
+
*.ipynb_checkpoints/
|
|
76
|
+
notebook.ipynb
|
|
77
|
+
|
|
78
|
+
# Secrets
|
|
79
|
+
*.pem
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: northstar-sdk
|
|
3
|
+
Version: 1.0
|
|
4
|
+
Summary: Thin HTTP client for the Northstar platform API
|
|
5
|
+
Project-URL: Homepage, https://github.com/nathan294/northstar
|
|
6
|
+
Project-URL: Repository, https://github.com/nathan294/northstar
|
|
7
|
+
Project-URL: Issues, https://github.com/nathan294/northstar/issues
|
|
8
|
+
Author: Nathan Boulogne
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
Keywords: api,data,http,northstar,sdk
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
18
|
+
Requires-Python: >=3.13
|
|
19
|
+
Requires-Dist: httpx>=0.28.0
|
|
20
|
+
Requires-Dist: loguru>=0.7.0
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# northstar-sdk
|
|
24
|
+
|
|
25
|
+
Thin HTTP client for the Northstar platform. Talks only to the **Next.js API** (`NORTHSTAR_API_URL`) with a **developer account API key**.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install northstar-sdk
|
|
31
|
+
# or
|
|
32
|
+
uv add northstar-sdk
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Configuration
|
|
36
|
+
|
|
37
|
+
| Variable | Description | Example |
|
|
38
|
+
| ----------------- | ---------------------------------- | --------------------------- |
|
|
39
|
+
| `NORTHSTAR_API_URL` | Next.js API base (includes `/api`) | `http://localhost:3000/api` |
|
|
40
|
+
| `NORTHSTAR_API_KEY` | Developer account API key | `northstar_…` |
|
|
41
|
+
|
|
42
|
+
## Quick start (100 % `Client`)
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from northstar_sdk import Client
|
|
46
|
+
|
|
47
|
+
client = Client()
|
|
48
|
+
|
|
49
|
+
# Teams: read + create on the same namespace
|
|
50
|
+
team = client.teams.create(name="Sales", description="Revenue ops")
|
|
51
|
+
record = client.teams.find_by_name("Sales")
|
|
52
|
+
|
|
53
|
+
# Users: resolve profile or get a handle for actions
|
|
54
|
+
guest = client.users.resolve("guest@example.com") # .id, .email, .name
|
|
55
|
+
guest_handle = client.users.for_ref("guest@example.com")
|
|
56
|
+
user = client.users.create(
|
|
57
|
+
first_name="Ada",
|
|
58
|
+
last_name="Lovelace",
|
|
59
|
+
email="ada@example.com",
|
|
60
|
+
password="secure-password",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# KPI → chain governance on the returned handle
|
|
64
|
+
my_kpi = client.kpis.create(
|
|
65
|
+
name="Monthly revenue",
|
|
66
|
+
dataset_id="…",
|
|
67
|
+
definition_type="MANUAL",
|
|
68
|
+
manual_config={"valueField": "amount", "aggregation": "SUM"},
|
|
69
|
+
)
|
|
70
|
+
my_kpi.assign_to_team("analytics")
|
|
71
|
+
|
|
72
|
+
# Ingestion (flat — no client.ingestion.*)
|
|
73
|
+
client.storage_systems.create(
|
|
74
|
+
slug="wh-pg",
|
|
75
|
+
name="Warehouse PostgreSQL",
|
|
76
|
+
kind="POSTGRESQL",
|
|
77
|
+
credentials={
|
|
78
|
+
"host": "db.internal",
|
|
79
|
+
"user": "northstar",
|
|
80
|
+
"password": "...",
|
|
81
|
+
"database": "analytics",
|
|
82
|
+
},
|
|
83
|
+
)
|
|
84
|
+
category = client.data_source_categories.create(
|
|
85
|
+
slug="saas", name="SaaS", color="#3366FF"
|
|
86
|
+
)
|
|
87
|
+
client.data_sources.create(slug="hubspot", name="HubSpot", category=category)
|
|
88
|
+
|
|
89
|
+
dataset = client.datasets.create(
|
|
90
|
+
location={"schema": "marts", "table": "nps"},
|
|
91
|
+
name="NPS",
|
|
92
|
+
type="TABLE",
|
|
93
|
+
)
|
|
94
|
+
dataset.analyze_and_check_alerts()
|
|
95
|
+
|
|
96
|
+
# Dashboard → create then chain governance on the returned handle
|
|
97
|
+
dashboard = client.dashboards.create(
|
|
98
|
+
title="Sales metrics",
|
|
99
|
+
icon="layout-dashboard",
|
|
100
|
+
embed_url="https://dashboards.example.com/sales/metrics",
|
|
101
|
+
)
|
|
102
|
+
dashboard.assign_to_team("Sales")
|
|
103
|
+
dashboard.assign_owner("nathan.boulogne@gmail.com")
|
|
104
|
+
dashboard.link_datasets(["NPS"])
|
|
105
|
+
dashboard.assign_editors(["editor@example.com"])
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Public exports
|
|
109
|
+
|
|
110
|
+
`Client`, catalogue exceptions (`NotFoundError`, `AmbiguousLookupError`), `Page`, orchestration helpers.
|
|
111
|
+
|
|
112
|
+
Advanced typing: `from northstar_sdk.handles import User, Kpi, Dataset` (not exported from top-level `northstar`).
|
|
113
|
+
|
|
114
|
+
This package ships [`py.typed`](src/northstar_sdk/py.typed) — use Pylance **basic** or stricter for parameter hints on `client.*.create` / `upsert`.
|
|
115
|
+
|
|
116
|
+
**Create / update / upsert payloads** (storage systems, datasets, data sources, collections, KPIs, dashboards): see [docs/register.md](docs/register.md). That page maps SDK calls to the FastAPI Pydantic models and Next.js Zod schemas that actually validate the body. **Document collections** (create + `ingest()` / RAG): [docs/document-collections.md](docs/document-collections.md).
|
|
117
|
+
|
|
118
|
+
## Client namespaces
|
|
119
|
+
|
|
120
|
+
| Property | Read | Write |
|
|
121
|
+
| -------- | ---- | ----- |
|
|
122
|
+
| `client.datasets` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
123
|
+
| `client.kpis` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
124
|
+
| `client.figures` | catalogue | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
125
|
+
| `client.dashboards` | browse, get, find_by_title | `create`, `upsert`, `update`, `delete`, `for_id`, `create_folder` |
|
|
126
|
+
| `client.dashboard_folders` | browse, list, get, find_by_id, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
127
|
+
| `client.data_sources` | list, get | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
128
|
+
| `client.data_source_categories` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
129
|
+
| `client.storage_systems` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
130
|
+
| `client.document_collections` | — | `create`, `upsert`, `update`, `delete`, `ingest`, `for_id` |
|
|
131
|
+
| `client.teams` | list, find_by_name, find_by_slug | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
132
|
+
| `client.users` | — | `create`, `upsert`, `update`, `resolve`, `for_ref`, `for_id` |
|
|
133
|
+
| `client.organisations` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
134
|
+
| `client.search` | global search | — |
|
|
135
|
+
|
|
136
|
+
## Orchestration
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
from northstar_sdk import Client, northstar_flow
|
|
140
|
+
|
|
141
|
+
client = Client()
|
|
142
|
+
|
|
143
|
+
@northstar_flow(client)
|
|
144
|
+
def my_flow():
|
|
145
|
+
client.data_sources.create(...)
|
|
146
|
+
```
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Publication sur PyPI
|
|
2
|
+
|
|
3
|
+
Le package `northstar-sdk` partage la **même version** que la plateforme Northstar ([`VERSION`](../../VERSION) à la racine du dépôt). Les scripts [`scripts/bump_version.py`](../../scripts/bump_version.py) et [`scripts/release.py`](../../scripts/release.py) synchronisent `packages/python_sdk/pyproject.toml` avec ce fichier.
|
|
4
|
+
|
|
5
|
+
## Workflow de release
|
|
6
|
+
|
|
7
|
+
Depuis la racine du monorepo :
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
task bump:minor # ou bump:major / bump:rc — met à jour VERSION + tous les manifests
|
|
11
|
+
task release # commit, tag, push, build SDK, publication PyPI
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
`task release` exécute `uv build --package northstar-sdk` puis `uv publish --index pypi` après le push Git. Un token PyPI doit être disponible dans l'environnement.
|
|
15
|
+
|
|
16
|
+
## Prérequis
|
|
17
|
+
|
|
18
|
+
- [uv](https://docs.astral.sh/uv/getting-started/installation/) installé
|
|
19
|
+
- Un compte sur [pypi.org](https://pypi.org/)
|
|
20
|
+
- Un token API PyPI avec accès au projet `northstar-sdk`
|
|
21
|
+
|
|
22
|
+
## Token PyPI
|
|
23
|
+
|
|
24
|
+
1. PyPI → **Account settings → API tokens**
|
|
25
|
+
2. Crée un token (scope « Project: northstar-sdk » une fois le projet créé, sinon « Entire account » pour la première release)
|
|
26
|
+
3. Exporte-le **sans le commiter** :
|
|
27
|
+
|
|
28
|
+
```powershell
|
|
29
|
+
$env:UV_PUBLISH_TOKEN = "pypi-..."
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
export UV_PUBLISH_TOKEN="pypi-..."
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`uv publish` lit automatiquement `UV_PUBLISH_TOKEN`.
|
|
37
|
+
|
|
38
|
+
## Build manuel (debug)
|
|
39
|
+
|
|
40
|
+
Depuis la racine du monorepo :
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
python scripts/bump_version.py --sync-only
|
|
44
|
+
uv build --package northstar-sdk --clear
|
|
45
|
+
uv publish --index pypi dist/northstar_sdk-*
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Les artefacts sont écrits dans `dist/` à la racine du dépôt.
|
|
49
|
+
|
|
50
|
+
## TestPyPI (optionnel)
|
|
51
|
+
|
|
52
|
+
Les index `pypi` et `testpypi` sont déclarés dans le [`pyproject.toml` racine](../../pyproject.toml). Pour un essai sur [TestPyPI](https://test.pypi.org/) :
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
uv build --package northstar-sdk --clear
|
|
56
|
+
uv publish --index testpypi dist/northstar_sdk-*
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Utilise un token TestPyPI dans `UV_PUBLISH_TOKEN`.
|
|
60
|
+
|
|
61
|
+
## Dépannage
|
|
62
|
+
|
|
63
|
+
- **UV_PUBLISH_TOKEN is not set** : exporte le token avant `task release`.
|
|
64
|
+
- **403 / invalid credentials** : vérifie le token et le scope du projet.
|
|
65
|
+
- **File already exists** : la version est déjà sur PyPI — bump avec `task bump:minor` (ou major/rc) puis relance la release.
|
|
66
|
+
|
|
67
|
+
## Références
|
|
68
|
+
|
|
69
|
+
- [uv — Building and publishing a package](https://docs.astral.sh/uv/guides/package/)
|
|
70
|
+
- [PyPI — API tokens](https://pypi.org/help/#apitoken)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# northstar-sdk
|
|
2
|
+
|
|
3
|
+
Thin HTTP client for the Northstar platform. Talks only to the **Next.js API** (`NORTHSTAR_API_URL`) with a **developer account API key**.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install northstar-sdk
|
|
9
|
+
# or
|
|
10
|
+
uv add northstar-sdk
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Configuration
|
|
14
|
+
|
|
15
|
+
| Variable | Description | Example |
|
|
16
|
+
| ----------------- | ---------------------------------- | --------------------------- |
|
|
17
|
+
| `NORTHSTAR_API_URL` | Next.js API base (includes `/api`) | `http://localhost:3000/api` |
|
|
18
|
+
| `NORTHSTAR_API_KEY` | Developer account API key | `northstar_…` |
|
|
19
|
+
|
|
20
|
+
## Quick start (100 % `Client`)
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from northstar_sdk import Client
|
|
24
|
+
|
|
25
|
+
client = Client()
|
|
26
|
+
|
|
27
|
+
# Teams: read + create on the same namespace
|
|
28
|
+
team = client.teams.create(name="Sales", description="Revenue ops")
|
|
29
|
+
record = client.teams.find_by_name("Sales")
|
|
30
|
+
|
|
31
|
+
# Users: resolve profile or get a handle for actions
|
|
32
|
+
guest = client.users.resolve("guest@example.com") # .id, .email, .name
|
|
33
|
+
guest_handle = client.users.for_ref("guest@example.com")
|
|
34
|
+
user = client.users.create(
|
|
35
|
+
first_name="Ada",
|
|
36
|
+
last_name="Lovelace",
|
|
37
|
+
email="ada@example.com",
|
|
38
|
+
password="secure-password",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
# KPI → chain governance on the returned handle
|
|
42
|
+
my_kpi = client.kpis.create(
|
|
43
|
+
name="Monthly revenue",
|
|
44
|
+
dataset_id="…",
|
|
45
|
+
definition_type="MANUAL",
|
|
46
|
+
manual_config={"valueField": "amount", "aggregation": "SUM"},
|
|
47
|
+
)
|
|
48
|
+
my_kpi.assign_to_team("analytics")
|
|
49
|
+
|
|
50
|
+
# Ingestion (flat — no client.ingestion.*)
|
|
51
|
+
client.storage_systems.create(
|
|
52
|
+
slug="wh-pg",
|
|
53
|
+
name="Warehouse PostgreSQL",
|
|
54
|
+
kind="POSTGRESQL",
|
|
55
|
+
credentials={
|
|
56
|
+
"host": "db.internal",
|
|
57
|
+
"user": "northstar",
|
|
58
|
+
"password": "...",
|
|
59
|
+
"database": "analytics",
|
|
60
|
+
},
|
|
61
|
+
)
|
|
62
|
+
category = client.data_source_categories.create(
|
|
63
|
+
slug="saas", name="SaaS", color="#3366FF"
|
|
64
|
+
)
|
|
65
|
+
client.data_sources.create(slug="hubspot", name="HubSpot", category=category)
|
|
66
|
+
|
|
67
|
+
dataset = client.datasets.create(
|
|
68
|
+
location={"schema": "marts", "table": "nps"},
|
|
69
|
+
name="NPS",
|
|
70
|
+
type="TABLE",
|
|
71
|
+
)
|
|
72
|
+
dataset.analyze_and_check_alerts()
|
|
73
|
+
|
|
74
|
+
# Dashboard → create then chain governance on the returned handle
|
|
75
|
+
dashboard = client.dashboards.create(
|
|
76
|
+
title="Sales metrics",
|
|
77
|
+
icon="layout-dashboard",
|
|
78
|
+
embed_url="https://dashboards.example.com/sales/metrics",
|
|
79
|
+
)
|
|
80
|
+
dashboard.assign_to_team("Sales")
|
|
81
|
+
dashboard.assign_owner("nathan.boulogne@gmail.com")
|
|
82
|
+
dashboard.link_datasets(["NPS"])
|
|
83
|
+
dashboard.assign_editors(["editor@example.com"])
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Public exports
|
|
87
|
+
|
|
88
|
+
`Client`, catalogue exceptions (`NotFoundError`, `AmbiguousLookupError`), `Page`, orchestration helpers.
|
|
89
|
+
|
|
90
|
+
Advanced typing: `from northstar_sdk.handles import User, Kpi, Dataset` (not exported from top-level `northstar`).
|
|
91
|
+
|
|
92
|
+
This package ships [`py.typed`](src/northstar_sdk/py.typed) — use Pylance **basic** or stricter for parameter hints on `client.*.create` / `upsert`.
|
|
93
|
+
|
|
94
|
+
**Create / update / upsert payloads** (storage systems, datasets, data sources, collections, KPIs, dashboards): see [docs/register.md](docs/register.md). That page maps SDK calls to the FastAPI Pydantic models and Next.js Zod schemas that actually validate the body. **Document collections** (create + `ingest()` / RAG): [docs/document-collections.md](docs/document-collections.md).
|
|
95
|
+
|
|
96
|
+
## Client namespaces
|
|
97
|
+
|
|
98
|
+
| Property | Read | Write |
|
|
99
|
+
| -------- | ---- | ----- |
|
|
100
|
+
| `client.datasets` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
101
|
+
| `client.kpis` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
102
|
+
| `client.figures` | catalogue | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
103
|
+
| `client.dashboards` | browse, get, find_by_title | `create`, `upsert`, `update`, `delete`, `for_id`, `create_folder` |
|
|
104
|
+
| `client.dashboard_folders` | browse, list, get, find_by_id, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
105
|
+
| `client.data_sources` | list, get | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
106
|
+
| `client.data_source_categories` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
107
|
+
| `client.storage_systems` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
108
|
+
| `client.document_collections` | — | `create`, `upsert`, `update`, `delete`, `ingest`, `for_id` |
|
|
109
|
+
| `client.teams` | list, find_by_name, find_by_slug | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
110
|
+
| `client.users` | — | `create`, `upsert`, `update`, `resolve`, `for_ref`, `for_id` |
|
|
111
|
+
| `client.organisations` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
|
|
112
|
+
| `client.search` | global search | — |
|
|
113
|
+
|
|
114
|
+
## Orchestration
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from northstar_sdk import Client, northstar_flow
|
|
118
|
+
|
|
119
|
+
client = Client()
|
|
120
|
+
|
|
121
|
+
@northstar_flow(client)
|
|
122
|
+
def my_flow():
|
|
123
|
+
client.data_sources.create(...)
|
|
124
|
+
```
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# Document collections (`northstar-sdk`)
|
|
2
|
+
|
|
3
|
+
A **document collection** is a catalogue folder of files (today: **Google Drive PDFs**) that
|
|
4
|
+
the platform can list, extract, embed, and search in Chat. Datasets are warehouse
|
|
5
|
+
tables; collections are document stores.
|
|
6
|
+
|
|
7
|
+
Create / update / upsert payloads are in [register.md](register.md). This page is the
|
|
8
|
+
**lifecycle**: what a collection is, how `ingest()` works, and what you need for RAG.
|
|
9
|
+
|
|
10
|
+
The SDK talks only to Next.js (`NORTHSTAR_API_URL`, including `/api`) with
|
|
11
|
+
`NORTHSTAR_API_KEY`. Organization is inferred from the key.
|
|
12
|
+
|
|
13
|
+
## Where the code lives
|
|
14
|
+
|
|
15
|
+
```mermaid
|
|
16
|
+
flowchart LR
|
|
17
|
+
Sdk["northstar-sdk Client"]
|
|
18
|
+
Bff["Next.js /api/feature/document-collections/…"]
|
|
19
|
+
FastApi["FastAPI /v1/feature/document-collection/…"]
|
|
20
|
+
Compute["northstar-compute ingest"]
|
|
21
|
+
Drive["Google Drive"]
|
|
22
|
+
Pg["Postgres document_collection + document"]
|
|
23
|
+
Qdrant["Qdrant document_chunk"]
|
|
24
|
+
Sdk -->|"X-API-KEY"| Bff
|
|
25
|
+
Bff -->|"proxyAuthenticatedJson"| FastApi
|
|
26
|
+
FastApi -->|"Prefect ingest-document-collection or in-process"| Compute
|
|
27
|
+
Compute --> Drive
|
|
28
|
+
Compute --> Pg
|
|
29
|
+
Compute --> Qdrant
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
| Step | SDK | Next.js | Canonical implementation |
|
|
33
|
+
| --- | --- | --- | --- |
|
|
34
|
+
| Create / upsert | `client.document_collections.create` / `.upsert` | [`create/route.ts`](../../../front/src/app/api/feature/document-collections/create/route.ts) → `POST /v1/feature/document-collection/create` | [`packages/northstar_compute/.../sync/sync_document_collection.py`](../../northstar_compute/src/northstar_compute/document_collections/sync/sync_document_collection.py) `upsert_document_collection` |
|
|
35
|
+
| Ingest | `collection.ingest()` | [`ingest/route.ts`](../../../front/src/app/api/feature/document-collections/ingest/route.ts) → `POST /v1/feature/document-collection/ingest` | [`ingest_document_collection.py`](../../northstar_compute/src/northstar_compute/document_collections/ingest/ingest_document_collection.py) |
|
|
36
|
+
| Location keys | `location={"folderId": "…"}` | — | [`location.py`](../../northstar_compute/src/northstar_compute/document_collections/location.py) `LOCATION_KEY_PARTS` |
|
|
37
|
+
| Prefect job | — | — | [`jobs/src/jobs/flows/ingest_document_collection.py`](../../../jobs/src/jobs/flows/ingest_document_collection.py) (`ingest-document-collection`) |
|
|
38
|
+
|
|
39
|
+
**Naming:** compute `upsert_document_collection` writes the **catalogue row**.
|
|
40
|
+
SDK/API `ingest()` **pulls files** (Drive → Postgres → Qdrant). Create does not
|
|
41
|
+
download PDFs.
|
|
42
|
+
|
|
43
|
+
## What you get
|
|
44
|
+
|
|
45
|
+
| Layer | Role |
|
|
46
|
+
| --- | --- |
|
|
47
|
+
| `document_collection` | One Drive folder. Unique per org on **slug** and on **storage system + `locationKey`** (`folderId`). |
|
|
48
|
+
| `document` | One remote PDF. Unique per collection + Drive file id (`externalId`). |
|
|
49
|
+
| Qdrant `document_chunk` | Embedded text chunks, used by platform Chat. |
|
|
50
|
+
|
|
51
|
+
The UI (`/feature/documents`) lists collections and files. Chat can attach a
|
|
52
|
+
collection and retrieve passages (`kind=document_chunk`).
|
|
53
|
+
|
|
54
|
+
Ingestion is implemented **only** for storage kind `GOOGLE_DRIVE`.
|
|
55
|
+
|
|
56
|
+
## Prerequisites
|
|
57
|
+
|
|
58
|
+
1. A **Google Drive storage system** in the same org, with extracted service-account
|
|
59
|
+
fields (not a raw JSON blob). See [register.md](register.md).
|
|
60
|
+
2. Share the Drive **folder** with the service-account email (`client_email`) as
|
|
61
|
+
a reader (or use a Shared Drive the account can access).
|
|
62
|
+
3. For **indexed** files and Chat RAG, the process that runs ingest
|
|
63
|
+
(FastAPI in-process, or Prefect jobs) needs:
|
|
64
|
+
- `QDRANT_URL` (optional `QDRANT_API_KEY`)
|
|
65
|
+
- `OPENAI_API_KEY` (embeddings)
|
|
66
|
+
- the same `CREDENTIALS_ENCRYPTION_KEY` as the API (to decrypt Drive credentials)
|
|
67
|
+
|
|
68
|
+
Without Qdrant, `ingest()` still lists files and writes `document` rows with
|
|
69
|
+
`indexStatus=PENDING`. Chat retrieval then returns nothing.
|
|
70
|
+
|
|
71
|
+
## Create / upsert
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from northstar_sdk import Client
|
|
75
|
+
|
|
76
|
+
client = Client()
|
|
77
|
+
|
|
78
|
+
drive = client.storage_systems.create(
|
|
79
|
+
slug="gdrive-legal",
|
|
80
|
+
name="Legal Drive",
|
|
81
|
+
kind="GOOGLE_DRIVE",
|
|
82
|
+
credentials={
|
|
83
|
+
key: value
|
|
84
|
+
for key, value in sa.items()
|
|
85
|
+
if key not in ("service_account", "service_account_json")
|
|
86
|
+
},
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
collection = client.document_collections.create(
|
|
90
|
+
name="Contracts",
|
|
91
|
+
storage_system_slug="gdrive-legal",
|
|
92
|
+
location={"folderId": "1abc…"}, # Drive folder id from the URL
|
|
93
|
+
description="Executed contracts",
|
|
94
|
+
tags=["legal"],
|
|
95
|
+
)
|
|
96
|
+
# collection.collection_id is set
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`folder_id` is accepted and rewritten to `folderId`. FastAPI default slug is
|
|
100
|
+
`"gdrive"` if omitted; the SDK always sends `storage_system_slug`.
|
|
101
|
+
|
|
102
|
+
- **`create()`** inserts only. Same org slug or Drive folder → **409** with
|
|
103
|
+
`existing_id` / `conflict_key` (`slug` or `folderId`).
|
|
104
|
+
- **`upsert()`** updates the existing row if either identity matches:
|
|
105
|
+
- `(organizationId, storageSystemId, locationKey)` — same Drive folder
|
|
106
|
+
- `(organizationId, slug)` — same catalogue name/slug
|
|
107
|
+
- If the slug belongs to one collection and the folder to another, the API
|
|
108
|
+
returns **409**.
|
|
109
|
+
|
|
110
|
+
Optional: `slug` (else slugified from `name`), `chat_question_examples`.
|
|
111
|
+
|
|
112
|
+
## Ingest
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
result = collection.ingest()
|
|
116
|
+
# Prefect:
|
|
117
|
+
# {
|
|
118
|
+
# "collection_id": "…",
|
|
119
|
+
# "deployment_name": "ingest-document-collection",
|
|
120
|
+
# "flow_run_id": "…",
|
|
121
|
+
# "message": "Successfully triggered flow run for deployment ingest-document-collection",
|
|
122
|
+
# "already_running": False,
|
|
123
|
+
# }
|
|
124
|
+
# In-process fallback (no Prefect deployment) also includes file_count, indexed_count, …
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Requires `collection.collection_id` (call `create()` or `upsert()` first).
|
|
128
|
+
|
|
129
|
+
HTTP: `POST /feature/document-collections/ingest` with `{ "collection_id": "…" }`.
|
|
130
|
+
|
|
131
|
+
### What FastAPI does
|
|
132
|
+
|
|
133
|
+
1. If Prefect deployment `ingest-document-collection` exists, enqueue it with
|
|
134
|
+
`collection_ids: [id]`. The HTTP response has `flow_run_id` + `message` —
|
|
135
|
+
the job fills `fileCount` / `indexedCount` / `lastSyncedAt` asynchronously.
|
|
136
|
+
Poll the UI or re-fetch the collection.
|
|
137
|
+
2. If that deployment is **missing** (typical local `task api` without Prefect),
|
|
138
|
+
ingest runs **in-process** and the response includes real counts.
|
|
139
|
+
|
|
140
|
+
If a run is already `RUNNING` / `PENDING` for that collection, FastAPI returns
|
|
141
|
+
`already_running=True` and the existing `flow_run_id`.
|
|
142
|
+
|
|
143
|
+
### What ingest does
|
|
144
|
+
|
|
145
|
+
1. Load the collection + Drive credentials.
|
|
146
|
+
2. List **PDFs directly in that folder** (not subfolders, not other MIME types).
|
|
147
|
+
Shared Drives: set `driveId` on the storage `config` or on `location`.
|
|
148
|
+
3. For each file:
|
|
149
|
+
- Fingerprint = Drive `md5Checksum`, else `modifiedTime`, else name.
|
|
150
|
+
- Skip download if the row already has the same hash and `indexStatus=INDEXED`.
|
|
151
|
+
- Else download, extract text with PyMuPDF, chunk (~700 tokens, 80 overlap),
|
|
152
|
+
embed, upsert Qdrant points (`kind=document_chunk`).
|
|
153
|
+
4. Delete Postgres rows **and** Qdrant points for files that disappeared from Drive.
|
|
154
|
+
5. Update collection counters and `lastSyncedAt`.
|
|
155
|
+
|
|
156
|
+
### Document `indexStatus`
|
|
157
|
+
|
|
158
|
+
| Status | Meaning |
|
|
159
|
+
| --- | --- |
|
|
160
|
+
| `INDEXED` | Text extracted and vectors written (or unchanged hash). |
|
|
161
|
+
| `PENDING` | Catalogue row exists; Qdrant/OpenAI not configured, so no vectors. |
|
|
162
|
+
| `FAILED` | Download / extract / embed error (`indexError` set), or empty PDF text. |
|
|
163
|
+
| `SKIPPED` | Non-PDF (should not appear in the Drive listing today). |
|
|
164
|
+
|
|
165
|
+
Re-run `ingest()` after adding files, replacing a PDF, or enabling Qdrant.
|
|
166
|
+
|
|
167
|
+
## Chat
|
|
168
|
+
|
|
169
|
+
Indexed chunks are searched when Chat is scoped to the collection. The UI
|
|
170
|
+
**Ask** link is `/feature/chat?collectionId=<id>`. Retrieval filters Qdrant by
|
|
171
|
+
`organization_id` + `collection_id`.
|
|
172
|
+
|
|
173
|
+
No vectors (`vectors_enabled: false` or all `PENDING`) → no document passages.
|
|
174
|
+
|
|
175
|
+
## End-to-end
|
|
176
|
+
|
|
177
|
+
```python
|
|
178
|
+
import json
|
|
179
|
+
from northstar_sdk import Client
|
|
180
|
+
|
|
181
|
+
client = Client()
|
|
182
|
+
|
|
183
|
+
with open("service-account.json") as fh:
|
|
184
|
+
sa = json.load(fh)
|
|
185
|
+
|
|
186
|
+
client.storage_systems.create(
|
|
187
|
+
slug="gdrive-legal",
|
|
188
|
+
name="Legal Drive",
|
|
189
|
+
kind="GOOGLE_DRIVE",
|
|
190
|
+
credentials={
|
|
191
|
+
key: value
|
|
192
|
+
for key, value in sa.items()
|
|
193
|
+
if key not in ("service_account", "service_account_json")
|
|
194
|
+
},
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
collection = client.document_collections.create(
|
|
198
|
+
name="Contracts",
|
|
199
|
+
storage_system_slug="gdrive-legal",
|
|
200
|
+
location={"folderId": "1abc…"},
|
|
201
|
+
)
|
|
202
|
+
print(collection.ingest())
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Share the folder with the service-account email before `ingest()`, or Drive returns
|
|
206
|
+
an empty list / a 403 from the Drive API.
|
|
207
|
+
|
|
208
|
+
## Limits (current)
|
|
209
|
+
|
|
210
|
+
- Google Drive only.
|
|
211
|
+
- PDFs only, **one folder** (no recursion).
|
|
212
|
+
- Ingest is not exposed as a button in the UI; the detail page tells you to run
|
|
213
|
+
`collection.ingest()` from the SDK (or `POST /feature/document-collections/ingest`).
|
|
214
|
+
- Service-account JSON must be **parsed locally**; the API rejects
|
|
215
|
+
`service_account` / `service_account_json` blobs.
|