northstar-sdk 1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. northstar_sdk-1.0/.env.example +4 -0
  2. northstar_sdk-1.0/.gitignore +79 -0
  3. northstar_sdk-1.0/.python-version +1 -0
  4. northstar_sdk-1.0/PKG-INFO +146 -0
  5. northstar_sdk-1.0/PUBLISHING.md +70 -0
  6. northstar_sdk-1.0/README.md +124 -0
  7. northstar_sdk-1.0/docs/document-collections.md +215 -0
  8. northstar_sdk-1.0/docs/register.md +409 -0
  9. northstar_sdk-1.0/pyproject.toml +42 -0
  10. northstar_sdk-1.0/src/northstar_sdk/__init__.py +44 -0
  11. northstar_sdk-1.0/src/northstar_sdk/catalogue/__init__.py +46 -0
  12. northstar_sdk-1.0/src/northstar_sdk/catalogue/base.py +200 -0
  13. northstar_sdk-1.0/src/northstar_sdk/catalogue/dashboard_folders.py +116 -0
  14. northstar_sdk-1.0/src/northstar_sdk/catalogue/dashboards.py +187 -0
  15. northstar_sdk-1.0/src/northstar_sdk/catalogue/data_sources.py +75 -0
  16. northstar_sdk-1.0/src/northstar_sdk/catalogue/datasets.py +73 -0
  17. northstar_sdk-1.0/src/northstar_sdk/catalogue/exceptions.py +13 -0
  18. northstar_sdk-1.0/src/northstar_sdk/catalogue/figures.py +93 -0
  19. northstar_sdk-1.0/src/northstar_sdk/catalogue/kpis.py +82 -0
  20. northstar_sdk-1.0/src/northstar_sdk/catalogue/page.py +50 -0
  21. northstar_sdk-1.0/src/northstar_sdk/catalogue/search.py +83 -0
  22. northstar_sdk-1.0/src/northstar_sdk/catalogue/teams.py +69 -0
  23. northstar_sdk-1.0/src/northstar_sdk/client.py +299 -0
  24. northstar_sdk-1.0/src/northstar_sdk/handles/__init__.py +72 -0
  25. northstar_sdk-1.0/src/northstar_sdk/handles/_access.py +74 -0
  26. northstar_sdk-1.0/src/northstar_sdk/handles/dashboard.py +372 -0
  27. northstar_sdk-1.0/src/northstar_sdk/handles/dashboard_folder.py +150 -0
  28. northstar_sdk-1.0/src/northstar_sdk/handles/data_source.py +136 -0
  29. northstar_sdk-1.0/src/northstar_sdk/handles/data_source_category.py +115 -0
  30. northstar_sdk-1.0/src/northstar_sdk/handles/dataset.py +483 -0
  31. northstar_sdk-1.0/src/northstar_sdk/handles/dataset_ref.py +49 -0
  32. northstar_sdk-1.0/src/northstar_sdk/handles/document_collection.py +153 -0
  33. northstar_sdk-1.0/src/northstar_sdk/handles/figure.py +179 -0
  34. northstar_sdk-1.0/src/northstar_sdk/handles/kpi.py +206 -0
  35. northstar_sdk-1.0/src/northstar_sdk/handles/organisation.py +115 -0
  36. northstar_sdk-1.0/src/northstar_sdk/handles/storage_system.py +182 -0
  37. northstar_sdk-1.0/src/northstar_sdk/handles/team.py +99 -0
  38. northstar_sdk-1.0/src/northstar_sdk/handles/team_ref.py +32 -0
  39. northstar_sdk-1.0/src/northstar_sdk/handles/user.py +364 -0
  40. northstar_sdk-1.0/src/northstar_sdk/handles/user_ref.py +44 -0
  41. northstar_sdk-1.0/src/northstar_sdk/namespaces/__init__.py +27 -0
  42. northstar_sdk-1.0/src/northstar_sdk/namespaces/dashboard_folders.py +67 -0
  43. northstar_sdk-1.0/src/northstar_sdk/namespaces/dashboards.py +126 -0
  44. northstar_sdk-1.0/src/northstar_sdk/namespaces/data_source_categories.py +57 -0
  45. northstar_sdk-1.0/src/northstar_sdk/namespaces/data_sources.py +104 -0
  46. northstar_sdk-1.0/src/northstar_sdk/namespaces/datasets.py +137 -0
  47. northstar_sdk-1.0/src/northstar_sdk/namespaces/document_collections.py +118 -0
  48. northstar_sdk-1.0/src/northstar_sdk/namespaces/figures.py +113 -0
  49. northstar_sdk-1.0/src/northstar_sdk/namespaces/kpis.py +140 -0
  50. northstar_sdk-1.0/src/northstar_sdk/namespaces/organisations.py +78 -0
  51. northstar_sdk-1.0/src/northstar_sdk/namespaces/storage_systems.py +102 -0
  52. northstar_sdk-1.0/src/northstar_sdk/namespaces/teams.py +47 -0
  53. northstar_sdk-1.0/src/northstar_sdk/namespaces/users.py +63 -0
  54. northstar_sdk-1.0/src/northstar_sdk/orchestration.py +326 -0
  55. northstar_sdk-1.0/src/northstar_sdk/py.typed +0 -0
  56. northstar_sdk-1.0/src/northstar_sdk/types/__init__.py +62 -0
  57. northstar_sdk-1.0/src/northstar_sdk/types/json.py +8 -0
  58. northstar_sdk-1.0/src/northstar_sdk/types/params.py +134 -0
  59. northstar_sdk-1.0/src/northstar_sdk/types/responses.py +59 -0
  60. northstar_sdk-1.0/tests/catalogue/__init__.py +0 -0
  61. northstar_sdk-1.0/tests/catalogue/test_catalogue.py +127 -0
  62. northstar_sdk-1.0/tests/handles/__init__.py +0 -0
  63. northstar_sdk-1.0/tests/handles/test_user.py +122 -0
  64. northstar_sdk-1.0/tests/namespaces/__init__.py +0 -0
  65. northstar_sdk-1.0/tests/namespaces/test_dashboards.py +181 -0
  66. northstar_sdk-1.0/tests/namespaces/test_datasets.py +62 -0
  67. northstar_sdk-1.0/tests/namespaces/test_document_collections.py +98 -0
  68. northstar_sdk-1.0/tests/namespaces/test_kpis.py +53 -0
  69. northstar_sdk-1.0/tests/test_typing_params.py +133 -0
  70. northstar_sdk-1.0/uv.lock +81 -0
@@ -0,0 +1,4 @@
1
+ # Northstar platform (Next.js API) — required for the HTTP SDK
2
+ NORTHSTAR_API_URL=http://localhost:3000/api
3
+ # Developer account API key from the app: Admin → Developers
4
+ NORTHSTAR_API_KEY=northstar_...
@@ -0,0 +1,79 @@
1
+ # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2
+
3
+ # Dependencies
4
+ **/node_modules/
5
+ **/.pnp
6
+ .pnp.*
7
+ .yarn/*
8
+ !.yarn/patches
9
+ !.yarn/plugins
10
+ !.yarn/releases
11
+ !.yarn/versions
12
+
13
+ # Testing
14
+ **/coverage/
15
+
16
+ # Next.js
17
+ **/.next/
18
+ **/out/
19
+
20
+ # Production
21
+ **/build/
22
+
23
+ # Python
24
+ **/__pycache__/
25
+ **/*.py[cod]
26
+ **/.venv/
27
+ **/dist/
28
+ **/wheels/
29
+ **/*.egg-info/
30
+ **/.ruff_cache/
31
+ **/.pytest_cache/
32
+
33
+ # TypeScript
34
+ **/*.tsbuildinfo
35
+ **/next-env.d.ts
36
+
37
+ # Prisma
38
+ **/prisma/generated/
39
+
40
+ # Environment variables
41
+ .env
42
+ .env.*
43
+ !.env.example
44
+ !.env.*.example
45
+
46
+ # Debug logs
47
+ npm-debug.log*
48
+ yarn-debug.log*
49
+ yarn-error.log*
50
+ .pnpm-debug.log*
51
+
52
+ # Deployment
53
+ **/.vercel/
54
+
55
+ # OS
56
+ **/.DS_Store
57
+ **/Thumbs.db
58
+
59
+ # IDE — ignore personal settings, keep shared workspace config
60
+ **/.vscode/*
61
+ !**/.vscode/settings.json
62
+ !**/.vscode/extensions.json
63
+ !**/.vscode/tasks.json
64
+ !**/.vscode/launch.json
65
+ **/.idea/
66
+
67
+ # Cursor
68
+ **/.cursor/
69
+
70
+ # Local themes & branding
71
+ **/src/config/themes/
72
+ **/public/branding/
73
+
74
+ # Notebooks
75
+ *.ipynb_checkpoints/
76
+ notebook.ipynb
77
+
78
+ # Secrets
79
+ *.pem
@@ -0,0 +1 @@
1
+ 3.13
@@ -0,0 +1,146 @@
1
+ Metadata-Version: 2.5
2
+ Name: northstar-sdk
3
+ Version: 1.0
4
+ Summary: Thin HTTP client for the Northstar platform API
5
+ Project-URL: Homepage, https://github.com/nathan294/northstar
6
+ Project-URL: Repository, https://github.com/nathan294/northstar
7
+ Project-URL: Issues, https://github.com/nathan294/northstar/issues
8
+ Author: Nathan Boulogne
9
+ License-Expression: Apache-2.0
10
+ Keywords: api,data,http,northstar,sdk
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
18
+ Requires-Python: >=3.13
19
+ Requires-Dist: httpx>=0.28.0
20
+ Requires-Dist: loguru>=0.7.0
21
+ Description-Content-Type: text/markdown
22
+
23
+ # northstar-sdk
24
+
25
+ Thin HTTP client for the Northstar platform. Talks only to the **Next.js API** (`NORTHSTAR_API_URL`) with a **developer account API key**.
26
+
27
+ ## Install
28
+
29
+ ```bash
30
+ pip install northstar-sdk
31
+ # or
32
+ uv add northstar-sdk
33
+ ```
34
+
35
+ ## Configuration
36
+
37
+ | Variable | Description | Example |
38
+ | ----------------- | ---------------------------------- | --------------------------- |
39
+ | `NORTHSTAR_API_URL` | Next.js API base (includes `/api`) | `http://localhost:3000/api` |
40
+ | `NORTHSTAR_API_KEY` | Developer account API key | `northstar_…` |
41
+
42
+ ## Quick start (100 % `Client`)
43
+
44
+ ```python
45
+ from northstar_sdk import Client
46
+
47
+ client = Client()
48
+
49
+ # Teams: read + create on the same namespace
50
+ team = client.teams.create(name="Sales", description="Revenue ops")
51
+ record = client.teams.find_by_name("Sales")
52
+
53
+ # Users: resolve profile or get a handle for actions
54
+ guest = client.users.resolve("guest@example.com") # .id, .email, .name
55
+ guest_handle = client.users.for_ref("guest@example.com")
56
+ user = client.users.create(
57
+ first_name="Ada",
58
+ last_name="Lovelace",
59
+ email="ada@example.com",
60
+ password="secure-password",
61
+ )
62
+
63
+ # KPI → chain governance on the returned handle
64
+ my_kpi = client.kpis.create(
65
+ name="Monthly revenue",
66
+ dataset_id="…",
67
+ definition_type="MANUAL",
68
+ manual_config={"valueField": "amount", "aggregation": "SUM"},
69
+ )
70
+ my_kpi.assign_to_team("analytics")
71
+
72
+ # Ingestion (flat — no client.ingestion.*)
73
+ client.storage_systems.create(
74
+ slug="wh-pg",
75
+ name="Warehouse PostgreSQL",
76
+ kind="POSTGRESQL",
77
+ credentials={
78
+ "host": "db.internal",
79
+ "user": "northstar",
80
+ "password": "...",
81
+ "database": "analytics",
82
+ },
83
+ )
84
+ category = client.data_source_categories.create(
85
+ slug="saas", name="SaaS", color="#3366FF"
86
+ )
87
+ client.data_sources.create(slug="hubspot", name="HubSpot", category=category)
88
+
89
+ dataset = client.datasets.create(
90
+ location={"schema": "marts", "table": "nps"},
91
+ name="NPS",
92
+ type="TABLE",
93
+ )
94
+ dataset.analyze_and_check_alerts()
95
+
96
+ # Dashboard → create then chain governance on the returned handle
97
+ dashboard = client.dashboards.create(
98
+ title="Sales metrics",
99
+ icon="layout-dashboard",
100
+ embed_url="https://dashboards.example.com/sales/metrics",
101
+ )
102
+ dashboard.assign_to_team("Sales")
103
+ dashboard.assign_owner("nathan.boulogne@gmail.com")
104
+ dashboard.link_datasets(["NPS"])
105
+ dashboard.assign_editors(["editor@example.com"])
106
+ ```
107
+
108
+ ## Public exports
109
+
110
+ `Client`, catalogue exceptions (`NotFoundError`, `AmbiguousLookupError`), `Page`, orchestration helpers.
111
+
112
+ Advanced typing: `from northstar_sdk.handles import User, Kpi, Dataset` (not exported from top-level `northstar`).
113
+
114
+ This package ships [`py.typed`](src/northstar_sdk/py.typed) — use Pylance **basic** or stricter for parameter hints on `client.*.create` / `upsert`.
115
+
116
+ **Create / update / upsert payloads** (storage systems, datasets, data sources, collections, KPIs, dashboards): see [docs/register.md](docs/register.md). That page maps SDK calls to the FastAPI Pydantic models and Next.js Zod schemas that actually validate the body. **Document collections** (create + `ingest()` / RAG): [docs/document-collections.md](docs/document-collections.md).
117
+
118
+ ## Client namespaces
119
+
120
+ | Property | Read | Write |
121
+ | -------- | ---- | ----- |
122
+ | `client.datasets` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
123
+ | `client.kpis` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
124
+ | `client.figures` | catalogue | `create`, `upsert`, `update`, `delete`, `for_id` |
125
+ | `client.dashboards` | browse, get, find_by_title | `create`, `upsert`, `update`, `delete`, `for_id`, `create_folder` |
126
+ | `client.dashboard_folders` | browse, list, get, find_by_id, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
127
+ | `client.data_sources` | list, get | `create`, `upsert`, `update`, `delete`, `for_id` |
128
+ | `client.data_source_categories` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
129
+ | `client.storage_systems` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
130
+ | `client.document_collections` | — | `create`, `upsert`, `update`, `delete`, `ingest`, `for_id` |
131
+ | `client.teams` | list, find_by_name, find_by_slug | `create`, `upsert`, `update`, `delete`, `for_id` |
132
+ | `client.users` | — | `create`, `upsert`, `update`, `resolve`, `for_ref`, `for_id` |
133
+ | `client.organisations` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
134
+ | `client.search` | global search | — |
135
+
136
+ ## Orchestration
137
+
138
+ ```python
139
+ from northstar_sdk import Client, northstar_flow
140
+
141
+ client = Client()
142
+
143
+ @northstar_flow(client)
144
+ def my_flow():
145
+ client.data_sources.create(...)
146
+ ```
@@ -0,0 +1,70 @@
1
+ # Publication sur PyPI
2
+
3
+ Le package `northstar-sdk` partage la **même version** que la plateforme Northstar ([`VERSION`](../../VERSION) à la racine du dépôt). Les scripts [`scripts/bump_version.py`](../../scripts/bump_version.py) et [`scripts/release.py`](../../scripts/release.py) synchronisent `packages/python_sdk/pyproject.toml` avec ce fichier.
4
+
5
+ ## Workflow de release
6
+
7
+ Depuis la racine du monorepo :
8
+
9
+ ```bash
10
+ task bump:minor # ou bump:major / bump:rc — met à jour VERSION + tous les manifests
11
+ task release # commit, tag, push, build SDK, publication PyPI
12
+ ```
13
+
14
+ `task release` exécute `uv build --package northstar-sdk` puis `uv publish --index pypi` après le push Git. Un token PyPI doit être disponible dans l'environnement.
15
+
16
+ ## Prérequis
17
+
18
+ - [uv](https://docs.astral.sh/uv/getting-started/installation/) installé
19
+ - Un compte sur [pypi.org](https://pypi.org/)
20
+ - Un token API PyPI avec accès au projet `northstar-sdk`
21
+
22
+ ## Token PyPI
23
+
24
+ 1. PyPI → **Account settings → API tokens**
25
+ 2. Crée un token (scope « Project: northstar-sdk » une fois le projet créé, sinon « Entire account » pour la première release)
26
+ 3. Exporte-le **sans le commiter** :
27
+
28
+ ```powershell
29
+ $env:UV_PUBLISH_TOKEN = "pypi-..."
30
+ ```
31
+
32
+ ```bash
33
+ export UV_PUBLISH_TOKEN="pypi-..."
34
+ ```
35
+
36
+ `uv publish` lit automatiquement `UV_PUBLISH_TOKEN`.
37
+
38
+ ## Build manuel (debug)
39
+
40
+ Depuis la racine du monorepo :
41
+
42
+ ```bash
43
+ python scripts/bump_version.py --sync-only
44
+ uv build --package northstar-sdk --clear
45
+ uv publish --index pypi dist/northstar_sdk-*
46
+ ```
47
+
48
+ Les artefacts sont écrits dans `dist/` à la racine du dépôt.
49
+
50
+ ## TestPyPI (optionnel)
51
+
52
+ Les index `pypi` et `testpypi` sont déclarés dans le [`pyproject.toml` racine](../../pyproject.toml). Pour un essai sur [TestPyPI](https://test.pypi.org/) :
53
+
54
+ ```bash
55
+ uv build --package northstar-sdk --clear
56
+ uv publish --index testpypi dist/northstar_sdk-*
57
+ ```
58
+
59
+ Utilise un token TestPyPI dans `UV_PUBLISH_TOKEN`.
60
+
61
+ ## Dépannage
62
+
63
+ - **UV_PUBLISH_TOKEN is not set** : exporte le token avant `task release`.
64
+ - **403 / invalid credentials** : vérifie le token et le scope du projet.
65
+ - **File already exists** : la version est déjà sur PyPI — bump avec `task bump:minor` (ou major/rc) puis relance la release.
66
+
67
+ ## Références
68
+
69
+ - [uv — Building and publishing a package](https://docs.astral.sh/uv/guides/package/)
70
+ - [PyPI — API tokens](https://pypi.org/help/#apitoken)
@@ -0,0 +1,124 @@
1
+ # northstar-sdk
2
+
3
+ Thin HTTP client for the Northstar platform. Talks only to the **Next.js API** (`NORTHSTAR_API_URL`) with a **developer account API key**.
4
+
5
+ ## Install
6
+
7
+ ```bash
8
+ pip install northstar-sdk
9
+ # or
10
+ uv add northstar-sdk
11
+ ```
12
+
13
+ ## Configuration
14
+
15
+ | Variable | Description | Example |
16
+ | ----------------- | ---------------------------------- | --------------------------- |
17
+ | `NORTHSTAR_API_URL` | Next.js API base (includes `/api`) | `http://localhost:3000/api` |
18
+ | `NORTHSTAR_API_KEY` | Developer account API key | `northstar_…` |
19
+
20
+ ## Quick start (100 % `Client`)
21
+
22
+ ```python
23
+ from northstar_sdk import Client
24
+
25
+ client = Client()
26
+
27
+ # Teams: read + create on the same namespace
28
+ team = client.teams.create(name="Sales", description="Revenue ops")
29
+ record = client.teams.find_by_name("Sales")
30
+
31
+ # Users: resolve profile or get a handle for actions
32
+ guest = client.users.resolve("guest@example.com") # .id, .email, .name
33
+ guest_handle = client.users.for_ref("guest@example.com")
34
+ user = client.users.create(
35
+ first_name="Ada",
36
+ last_name="Lovelace",
37
+ email="ada@example.com",
38
+ password="secure-password",
39
+ )
40
+
41
+ # KPI → chain governance on the returned handle
42
+ my_kpi = client.kpis.create(
43
+ name="Monthly revenue",
44
+ dataset_id="…",
45
+ definition_type="MANUAL",
46
+ manual_config={"valueField": "amount", "aggregation": "SUM"},
47
+ )
48
+ my_kpi.assign_to_team("analytics")
49
+
50
+ # Ingestion (flat — no client.ingestion.*)
51
+ client.storage_systems.create(
52
+ slug="wh-pg",
53
+ name="Warehouse PostgreSQL",
54
+ kind="POSTGRESQL",
55
+ credentials={
56
+ "host": "db.internal",
57
+ "user": "northstar",
58
+ "password": "...",
59
+ "database": "analytics",
60
+ },
61
+ )
62
+ category = client.data_source_categories.create(
63
+ slug="saas", name="SaaS", color="#3366FF"
64
+ )
65
+ client.data_sources.create(slug="hubspot", name="HubSpot", category=category)
66
+
67
+ dataset = client.datasets.create(
68
+ location={"schema": "marts", "table": "nps"},
69
+ name="NPS",
70
+ type="TABLE",
71
+ )
72
+ dataset.analyze_and_check_alerts()
73
+
74
+ # Dashboard → create then chain governance on the returned handle
75
+ dashboard = client.dashboards.create(
76
+ title="Sales metrics",
77
+ icon="layout-dashboard",
78
+ embed_url="https://dashboards.example.com/sales/metrics",
79
+ )
80
+ dashboard.assign_to_team("Sales")
81
+ dashboard.assign_owner("nathan.boulogne@gmail.com")
82
+ dashboard.link_datasets(["NPS"])
83
+ dashboard.assign_editors(["editor@example.com"])
84
+ ```
85
+
86
+ ## Public exports
87
+
88
+ `Client`, catalogue exceptions (`NotFoundError`, `AmbiguousLookupError`), `Page`, orchestration helpers.
89
+
90
+ Advanced typing: `from northstar_sdk.handles import User, Kpi, Dataset` (not exported from top-level `northstar`).
91
+
92
+ This package ships [`py.typed`](src/northstar_sdk/py.typed) — use Pylance **basic** or stricter for parameter hints on `client.*.create` / `upsert`.
93
+
94
+ **Create / update / upsert payloads** (storage systems, datasets, data sources, collections, KPIs, dashboards): see [docs/register.md](docs/register.md). That page maps SDK calls to the FastAPI Pydantic models and Next.js Zod schemas that actually validate the body. **Document collections** (create + `ingest()` / RAG): [docs/document-collections.md](docs/document-collections.md).
95
+
96
+ ## Client namespaces
97
+
98
+ | Property | Read | Write |
99
+ | -------- | ---- | ----- |
100
+ | `client.datasets` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
101
+ | `client.kpis` | list, get, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
102
+ | `client.figures` | catalogue | `create`, `upsert`, `update`, `delete`, `for_id` |
103
+ | `client.dashboards` | browse, get, find_by_title | `create`, `upsert`, `update`, `delete`, `for_id`, `create_folder` |
104
+ | `client.dashboard_folders` | browse, list, get, find_by_id, find_by_name | `create`, `upsert`, `update`, `delete`, `for_id` |
105
+ | `client.data_sources` | list, get | `create`, `upsert`, `update`, `delete`, `for_id` |
106
+ | `client.data_source_categories` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
107
+ | `client.storage_systems` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
108
+ | `client.document_collections` | — | `create`, `upsert`, `update`, `delete`, `ingest`, `for_id` |
109
+ | `client.teams` | list, find_by_name, find_by_slug | `create`, `upsert`, `update`, `delete`, `for_id` |
110
+ | `client.users` | — | `create`, `upsert`, `update`, `resolve`, `for_ref`, `for_id` |
111
+ | `client.organisations` | — | `create`, `upsert`, `update`, `delete`, `for_id` |
112
+ | `client.search` | global search | — |
113
+
114
+ ## Orchestration
115
+
116
+ ```python
117
+ from northstar_sdk import Client, northstar_flow
118
+
119
+ client = Client()
120
+
121
+ @northstar_flow(client)
122
+ def my_flow():
123
+ client.data_sources.create(...)
124
+ ```
@@ -0,0 +1,215 @@
1
+ # Document collections (`northstar-sdk`)
2
+
3
+ A **document collection** is a catalogue folder of files (today: **Google Drive PDFs**) that
4
+ the platform can list, extract, embed, and search in Chat. Datasets are warehouse
5
+ tables; collections are document stores.
6
+
7
+ Create / update / upsert payloads are in [register.md](register.md). This page is the
8
+ **lifecycle**: what a collection is, how `ingest()` works, and what you need for RAG.
9
+
10
+ The SDK talks only to Next.js (`NORTHSTAR_API_URL`, including `/api`) with
11
+ `NORTHSTAR_API_KEY`. Organization is inferred from the key.
12
+
13
+ ## Where the code lives
14
+
15
+ ```mermaid
16
+ flowchart LR
17
+ Sdk["northstar-sdk Client"]
18
+ Bff["Next.js /api/feature/document-collections/…"]
19
+ FastApi["FastAPI /v1/feature/document-collection/…"]
20
+ Compute["northstar-compute ingest"]
21
+ Drive["Google Drive"]
22
+ Pg["Postgres document_collection + document"]
23
+ Qdrant["Qdrant document_chunk"]
24
+ Sdk -->|"X-API-KEY"| Bff
25
+ Bff -->|"proxyAuthenticatedJson"| FastApi
26
+ FastApi -->|"Prefect ingest-document-collection or in-process"| Compute
27
+ Compute --> Drive
28
+ Compute --> Pg
29
+ Compute --> Qdrant
30
+ ```
31
+
32
+ | Step | SDK | Next.js | Canonical implementation |
33
+ | --- | --- | --- | --- |
34
+ | Create / upsert | `client.document_collections.create` / `.upsert` | [`create/route.ts`](../../../front/src/app/api/feature/document-collections/create/route.ts) → `POST /v1/feature/document-collection/create` | [`packages/northstar_compute/.../sync/sync_document_collection.py`](../../northstar_compute/src/northstar_compute/document_collections/sync/sync_document_collection.py) `upsert_document_collection` |
35
+ | Ingest | `collection.ingest()` | [`ingest/route.ts`](../../../front/src/app/api/feature/document-collections/ingest/route.ts) → `POST /v1/feature/document-collection/ingest` | [`ingest_document_collection.py`](../../northstar_compute/src/northstar_compute/document_collections/ingest/ingest_document_collection.py) |
36
+ | Location keys | `location={"folderId": "…"}` | — | [`location.py`](../../northstar_compute/src/northstar_compute/document_collections/location.py) `LOCATION_KEY_PARTS` |
37
+ | Prefect job | — | — | [`jobs/src/jobs/flows/ingest_document_collection.py`](../../../jobs/src/jobs/flows/ingest_document_collection.py) (`ingest-document-collection`) |
38
+
39
+ **Naming:** compute `upsert_document_collection` writes the **catalogue row**.
40
+ SDK/API `ingest()` **pulls files** (Drive → Postgres → Qdrant). Create does not
41
+ download PDFs.
42
+
43
+ ## What you get
44
+
45
+ | Layer | Role |
46
+ | --- | --- |
47
+ | `document_collection` | One Drive folder. Unique per org on **slug** and on **storage system + `locationKey`** (`folderId`). |
48
+ | `document` | One remote PDF. Unique per collection + Drive file id (`externalId`). |
49
+ | Qdrant `document_chunk` | Embedded text chunks, used by platform Chat. |
50
+
51
+ The UI (`/feature/documents`) lists collections and files. Chat can attach a
52
+ collection and retrieve passages (`kind=document_chunk`).
53
+
54
+ Ingestion is implemented **only** for storage kind `GOOGLE_DRIVE`.
55
+
56
+ ## Prerequisites
57
+
58
+ 1. A **Google Drive storage system** in the same org, with extracted service-account
59
+ fields (not a raw JSON blob). See [register.md](register.md).
60
+ 2. Share the Drive **folder** with the service-account email (`client_email`) as
61
+ a reader (or use a Shared Drive the account can access).
62
+ 3. For **indexed** files and Chat RAG, the process that runs ingest
63
+ (FastAPI in-process, or Prefect jobs) needs:
64
+ - `QDRANT_URL` (optional `QDRANT_API_KEY`)
65
+ - `OPENAI_API_KEY` (embeddings)
66
+ - the same `CREDENTIALS_ENCRYPTION_KEY` as the API (to decrypt Drive credentials)
67
+
68
+ Without Qdrant, `ingest()` still lists files and writes `document` rows with
69
+ `indexStatus=PENDING`. Chat retrieval then returns nothing.
70
+
71
+ ## Create / upsert
72
+
73
+ ```python
74
+ from northstar_sdk import Client
75
+
76
+ client = Client()
77
+
78
+ drive = client.storage_systems.create(
79
+ slug="gdrive-legal",
80
+ name="Legal Drive",
81
+ kind="GOOGLE_DRIVE",
82
+ credentials={
83
+ key: value
84
+ for key, value in sa.items()
85
+ if key not in ("service_account", "service_account_json")
86
+ },
87
+ )
88
+
89
+ collection = client.document_collections.create(
90
+ name="Contracts",
91
+ storage_system_slug="gdrive-legal",
92
+ location={"folderId": "1abc…"}, # Drive folder id from the URL
93
+ description="Executed contracts",
94
+ tags=["legal"],
95
+ )
96
+ # collection.collection_id is set
97
+ ```
98
+
99
+ `folder_id` is accepted and rewritten to `folderId`. FastAPI default slug is
100
+ `"gdrive"` if omitted; the SDK always sends `storage_system_slug`.
101
+
102
+ - **`create()`** inserts only. Same org slug or Drive folder → **409** with
103
+ `existing_id` / `conflict_key` (`slug` or `folderId`).
104
+ - **`upsert()`** updates the existing row if either identity matches:
105
+ - `(organizationId, storageSystemId, locationKey)` — same Drive folder
106
+ - `(organizationId, slug)` — same catalogue name/slug
107
+ - If the slug belongs to one collection and the folder to another, the API
108
+ returns **409**.
109
+
110
+ Optional: `slug` (else slugified from `name`), `chat_question_examples`.
111
+
112
+ ## Ingest
113
+
114
+ ```python
115
+ result = collection.ingest()
116
+ # Prefect:
117
+ # {
118
+ # "collection_id": "…",
119
+ # "deployment_name": "ingest-document-collection",
120
+ # "flow_run_id": "…",
121
+ # "message": "Successfully triggered flow run for deployment ingest-document-collection",
122
+ # "already_running": False,
123
+ # }
124
+ # In-process fallback (no Prefect deployment) also includes file_count, indexed_count, …
125
+ ```
126
+
127
+ Requires `collection.collection_id` (call `create()` or `upsert()` first).
128
+
129
+ HTTP: `POST /feature/document-collections/ingest` with `{ "collection_id": "…" }`.
130
+
131
+ ### What FastAPI does
132
+
133
+ 1. If Prefect deployment `ingest-document-collection` exists, enqueue it with
134
+ `collection_ids: [id]`. The HTTP response has `flow_run_id` + `message` —
135
+ the job fills `fileCount` / `indexedCount` / `lastSyncedAt` asynchronously.
136
+ Poll the UI or re-fetch the collection.
137
+ 2. If that deployment is **missing** (typical local `task api` without Prefect),
138
+ ingest runs **in-process** and the response includes real counts.
139
+
140
+ If a run is already `RUNNING` / `PENDING` for that collection, FastAPI returns
141
+ `already_running=True` and the existing `flow_run_id`.
142
+
143
+ ### What ingest does
144
+
145
+ 1. Load the collection + Drive credentials.
146
+ 2. List **PDFs directly in that folder** (not subfolders, not other MIME types).
147
+ Shared Drives: set `driveId` on the storage `config` or on `location`.
148
+ 3. For each file:
149
+ - Fingerprint = Drive `md5Checksum`, else `modifiedTime`, else name.
150
+ - Skip download if the row already has the same hash and `indexStatus=INDEXED`.
151
+ - Else download, extract text with PyMuPDF, chunk (~700 tokens, 80 overlap),
152
+ embed, upsert Qdrant points (`kind=document_chunk`).
153
+ 4. Delete Postgres rows **and** Qdrant points for files that disappeared from Drive.
154
+ 5. Update collection counters and `lastSyncedAt`.
155
+
156
+ ### Document `indexStatus`
157
+
158
+ | Status | Meaning |
159
+ | --- | --- |
160
+ | `INDEXED` | Text extracted and vectors written (or unchanged hash). |
161
+ | `PENDING` | Catalogue row exists; Qdrant/OpenAI not configured, so no vectors. |
162
+ | `FAILED` | Download / extract / embed error (`indexError` set), or empty PDF text. |
163
+ | `SKIPPED` | Non-PDF (should not appear in the Drive listing today). |
164
+
165
+ Re-run `ingest()` after adding files, replacing a PDF, or enabling Qdrant.
166
+
167
+ ## Chat
168
+
169
+ Indexed chunks are searched when Chat is scoped to the collection. The UI
170
+ **Ask** link is `/feature/chat?collectionId=<id>`. Retrieval filters Qdrant by
171
+ `organization_id` + `collection_id`.
172
+
173
+ No vectors (`vectors_enabled: false` or all `PENDING`) → no document passages.
174
+
175
+ ## End-to-end
176
+
177
+ ```python
178
+ import json
179
+ from northstar_sdk import Client
180
+
181
+ client = Client()
182
+
183
+ with open("service-account.json") as fh:
184
+ sa = json.load(fh)
185
+
186
+ client.storage_systems.create(
187
+ slug="gdrive-legal",
188
+ name="Legal Drive",
189
+ kind="GOOGLE_DRIVE",
190
+ credentials={
191
+ key: value
192
+ for key, value in sa.items()
193
+ if key not in ("service_account", "service_account_json")
194
+ },
195
+ )
196
+
197
+ collection = client.document_collections.create(
198
+ name="Contracts",
199
+ storage_system_slug="gdrive-legal",
200
+ location={"folderId": "1abc…"},
201
+ )
202
+ print(collection.ingest())
203
+ ```
204
+
205
+ Share the folder with the service-account email before `ingest()`, or Drive returns
206
+ an empty list / a 403 from the Drive API.
207
+
208
+ ## Limits (current)
209
+
210
+ - Google Drive only.
211
+ - PDFs only, **one folder** (no recursion).
212
+ - Ingest is not exposed as a button in the UI; the detail page tells you to run
213
+ `collection.ingest()` from the SDK (or `POST /feature/document-collections/ingest`).
214
+ - Service-account JSON must be **parsed locally**; the API rejects
215
+ `service_account` / `service_account_json` blobs.