terria-catalog 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- terria_catalog-0.1.0/.gitignore +38 -0
- terria_catalog-0.1.0/AGENTS.md +44 -0
- terria_catalog-0.1.0/CHANGELOG.md +36 -0
- terria_catalog-0.1.0/LICENSE +21 -0
- terria_catalog-0.1.0/PKG-INFO +277 -0
- terria_catalog-0.1.0/README.md +236 -0
- terria_catalog-0.1.0/docs/AI_USAGE.md +111 -0
- terria_catalog-0.1.0/docs/ARCHITECTURE.md +164 -0
- terria_catalog-0.1.0/docs/COORDINATION.md +106 -0
- terria_catalog-0.1.0/docs/system_prompt.md +43 -0
- terria_catalog-0.1.0/examples/01_load_and_inspect.py +40 -0
- terria_catalog-0.1.0/examples/02_add_items.py +61 -0
- terria_catalog-0.1.0/examples/03_move_update_remove.py +38 -0
- terria_catalog-0.1.0/examples/04_nested_groups.py +31 -0
- terria_catalog-0.1.0/examples/05_publish_local.py +35 -0
- terria_catalog-0.1.0/examples/06_publish_s3.py +49 -0
- terria_catalog-0.1.0/examples/07_release_workflow.py +77 -0
- terria_catalog-0.1.0/examples/README.md +21 -0
- terria_catalog-0.1.0/llms.txt +34 -0
- terria_catalog-0.1.0/pyproject.toml +149 -0
- terria_catalog-0.1.0/skills/terria-catalog/SKILL.md +73 -0
- terria_catalog-0.1.0/src/terria_catalog/__init__.py +151 -0
- terria_catalog-0.1.0/src/terria_catalog/agent/__init__.py +32 -0
- terria_catalog-0.1.0/src/terria_catalog/agent/cli.py +167 -0
- terria_catalog-0.1.0/src/terria_catalog/agent/diff.py +125 -0
- terria_catalog-0.1.0/src/terria_catalog/agent/mcp_server.py +103 -0
- terria_catalog-0.1.0/src/terria_catalog/agent/session.py +303 -0
- terria_catalog-0.1.0/src/terria_catalog/agent/tools.py +214 -0
- terria_catalog-0.1.0/src/terria_catalog/catalog.py +279 -0
- terria_catalog-0.1.0/src/terria_catalog/config.py +45 -0
- terria_catalog-0.1.0/src/terria_catalog/exceptions.py +141 -0
- terria_catalog-0.1.0/src/terria_catalog/items/__init__.py +55 -0
- terria_catalog-0.1.0/src/terria_catalog/items/arcgis.py +49 -0
- terria_catalog-0.1.0/src/terria_catalog/items/cesium_3d_tiles.py +30 -0
- terria_catalog-0.1.0/src/terria_catalog/items/cesium_terrain.py +28 -0
- terria_catalog-0.1.0/src/terria_catalog/items/cog.py +31 -0
- terria_catalog-0.1.0/src/terria_catalog/items/csv.py +35 -0
- terria_catalog-0.1.0/src/terria_catalog/items/geojson.py +40 -0
- terria_catalog-0.1.0/src/terria_catalog/items/ion_imagery.py +22 -0
- terria_catalog-0.1.0/src/terria_catalog/items/placeholders.py +74 -0
- terria_catalog-0.1.0/src/terria_catalog/items/shapefile.py +27 -0
- terria_catalog-0.1.0/src/terria_catalog/items/wms.py +35 -0
- terria_catalog-0.1.0/src/terria_catalog/items/wmts.py +24 -0
- terria_catalog-0.1.0/src/terria_catalog/models/__init__.py +43 -0
- terria_catalog-0.1.0/src/terria_catalog/models/common.py +192 -0
- terria_catalog-0.1.0/src/terria_catalog/models/group.py +106 -0
- terria_catalog-0.1.0/src/terria_catalog/models/item.py +60 -0
- terria_catalog-0.1.0/src/terria_catalog/models/member.py +130 -0
- terria_catalog-0.1.0/src/terria_catalog/operations/__init__.py +36 -0
- terria_catalog-0.1.0/src/terria_catalog/operations/_tree.py +159 -0
- terria_catalog-0.1.0/src/terria_catalog/operations/add.py +26 -0
- terria_catalog-0.1.0/src/terria_catalog/operations/move.py +44 -0
- terria_catalog-0.1.0/src/terria_catalog/operations/remove.py +25 -0
- terria_catalog-0.1.0/src/terria_catalog/operations/reorder.py +47 -0
- terria_catalog-0.1.0/src/terria_catalog/operations/update.py +31 -0
- terria_catalog-0.1.0/src/terria_catalog/py.typed +0 -0
- terria_catalog-0.1.0/src/terria_catalog/registry.py +41 -0
- terria_catalog-0.1.0/src/terria_catalog/release.py +208 -0
- terria_catalog-0.1.0/src/terria_catalog/serializers/__init__.py +11 -0
- terria_catalog-0.1.0/src/terria_catalog/serializers/terria_json.py +319 -0
- terria_catalog-0.1.0/src/terria_catalog/storage/__init__.py +21 -0
- terria_catalog-0.1.0/src/terria_catalog/storage/base.py +160 -0
- terria_catalog-0.1.0/src/terria_catalog/storage/local.py +81 -0
- terria_catalog-0.1.0/src/terria_catalog/storage/s3.py +214 -0
- terria_catalog-0.1.0/src/terria_catalog/transaction.py +60 -0
- terria_catalog-0.1.0/src/terria_catalog/utils/__init__.py +20 -0
- terria_catalog-0.1.0/src/terria_catalog/utils/naming.py +38 -0
- terria_catalog-0.1.0/src/terria_catalog/utils/paths.py +60 -0
- terria_catalog-0.1.0/src/terria_catalog/validators/__init__.py +31 -0
- terria_catalog-0.1.0/src/terria_catalog/validators/base.py +85 -0
- terria_catalog-0.1.0/src/terria_catalog/validators/hierarchy.py +93 -0
- terria_catalog-0.1.0/src/terria_catalog/validators/ids.py +38 -0
- terria_catalog-0.1.0/src/terria_catalog/validators/schema.py +75 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
*.egg
|
|
9
|
+
|
|
10
|
+
# Virtual envs
|
|
11
|
+
.venv/
|
|
12
|
+
venv/
|
|
13
|
+
env/
|
|
14
|
+
|
|
15
|
+
# Tooling caches
|
|
16
|
+
.mypy_cache/
|
|
17
|
+
.ruff_cache/
|
|
18
|
+
.pytest_cache/
|
|
19
|
+
.coverage
|
|
20
|
+
.coverage.*
|
|
21
|
+
htmlcov/
|
|
22
|
+
coverage.xml
|
|
23
|
+
|
|
24
|
+
# IDE / OS
|
|
25
|
+
.idea/
|
|
26
|
+
.vscode/
|
|
27
|
+
.DS_Store
|
|
28
|
+
|
|
29
|
+
# Local secrets / scratch
|
|
30
|
+
*.local
|
|
31
|
+
.env
|
|
32
|
+
|
|
33
|
+
# Vendored reference material (not part of the package)
|
|
34
|
+
/terriajs-main/
|
|
35
|
+
/terria-docs/
|
|
36
|
+
|
|
37
|
+
# Generated report artifacts (Word/PDF exports)
|
|
38
|
+
/dist-reports/
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# AGENTS.md
|
|
2
|
+
|
|
3
|
+
Guidance for AI agents working in or with this repository.
|
|
4
|
+
|
|
5
|
+
## What this project is
|
|
6
|
+
|
|
7
|
+
`terria-catalog` is a Python SDK + AI toolset for editing **TerriaJS
|
|
8
|
+
initialization catalogs** as typed objects (not raw JSON), with safe versioned
|
|
9
|
+
publishing to S3. If you are helping a user *manage a catalog* (add/remove/move
|
|
10
|
+
datasets, organize groups, publish), use the tools — do not hand-edit JSON.
|
|
11
|
+
|
|
12
|
+
## Two modes
|
|
13
|
+
|
|
14
|
+
**A. Editing a catalog (most common).** Use the agent tools via the MCP server,
|
|
15
|
+
the `terria-catalog` CLI, or `terria_catalog.agent.CatalogSession`. Follow the
|
|
16
|
+
workflow in [`docs/system_prompt.md`](docs/system_prompt.md):
|
|
17
|
+
reload → describe/search → edit → validate → preview → confirm → publish.
|
|
18
|
+
`publish`/`rollback` require `confirm: true`.
|
|
19
|
+
|
|
20
|
+
**B. Developing this package.** See [`CONTRIBUTING.md`](CONTRIBUTING.md). Quality
|
|
21
|
+
gates (all must pass): `ruff check .`, `ruff format --check .`, `black --check .`,
|
|
22
|
+
`mypy`, `pytest`. Golden rules: models are pure data; JSON lives only in the
|
|
23
|
+
serializer; storage is decoupled; adding an item type is additive (create a
|
|
24
|
+
dataclass with `terria_type`, export it, add a test).
|
|
25
|
+
|
|
26
|
+
## Item types
|
|
27
|
+
|
|
28
|
+
`geojson`, `wms`, `wmts`, `csv`, `shp`, `cog`, `cesium-terrain`, `3d-tiles`,
|
|
29
|
+
`ion-imagery`, `esri-mapServer`, `esri-featureServer`, `group`. Items accept any
|
|
30
|
+
Terria trait; unmodeled traits round-trip via a passthrough, so you never lose
|
|
31
|
+
data.
|
|
32
|
+
|
|
33
|
+
## Key paths
|
|
34
|
+
|
|
35
|
+
- `src/terria_catalog/agent/` — the AI surface (session, tools, MCP server, CLI).
|
|
36
|
+
- `src/terria_catalog/` — the core SDK (models, serializer, storage, release).
|
|
37
|
+
- `docs/AI_USAGE.md` — how to wire up MCP / CLI / Python.
|
|
38
|
+
- `skills/terria-catalog/` — a Claude Skill packaging the workflow.
|
|
39
|
+
- `examples/` — runnable scripts.
|
|
40
|
+
|
|
41
|
+
## Safety
|
|
42
|
+
|
|
43
|
+
Prefer `--dry-run` (CLI) or `preview_changes` before writing. Never publish
|
|
44
|
+
without showing the human the diff. To undo, use `history` + `rollback`.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format is based on
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres
|
|
5
|
+
to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.1.0] - 2026-07-18
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Typed Terria object model** mirroring the TerriaJS source traits: `Catalog`,
|
|
14
|
+
`CatalogGroup`, and item types `GeoJsonCatalogItem`, `WebMapServiceCatalogItem`,
|
|
15
|
+
`WebMapTileServiceCatalogItem`, `CsvCatalogItem`, `ShapefileCatalogItem`,
|
|
16
|
+
`CogCatalogItem`, `CesiumTerrainCatalogItem`, `Cesium3DTilesCatalogItem`,
|
|
17
|
+
`IonImageryCatalogItem`, `ArcGisMapServerCatalogItem`,
|
|
18
|
+
`ArcGisFeatureServerCatalogItem`, plus placeholders for further types.
|
|
19
|
+
- **Sparse, lossless serializer** — the only place JSON exists; unknown types and
|
|
20
|
+
traits round-trip via `GenericCatalogMember` and `extra` passthrough.
|
|
21
|
+
- **Path- and id-based operations**: add, remove, update, move, reorder, with
|
|
22
|
+
first-class `CatalogGroup` objects and auto-created group paths.
|
|
23
|
+
- **Validation** (duplicate ids/paths, hierarchy, required properties, member
|
|
24
|
+
types) and **transactions** with validate-or-rollback.
|
|
25
|
+
- **Pluggable storage**: `LocalStorage` and `S3Storage` (boto3 optional), behind a
|
|
26
|
+
narrow `StorageBackend` interface.
|
|
27
|
+
- **Release layer** (`ReleaseManager`): validated, versioned publishing;
|
|
28
|
+
optimistic-concurrency conditional writes; history and rollback; a pluggable
|
|
29
|
+
`CacheInvalidator` hook.
|
|
30
|
+
- **AI-native surface** (`terria_catalog.agent`): a safe `CatalogSession`, a shared
|
|
31
|
+
tool catalogue, an **MCP server** (`terria-catalog-mcp`), a **CLI**
|
|
32
|
+
(`terria-catalog`), and exportable function-calling schemas.
|
|
33
|
+
- Documentation, runnable examples, and a Claude Skill.
|
|
34
|
+
|
|
35
|
+
[Unreleased]: https://github.com/iwmihq/terria-catalog/compare/v0.1.0...HEAD
|
|
36
|
+
[0.1.0]: https://github.com/iwmihq/terria-catalog/releases/tag/v0.1.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 International Water Management Institute (IWMI)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: terria-catalog
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A Pythonic SDK for creating, modifying, validating and publishing TerriaJS initialization catalogs.
|
|
5
|
+
Project-URL: Homepage, https://github.com/iwmihq/terria-catalog
|
|
6
|
+
Project-URL: Repository, https://github.com/iwmihq/terria-catalog
|
|
7
|
+
Project-URL: Documentation, https://github.com/iwmihq/terria-catalog/blob/main/docs/ARCHITECTURE.md
|
|
8
|
+
Author: IWMI Digital Twin Team
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: catalog,digital-twin,geospatial,gis,terria,terriajs
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: anyio>=4.0; extra == 'dev'
|
|
24
|
+
Requires-Dist: black>=24.8; extra == 'dev'
|
|
25
|
+
Requires-Dist: boto3-stubs[s3]>=1.34; extra == 'dev'
|
|
26
|
+
Requires-Dist: boto3>=1.34; extra == 'dev'
|
|
27
|
+
Requires-Dist: coverage>=7.6; extra == 'dev'
|
|
28
|
+
Requires-Dist: mcp>=1.2; extra == 'dev'
|
|
29
|
+
Requires-Dist: moto[s3]>=5.0; extra == 'dev'
|
|
30
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
31
|
+
Requires-Dist: pre-commit>=3.8; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
34
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
35
|
+
Provides-Extra: mcp
|
|
36
|
+
Requires-Dist: anyio>=4.0; extra == 'mcp'
|
|
37
|
+
Requires-Dist: mcp>=1.2; extra == 'mcp'
|
|
38
|
+
Provides-Extra: s3
|
|
39
|
+
Requires-Dist: boto3>=1.34; extra == 's3'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# terria-catalog
|
|
43
|
+
|
|
44
|
+
A Pythonic SDK for creating, modifying, validating and publishing
|
|
45
|
+
[**TerriaJS**](https://terria.io) *initialization catalogs*.
|
|
46
|
+
|
|
47
|
+
`terria-catalog` lets you build and edit Terria init files as **strongly-typed
|
|
48
|
+
Python objects** — `CatalogGroup`, `GeoJsonCatalogItem`,
|
|
49
|
+
`WebMapServiceCatalogItem`, and friends — instead of hand-editing JSON or juggling
|
|
50
|
+
dictionaries. It mirrors Terria's own terminology and object model (derived
|
|
51
|
+
directly from the TerriaJS source `Traits` classes), and produces valid Terria
|
|
52
|
+
init JSON that is compatible with the official specification.
|
|
53
|
+
|
|
54
|
+
It is designed to be the canonical way our Digital Twin ecosystem creates and
|
|
55
|
+
modifies catalogs stored in Amazon S3 — but it has no hard dependency on S3 (or
|
|
56
|
+
even on any I/O): storage and serialization are cleanly decoupled and injectable.
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from terria_catalog import Catalog, GeoJsonCatalogItem
|
|
60
|
+
|
|
61
|
+
catalog = Catalog.from_s3(bucket="my-bucket", key="init/catalog.json")
|
|
62
|
+
|
|
63
|
+
catalog.add(
|
|
64
|
+
GeoJsonCatalogItem(
|
|
65
|
+
name="Reservoirs",
|
|
66
|
+
id="reservoirs",
|
|
67
|
+
url="https://example.org/reservoirs.geojson",
|
|
68
|
+
),
|
|
69
|
+
path="/Water Resources/Reservoirs", # missing groups are auto-created
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
catalog.move("reservoirs", "/Water Resources")
|
|
73
|
+
catalog.update("reservoirs", description="Major reservoirs in the basin")
|
|
74
|
+
catalog.remove("some-old-dataset")
|
|
75
|
+
|
|
76
|
+
catalog.publish() # validates, serializes, and writes back to S3
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
You never touch JSON. You never touch dictionaries. The JSON exists only inside
|
|
80
|
+
the serializer.
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## Installation
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install terria-catalog # core (no cloud dependencies)
|
|
88
|
+
pip install 'terria-catalog[s3]' # + boto3 for the S3 storage backend
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Requires **Python 3.10+**.
|
|
92
|
+
|
|
93
|
+
---
|
|
94
|
+
|
|
95
|
+
## Core concepts
|
|
96
|
+
|
|
97
|
+
| Concept | Type | Notes |
|
|
98
|
+
| --- | --- | --- |
|
|
99
|
+
| The whole init file | `Catalog` | Facade: holds the member tree, `homeCamera`, and other init-level properties. |
|
|
100
|
+
| A folder | `CatalogGroup` (`"group"`) | First-class object; nestable; add/remove members through it. |
|
|
101
|
+
| A dataset | `CatalogItem` subclasses | e.g. `GeoJsonCatalogItem`, `WebMapServiceCatalogItem`. |
|
|
102
|
+
| A shared trait | `Rectangle`, `InfoSection`, `GeoJsonStyle`, `TableStyle`, … | Plain typed value objects. |
|
|
103
|
+
| Where it's stored | `StorageBackend` (`LocalStorage`, `S3Storage`) | Injected; the catalog doesn't care where it lives. |
|
|
104
|
+
| Turning objects into JSON | `TerriaJsonSerializer` | The **only** place JSON exists. |
|
|
105
|
+
|
|
106
|
+
### Supported item types
|
|
107
|
+
|
|
108
|
+
Fully modeled (with type-specific traits):
|
|
109
|
+
|
|
110
|
+
`GeoJsonCatalogItem` · `WebMapServiceCatalogItem` · `WebMapTileServiceCatalogItem`
|
|
111
|
+
· `CsvCatalogItem` · `ShapefileCatalogItem` · `CogCatalogItem` ·
|
|
112
|
+
`CesiumTerrainCatalogItem` · `Cesium3DTilesCatalogItem` · `IonImageryCatalogItem`
|
|
113
|
+
· `ArcGisMapServerCatalogItem` · `ArcGisFeatureServerCatalogItem`
|
|
114
|
+
|
|
115
|
+
Placeholders (correct `type`, traits not yet modeled — data still round-trips):
|
|
116
|
+
`KmlCatalogItem`, `CzmlCatalogItem`, `GpxCatalogItem`, `GltfCatalogItem`,
|
|
117
|
+
`WebFeatureServiceCatalogItem`, `OpenStreetMapCatalogItem`,
|
|
118
|
+
`MapboxVectorTileCatalogItem`.
|
|
119
|
+
|
|
120
|
+
Any **unknown** Terria type encountered when loading a catalog becomes a
|
|
121
|
+
`GenericCatalogMember` so no data is ever lost on a load/save round-trip.
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## The public API
|
|
126
|
+
|
|
127
|
+
### Loading
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
from terria_catalog import Catalog
|
|
131
|
+
|
|
132
|
+
Catalog.from_s3(bucket="b", key="init/catalog.json") # needs [s3] extra
|
|
133
|
+
Catalog.from_file("wwwroot/init/catalog.json")
|
|
134
|
+
Catalog.from_json(json_string)
|
|
135
|
+
Catalog() # a new, empty catalog
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Editing (paths *or* ids everywhere)
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
catalog.add(item, path="/Water Resources/Reservoirs") # returns the item
|
|
142
|
+
catalog.remove("dataset-id") # id, name, or path
|
|
143
|
+
catalog.update("dataset-id", description="...", opacity=0.6)
|
|
144
|
+
catalog.move("dataset-id", "/Infrastructure/Dams")
|
|
145
|
+
catalog.reorder("/Water Resources", ["reservoirs", "gauges"])
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Identifiers are resolved as a **path** if they contain `/`, otherwise by **id**
|
|
149
|
+
(then by **name**). Unknown keyword arguments to `update()` are stored in the
|
|
150
|
+
member's `extra` mapping, so you can set traits the SDK doesn't model yet.
|
|
151
|
+
|
|
152
|
+
### Groups as first-class objects
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
water = catalog.group("/Water Resources") # created if missing
|
|
156
|
+
water.add(GeoJsonCatalogItem(name="Reservoirs", id="r", url="..."))
|
|
157
|
+
water.create_group("Rivers") # idempotent
|
|
158
|
+
water.group("Rivers/Perennial") # relative, auto-creating
|
|
159
|
+
water.remove("r")
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### Validation & transactions
|
|
163
|
+
|
|
164
|
+
```python
|
|
165
|
+
catalog.validate() # raises ValidationFailedError on problems
|
|
166
|
+
errors = catalog.validate(raise_on_error=False)
|
|
167
|
+
|
|
168
|
+
with catalog.transaction(): # atomic: rolls back on error…
|
|
169
|
+
catalog.add(...)
|
|
170
|
+
catalog.move(...)
|
|
171
|
+
catalog.remove(...)
|
|
172
|
+
# …or if validation fails at the end of the block
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Built-in validators cover duplicate ids, duplicate paths, invalid hierarchy,
|
|
176
|
+
missing required properties, and unknown Terria types.
|
|
177
|
+
|
|
178
|
+
### Publishing
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
catalog.publish() # to the bound storage backend
|
|
182
|
+
catalog.publish(LocalStorage("out.json"))
|
|
183
|
+
catalog.to_json() # -> str (no I/O)
|
|
184
|
+
catalog.to_dict() # -> dict (no I/O)
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
### Safe releases: versioning, rollback & concurrency
|
|
188
|
+
|
|
189
|
+
`catalog.publish()` is a plain overwrite (last-write-wins). When several pipelines
|
|
190
|
+
edit the same S3 catalog and you need rollback + a CDN cache refresh, use
|
|
191
|
+
`ReleaseManager`:
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
from terria_catalog import ReleaseManager, S3Storage
|
|
195
|
+
|
|
196
|
+
manager = ReleaseManager(
|
|
197
|
+
S3Storage("my-bucket", "init/catalog.json"), # enable S3 bucket versioning
|
|
198
|
+
cache_invalidator=my_cloudfront_invalidator, # your CacheInvalidator impl
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# Concurrency-safe read-modify-write: reloads & retries if another pipeline wins.
|
|
202
|
+
manager.update(lambda cat: cat.add(item, path="/Climate"), message="add rainfall")
|
|
203
|
+
|
|
204
|
+
# Rollback to a previous version.
|
|
205
|
+
versions = manager.history() # newest first
|
|
206
|
+
manager.rollback(to=versions[1].version_id)
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
`ReleaseManager.publish()` validates first and returns a `Release` handle (ETag,
|
|
210
|
+
version id, checksum, timestamp). Optimistic concurrency uses conditional writes
|
|
211
|
+
(`If-Match`), so competing pipelines retry instead of clobbering each other. The
|
|
212
|
+
`CacheInvalidator` interface is where CloudFront (or any CDN) plugs in — the
|
|
213
|
+
library itself has no CDN dependency. Environment specifics (bucket, IAM, the
|
|
214
|
+
actual CloudFront call) stay in your deployment code.
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## Use it with your AI (no JSON, no code study)
|
|
219
|
+
|
|
220
|
+
Colleagues can manage the catalog in natural language through their assistant of
|
|
221
|
+
choice. Three surfaces, one safe tool set (validate → preview → confirm → publish,
|
|
222
|
+
with version history and rollback):
|
|
223
|
+
|
|
224
|
+
- **MCP server** — for Claude Desktop/Code, Cursor, etc. `pip install
|
|
225
|
+
'terria-catalog[mcp]'`, point it at a catalog with env vars, run
|
|
226
|
+
`terria-catalog-mcp`. Then just chat: *"add the 2026 flood COG under Flood
|
|
227
|
+
Forecasting and publish."*
|
|
228
|
+
- **CLI** — `terria-catalog --s3 bucket/init/catalog.json add --type cog …`
|
|
229
|
+
(with `--json` and `--dry-run`), ideal for shell-driving agents.
|
|
230
|
+
- **Python** — `terria_catalog.agent.CatalogSession` + `run_tool` /
|
|
231
|
+
`export_schemas` for custom function-calling agents.
|
|
232
|
+
|
|
233
|
+
There's also a **Claude Skill** ([`skills/terria-catalog/`](skills/terria-catalog/SKILL.md))
|
|
234
|
+
and a drop-in [system prompt](docs/system_prompt.md). Full guide:
|
|
235
|
+
[`docs/AI_USAGE.md`](docs/AI_USAGE.md).
|
|
236
|
+
|
|
237
|
+
## Design philosophy
|
|
238
|
+
|
|
239
|
+
- **Objects, not JSON.** Consumers manipulate typed Terria objects. JSON lives
|
|
240
|
+
only in `serializers/terria_json.py`.
|
|
241
|
+
- **Decoupled storage.** The catalog talks to a `StorageBackend` interface; S3 is
|
|
242
|
+
just one implementation and is imported lazily so core has no cloud deps.
|
|
243
|
+
- **Injectable everything.** Serializer, storage, config and validators are all
|
|
244
|
+
passed in, never hard-wired.
|
|
245
|
+
- **Open for extension.** Adding a new item type is: create a dataclass that sets
|
|
246
|
+
`terria_type`, export it, add a test. It self-registers — no serializer,
|
|
247
|
+
registry, or facade changes required.
|
|
248
|
+
|
|
249
|
+
See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the full design and
|
|
250
|
+
[`examples/`](examples/) for runnable scripts.
|
|
251
|
+
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
## Documentation
|
|
255
|
+
|
|
256
|
+
- [Use it with your AI](docs/AI_USAGE.md) — MCP / CLI / Python for assistants.
|
|
257
|
+
- [Coordinating multiple writers](docs/COORDINATION.md) — the pipeline-vs-human plan.
|
|
258
|
+
- [Architecture](docs/ARCHITECTURE.md) — how the library is built.
|
|
259
|
+
- [System prompt](docs/system_prompt.md) · [Claude Skill](skills/terria-catalog/SKILL.md) · [AGENTS.md](AGENTS.md) · [llms.txt](llms.txt)
|
|
260
|
+
|
|
261
|
+
## Development
|
|
262
|
+
|
|
263
|
+
```bash
|
|
264
|
+
pip install -e '.[dev]'
|
|
265
|
+
pre-commit install
|
|
266
|
+
|
|
267
|
+
ruff check . && ruff format --check .
|
|
268
|
+
black --check .
|
|
269
|
+
mypy
|
|
270
|
+
pytest --cov=terria_catalog
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
See [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
|
274
|
+
|
|
275
|
+
## License
|
|
276
|
+
|
|
277
|
+
[MIT](LICENSE) © International Water Management Institute (IWMI).
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
# terria-catalog
|
|
2
|
+
|
|
3
|
+
A Pythonic SDK for creating, modifying, validating and publishing
|
|
4
|
+
[**TerriaJS**](https://terria.io) *initialization catalogs*.
|
|
5
|
+
|
|
6
|
+
`terria-catalog` lets you build and edit Terria init files as **strongly-typed
|
|
7
|
+
Python objects** — `CatalogGroup`, `GeoJsonCatalogItem`,
|
|
8
|
+
`WebMapServiceCatalogItem`, and friends — instead of hand-editing JSON or juggling
|
|
9
|
+
dictionaries. It mirrors Terria's own terminology and object model (derived
|
|
10
|
+
directly from the TerriaJS source `Traits` classes), and produces valid Terria
|
|
11
|
+
init JSON that is compatible with the official specification.
|
|
12
|
+
|
|
13
|
+
It is designed to be the canonical way our Digital Twin ecosystem creates and
|
|
14
|
+
modifies catalogs stored in Amazon S3 — but it has no hard dependency on S3 (or
|
|
15
|
+
even on any I/O): storage and serialization are cleanly decoupled and injectable.
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from terria_catalog import Catalog, GeoJsonCatalogItem
|
|
19
|
+
|
|
20
|
+
catalog = Catalog.from_s3(bucket="my-bucket", key="init/catalog.json")
|
|
21
|
+
|
|
22
|
+
catalog.add(
|
|
23
|
+
GeoJsonCatalogItem(
|
|
24
|
+
name="Reservoirs",
|
|
25
|
+
id="reservoirs",
|
|
26
|
+
url="https://example.org/reservoirs.geojson",
|
|
27
|
+
),
|
|
28
|
+
path="/Water Resources/Reservoirs", # missing groups are auto-created
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
catalog.move("reservoirs", "/Water Resources")
|
|
32
|
+
catalog.update("reservoirs", description="Major reservoirs in the basin")
|
|
33
|
+
catalog.remove("some-old-dataset")
|
|
34
|
+
|
|
35
|
+
catalog.publish() # validates, serializes, and writes back to S3
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
You never touch JSON. You never touch dictionaries. The JSON exists only inside
|
|
39
|
+
the serializer.
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Installation
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install terria-catalog # core (no cloud dependencies)
|
|
47
|
+
pip install 'terria-catalog[s3]' # + boto3 for the S3 storage backend
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Requires **Python 3.10+**.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## Core concepts
|
|
55
|
+
|
|
56
|
+
| Concept | Type | Notes |
|
|
57
|
+
| --- | --- | --- |
|
|
58
|
+
| The whole init file | `Catalog` | Facade: holds the member tree, `homeCamera`, and other init-level properties. |
|
|
59
|
+
| A folder | `CatalogGroup` (`"group"`) | First-class object; nestable; add/remove members through it. |
|
|
60
|
+
| A dataset | `CatalogItem` subclasses | e.g. `GeoJsonCatalogItem`, `WebMapServiceCatalogItem`. |
|
|
61
|
+
| A shared trait | `Rectangle`, `InfoSection`, `GeoJsonStyle`, `TableStyle`, … | Plain typed value objects. |
|
|
62
|
+
| Where it's stored | `StorageBackend` (`LocalStorage`, `S3Storage`) | Injected; the catalog doesn't care where it lives. |
|
|
63
|
+
| Turning objects into JSON | `TerriaJsonSerializer` | The **only** place JSON exists. |
|
|
64
|
+
|
|
65
|
+
### Supported item types
|
|
66
|
+
|
|
67
|
+
Fully modeled (with type-specific traits):
|
|
68
|
+
|
|
69
|
+
`GeoJsonCatalogItem` · `WebMapServiceCatalogItem` · `WebMapTileServiceCatalogItem`
|
|
70
|
+
· `CsvCatalogItem` · `ShapefileCatalogItem` · `CogCatalogItem` ·
|
|
71
|
+
`CesiumTerrainCatalogItem` · `Cesium3DTilesCatalogItem` · `IonImageryCatalogItem`
|
|
72
|
+
· `ArcGisMapServerCatalogItem` · `ArcGisFeatureServerCatalogItem`
|
|
73
|
+
|
|
74
|
+
Placeholders (correct `type`, traits not yet modeled — data still round-trips):
|
|
75
|
+
`KmlCatalogItem`, `CzmlCatalogItem`, `GpxCatalogItem`, `GltfCatalogItem`,
|
|
76
|
+
`WebFeatureServiceCatalogItem`, `OpenStreetMapCatalogItem`,
|
|
77
|
+
`MapboxVectorTileCatalogItem`.
|
|
78
|
+
|
|
79
|
+
Any **unknown** Terria type encountered when loading a catalog becomes a
|
|
80
|
+
`GenericCatalogMember` so no data is ever lost on a load/save round-trip.
|
|
81
|
+
|
|
82
|
+
---
|
|
83
|
+
|
|
84
|
+
## The public API
|
|
85
|
+
|
|
86
|
+
### Loading
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from terria_catalog import Catalog
|
|
90
|
+
|
|
91
|
+
Catalog.from_s3(bucket="b", key="init/catalog.json") # needs [s3] extra
|
|
92
|
+
Catalog.from_file("wwwroot/init/catalog.json")
|
|
93
|
+
Catalog.from_json(json_string)
|
|
94
|
+
Catalog() # a new, empty catalog
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
### Editing (paths *or* ids everywhere)
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
catalog.add(item, path="/Water Resources/Reservoirs") # returns the item
|
|
101
|
+
catalog.remove("dataset-id") # id, name, or path
|
|
102
|
+
catalog.update("dataset-id", description="...", opacity=0.6)
|
|
103
|
+
catalog.move("dataset-id", "/Infrastructure/Dams")
|
|
104
|
+
catalog.reorder("/Water Resources", ["reservoirs", "gauges"])
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Identifiers are resolved as a **path** if they contain `/`, otherwise by **id**
|
|
108
|
+
(then by **name**). Unknown keyword arguments to `update()` are stored in the
|
|
109
|
+
member's `extra` mapping, so you can set traits the SDK doesn't model yet.
|
|
110
|
+
|
|
111
|
+
### Groups as first-class objects
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
water = catalog.group("/Water Resources") # created if missing
|
|
115
|
+
water.add(GeoJsonCatalogItem(name="Reservoirs", id="r", url="..."))
|
|
116
|
+
water.create_group("Rivers") # idempotent
|
|
117
|
+
water.group("Rivers/Perennial") # relative, auto-creating
|
|
118
|
+
water.remove("r")
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
### Validation & transactions
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
catalog.validate() # raises ValidationFailedError on problems
|
|
125
|
+
errors = catalog.validate(raise_on_error=False)
|
|
126
|
+
|
|
127
|
+
with catalog.transaction(): # atomic: rolls back on error…
|
|
128
|
+
catalog.add(...)
|
|
129
|
+
catalog.move(...)
|
|
130
|
+
catalog.remove(...)
|
|
131
|
+
# …or if validation fails at the end of the block
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Built-in validators cover duplicate ids, duplicate paths, invalid hierarchy,
|
|
135
|
+
missing required properties, and unknown Terria types.
|
|
136
|
+
|
|
137
|
+
### Publishing
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
catalog.publish() # to the bound storage backend
|
|
141
|
+
catalog.publish(LocalStorage("out.json"))
|
|
142
|
+
catalog.to_json() # -> str (no I/O)
|
|
143
|
+
catalog.to_dict() # -> dict (no I/O)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
### Safe releases: versioning, rollback & concurrency
|
|
147
|
+
|
|
148
|
+
`catalog.publish()` is a plain overwrite (last-write-wins). When several pipelines
|
|
149
|
+
edit the same S3 catalog and you need rollback + a CDN cache refresh, use
|
|
150
|
+
`ReleaseManager`:
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
from terria_catalog import ReleaseManager, S3Storage
|
|
154
|
+
|
|
155
|
+
manager = ReleaseManager(
|
|
156
|
+
S3Storage("my-bucket", "init/catalog.json"), # enable S3 bucket versioning
|
|
157
|
+
cache_invalidator=my_cloudfront_invalidator, # your CacheInvalidator impl
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
# Concurrency-safe read-modify-write: reloads & retries if another pipeline wins.
|
|
161
|
+
manager.update(lambda cat: cat.add(item, path="/Climate"), message="add rainfall")
|
|
162
|
+
|
|
163
|
+
# Rollback to a previous version.
|
|
164
|
+
versions = manager.history() # newest first
|
|
165
|
+
manager.rollback(to=versions[1].version_id)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
`ReleaseManager.publish()` validates first and returns a `Release` handle (ETag,
|
|
169
|
+
version id, checksum, timestamp). Optimistic concurrency uses conditional writes
|
|
170
|
+
(`If-Match`), so competing pipelines retry instead of clobbering each other. The
|
|
171
|
+
`CacheInvalidator` interface is where CloudFront (or any CDN) plugs in — the
|
|
172
|
+
library itself has no CDN dependency. Environment specifics (bucket, IAM, the
|
|
173
|
+
actual CloudFront call) stay in your deployment code.
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
## Use it with your AI (no JSON, no code study)
|
|
178
|
+
|
|
179
|
+
Colleagues can manage the catalog in natural language through their assistant of
|
|
180
|
+
choice. Three surfaces, one safe tool set (validate → preview → confirm → publish,
|
|
181
|
+
with version history and rollback):
|
|
182
|
+
|
|
183
|
+
- **MCP server** — for Claude Desktop/Code, Cursor, etc. `pip install
|
|
184
|
+
'terria-catalog[mcp]'`, point it at a catalog with env vars, run
|
|
185
|
+
`terria-catalog-mcp`. Then just chat: *"add the 2026 flood COG under Flood
|
|
186
|
+
Forecasting and publish."*
|
|
187
|
+
- **CLI** — `terria-catalog --s3 bucket/init/catalog.json add --type cog …`
|
|
188
|
+
(with `--json` and `--dry-run`), ideal for shell-driving agents.
|
|
189
|
+
- **Python** — `terria_catalog.agent.CatalogSession` + `run_tool` /
|
|
190
|
+
`export_schemas` for custom function-calling agents.
|
|
191
|
+
|
|
192
|
+
There's also a **Claude Skill** ([`skills/terria-catalog/`](skills/terria-catalog/SKILL.md))
|
|
193
|
+
and a drop-in [system prompt](docs/system_prompt.md). Full guide:
|
|
194
|
+
[`docs/AI_USAGE.md`](docs/AI_USAGE.md).
|
|
195
|
+
|
|
196
|
+
## Design philosophy
|
|
197
|
+
|
|
198
|
+
- **Objects, not JSON.** Consumers manipulate typed Terria objects. JSON lives
|
|
199
|
+
only in `serializers/terria_json.py`.
|
|
200
|
+
- **Decoupled storage.** The catalog talks to a `StorageBackend` interface; S3 is
|
|
201
|
+
just one implementation and is imported lazily so core has no cloud deps.
|
|
202
|
+
- **Injectable everything.** Serializer, storage, config and validators are all
|
|
203
|
+
passed in, never hard-wired.
|
|
204
|
+
- **Open for extension.** Adding a new item type is: create a dataclass that sets
|
|
205
|
+
`terria_type`, export it, add a test. It self-registers — no serializer,
|
|
206
|
+
registry, or facade changes required.
|
|
207
|
+
|
|
208
|
+
See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the full design and
|
|
209
|
+
[`examples/`](examples/) for runnable scripts.
|
|
210
|
+
|
|
211
|
+
---
|
|
212
|
+
|
|
213
|
+
## Documentation
|
|
214
|
+
|
|
215
|
+
- [Use it with your AI](docs/AI_USAGE.md) — MCP / CLI / Python for assistants.
|
|
216
|
+
- [Coordinating multiple writers](docs/COORDINATION.md) — the pipeline-vs-human plan.
|
|
217
|
+
- [Architecture](docs/ARCHITECTURE.md) — how the library is built.
|
|
218
|
+
- [System prompt](docs/system_prompt.md) · [Claude Skill](skills/terria-catalog/SKILL.md) · [AGENTS.md](AGENTS.md) · [llms.txt](llms.txt)
|
|
219
|
+
|
|
220
|
+
## Development
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
pip install -e '.[dev]'
|
|
224
|
+
pre-commit install
|
|
225
|
+
|
|
226
|
+
ruff check . && ruff format --check .
|
|
227
|
+
black --check .
|
|
228
|
+
mypy
|
|
229
|
+
pytest --cov=terria_catalog
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
See [`CONTRIBUTING.md`](CONTRIBUTING.md).
|
|
233
|
+
|
|
234
|
+
## License
|
|
235
|
+
|
|
236
|
+
[MIT](LICENSE) © International Water Management Institute (IWMI).
|