overture-core 0.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- overture_core-0.0.0/.python-version +1 -0
- overture_core-0.0.0/LICENSE +21 -0
- overture_core-0.0.0/PKG-INFO +44 -0
- overture_core-0.0.0/README.md +17 -0
- overture_core-0.0.0/overture_core/__init__.py +0 -0
- overture_core-0.0.0/overture_core/stac/__init__.py +0 -0
- overture_core-0.0.0/overture_core/stac/job.py +160 -0
- overture_core-0.0.0/overture_core/stac/publisher.py +178 -0
- overture_core-0.0.0/overture_core.egg-info/PKG-INFO +44 -0
- overture_core-0.0.0/overture_core.egg-info/SOURCES.txt +20 -0
- overture_core-0.0.0/overture_core.egg-info/dependency_links.txt +1 -0
- overture_core-0.0.0/overture_core.egg-info/requires.txt +8 -0
- overture_core-0.0.0/overture_core.egg-info/scm_file_list.json +17 -0
- overture_core-0.0.0/overture_core.egg-info/scm_version.json +8 -0
- overture_core-0.0.0/overture_core.egg-info/top_level.txt +1 -0
- overture_core-0.0.0/pyproject.toml +83 -0
- overture_core-0.0.0/setup.cfg +4 -0
- overture_core-0.0.0/tests/__init__.py +0 -0
- overture_core-0.0.0/tests/stac/__init__.py +0 -0
- overture_core-0.0.0/tests/stac/test_job.py +160 -0
- overture_core-0.0.0/tests/stac/test_publisher.py +409 -0
- overture_core-0.0.0/uv.lock +1378 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.11
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Overture Maps
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: overture-core
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: Shared Overture business logic — portable, framework-agnostic job classes.
|
|
5
|
+
Author: Overture Maps Foundation
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/OvertureMaps/overture-core
|
|
8
|
+
Project-URL: Source, https://github.com/OvertureMaps/overture-core/tree/main/packages/overture_core
|
|
9
|
+
Project-URL: Issues, https://github.com/OvertureMaps/overture-core/issues
|
|
10
|
+
Keywords: overture,overturemaps,stac,gis
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: overture-serverless>=0.1.0
|
|
20
|
+
Requires-Dist: overture-stac
|
|
21
|
+
Requires-Dist: pystac>=1.8.4
|
|
22
|
+
Requires-Dist: boto3>=1.35
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=9.1.1; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov>=7.0.0; extra == "dev"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# overture_core
|
|
29
|
+
|
|
30
|
+
[](https://pypi.org/project/overture-core/)
|
|
31
|
+
[](https://pypi.org/project/overture-core/)
|
|
32
|
+
|
|
33
|
+
Shared, framework-agnostic business logic — portable job classes built on [`overture-serverless`](../overture_serverless).
|
|
34
|
+
|
|
35
|
+
## Jobs
|
|
36
|
+
|
|
37
|
+
- **`stac.job.PublishStac`** — publishes the Overture STAC catalog. Two modes selected by the presence of the `release` param:
|
|
38
|
+
- *single-release* (`release` + `source_path` + `scratch_bucket`): publish just that release, mirror scoped to `stac/{release}/`. Schema comes from the RC bundle's `metadata.json`.
|
|
39
|
+
- *walk* (no `release`, requires `scratch_bucket`): rebuild the catalog for every release currently in the public bucket, mirror the whole `stac/` prefix with delete-orphans. Schema per release comes from the existing STAC (`schema:version` in each `catalog.json`); on cache miss the job self-heals by reading schema from the released RC bundle in `scratch_bucket` and raises if it can't recover.
|
|
40
|
+
|
|
41
|
+
## Future work
|
|
42
|
+
|
|
43
|
+
- **Bundle-owned schema accessor** — when `ReleaseCandidateBundle` becomes publishable, move `read_schema_version_from_rc_bundle` onto it as `resolved_schema_version()`; single-release mode's schema read shrinks to one line.
|
|
44
|
+
- **Upstream `build_release_catalog`** — file a PR against [OvertureMaps/stac](https://github.com/OvertureMaps/stac) exposing this as a public API (their CLI already does the exact call sequence). When it lands, our wrapper collapses.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# overture_core
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/overture-core/)
|
|
4
|
+
[](https://pypi.org/project/overture-core/)
|
|
5
|
+
|
|
6
|
+
Shared, framework-agnostic business logic — portable job classes built on [`overture-serverless`](../overture_serverless).
|
|
7
|
+
|
|
8
|
+
## Jobs
|
|
9
|
+
|
|
10
|
+
- **`stac.job.PublishStac`** — publishes the Overture STAC catalog. Two modes selected by the presence of the `release` param:
|
|
11
|
+
- *single-release* (`release` + `source_path` + `scratch_bucket`): publish just that release, mirror scoped to `stac/{release}/`. Schema comes from the RC bundle's `metadata.json`.
|
|
12
|
+
- *walk* (no `release`, requires `scratch_bucket`): rebuild the catalog for every release currently in the public bucket, mirror the whole `stac/` prefix with delete-orphans. Schema per release comes from the existing STAC (`schema:version` in each `catalog.json`); on cache miss the job self-heals by reading schema from the released RC bundle in `scratch_bucket` and raises if it can't recover.
|
|
13
|
+
|
|
14
|
+
## Future work
|
|
15
|
+
|
|
16
|
+
- **Bundle-owned schema accessor** — when `ReleaseCandidateBundle` becomes publishable, move `read_schema_version_from_rc_bundle` onto it as `resolved_schema_version()`; single-release mode's schema read shrinks to one line.
|
|
17
|
+
- **Upstream `build_release_catalog`** — file a PR against [OvertureMaps/stac](https://github.com/OvertureMaps/stac) exposing this as a public API (their CLI already does the exact call sequence). When it lands, our wrapper collapses.
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""``PublishStac`` — composes ``publisher.py`` helpers into a serverless job.
|
|
2
|
+
|
|
3
|
+
Two modes, selected by presence of the ``release`` parameter:
|
|
4
|
+
|
|
5
|
+
- **single-release** (``release`` + ``source_path`` + ``scratch_bucket``
|
|
6
|
+
supplied): publish exactly that release. Schema resolved from the RC bundle
|
|
7
|
+
at ``source_path``. Mirror scoped to ``stac/{release}/`` only — no other
|
|
8
|
+
releases are touched.
|
|
9
|
+
|
|
10
|
+
- **walk** (``scratch_bucket`` supplied, no ``release``): rebuild the catalog
|
|
11
|
+
for every release currently present in the public release bucket. Schema
|
|
12
|
+
per release comes from the existing STAC (``schema:version`` in each
|
|
13
|
+
release's ``catalog.json``); on cache miss the job **self-heals** by finding
|
|
14
|
+
the RC bundle with the ``released`` marker in ``scratch_bucket`` and
|
|
15
|
+
reading its ``metadata.json``. Self-heal is LOUD — it raises if it can't
|
|
16
|
+
recover the schema for a release, aborting the walk. Mirror covers the
|
|
17
|
+
whole ``stac/`` prefix with delete-orphans, so releases removed by bucket
|
|
18
|
+
lifecycle policy get their STAC items removed.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import tempfile
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
from overture_serverless.job import ServerlessPythonJob
|
|
27
|
+
|
|
28
|
+
from overture_core.stac.publisher import (
|
|
29
|
+
PROD_ROOT_HREF,
|
|
30
|
+
PUBLIC_RELEASE_BUCKET,
|
|
31
|
+
STAC_S3_PREFIX,
|
|
32
|
+
build_release_catalog,
|
|
33
|
+
list_public_releases,
|
|
34
|
+
mirror_directory_to_s3,
|
|
35
|
+
read_existing_stac_schemas,
|
|
36
|
+
read_schema_from_released_rc,
|
|
37
|
+
read_schema_version_from_rc_bundle,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class PublishStac(ServerlessPythonJob):
|
|
42
|
+
def execute_job(self) -> None:
|
|
43
|
+
extras_bucket = self.get_param("extras_bucket")
|
|
44
|
+
root_href = self.get_param(
|
|
45
|
+
"root_href", default=PROD_ROOT_HREF, is_required=False
|
|
46
|
+
)
|
|
47
|
+
workers = int(self.get_param("workers", default=4, is_required=False))
|
|
48
|
+
|
|
49
|
+
release = self.get_param("release", is_required=False)
|
|
50
|
+
|
|
51
|
+
if release:
|
|
52
|
+
# single-release mode requires the coupled trio.
|
|
53
|
+
source_path = self.get_param("source_path", is_required=False)
|
|
54
|
+
scratch_bucket = self.get_param("scratch_bucket", is_required=False)
|
|
55
|
+
missing = [
|
|
56
|
+
name
|
|
57
|
+
for name, value in [
|
|
58
|
+
("source_path", source_path),
|
|
59
|
+
("scratch_bucket", scratch_bucket),
|
|
60
|
+
]
|
|
61
|
+
if not value
|
|
62
|
+
]
|
|
63
|
+
if missing:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"single-release mode requires release + source_path + scratch_bucket; "
|
|
66
|
+
f"missing: {missing}"
|
|
67
|
+
)
|
|
68
|
+
self._publish_single(
|
|
69
|
+
release=release,
|
|
70
|
+
source_path=source_path,
|
|
71
|
+
scratch_bucket=scratch_bucket,
|
|
72
|
+
extras_bucket=extras_bucket,
|
|
73
|
+
root_href=root_href,
|
|
74
|
+
workers=workers,
|
|
75
|
+
)
|
|
76
|
+
else:
|
|
77
|
+
scratch_bucket = self.get_param("scratch_bucket")
|
|
78
|
+
release_bucket = self.get_param(
|
|
79
|
+
"release_bucket", default=PUBLIC_RELEASE_BUCKET, is_required=False
|
|
80
|
+
)
|
|
81
|
+
self._publish_walk(
|
|
82
|
+
scratch_bucket=scratch_bucket,
|
|
83
|
+
extras_bucket=extras_bucket,
|
|
84
|
+
release_bucket=release_bucket,
|
|
85
|
+
root_href=root_href,
|
|
86
|
+
workers=workers,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
def _publish_single(
|
|
90
|
+
self,
|
|
91
|
+
release: str,
|
|
92
|
+
source_path: str,
|
|
93
|
+
scratch_bucket: str,
|
|
94
|
+
extras_bucket: str,
|
|
95
|
+
root_href: str,
|
|
96
|
+
workers: int,
|
|
97
|
+
) -> None:
|
|
98
|
+
schema = read_schema_version_from_rc_bundle(scratch_bucket, source_path)
|
|
99
|
+
self.log(f"Publishing single release {release} with schema {schema}")
|
|
100
|
+
|
|
101
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
102
|
+
output = Path(tmp) / "stac"
|
|
103
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
104
|
+
build_release_catalog(
|
|
105
|
+
release=release,
|
|
106
|
+
schema_version=schema,
|
|
107
|
+
output=output,
|
|
108
|
+
root_href=root_href,
|
|
109
|
+
workers=workers,
|
|
110
|
+
)
|
|
111
|
+
release_prefix = f"{STAC_S3_PREFIX}{release}/"
|
|
112
|
+
deleted = mirror_directory_to_s3(
|
|
113
|
+
output / release, extras_bucket, release_prefix
|
|
114
|
+
)
|
|
115
|
+
self.log(
|
|
116
|
+
f"Mirrored {release} to s3://{extras_bucket}/{release_prefix}; "
|
|
117
|
+
f"deleted {deleted} orphans within that release"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
def _publish_walk(
|
|
121
|
+
self,
|
|
122
|
+
scratch_bucket: str,
|
|
123
|
+
extras_bucket: str,
|
|
124
|
+
release_bucket: str,
|
|
125
|
+
root_href: str,
|
|
126
|
+
workers: int,
|
|
127
|
+
) -> None:
|
|
128
|
+
known_schemas = read_existing_stac_schemas(extras_bucket, STAC_S3_PREFIX)
|
|
129
|
+
self.log(f"Cached {len(known_schemas)} schemas from existing STAC")
|
|
130
|
+
|
|
131
|
+
releases = list_public_releases(release_bucket)
|
|
132
|
+
self.log(
|
|
133
|
+
f"Discovered {len(releases)} releases in s3://{release_bucket}/release/"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
137
|
+
output = Path(tmp) / "stac"
|
|
138
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
139
|
+
for release in releases:
|
|
140
|
+
schema = known_schemas.get(release)
|
|
141
|
+
if schema is None:
|
|
142
|
+
self.log(
|
|
143
|
+
f"No cached schema for {release}; self-healing from RC bundle in "
|
|
144
|
+
f"s3://{scratch_bucket}"
|
|
145
|
+
)
|
|
146
|
+
schema = read_schema_from_released_rc(scratch_bucket, release)
|
|
147
|
+
self.log(f"Self-healed {release} -> {schema}")
|
|
148
|
+
build_release_catalog(
|
|
149
|
+
release=release,
|
|
150
|
+
schema_version=schema,
|
|
151
|
+
output=output,
|
|
152
|
+
root_href=root_href,
|
|
153
|
+
workers=workers,
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
deleted = mirror_directory_to_s3(output, extras_bucket, STAC_S3_PREFIX)
|
|
157
|
+
self.log(
|
|
158
|
+
f"Mirrored {len(releases)} releases to s3://{extras_bucket}/{STAC_S3_PREFIX}; "
|
|
159
|
+
f"deleted {deleted} orphans"
|
|
160
|
+
)
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""STAC publishing operations. Composed by ``job.py``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import boto3
|
|
10
|
+
import pystac
|
|
11
|
+
from botocore import UNSIGNED
|
|
12
|
+
from botocore.config import Config
|
|
13
|
+
from overture_stac.overture_stac import OvertureRelease
|
|
14
|
+
|
|
15
|
+
PROD_ROOT_HREF = "https://stac.overturemaps.org"
|
|
16
|
+
STAC_S3_PREFIX = "stac/"
|
|
17
|
+
PUBLIC_RELEASE_BUCKET = "overturemaps-us-west-2"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def read_schema_version_from_rc_bundle(
|
|
21
|
+
scratch_bucket: str, input_source_path: str
|
|
22
|
+
) -> str:
|
|
23
|
+
"""Read ``schema_version`` from an RC bundle's ``metadata.json``."""
|
|
24
|
+
key = f"{input_source_path.rstrip('/')}/metadata.json"
|
|
25
|
+
obj = boto3.client("s3").get_object(Bucket=scratch_bucket, Key=key)
|
|
26
|
+
data: dict[str, Any] = json.loads(obj["Body"].read())
|
|
27
|
+
for inp in data.get("cdp_release_candidate_dag", {}).get("inputs", []):
|
|
28
|
+
for section_val in inp.values():
|
|
29
|
+
if isinstance(section_val, dict):
|
|
30
|
+
schema_version = section_val.get("parameters", {}).get("schema_version")
|
|
31
|
+
if schema_version:
|
|
32
|
+
return schema_version
|
|
33
|
+
raise RuntimeError(f"schema_version not found in s3://{scratch_bucket}/{key}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def read_schema_from_released_rc(scratch_bucket: str, release: str) -> str:
|
|
37
|
+
"""Find the RC bundle marked ``released`` for *release* and read its schema.
|
|
38
|
+
|
|
39
|
+
Walks ``release_candidate/release={release}/run=*/`` in *scratch_bucket*,
|
|
40
|
+
picks the run(s) carrying a ``released`` marker (the one that became a real
|
|
41
|
+
release), and returns its schema_version. Raises on any failure — used as a
|
|
42
|
+
LOUD self-heal fallback from walk mode.
|
|
43
|
+
"""
|
|
44
|
+
import botocore.exceptions
|
|
45
|
+
|
|
46
|
+
s3 = boto3.client("s3")
|
|
47
|
+
prefix = f"release_candidate/release={release}/"
|
|
48
|
+
|
|
49
|
+
runs: list[str] = []
|
|
50
|
+
paginator = s3.get_paginator("list_objects_v2")
|
|
51
|
+
for page in paginator.paginate(Bucket=scratch_bucket, Prefix=prefix, Delimiter="/"):
|
|
52
|
+
for cp in page.get("CommonPrefixes", []):
|
|
53
|
+
run = cp["Prefix"][len(prefix) :].rstrip("/")
|
|
54
|
+
if run.startswith("run="):
|
|
55
|
+
runs.append(run)
|
|
56
|
+
|
|
57
|
+
if not runs:
|
|
58
|
+
raise RuntimeError(f"No RC runs found under s3://{scratch_bucket}/{prefix}")
|
|
59
|
+
|
|
60
|
+
released_runs: list[str] = []
|
|
61
|
+
for run in runs:
|
|
62
|
+
try:
|
|
63
|
+
s3.head_object(Bucket=scratch_bucket, Key=f"{prefix}{run}/released")
|
|
64
|
+
released_runs.append(run)
|
|
65
|
+
except botocore.exceptions.ClientError as exc:
|
|
66
|
+
if exc.response["Error"]["Code"] in ("404", "NoSuchKey"):
|
|
67
|
+
continue
|
|
68
|
+
raise
|
|
69
|
+
|
|
70
|
+
if not released_runs:
|
|
71
|
+
raise RuntimeError(
|
|
72
|
+
f"No RC run under s3://{scratch_bucket}/{prefix} has a 'released' marker"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
chosen_run = sorted(released_runs)[-1]
|
|
76
|
+
return read_schema_version_from_rc_bundle(scratch_bucket, f"{prefix}{chosen_run}")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def list_public_releases(
|
|
80
|
+
release_bucket: str = PUBLIC_RELEASE_BUCKET, prefix: str = "release/"
|
|
81
|
+
) -> list[str]:
|
|
82
|
+
"""List release versions currently present under ``s3://{release_bucket}/{prefix}``."""
|
|
83
|
+
s3 = boto3.client("s3", config=Config(signature_version=UNSIGNED))
|
|
84
|
+
releases: list[str] = []
|
|
85
|
+
paginator = s3.get_paginator("list_objects_v2")
|
|
86
|
+
for page in paginator.paginate(Bucket=release_bucket, Prefix=prefix, Delimiter="/"):
|
|
87
|
+
for cp in page.get("CommonPrefixes", []):
|
|
88
|
+
name = cp["Prefix"][len(prefix) :].rstrip("/")
|
|
89
|
+
if name:
|
|
90
|
+
releases.append(name)
|
|
91
|
+
return sorted(releases)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def read_existing_stac_schemas(
|
|
95
|
+
extras_bucket: str, prefix: str = STAC_S3_PREFIX
|
|
96
|
+
) -> dict[str, str]:
|
|
97
|
+
"""Return ``{release: schema_version}`` extracted from any existing per-release
|
|
98
|
+
``catalog.json`` at ``s3://{extras_bucket}/{prefix}<release>/catalog.json``.
|
|
99
|
+
|
|
100
|
+
Missing / unreadable / schema-less catalogs are silently skipped — this
|
|
101
|
+
map is a best-effort cache, not authoritative.
|
|
102
|
+
"""
|
|
103
|
+
s3 = boto3.client("s3")
|
|
104
|
+
schemas: dict[str, str] = {}
|
|
105
|
+
paginator = s3.get_paginator("list_objects_v2")
|
|
106
|
+
for page in paginator.paginate(Bucket=extras_bucket, Prefix=prefix, Delimiter="/"):
|
|
107
|
+
for cp in page.get("CommonPrefixes", []):
|
|
108
|
+
release = cp["Prefix"][len(prefix) :].rstrip("/")
|
|
109
|
+
if not release:
|
|
110
|
+
continue
|
|
111
|
+
try:
|
|
112
|
+
obj = s3.get_object(
|
|
113
|
+
Bucket=extras_bucket, Key=f"{prefix}{release}/catalog.json"
|
|
114
|
+
)
|
|
115
|
+
data = json.loads(obj["Body"].read())
|
|
116
|
+
schema = data.get("schema:version")
|
|
117
|
+
if schema:
|
|
118
|
+
schemas[release] = schema
|
|
119
|
+
except Exception:
|
|
120
|
+
continue
|
|
121
|
+
return schemas
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def build_release_catalog(
|
|
125
|
+
release: str,
|
|
126
|
+
schema_version: str | None,
|
|
127
|
+
output: Path,
|
|
128
|
+
root_href: str = PROD_ROOT_HREF,
|
|
129
|
+
workers: int = 4,
|
|
130
|
+
) -> None:
|
|
131
|
+
"""Build one release's STAC catalog into *output*.
|
|
132
|
+
|
|
133
|
+
TODO: move upstream into ``overture-stac`` — this is the exact call
|
|
134
|
+
sequence in that library's CLI; belongs as a public API there.
|
|
135
|
+
"""
|
|
136
|
+
catalog = OvertureRelease(
|
|
137
|
+
release=release,
|
|
138
|
+
schema=schema_version,
|
|
139
|
+
output=output,
|
|
140
|
+
debug=False,
|
|
141
|
+
)
|
|
142
|
+
catalog.build_release_catalog(
|
|
143
|
+
title=f"{release} Overture Release", max_workers=workers
|
|
144
|
+
)
|
|
145
|
+
catalog.release_catalog.normalize_hrefs(f"{root_href}/{release}/")
|
|
146
|
+
catalog.release_catalog.save(
|
|
147
|
+
catalog_type=pystac.CatalogType.ABSOLUTE_PUBLISHED,
|
|
148
|
+
dest_href=str(output / release),
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def mirror_directory_to_s3(local_root: Path, bucket: str, prefix: str) -> int:
|
|
153
|
+
"""Mirror *local_root* to ``s3://{bucket}/{prefix}``; return orphan-delete count."""
|
|
154
|
+
s3 = boto3.client("s3")
|
|
155
|
+
|
|
156
|
+
local_keys: set[str] = set()
|
|
157
|
+
for path in local_root.rglob("*"):
|
|
158
|
+
if not path.is_file():
|
|
159
|
+
continue
|
|
160
|
+
rel = path.relative_to(local_root).as_posix()
|
|
161
|
+
key = f"{prefix}{rel}"
|
|
162
|
+
local_keys.add(key)
|
|
163
|
+
s3.upload_file(str(path), bucket, key)
|
|
164
|
+
|
|
165
|
+
remote_keys: set[str] = set()
|
|
166
|
+
paginator = s3.get_paginator("list_objects_v2")
|
|
167
|
+
for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
|
|
168
|
+
for obj in page.get("Contents", []):
|
|
169
|
+
remote_keys.add(obj["Key"])
|
|
170
|
+
|
|
171
|
+
orphans = list(remote_keys - local_keys)
|
|
172
|
+
for i in range(0, len(orphans), 1000):
|
|
173
|
+
batch = orphans[i : i + 1000]
|
|
174
|
+
s3.delete_objects(
|
|
175
|
+
Bucket=bucket,
|
|
176
|
+
Delete={"Objects": [{"Key": k} for k in batch]},
|
|
177
|
+
)
|
|
178
|
+
return len(orphans)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: overture-core
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: Shared Overture business logic — portable, framework-agnostic job classes.
|
|
5
|
+
Author: Overture Maps Foundation
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/OvertureMaps/overture-core
|
|
8
|
+
Project-URL: Source, https://github.com/OvertureMaps/overture-core/tree/main/packages/overture_core
|
|
9
|
+
Project-URL: Issues, https://github.com/OvertureMaps/overture-core/issues
|
|
10
|
+
Keywords: overture,overturemaps,stac,gis
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: overture-serverless>=0.1.0
|
|
20
|
+
Requires-Dist: overture-stac
|
|
21
|
+
Requires-Dist: pystac>=1.8.4
|
|
22
|
+
Requires-Dist: boto3>=1.35
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=9.1.1; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov>=7.0.0; extra == "dev"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# overture_core
|
|
29
|
+
|
|
30
|
+
[](https://pypi.org/project/overture-core/)
|
|
31
|
+
[](https://pypi.org/project/overture-core/)
|
|
32
|
+
|
|
33
|
+
Shared, framework-agnostic business logic — portable job classes built on [`overture-serverless`](../overture_serverless).
|
|
34
|
+
|
|
35
|
+
## Jobs
|
|
36
|
+
|
|
37
|
+
- **`stac.job.PublishStac`** — publishes the Overture STAC catalog. Two modes selected by the presence of the `release` param:
|
|
38
|
+
- *single-release* (`release` + `source_path` + `scratch_bucket`): publish just that release, mirror scoped to `stac/{release}/`. Schema comes from the RC bundle's `metadata.json`.
|
|
39
|
+
- *walk* (no `release`, requires `scratch_bucket`): rebuild the catalog for every release currently in the public bucket, mirror the whole `stac/` prefix with delete-orphans. Schema per release comes from the existing STAC (`schema:version` in each `catalog.json`); on cache miss the job self-heals by reading schema from the released RC bundle in `scratch_bucket` and raises if it can't recover.
|
|
40
|
+
|
|
41
|
+
## Future work
|
|
42
|
+
|
|
43
|
+
- **Bundle-owned schema accessor** — when `ReleaseCandidateBundle` becomes publishable, move `read_schema_version_from_rc_bundle` onto it as `resolved_schema_version()`; single-release mode's schema read shrinks to one line.
|
|
44
|
+
- **Upstream `build_release_catalog`** — file a PR against [OvertureMaps/stac](https://github.com/OvertureMaps/stac) exposing this as a public API (their CLI already does the exact call sequence). When it lands, our wrapper collapses.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
.python-version
|
|
2
|
+
LICENSE
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
uv.lock
|
|
6
|
+
overture_core/__init__.py
|
|
7
|
+
overture_core.egg-info/PKG-INFO
|
|
8
|
+
overture_core.egg-info/SOURCES.txt
|
|
9
|
+
overture_core.egg-info/dependency_links.txt
|
|
10
|
+
overture_core.egg-info/requires.txt
|
|
11
|
+
overture_core.egg-info/scm_file_list.json
|
|
12
|
+
overture_core.egg-info/scm_version.json
|
|
13
|
+
overture_core.egg-info/top_level.txt
|
|
14
|
+
overture_core/stac/__init__.py
|
|
15
|
+
overture_core/stac/job.py
|
|
16
|
+
overture_core/stac/publisher.py
|
|
17
|
+
tests/__init__.py
|
|
18
|
+
tests/stac/__init__.py
|
|
19
|
+
tests/stac/test_job.py
|
|
20
|
+
tests/stac/test_publisher.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
{
|
|
2
|
+
"files": [
|
|
3
|
+
".python-version",
|
|
4
|
+
"LICENSE",
|
|
5
|
+
"README.md",
|
|
6
|
+
"overture_core/__init__.py",
|
|
7
|
+
"overture_core/stac/__init__.py",
|
|
8
|
+
"overture_core/stac/job.py",
|
|
9
|
+
"overture_core/stac/publisher.py",
|
|
10
|
+
"pyproject.toml",
|
|
11
|
+
"tests/__init__.py",
|
|
12
|
+
"tests/stac/__init__.py",
|
|
13
|
+
"tests/stac/test_job.py",
|
|
14
|
+
"tests/stac/test_publisher.py",
|
|
15
|
+
"uv.lock"
|
|
16
|
+
]
|
|
17
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
overture_core
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "overture-core"
|
|
3
|
+
dynamic = ["version"]
|
|
4
|
+
description = "Shared Overture business logic — portable, framework-agnostic job classes."
|
|
5
|
+
authors = [
|
|
6
|
+
{name = "Overture Maps Foundation"}
|
|
7
|
+
]
|
|
8
|
+
license = "MIT"
|
|
9
|
+
# Vendored copy of the repo-root LICENSE: setuptools copies license-files
|
|
10
|
+
# sdist entries to the exact relative path given, so "../../LICENSE" landed
|
|
11
|
+
# outside the sdist root and PyPI rejected the upload. A package-local path
|
|
12
|
+
# is the only one that survives packaging.
|
|
13
|
+
license-files = ["LICENSE"]
|
|
14
|
+
readme = "README.md"
|
|
15
|
+
requires-python = ">=3.11"
|
|
16
|
+
keywords = ["overture", "overturemaps", "stac", "gis"]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 4 - Beta",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
]
|
|
24
|
+
dependencies = [
|
|
25
|
+
"overture-serverless>=0.1.0",
|
|
26
|
+
"overture-stac",
|
|
27
|
+
"pystac>=1.8.4",
|
|
28
|
+
"boto3>=1.35",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/OvertureMaps/overture-core"
|
|
33
|
+
Source = "https://github.com/OvertureMaps/overture-core/tree/main/packages/overture_core"
|
|
34
|
+
Issues = "https://github.com/OvertureMaps/overture-core/issues"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
dev = [
|
|
38
|
+
"pytest>=9.1.1",
|
|
39
|
+
"pytest-cov>=7.0.0",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[build-system]
|
|
43
|
+
requires = ["setuptools>=77.0.3", "setuptools_scm>=8.0", "wheel"]
|
|
44
|
+
build-backend = "setuptools.build_meta"
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.packages.find]
|
|
47
|
+
include = ["overture_core*"]
|
|
48
|
+
|
|
49
|
+
[tool.setuptools_scm]
|
|
50
|
+
root = "../.."
|
|
51
|
+
tag_regex = "^overture_core-v(?P<version>.*)$"
|
|
52
|
+
git_describe_command = "git describe --dirty --tags --long --match 'overture_core-v*'"
|
|
53
|
+
# PyPI (and Test PyPI) rejects local version identifiers (the "+g<hash>"
|
|
54
|
+
# suffix setuptools_scm appends between tags), so drop it. Only affects
|
|
55
|
+
# unreleased/dirty builds (e.g. the workflow_dispatch smoke test) — a real
|
|
56
|
+
# release always builds from a clean tag.
|
|
57
|
+
local_scheme = "no-local-version"
|
|
58
|
+
|
|
59
|
+
[tool.uv.sources]
|
|
60
|
+
overture-serverless = { path = "../overture_serverless" }
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
|
|
65
|
+
[tool.coverage.run]
|
|
66
|
+
source = ["overture_core"]
|
|
67
|
+
|
|
68
|
+
[tool.coverage.report]
|
|
69
|
+
fail_under = 95
|
|
70
|
+
show_missing = true
|
|
71
|
+
|
|
72
|
+
[tool.semantic_release]
|
|
73
|
+
commit_parser = "conventional-monorepo"
|
|
74
|
+
tag_format = "overture_core-v{version}"
|
|
75
|
+
allow_zero_version = true
|
|
76
|
+
|
|
77
|
+
# path_filters (relative to this directory, since the release workflow cds
|
|
78
|
+
# here) is what attributes a commit to this package: PSR checks the commit's
|
|
79
|
+
# actual changed files, not its scope. No scope_prefix is set, so a plain
|
|
80
|
+
# `fix: ...`/`feat: ...` with no scope works, as long as the commit touches
|
|
81
|
+
# files under packages/overture_core/.
|
|
82
|
+
[tool.semantic_release.commit_parser_options]
|
|
83
|
+
path_filters = ["."]
|
|
File without changes
|
|
File without changes
|