coglite 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coglite-0.1.0/CHANGELOG.md +24 -0
- coglite-0.1.0/CITATION.cff +19 -0
- coglite-0.1.0/LICENSE +21 -0
- coglite-0.1.0/MANIFEST.in +9 -0
- coglite-0.1.0/PKG-INFO +262 -0
- coglite-0.1.0/README.md +222 -0
- coglite-0.1.0/pyproject.toml +122 -0
- coglite-0.1.0/setup.cfg +4 -0
- coglite-0.1.0/src/coglite/__init__.py +58 -0
- coglite-0.1.0/src/coglite/cli.py +152 -0
- coglite-0.1.0/src/coglite/codecs.py +290 -0
- coglite-0.1.0/src/coglite/cog.py +591 -0
- coglite-0.1.0/src/coglite/errors.py +36 -0
- coglite-0.1.0/src/coglite/geo.py +303 -0
- coglite-0.1.0/src/coglite/io.py +484 -0
- coglite-0.1.0/src/coglite/png.py +80 -0
- coglite-0.1.0/src/coglite/py.typed +0 -0
- coglite-0.1.0/src/coglite/tiff.py +440 -0
- coglite-0.1.0/src/coglite/validate.py +189 -0
- coglite-0.1.0/src/coglite.egg-info/PKG-INFO +262 -0
- coglite-0.1.0/src/coglite.egg-info/SOURCES.txt +35 -0
- coglite-0.1.0/src/coglite.egg-info/dependency_links.txt +1 -0
- coglite-0.1.0/src/coglite.egg-info/entry_points.txt +2 -0
- coglite-0.1.0/src/coglite.egg-info/requires.txt +13 -0
- coglite-0.1.0/src/coglite.egg-info/top_level.txt +1 -0
- coglite-0.1.0/tests/conftest.py +185 -0
- coglite-0.1.0/tests/data/cog_f32_lzw_pred3_bigtiff_nan.tif +0 -0
- coglite-0.1.0/tests/data/cog_u16_lzw_pred2.tif +0 -0
- coglite-0.1.0/tests/data/cog_u8_rgb_mask_deflate.tif +0 -0
- coglite-0.1.0/tests/data/gtiff_i16_be_packbits_planar_strips.tif +0 -0
- coglite-0.1.0/tests/data/make_fixtures.py +104 -0
- coglite-0.1.0/tests/test_api.py +310 -0
- coglite-0.1.0/tests/test_fixtures.py +83 -0
- coglite-0.1.0/tests/test_formats.py +316 -0
- coglite-0.1.0/tests/test_gdal_crossval.py +305 -0
- coglite-0.1.0/tests/test_http.py +234 -0
- coglite-0.1.0/tests/tiffwriter.py +400 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
## [Unreleased]
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-11-09
|
|
8
|
+
### Added
|
|
9
|
+
- `coglite.open()` and the `COG` reader: `profile`, `info()`, `bounds`, `crs_epsg`,
|
|
10
|
+
`transform`, `nodata`, `scales` / `offsets`, `overviews`, `levels`, `read()`,
|
|
11
|
+
`read_mask()`, `read_bbox()` (best-overview selection), `point()`, `tile()` (EPSG:3857
|
|
12
|
+
COGs only) and `validate()`.
|
|
13
|
+
- TIFF / BigTIFF parser: both byte orders, IFD chains, SubIFDs, tiled and stripped
|
|
14
|
+
layouts, chunky and separate planar configuration, overviews, GDAL internal masks,
|
|
15
|
+
sparse blocks, 1/8/16/32/64-bit integer and floating-point samples.
|
|
16
|
+
- GeoTIFF support: ModelPixelScale, ModelTiepoint, ModelTransformation, GeoKeyDirectory
|
|
17
|
+
to EPSG, PixelIsPoint, GDAL_NODATA and GDAL_METADATA scale/offset.
|
|
18
|
+
- Built-in codecs: none, Deflate, LZW (pure Python), PackBits, LZMA; predictors 2 and 3.
|
|
19
|
+
Optional JPEG, WebP, ZSTD and LERC through the `imagecodecs` extra.
|
|
20
|
+
- Range readers: local files (mmap), bytes, HTTP(S) with Range requests, retries,
|
|
21
|
+
timeouts, request coalescing, LRU block cache and a thread pool; `CallableReader`
|
|
22
|
+
and `register_scheme()` for S3 and other storage.
|
|
23
|
+
- COG layout validation modelled on GDAL's `validate_cloud_optimized_geotiff.py`.
|
|
24
|
+
- Command line: `coglite info`, `coglite read` (.npy / .png) and `coglite validate`.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use this software, please cite it as below."
|
|
3
|
+
authors:
|
|
4
|
+
- family-names: "Eminoğlu"
|
|
5
|
+
given-names: "Yusuf"
|
|
6
|
+
email: "yusufeminoglu@gmail.com"
|
|
7
|
+
orcid: "https://orcid.org/0009-0005-6000-2934"
|
|
8
|
+
title: "coglite"
|
|
9
|
+
version: "0.1.0"
|
|
10
|
+
date-released: "2026-11-09"
|
|
11
|
+
license: "MIT"
|
|
12
|
+
repository-code: "https://github.com/YusufEminoglu/coglite_sdk"
|
|
13
|
+
url: "https://pypi.org/project/coglite/"
|
|
14
|
+
abstract: "Pure-Python (numpy only) reader for Cloud-Optimized GeoTIFFs over HTTP(S) range requests or local files: TIFF/BigTIFF parsing, GeoTIFF georeferencing, windowed and overview reads, request coalescing and COG validation without GDAL."
|
|
15
|
+
keywords:
|
|
16
|
+
- cloud-optimized-geotiff
|
|
17
|
+
- geotiff
|
|
18
|
+
- raster
|
|
19
|
+
- gis
|
coglite-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yusuf Eminoğlu
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
coglite-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: coglite
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Read windows, pixels and overviews from Cloud-Optimized GeoTIFFs over HTTP(S) or local files with numpy only - no GDAL, no rasterio.
|
|
5
|
+
Author-email: Yusuf Eminoğlu <yusufeminoglu@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/YusufEminoglu/coglite_sdk
|
|
8
|
+
Project-URL: Repository, https://github.com/YusufEminoglu/coglite_sdk
|
|
9
|
+
Project-URL: Issues, https://github.com/YusufEminoglu/coglite_sdk/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/YusufEminoglu/coglite_sdk/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: cog,cloud-optimized-geotiff,geotiff,tiff,bigtiff,raster,gis,range-request,serverless,pyodide,remote-sensing,web-mercator
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Operating System :: OS Independent
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: GIS
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Image Processing
|
|
23
|
+
Classifier: Intended Audience :: Science/Research
|
|
24
|
+
Classifier: Intended Audience :: Developers
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.9
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
Requires-Dist: numpy>=1.20.0
|
|
30
|
+
Provides-Extra: imagecodecs
|
|
31
|
+
Requires-Dist: imagecodecs>=2021.11.20; extra == "imagecodecs"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
35
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
36
|
+
Requires-Dist: mypy>=1.0.0; extra == "dev"
|
|
37
|
+
Provides-Extra: crossval
|
|
38
|
+
Requires-Dist: rasterio>=1.3.0; extra == "crossval"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# coglite
|
|
42
|
+
|
|
43
|
+
[](https://github.com/YusufEminoglu/coglite_sdk/actions/workflows/ci.yml)
|
|
44
|
+
[](https://pypi.org/project/coglite/)
|
|
45
|
+
[](https://pypi.org/project/coglite/)
|
|
46
|
+
[](LICENSE)
|
|
47
|
+
|
|
48
|
+
**Read windows, pixels and overviews from Cloud-Optimized GeoTIFFs over HTTP(S) or
|
|
49
|
+
local files, with numpy as the only dependency.** No GDAL, no rasterio, no PROJ,
|
|
50
|
+
no compiled extensions.
|
|
51
|
+
|
|
52
|
+
coglite is meant for places where installing GDAL is painful or impossible:
|
|
53
|
+
AWS Lambda / Cloud Functions, slim Docker images, Pyodide / JupyterLite, CI jobs
|
|
54
|
+
and small tools that only need to pull a few tiles out of a large raster.
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
import coglite
|
|
58
|
+
|
|
59
|
+
url = "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/Q/WD/2020/7/S2A_36QWD_20200701_0_L2A/B04.tif"
|
|
60
|
+
with coglite.open(url) as cog:
|
|
61
|
+
print(cog.profile) # width, height, dtype, compression, block size, overviews ...
|
|
62
|
+
print(cog.crs_epsg, cog.bounds)
|
|
63
|
+
arr = cog.read(window=(4096, 4096, 1024, 1024)) # (bands, rows, cols) numpy array
|
|
64
|
+
thumb = cog.read_bbox(cog.bounds, max_size=512) # picks the best overview
|
|
65
|
+
print(thumb.level, thumb.data.shape, thumb.transform)
|
|
66
|
+
print(cog.point(550000, 1850000)) # values at a map coordinate
|
|
67
|
+
print(cog.reader.stats) # requests / bytes actually fetched
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Opening that 10980 x 10980 Sentinel-2 band costs one 16 KiB request; the
|
|
71
|
+
1024 x 1024 window above is one more request of 1.3 MB.
|
|
72
|
+
|
|
73
|
+
## Installation
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
pip install coglite # numpy only
|
|
77
|
+
pip install "coglite[imagecodecs]" # + JPEG, WebP, ZSTD and LERC decoding
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Python 3.9 or newer.
|
|
81
|
+
|
|
82
|
+
## What is supported
|
|
83
|
+
|
|
84
|
+
| Area | Supported |
|
|
85
|
+
| --- | --- |
|
|
86
|
+
| Containers | classic TIFF and BigTIFF, little- and big-endian |
|
|
87
|
+
| Structure | IFD chains, SubIFDs, overviews (`NewSubfileType` reduced resolution), GDAL internal masks (1-bit or 8-bit, per overview), sparse (empty) blocks |
|
|
88
|
+
| Layout | tiled and stripped, `PlanarConfiguration` chunky (pixel interleaved) and separate (band interleaved), any number of bands |
|
|
89
|
+
| Sample types | unsigned and signed integers of 8, 16, 32, 64 bits; float 16/32/64; 1-bit masks |
|
|
90
|
+
| Built-in codecs | none, Deflate/zlib (8 and 32946), LZW (pure Python), PackBits, LZMA (standard library) |
|
|
91
|
+
| Optional codecs | JPEG (incl. YCbCr), WebP, ZSTD, LERC (+Deflate / +ZSTD) through `imagecodecs`; ZSTD also through the standard library on Python 3.14+ |
|
|
92
|
+
| Predictors | 1 (none), 2 (horizontal, integers and floats), 3 (floating point) |
|
|
93
|
+
| GeoTIFF | `ModelPixelScale` + `ModelTiepoint`, `ModelTransformation`, `GeoKeyDirectory` (+ double and ASCII params) to an EPSG code, `PixelIsPoint` half-pixel shift (as GDAL), `GDAL_NODATA` (incl. `nan`), `GDAL_METADATA` scale/offset and items |
|
|
94
|
+
| COG layout | GDAL ghost area (`GDAL_STRUCTURAL_METADATA`), block leader/trailer checks, validation of IFD and data order |
|
|
95
|
+
| I/O | local files (mmap, falls back to seek/read), `bytes`, HTTP(S) via `urllib` with Range requests, presigned URLs, any custom `fetch(offset, length)` |
|
|
96
|
+
|
|
97
|
+
Pixel values are verified to be identical to GDAL's (read through rasterio) for
|
|
98
|
+
every combination above, including overviews and masks; see
|
|
99
|
+
[`tests/test_gdal_crossval.py`](tests/test_gdal_crossval.py).
|
|
100
|
+
|
|
101
|
+
## API
|
|
102
|
+
|
|
103
|
+
### `coglite.open(source, **options) -> COG`
|
|
104
|
+
|
|
105
|
+
`source` may be a path, an `http(s)://` or `file://` URL, `bytes`, or a
|
|
106
|
+
`RangeReader`. HTTP options: `headers`, `timeout` (s, default 30), `retries`
|
|
107
|
+
(default 3), `max_workers` (default 4), `merge_gap` (bytes, default 64 KiB),
|
|
108
|
+
`cache_bytes` (default 32 MiB). `prefetch` (default 16 KiB) is the size of the
|
|
109
|
+
first header request.
|
|
110
|
+
|
|
111
|
+
### `COG`
|
|
112
|
+
|
|
113
|
+
| Member | Description |
|
|
114
|
+
| --- | --- |
|
|
115
|
+
| `width`, `height`, `count`, `dtype` | raster size, band count, numpy dtype name |
|
|
116
|
+
| `transform` | `Affine(a, b, c, d, e, f)` (rasterio order); `.apply(col, row)`, `.inverse()`, `.to_gdal()` |
|
|
117
|
+
| `bounds` | `(left, bottom, right, top)` in the raster CRS |
|
|
118
|
+
| `crs_epsg` | EPSG code from the GeoKeys (None for user-defined CRSs) |
|
|
119
|
+
| `nodata`, `scales`, `offsets`, `metadata`, `band_metadata`, `geokeys` | GDAL metadata |
|
|
120
|
+
| `profile`, `info()` | summary dicts (`info()` is JSON-serialisable) |
|
|
121
|
+
| `overviews`, `levels` | `Level(index, width, height, block_width, block_height, transform, decimation)`; `levels[0]` is full resolution |
|
|
122
|
+
| `has_mask` | an internal mask is present |
|
|
123
|
+
| `read(window=None, bands=None, overview=None, *, masked=False, fill_value=None, apply_scale=False)` | `(bands, rows, cols)` array. `window` is `Window(col_off, row_off, width, height)` or a 4-tuple in pixels of the chosen level; parts outside the image are filled. `bands` is 1-based (an int returns a 2-D array). `overview=0` is the first overview. `masked=True` returns a `numpy.ma.MaskedArray` (internal mask, nodata, outside pixels). `apply_scale=True` returns `value * scale + offset` as float64. |
|
|
124
|
+
| `read_mask(window=None, overview=None)` | uint8 validity mask (255 valid, 0 invalid) |
|
|
125
|
+
| `read_bbox(bbox, crs=None, max_size=1024, bands=None, *, masked=False)` | reads the pixels covering `bbox`, choosing the finest level whose window fits in `max_size` (longest side); decimates with nearest neighbour only when even the coarsest overview is too large. Returns `RasterResult(data, transform, crs_epsg, level, window)` with `.bounds`. |
|
|
126
|
+
| `point(x, y, crs=None, bands=None, overview=None)` | values at a map coordinate, one per band |
|
|
127
|
+
| `tile(z, x, y, tilesize=256, bands=None, *, masked=False, fill_value=None)` | XYZ Web Mercator tile with nearest-neighbour sampling, **only for COGs that are themselves in EPSG:3857** |
|
|
128
|
+
| `validate(check_blocks=True)` | `ValidationReport(errors, warnings, details)`; `.is_valid`, `.to_dict()` |
|
|
129
|
+
| `reader.stats` | `requests`, `bytes_fetched`, `cache_hits`, `ranges_requested`, `retries` |
|
|
130
|
+
|
|
131
|
+
### Coordinate systems: know the limits
|
|
132
|
+
|
|
133
|
+
coglite has no PROJ. `crs=` arguments accept only the raster's own EPSG code or
|
|
134
|
+
the EPSG:4326 / EPSG:3857 pair (converted with the closed-form spherical Mercator
|
|
135
|
+
formulas). Anything else raises `CRSError`; reproject your bbox first, e.g. with
|
|
136
|
+
`pyproj`. `tile()` performs no reprojection, so it only works on EPSG:3857 COGs
|
|
137
|
+
(also 900913, 3785, 102100, 102113), and it never resamples other than nearest
|
|
138
|
+
neighbour. Rotated or sheared rasters can be read with `read()`, but
|
|
139
|
+
`read_bbox()` and `tile()` need a north-up transform.
|
|
140
|
+
|
|
141
|
+
## Range requests, coalescing, caching
|
|
142
|
+
|
|
143
|
+
All block reads of one `read()` call are gathered and handed to the reader in
|
|
144
|
+
one batch. The reader
|
|
145
|
+
|
|
146
|
+
1. returns blocks already in its LRU cache (32 MiB by default),
|
|
147
|
+
2. merges ranges whose gap is at most `merge_gap` bytes (64 KiB by default for
|
|
148
|
+
HTTP) into one request of at most `max_request` bytes (16 MiB),
|
|
149
|
+
3. issues the merged requests concurrently with up to `max_workers` threads
|
|
150
|
+
(4 by default, a polite value for public buckets).
|
|
151
|
+
|
|
152
|
+
COGs store the tiles of a row next to each other, so a typical window costs one
|
|
153
|
+
request per few tile rows, or a single request. Measured against the local test
|
|
154
|
+
server (512 x 512 COG, 32 x 32 tiles):
|
|
155
|
+
|
|
156
|
+
| Read | coalescing off (`merge_gap=-1`) | default |
|
|
157
|
+
| --- | --- | --- |
|
|
158
|
+
| 3 x 3 tile window | 9 requests | 1 request |
|
|
159
|
+
| full image (256 tiles) | 256 requests | 1 request |
|
|
160
|
+
| repeated window | 0 requests (cache) | 0 requests (cache) |
|
|
161
|
+
|
|
162
|
+
The header is probed with one request of `prefetch` bytes; if the IFDs are
|
|
163
|
+
bigger, the buffer grows (at least doubling) instead of issuing many tiny
|
|
164
|
+
requests. GDAL-written COGs keep all IFDs at the start of the file, so opening
|
|
165
|
+
costs one request, and blocks that already arrived with the header buffer are
|
|
166
|
+
never fetched again (a COG smaller than `prefetch` is read with one request).
|
|
167
|
+
|
|
168
|
+
HTTP behaviour: only `GET` with a `Range` header is used (never `HEAD`), so URLs
|
|
169
|
+
presigned for `GET` work unchanged. Timeouts, connection errors, HTTP 429 and
|
|
170
|
+
5xx are retried with exponential backoff; other HTTP errors raise
|
|
171
|
+
`RangeRequestError` at once. A server that ignores `Range` and answers `200` is
|
|
172
|
+
handled by keeping the body in memory.
|
|
173
|
+
|
|
174
|
+
### Plugging in other storage
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
# S3 through boto3 (or anything that can read a byte range)
|
|
178
|
+
import boto3, coglite
|
|
179
|
+
s3 = boto3.client("s3")
|
|
180
|
+
|
|
181
|
+
def fetch(offset, length):
|
|
182
|
+
r = s3.get_object(Bucket="my-bucket", Key="dem.tif",
|
|
183
|
+
Range=f"bytes={offset}-{offset + length - 1}")
|
|
184
|
+
return r["Body"].read()
|
|
185
|
+
|
|
186
|
+
cog = coglite.open(coglite.CallableReader(fetch, name="s3://my-bucket/dem.tif"))
|
|
187
|
+
|
|
188
|
+
# or presign and use plain HTTPS
|
|
189
|
+
url = s3.generate_presigned_url("get_object", Params={"Bucket": "my-bucket", "Key": "dem.tif"})
|
|
190
|
+
cog = coglite.open(url)
|
|
191
|
+
|
|
192
|
+
# or register a URL scheme once
|
|
193
|
+
coglite.register_scheme("s3", lambda url, **kw: coglite.HTTPReader(presign(url), **kw))
|
|
194
|
+
cog = coglite.open("s3://my-bucket/dem.tif")
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Subclass `coglite.RangeReader` and implement `_fetch(offset, length)` (and
|
|
198
|
+
optionally `size`) for anything else; caching, coalescing and concurrency come
|
|
199
|
+
for free.
|
|
200
|
+
|
|
201
|
+
### Pyodide / WebAssembly
|
|
202
|
+
|
|
203
|
+
Threads are not available in Pyodide: pass `max_workers=1`. Synchronous
|
|
204
|
+
`urllib` needs the `pyodide-http` patch (`pyodide_http.patch_all()`), or use a
|
|
205
|
+
`CallableReader` around a synchronous `XMLHttpRequest`, or fetch the file
|
|
206
|
+
yourself and pass the `bytes`.
|
|
207
|
+
|
|
208
|
+
## Command line
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
coglite info https://example.com/cog.tif # profile, bounds, overviews, I/O used
|
|
212
|
+
coglite info cog.tif --json
|
|
213
|
+
coglite read cog.tif --bbox 500000 4090000 501000 4100000 --max-size 512 -o out.png
|
|
214
|
+
coglite read cog.tif --bbox 28.9 41.0 29.1 41.1 --crs EPSG:4326 -o out.npy # 3857 COGs
|
|
215
|
+
coglite read cog.tif --window 0 0 256 256 --overview 1 --bands 3,2,1 -o rgb.png
|
|
216
|
+
coglite validate https://example.com/cog.tif # exit code 1 when not a valid COG
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
PNG output is stretched to 8 bits per band (min/max of valid pixels); masked
|
|
220
|
+
pixels become transparent. `--header "Authorization: Bearer ..."` adds HTTP headers.
|
|
221
|
+
|
|
222
|
+
## COG validation rules
|
|
223
|
+
|
|
224
|
+
`COG.validate()` follows GDAL's `validate_cloud_optimized_geotiff.py`:
|
|
225
|
+
|
|
226
|
+
* errors: images (or overviews) larger than 512 x 512 that are not tiled; the
|
|
227
|
+
first IFD not at the start of the file; IFDs not ordered full resolution then
|
|
228
|
+
decreasing overviews; IFDs located after image data; image data not stored from
|
|
229
|
+
the smallest overview to full resolution; a ghost area declaring
|
|
230
|
+
`KNOWN_INCOMPATIBLE_EDITION=YES`; block leaders/trailers that contradict the
|
|
231
|
+
ghost area; non row-major block order when the ghost area promises it;
|
|
232
|
+
* warnings: no overviews on large images; tile sizes not multiples of 16 or not
|
|
233
|
+
powers of two; non row-major block order; overviews stored in SubIFDs.
|
|
234
|
+
|
|
235
|
+
## Performance notes and limitations
|
|
236
|
+
|
|
237
|
+
* Deflate, LZMA and PackBits decode at native speed. LZW is pure Python
|
|
238
|
+
(about 0.6 s per MiB of noisy data; much faster on smooth data). Prefer Deflate
|
|
239
|
+
or ZSTD when you write COGs for coglite.
|
|
240
|
+
* No resampling beyond nearest neighbour and no reprojection.
|
|
241
|
+
* Not supported: bit depths other than 1/8/16/32/64, palette expansion
|
|
242
|
+
(`ColorMap` is ignored; indices are returned), old-style JPEG (compression 6),
|
|
243
|
+
JPEG-XL, CCITT fax codecs, sample-format mixtures inside one image.
|
|
244
|
+
* Multi-page TIFFs: only the first page and its reduced-resolution pages are used.
|
|
245
|
+
|
|
246
|
+
## Development
|
|
247
|
+
|
|
248
|
+
```bash
|
|
249
|
+
pip install -e ".[dev]"
|
|
250
|
+
pytest # GDAL cross-validation runs when gdal_translate + rasterio exist
|
|
251
|
+
ruff check . && ruff format --check src tests && mypy src
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
`tests/data/` holds four tiny GDAL-written fixtures (78 KB in total, regenerated
|
|
255
|
+
with `tests/data/make_fixtures.py`). Larger cross-validation files are generated
|
|
256
|
+
at test time when `gdal_translate` is available (set `COGLITE_GDAL_TRANSLATE` to
|
|
257
|
+
its path, or `COGLITE_NO_GDAL=1` to skip them). HTTP tests use a local
|
|
258
|
+
standard-library server; the test suite never touches the internet.
|
|
259
|
+
|
|
260
|
+
## License
|
|
261
|
+
|
|
262
|
+
MIT, see [LICENSE](LICENSE). Citation metadata is in [CITATION.cff](CITATION.cff).
|
coglite-0.1.0/README.md
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# coglite
|
|
2
|
+
|
|
3
|
+
[](https://github.com/YusufEminoglu/coglite_sdk/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/coglite/)
|
|
5
|
+
[](https://pypi.org/project/coglite/)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
**Read windows, pixels and overviews from Cloud-Optimized GeoTIFFs over HTTP(S) or
|
|
9
|
+
local files, with numpy as the only dependency.** No GDAL, no rasterio, no PROJ,
|
|
10
|
+
no compiled extensions.
|
|
11
|
+
|
|
12
|
+
coglite is meant for places where installing GDAL is painful or impossible:
|
|
13
|
+
AWS Lambda / Cloud Functions, slim Docker images, Pyodide / JupyterLite, CI jobs
|
|
14
|
+
and small tools that only need to pull a few tiles out of a large raster.
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
import coglite
|
|
18
|
+
|
|
19
|
+
url = "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/36/Q/WD/2020/7/S2A_36QWD_20200701_0_L2A/B04.tif"
|
|
20
|
+
with coglite.open(url) as cog:
|
|
21
|
+
print(cog.profile) # width, height, dtype, compression, block size, overviews ...
|
|
22
|
+
print(cog.crs_epsg, cog.bounds)
|
|
23
|
+
arr = cog.read(window=(4096, 4096, 1024, 1024)) # (bands, rows, cols) numpy array
|
|
24
|
+
thumb = cog.read_bbox(cog.bounds, max_size=512) # picks the best overview
|
|
25
|
+
print(thumb.level, thumb.data.shape, thumb.transform)
|
|
26
|
+
print(cog.point(550000, 1850000)) # values at a map coordinate
|
|
27
|
+
print(cog.reader.stats) # requests / bytes actually fetched
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Opening that 10980 x 10980 Sentinel-2 band costs one 16 KiB request; the
|
|
31
|
+
1024 x 1024 window above is one more request of 1.3 MB.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install coglite # numpy only
|
|
37
|
+
pip install "coglite[imagecodecs]" # + JPEG, WebP, ZSTD and LERC decoding
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Python 3.9 or newer.
|
|
41
|
+
|
|
42
|
+
## What is supported
|
|
43
|
+
|
|
44
|
+
| Area | Supported |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| Containers | classic TIFF and BigTIFF, little- and big-endian |
|
|
47
|
+
| Structure | IFD chains, SubIFDs, overviews (`NewSubfileType` reduced resolution), GDAL internal masks (1-bit or 8-bit, per overview), sparse (empty) blocks |
|
|
48
|
+
| Layout | tiled and stripped, `PlanarConfiguration` chunky (pixel interleaved) and separate (band interleaved), any number of bands |
|
|
49
|
+
| Sample types | unsigned and signed integers of 8, 16, 32, 64 bits; float 16/32/64; 1-bit masks |
|
|
50
|
+
| Built-in codecs | none, Deflate/zlib (8 and 32946), LZW (pure Python), PackBits, LZMA (standard library) |
|
|
51
|
+
| Optional codecs | JPEG (incl. YCbCr), WebP, ZSTD, LERC (+Deflate / +ZSTD) through `imagecodecs`; ZSTD also through the standard library on Python 3.14+ |
|
|
52
|
+
| Predictors | 1 (none), 2 (horizontal, integers and floats), 3 (floating point) |
|
|
53
|
+
| GeoTIFF | `ModelPixelScale` + `ModelTiepoint`, `ModelTransformation`, `GeoKeyDirectory` (+ double and ASCII params) to an EPSG code, `PixelIsPoint` half-pixel shift (as GDAL), `GDAL_NODATA` (incl. `nan`), `GDAL_METADATA` scale/offset and items |
|
|
54
|
+
| COG layout | GDAL ghost area (`GDAL_STRUCTURAL_METADATA`), block leader/trailer checks, validation of IFD and data order |
|
|
55
|
+
| I/O | local files (mmap, falls back to seek/read), `bytes`, HTTP(S) via `urllib` with Range requests, presigned URLs, any custom `fetch(offset, length)` |
|
|
56
|
+
|
|
57
|
+
Pixel values are verified to be identical to GDAL's (read through rasterio) for
|
|
58
|
+
every combination above, including overviews and masks; see
|
|
59
|
+
[`tests/test_gdal_crossval.py`](tests/test_gdal_crossval.py).
|
|
60
|
+
|
|
61
|
+
## API
|
|
62
|
+
|
|
63
|
+
### `coglite.open(source, **options) -> COG`
|
|
64
|
+
|
|
65
|
+
`source` may be a path, an `http(s)://` or `file://` URL, `bytes`, or a
|
|
66
|
+
`RangeReader`. HTTP options: `headers`, `timeout` (s, default 30), `retries`
|
|
67
|
+
(default 3), `max_workers` (default 4), `merge_gap` (bytes, default 64 KiB),
|
|
68
|
+
`cache_bytes` (default 32 MiB). `prefetch` (default 16 KiB) is the size of the
|
|
69
|
+
first header request.
|
|
70
|
+
|
|
71
|
+
### `COG`
|
|
72
|
+
|
|
73
|
+
| Member | Description |
|
|
74
|
+
| --- | --- |
|
|
75
|
+
| `width`, `height`, `count`, `dtype` | raster size, band count, numpy dtype name |
|
|
76
|
+
| `transform` | `Affine(a, b, c, d, e, f)` (rasterio order); `.apply(col, row)`, `.inverse()`, `.to_gdal()` |
|
|
77
|
+
| `bounds` | `(left, bottom, right, top)` in the raster CRS |
|
|
78
|
+
| `crs_epsg` | EPSG code from the GeoKeys (None for user-defined CRSs) |
|
|
79
|
+
| `nodata`, `scales`, `offsets`, `metadata`, `band_metadata`, `geokeys` | GDAL metadata |
|
|
80
|
+
| `profile`, `info()` | summary dicts (`info()` is JSON-serialisable) |
|
|
81
|
+
| `overviews`, `levels` | `Level(index, width, height, block_width, block_height, transform, decimation)`; `levels[0]` is full resolution |
|
|
82
|
+
| `has_mask` | an internal mask is present |
|
|
83
|
+
| `read(window=None, bands=None, overview=None, *, masked=False, fill_value=None, apply_scale=False)` | `(bands, rows, cols)` array. `window` is `Window(col_off, row_off, width, height)` or a 4-tuple in pixels of the chosen level; parts outside the image are filled. `bands` is 1-based (an int returns a 2-D array). `overview=0` is the first overview. `masked=True` returns a `numpy.ma.MaskedArray` (internal mask, nodata, outside pixels). `apply_scale=True` returns `value * scale + offset` as float64. |
|
|
84
|
+
| `read_mask(window=None, overview=None)` | uint8 validity mask (255 valid, 0 invalid) |
|
|
85
|
+
| `read_bbox(bbox, crs=None, max_size=1024, bands=None, *, masked=False)` | reads the pixels covering `bbox`, choosing the finest level whose window fits in `max_size` (longest side); decimates with nearest neighbour only when even the coarsest overview is too large. Returns `RasterResult(data, transform, crs_epsg, level, window)` with `.bounds`. |
|
|
86
|
+
| `point(x, y, crs=None, bands=None, overview=None)` | values at a map coordinate, one per band |
|
|
87
|
+
| `tile(z, x, y, tilesize=256, bands=None, *, masked=False, fill_value=None)` | XYZ Web Mercator tile with nearest-neighbour sampling, **only for COGs that are themselves in EPSG:3857** |
|
|
88
|
+
| `validate(check_blocks=True)` | `ValidationReport(errors, warnings, details)`; `.is_valid`, `.to_dict()` |
|
|
89
|
+
| `reader.stats` | `requests`, `bytes_fetched`, `cache_hits`, `ranges_requested`, `retries` |
|
|
90
|
+
|
|
91
|
+
### Coordinate systems: know the limits
|
|
92
|
+
|
|
93
|
+
coglite has no PROJ. `crs=` arguments accept only the raster's own EPSG code or
|
|
94
|
+
the EPSG:4326 / EPSG:3857 pair (converted with the closed-form spherical Mercator
|
|
95
|
+
formulas). Anything else raises `CRSError`; reproject your bbox first, e.g. with
|
|
96
|
+
`pyproj`. `tile()` performs no reprojection, so it only works on EPSG:3857 COGs
|
|
97
|
+
(also 900913, 3785, 102100, 102113), and it never resamples other than nearest
|
|
98
|
+
neighbour. Rotated or sheared rasters can be read with `read()`, but
|
|
99
|
+
`read_bbox()` and `tile()` need a north-up transform.
|
|
100
|
+
|
|
101
|
+
## Range requests, coalescing, caching
|
|
102
|
+
|
|
103
|
+
All block reads of one `read()` call are gathered and handed to the reader in
|
|
104
|
+
one batch. The reader
|
|
105
|
+
|
|
106
|
+
1. returns blocks already in its LRU cache (32 MiB by default),
|
|
107
|
+
2. merges ranges whose gap is at most `merge_gap` bytes (64 KiB by default for
|
|
108
|
+
HTTP) into one request of at most `max_request` bytes (16 MiB),
|
|
109
|
+
3. issues the merged requests concurrently with up to `max_workers` threads
|
|
110
|
+
(4 by default, a polite value for public buckets).
|
|
111
|
+
|
|
112
|
+
COGs store the tiles of a row next to each other, so a typical window costs one
|
|
113
|
+
request per few tile rows, or a single request. Measured against the local test
|
|
114
|
+
server (512 x 512 COG, 32 x 32 tiles):
|
|
115
|
+
|
|
116
|
+
| Read | coalescing off (`merge_gap=-1`) | default |
|
|
117
|
+
| --- | --- | --- |
|
|
118
|
+
| 3 x 3 tile window | 9 requests | 1 request |
|
|
119
|
+
| full image (256 tiles) | 256 requests | 1 request |
|
|
120
|
+
| repeated window | 0 requests (cache) | 0 requests (cache) |
|
|
121
|
+
|
|
122
|
+
The header is probed with one request of `prefetch` bytes; if the IFDs are
|
|
123
|
+
bigger, the buffer grows (at least doubling) instead of issuing many tiny
|
|
124
|
+
requests. GDAL-written COGs keep all IFDs at the start of the file, so opening
|
|
125
|
+
costs one request, and blocks that already arrived with the header buffer are
|
|
126
|
+
never fetched again (a COG smaller than `prefetch` is read with one request).
|
|
127
|
+
|
|
128
|
+
HTTP behaviour: only `GET` with a `Range` header is used (never `HEAD`), so URLs
|
|
129
|
+
presigned for `GET` work unchanged. Timeouts, connection errors, HTTP 429 and
|
|
130
|
+
5xx are retried with exponential backoff; other HTTP errors raise
|
|
131
|
+
`RangeRequestError` at once. A server that ignores `Range` and answers `200` is
|
|
132
|
+
handled by keeping the body in memory.
|
|
133
|
+
|
|
134
|
+
### Plugging in other storage
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
# S3 through boto3 (or anything that can read a byte range)
|
|
138
|
+
import boto3, coglite
|
|
139
|
+
s3 = boto3.client("s3")
|
|
140
|
+
|
|
141
|
+
def fetch(offset, length):
|
|
142
|
+
r = s3.get_object(Bucket="my-bucket", Key="dem.tif",
|
|
143
|
+
Range=f"bytes={offset}-{offset + length - 1}")
|
|
144
|
+
return r["Body"].read()
|
|
145
|
+
|
|
146
|
+
cog = coglite.open(coglite.CallableReader(fetch, name="s3://my-bucket/dem.tif"))
|
|
147
|
+
|
|
148
|
+
# or presign and use plain HTTPS
|
|
149
|
+
url = s3.generate_presigned_url("get_object", Params={"Bucket": "my-bucket", "Key": "dem.tif"})
|
|
150
|
+
cog = coglite.open(url)
|
|
151
|
+
|
|
152
|
+
# or register a URL scheme once
|
|
153
|
+
coglite.register_scheme("s3", lambda url, **kw: coglite.HTTPReader(presign(url), **kw))
|
|
154
|
+
cog = coglite.open("s3://my-bucket/dem.tif")
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Subclass `coglite.RangeReader` and implement `_fetch(offset, length)` (and
|
|
158
|
+
optionally `size`) for anything else; caching, coalescing and concurrency come
|
|
159
|
+
for free.
|
|
160
|
+
|
|
161
|
+
### Pyodide / WebAssembly
|
|
162
|
+
|
|
163
|
+
Threads are not available in Pyodide: pass `max_workers=1`. Synchronous
|
|
164
|
+
`urllib` needs the `pyodide-http` patch (`pyodide_http.patch_all()`), or use a
|
|
165
|
+
`CallableReader` around a synchronous `XMLHttpRequest`, or fetch the file
|
|
166
|
+
yourself and pass the `bytes`.
|
|
167
|
+
|
|
168
|
+
## Command line
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
coglite info https://example.com/cog.tif # profile, bounds, overviews, I/O used
|
|
172
|
+
coglite info cog.tif --json
|
|
173
|
+
coglite read cog.tif --bbox 500000 4090000 501000 4100000 --max-size 512 -o out.png
|
|
174
|
+
coglite read cog.tif --bbox 28.9 41.0 29.1 41.1 --crs EPSG:4326 -o out.npy # 3857 COGs
|
|
175
|
+
coglite read cog.tif --window 0 0 256 256 --overview 1 --bands 3,2,1 -o rgb.png
|
|
176
|
+
coglite validate https://example.com/cog.tif # exit code 1 when not a valid COG
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
PNG output is stretched to 8 bits per band (min/max of valid pixels); masked
|
|
180
|
+
pixels become transparent. `--header "Authorization: Bearer ..."` adds HTTP headers.
|
|
181
|
+
|
|
182
|
+
## COG validation rules
|
|
183
|
+
|
|
184
|
+
`COG.validate()` follows GDAL's `validate_cloud_optimized_geotiff.py`:
|
|
185
|
+
|
|
186
|
+
* errors: images (or overviews) larger than 512 x 512 that are not tiled; the
|
|
187
|
+
first IFD not at the start of the file; IFDs not ordered full resolution then
|
|
188
|
+
decreasing overviews; IFDs located after image data; image data not stored from
|
|
189
|
+
the smallest overview to full resolution; a ghost area declaring
|
|
190
|
+
`KNOWN_INCOMPATIBLE_EDITION=YES`; block leaders/trailers that contradict the
|
|
191
|
+
ghost area; non row-major block order when the ghost area promises it;
|
|
192
|
+
* warnings: no overviews on large images; tile sizes not multiples of 16 or not
|
|
193
|
+
powers of two; non row-major block order; overviews stored in SubIFDs.
|
|
194
|
+
|
|
195
|
+
## Performance notes and limitations
|
|
196
|
+
|
|
197
|
+
* Deflate, LZMA and PackBits decode at native speed. LZW is pure Python
|
|
198
|
+
(about 0.6 s per MiB of noisy data; much faster on smooth data). Prefer Deflate
|
|
199
|
+
or ZSTD when you write COGs for coglite.
|
|
200
|
+
* No resampling beyond nearest neighbour and no reprojection.
|
|
201
|
+
* Not supported: bit depths other than 1/8/16/32/64, palette expansion
|
|
202
|
+
(`ColorMap` is ignored; indices are returned), old-style JPEG (compression 6),
|
|
203
|
+
JPEG-XL, CCITT fax codecs, sample-format mixtures inside one image.
|
|
204
|
+
* Multi-page TIFFs: only the first page and its reduced-resolution pages are used.
|
|
205
|
+
|
|
206
|
+
## Development
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
pip install -e ".[dev]"
|
|
210
|
+
pytest # GDAL cross-validation runs when gdal_translate + rasterio exist
|
|
211
|
+
ruff check . && ruff format --check src tests && mypy src
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
`tests/data/` holds four tiny GDAL-written fixtures (78 KB in total, regenerated
|
|
215
|
+
with `tests/data/make_fixtures.py`). Larger cross-validation files are generated
|
|
216
|
+
at test time when `gdal_translate` is available (set `COGLITE_GDAL_TRANSLATE` to
|
|
217
|
+
its path, or `COGLITE_NO_GDAL=1` to skip them). HTTP tests use a local
|
|
218
|
+
standard-library server; the test suite never touches the internet.
|
|
219
|
+
|
|
220
|
+
## License
|
|
221
|
+
|
|
222
|
+
MIT, see [LICENSE](LICENSE). Citation metadata is in [CITATION.cff](CITATION.cff).
|