lcfm-shapefiles 0.9.33__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lcfm_shapefiles-0.9.33/.gitattributes +3 -0
- lcfm_shapefiles-0.9.33/.github/copilot-instructions.md +114 -0
- lcfm_shapefiles-0.9.33/.github/workflows/auto-sync-github.yml +133 -0
- lcfm_shapefiles-0.9.33/.github/workflows/auto-sync.yml +148 -0
- lcfm_shapefiles-0.9.33/.github/workflows/format_test.yml +39 -0
- lcfm_shapefiles-0.9.33/.github/workflows/integration_tests.yml +28 -0
- lcfm_shapefiles-0.9.33/.github/workflows/publish_pypi.yml +53 -0
- lcfm_shapefiles-0.9.33/.github/workflows/publish_wheel.yml +68 -0
- lcfm_shapefiles-0.9.33/.gitignore +180 -0
- lcfm_shapefiles-0.9.33/.pre-commit-config.yaml +12 -0
- lcfm_shapefiles-0.9.33/.python-version +1 -0
- lcfm_shapefiles-0.9.33/Jenkinsfile +14 -0
- lcfm_shapefiles-0.9.33/LICENSE +27 -0
- lcfm_shapefiles-0.9.33/LICENSE-DATA +28 -0
- lcfm_shapefiles-0.9.33/PKG-INFO +145 -0
- lcfm_shapefiles-0.9.33/README-auto-sync.md +74 -0
- lcfm_shapefiles-0.9.33/README.md +122 -0
- lcfm_shapefiles-0.9.33/bucket_policy.json +15 -0
- lcfm_shapefiles-0.9.33/ci_cd_policy.json +69 -0
- lcfm_shapefiles-0.9.33/figures/grid.png +0 -0
- lcfm_shapefiles-0.9.33/makefile +63 -0
- lcfm_shapefiles-0.9.33/pyproject.toml +80 -0
- lcfm_shapefiles-0.9.33/shapefiles_sync.svg +4 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/__init__.py +61 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/_version.py +1 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/HRVPP_100p_S2-tiles.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_AOI-tropical_v1.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-reduced-tiles.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles-land_buffered-30km.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2018.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2019.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2020.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2021.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2022.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2023.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2024.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2025.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-1deg.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-30deg.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-3deg-TCD.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-3deg-land_buffered-30km.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-3deg.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_AOI-land_v5.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_AOI-tropical_v3.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_S2-tiles.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_latlon-30deg.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_latlon-3deg-TCD.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_latlon-3deg.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_1p_S2-tiles.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_2p_S2-tiles.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_global.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_10p-random.parquet +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_10p.parquet +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_1p.parquet +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_2p.parquet +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_global.parquet +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_tropics.parquet +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/keys_metadata.json +535 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/unclassified_2020/LCFM_100p_S2-tiles-unclassified.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/unclassified_2020/blocks_global_v12-unclassified.fgb +3 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data_availability.py +335 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/fix_antimeridian.py +74 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/labels.py +199 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/products.py +2233 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/readme.md +10 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/shapefiles.py +470 -0
- lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/tiling.py +132 -0
- lcfm_shapefiles-0.9.33/sync-shapefiles-terrascope.sh +40 -0
- lcfm_shapefiles-0.9.33/tests/pytest.ini +7 -0
- lcfm_shapefiles-0.9.33/tests/test_file.py +50 -0
- lcfm_shapefiles-0.9.33/tests/test_labels.py +54 -0
- lcfm_shapefiles-0.9.33/tests/test_packaging.py +48 -0
- lcfm_shapefiles-0.9.33/tests/test_products.py +637 -0
- lcfm_shapefiles-0.9.33/tests/test_s3_upload.py +133 -0
- lcfm_shapefiles-0.9.33/tests/test_shapefiles.py +416 -0
- lcfm_shapefiles-0.9.33/uv.lock +5492 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# LCFM Shapefiles - AI Coding Agent Instructions
|
|
2
|
+
|
|
3
|
+
## Project Overview
|
|
4
|
+
Python package providing wrapper functionality around FlatGeobuf (`.fgb`) shapefiles for the Land Cover Fraction Map (LCFM) project. The package handles downloading versioned shapefiles from S3 CloudFerro storage with MD5-based caching and provides product specifications in the form of classes. Product classes contain names, layers, data types, etc. and allow generation of metadata for LCFM datasets.
|
|
5
|
+
|
|
6
|
+
**Key pattern**: FlatGeobuf files are NOT stored in the repository due to size. Instead, they are stored in S3 (CloudFerro WAW4-1) with Git LFS pointers in the VM at `/vitodata/vegteam/auxdata/LCFM/shapefiles-lfs`. The package downloads them on-demand with version-based caching.
|
|
7
|
+
|
|
8
|
+
## Architecture & Core Components
|
|
9
|
+
|
|
10
|
+
### 1. Shapefile Management (`shapefiles.py`)
|
|
11
|
+
**Singleton pattern**: `ShapefileCacheManager` manages all shapefile downloads and caching
|
|
12
|
+
- **Cache location hierarchy**: `LCFM_SHAPEFILES_CACHE_HOME` → `XDG_CACHE_HOME/lcfm_shapefiles` → `~/.cache/lcfm_shapefiles`
|
|
13
|
+
- **Version-based caching**: Cache organized by package version (`~/.cache/lcfm_shapefiles/0.8.16/`)
|
|
14
|
+
- **Metadata driven**: `keys_metadata.json` contains S3 metadata (ETags for MD5 verification, version IDs, sizes)
|
|
15
|
+
- **S3 URI pattern**: `https://vito-lcf-shapefiles-waw4-1.s3.waw4-1.cloudferro.com/<filename>?versionId=<version_id>`
|
|
16
|
+
- **Integrity checks**: MD5 checksums verify downloaded files match expected ETags
|
|
17
|
+
|
|
18
|
+
**Typical usage pattern**:
|
|
19
|
+
```python
|
|
20
|
+
from lcfm_shapefiles import get_filepath
|
|
21
|
+
import geopandas as gpd
|
|
22
|
+
|
|
23
|
+
fp = get_filepath("LCFM_1p_S2-tiles.fgb") # Downloads if needed
|
|
24
|
+
gdf = gpd.read_file(fp)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
### 2. Product Abstraction (`products.py`)
|
|
28
|
+
**Registry pattern**: Products register themselves via `@Product.register_product()` decorator
|
|
29
|
+
- **Product hierarchy**: `Product` (abstract) → `LOSProduct`, `LSFProduct`, `LSCProduct`, `LCM10Product`, `LCM100Product`, etc.
|
|
30
|
+
- **Band-based routing**: Products register with band prefixes (e.g., `"LCM10:MAP"` for categorical map data)
|
|
31
|
+
- **Factory method**: `Product.create_product(product_name, version, band, year)` routes to correct subclass
|
|
32
|
+
- **Tile coordinate systems**: UTM zones determined from Sentinel-2 MGRS tile IDs (e.g., `34UCV` → EPSG:32634)
|
|
33
|
+
|
|
34
|
+
**Key functions**:
|
|
35
|
+
- `tile_to_epsg(tile)`: Convert S2 tile ID to EPSG code (row letter determines hemisphere)
|
|
36
|
+
- `utm_to_epsg(utm)`: Convert UTM zone string to EPSG code
|
|
37
|
+
|
|
38
|
+
### 3. Label Definitions (`labels.py`)
|
|
39
|
+
Enums defining land cover class labels, colors, and names for visualization
|
|
40
|
+
- `LCM10LabelsColors`: 10-meter resolution land cover classes (tree, shrub, grass, crop, wetland, etc.)
|
|
41
|
+
- Color schemes use normalized RGB arrays for matplotlib compatibility
|
|
42
|
+
|
|
43
|
+
### 4. Antimeridian Handling (`fix_antimeridian.py`)
|
|
44
|
+
Utility to split polygons crossing longitude ±180°
|
|
45
|
+
- `intersects_antimeridian()`: Detects large longitude jumps (>100° threshold)
|
|
46
|
+
- `fix_antimeridian_split()`: Splits into MultiPolygon with normalized coordinates
|
|
47
|
+
|
|
48
|
+
## Development Workflow
|
|
49
|
+
|
|
50
|
+
### Environment Setup
|
|
51
|
+
```bash
|
|
52
|
+
uv sync # Build venv with dependencies
|
|
53
|
+
uv run pytest # Run tests
|
|
54
|
+
uv run ruff format . # Format code (not ruff check!)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
**Important**: Use `uv` (not pip/conda) for all dependency management. Dev dependencies include geopandas, pandas[pyarrow], pytest, ruff.
|
|
58
|
+
|
|
59
|
+
### Testing
|
|
60
|
+
- Tests in `tests/` validate shapefile downloads (`test_shapefiles.py`), product creation (`test_products.py`), and labels (`test_labels.py`)
|
|
61
|
+
- Tests use `tmp_path` fixture and set `LCFM_SHAPEFILES_CACHE_HOME` to isolate cache
|
|
62
|
+
- **Critical**: Tests verify checksum validation and cached file integrity
|
|
63
|
+
- **When adding new Product classes**: Add corresponding tests to `tests/test_file.py`
|
|
64
|
+
|
|
65
|
+
### Deploying New/Updated Shapefiles
|
|
66
|
+
1. Add/update `.fgb` files in `src/lcfm_shapefiles/data/`
|
|
67
|
+
2. Update tests accordingly
|
|
68
|
+
3. Bump version in `pyproject.toml` → `src/lcfm_shapefiles/_version.py`
|
|
69
|
+
4. Push to `master` → if build passes, trigger promotion job
|
|
70
|
+
5. Auto-sync workflow uploads to S3 and updates VM cache
|
|
71
|
+
|
|
72
|
+
## Project-Specific Conventions
|
|
73
|
+
|
|
74
|
+
### File Naming
|
|
75
|
+
**Shapefiles**: `LCFM_<percent>_<grid/AOI>-<granularity>_v<version>.fgb`
|
|
76
|
+
- Example: `LCFM_100p_S2-tiles.fgb`, `LCFM_10p_latlon-3deg-TCD.fgb`
|
|
77
|
+
- Version tags removed in package to avoid conflicts with semantic versioning
|
|
78
|
+
|
|
79
|
+
### Version String Format
|
|
80
|
+
Products expect version as `vXXX` (3 digits), e.g., `v001`, `v127`. Constructor warns if format violated.
|
|
81
|
+
|
|
82
|
+
### Scripts Directory
|
|
83
|
+
`scripts/` contains data processing utilities (not part of package distribution):
|
|
84
|
+
- Block extraction/manipulation (`add_blocks.py`, `extract_block.py`)
|
|
85
|
+
- Tile selection/comparison (`check_tiles.py`, `select_tiles.py`)
|
|
86
|
+
- File format conversion (`compare_fgb_parquet_columns.py`)
|
|
87
|
+
|
|
88
|
+
These scripts often import from package: `from lcfm_shapefiles import get_filepath`
|
|
89
|
+
|
|
90
|
+
### Parquet Blocks
|
|
91
|
+
Since `v0.8.7`: Parquet-format grid blocks in `src/lcfm_shapefiles/data/blocks_parquet/`
|
|
92
|
+
- Source: `/vitodata/vegteam/auxdata/grid/blocks_global/` on Terrascope
|
|
93
|
+
|
|
94
|
+
### Blocks Grid Structure
|
|
95
|
+
`blocks_global.fgb` and `blocks_global.parquet` are **not** one row per tile — they contain one row per **block** (sub-tile subdivision), ~1.4M rows total.
|
|
96
|
+
- The unique `tile` column values in blocks files must match the tile IDs in `LCFM_100p_S2-tiles.fgb` exactly (currently **18,422 tiles** each)
|
|
97
|
+
- Columns: `tile_block_id`, `tile`, `block_id`, `epsg`, `landtype`, `antimeridian`, `xmin`, `ymin`, `xmax`, `ymax`
|
|
98
|
+
- `blocks_global.parquet` has no geometry column; `blocks_global.fgb` has geometry
|
|
99
|
+
- To verify consistency: compare `set(blocks['tile'])` against `set(s2_tiles['tile'])` — row counts will differ by design
|
|
100
|
+
|
|
101
|
+
## Critical Gotchas
|
|
102
|
+
|
|
103
|
+
1. **FGB files excluded from wheel**: `pyproject.toml` excludes `*.fgb` from built wheel to reduce package size
|
|
104
|
+
2. **Git LFS required**: On Terrascope VM, Git LFS pulls actual files; elsewhere package downloads from S3
|
|
105
|
+
3. **Cache invalidation**: Cache organized by package version; upgrading package creates new cache directory
|
|
106
|
+
4. **EPSG code logic**: S2 tile rows C-M are southern hemisphere (326XX), N-X are northern (327XX)
|
|
107
|
+
5. **Product registry**: Subclasses must define `product_name` ClassVar or `@register()` raises ValueError
|
|
108
|
+
6. **Metadata updates**: Call `update_cache_metadata()` to refresh `keys_metadata.json` from S3
|
|
109
|
+
|
|
110
|
+
## External Dependencies
|
|
111
|
+
|
|
112
|
+
- **S3 bucket**: `vito-lcf-shapefiles-waw4-1` on CloudFerro WAW4-1 (Warsaw region)
|
|
113
|
+
- **Artifactory**: `https://artifactory.vgt.vito.be/artifactory/api/pypi/python-packages-public/`
|
|
114
|
+
- **Terrascope paths**: `/vitodata/vegteam/auxdata/LCFM/shapefiles-lfs`, `/vitodata/vegteam_vol2/data/lcfm/shapefiles`
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
name: Auto-Sync LFS Files Github
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches:
|
|
6
|
+
- master # Change this to your default branch if different
|
|
7
|
+
paths:
|
|
8
|
+
- 'src/lcfm_shapefiles/data/**' # Only trigger when files in this path change
|
|
9
|
+
workflow_dispatch: # Allows manual triggering of the workflow
|
|
10
|
+
|
|
11
|
+
env:
|
|
12
|
+
SHAPEFILES_BUCKET: vito-lcf-shapefiles-waw4-1
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
sync:
|
|
16
|
+
name: Sync repository on VM cluster and S3
|
|
17
|
+
runs-on: ubuntu-latest # This will use your self-hosted runner
|
|
18
|
+
environment: CF_S3_ACCESS
|
|
19
|
+
env:
|
|
20
|
+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
21
|
+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
22
|
+
AWS_ENDPOINT_URL: ${{vars.AWS_ENDPOINT_URL}}
|
|
23
|
+
AWS_REGION: ${{vars.AWS_REGION}}
|
|
24
|
+
permissions:
|
|
25
|
+
contents: write
|
|
26
|
+
steps:
|
|
27
|
+
- name: Checkout repository
|
|
28
|
+
uses: actions/checkout@v4
|
|
29
|
+
with:
|
|
30
|
+
lfs: true
|
|
31
|
+
path: src/lcfm_shapefiles/data
|
|
32
|
+
|
|
33
|
+
- name: Configure AWS CLI for single-part uploads
|
|
34
|
+
run: |
|
|
35
|
+
# Set multipart threshold to 5GB to ensure files are uploaded as single objects
|
|
36
|
+
# This preserves MD5 ETags for integrity validation
|
|
37
|
+
aws configure set default.s3.multipart_threshold 5GB
|
|
38
|
+
aws configure set default.s3.multipart_chunksize 5GB
|
|
39
|
+
|
|
40
|
+
- name: push_s3
|
|
41
|
+
id: push_s3
|
|
42
|
+
working-directory: src/lcfm_shapefiles/data
|
|
43
|
+
run: |
|
|
44
|
+
|
|
45
|
+
echo "Starting intelligent S3 sync with ETag comparison..."
|
|
46
|
+
|
|
47
|
+
# Flag file to track if any uploads occurred (needed because while loop runs in subshell)
|
|
48
|
+
UPLOAD_FLAG=$(mktemp)
|
|
49
|
+
|
|
50
|
+
# Function to calculate MD5 hash of local file
|
|
51
|
+
calculate_local_etag() {
|
|
52
|
+
local file="$1"
|
|
53
|
+
md5sum "$file" | cut -d' ' -f1
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
# Function to get S3 object ETag
|
|
57
|
+
get_s3_etag() {
|
|
58
|
+
local s3_key="$1"
|
|
59
|
+
aws s3api head-object --bucket ${{ env.SHAPEFILES_BUCKET }} --key "$s3_key" --query 'ETag' --output text 2>/dev/null | tr -d '"' || echo ""
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
# Counters for statistics
|
|
63
|
+
uploaded_count=0
|
|
64
|
+
skipped_count=0
|
|
65
|
+
total_count=0
|
|
66
|
+
|
|
67
|
+
# Find all files to sync (excluding hidden files and directories)
|
|
68
|
+
find src/lcfm_shapefiles/data -type f ! -path '*/.*' ! -name 'keys_metadata.json' | while read -r file; do
|
|
69
|
+
# Extract s3_key as path relative to src/lcfm_shapefiles/data/
|
|
70
|
+
s3_key="${file#src/lcfm_shapefiles/data/}"
|
|
71
|
+
|
|
72
|
+
total_count=$((total_count + 1))
|
|
73
|
+
|
|
74
|
+
echo "Processing: $s3_key"
|
|
75
|
+
|
|
76
|
+
# Calculate local file ETag
|
|
77
|
+
local_etag=$(calculate_local_etag "$file")
|
|
78
|
+
|
|
79
|
+
# Get S3 object ETag
|
|
80
|
+
s3_etag=$(get_s3_etag "$s3_key")
|
|
81
|
+
|
|
82
|
+
if [ -z "$s3_etag" ]; then
|
|
83
|
+
echo " → File doesn't exist in S3, uploading..."
|
|
84
|
+
aws s3 cp "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
|
|
85
|
+
echo "uploaded" >> "$UPLOAD_FLAG"
|
|
86
|
+
echo " ✓ Uploaded"
|
|
87
|
+
elif [ "$local_etag" != "$s3_etag" ]; then
|
|
88
|
+
echo " → ETags differ (local: $local_etag, S3: $s3_etag), uploading..."
|
|
89
|
+
aws s3 cp "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
|
|
90
|
+
echo "uploaded" >> "$UPLOAD_FLAG"
|
|
91
|
+
echo " ✓ Updated"
|
|
92
|
+
else
|
|
93
|
+
echo " → ETags match, skipping upload"
|
|
94
|
+
echo " ⏭ Skipped"
|
|
95
|
+
fi
|
|
96
|
+
done
|
|
97
|
+
|
|
98
|
+
# Set output flag based on whether any files were uploaded
|
|
99
|
+
if [ -s "$UPLOAD_FLAG" ]; then
|
|
100
|
+
echo "files_uploaded=true" >> $GITHUB_OUTPUT
|
|
101
|
+
echo "Files were uploaded - metadata update required"
|
|
102
|
+
else
|
|
103
|
+
echo "files_uploaded=false" >> $GITHUB_OUTPUT
|
|
104
|
+
echo "No files uploaded - skipping metadata update"
|
|
105
|
+
fi
|
|
106
|
+
rm -f "$UPLOAD_FLAG"
|
|
107
|
+
|
|
108
|
+
- name: Update S3 metadata and commit
|
|
109
|
+
if: steps.push_s3.outputs.files_uploaded == 'true'
|
|
110
|
+
working-directory: src/lcfm_shapefiles/data
|
|
111
|
+
run: |
|
|
112
|
+
echo "Fetching S3 metadata for latest object versions..."
|
|
113
|
+
|
|
114
|
+
# Get all object versions from S3 and filter for latest versions only
|
|
115
|
+
aws s3api list-object-versions \
|
|
116
|
+
--endpoint-url ${{ vars.AWS_ENDPOINT_URL }} \
|
|
117
|
+
--bucket ${{ env.SHAPEFILES_BUCKET }} \
|
|
118
|
+
--query 'Versions[?IsLatest==`true`]' \
|
|
119
|
+
--output json > src/lcfm_shapefiles/data/keys_metadata.json
|
|
120
|
+
|
|
121
|
+
# Configure git identity for the commit
|
|
122
|
+
git config --local user.email "actions@github.com"
|
|
123
|
+
git config --local user.name "GitHub Actions"
|
|
124
|
+
|
|
125
|
+
# Commit and push the updated metadata file if there are changes
|
|
126
|
+
git add src/lcfm_shapefiles/data/keys_metadata.json
|
|
127
|
+
git commit -m "Update S3 keys metadata [skip ci]"
|
|
128
|
+
git push origin ${{ github.ref_name }}
|
|
129
|
+
echo "✓ Updated keys_metadata.json with latest S3 object versions"
|
|
130
|
+
aws s3 cp src/lcfm_shapefiles/data/keys_metadata.json "s3://${{ env.SHAPEFILES_BUCKET }}/keys_metadata.json"
|
|
131
|
+
echo "✓ Pushed keys_metadata.json to s3://${{ env.SHAPEFILES_BUCKET }}/keys_metadata.json"
|
|
132
|
+
|
|
133
|
+
echo "Metadata sync completed at $(date)"
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
name: Auto-Sync LFS Files Terrascope
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch: # Allows manual triggering of the workflow
|
|
5
|
+
|
|
6
|
+
env:
|
|
7
|
+
SHAPEFILES_BUCKET: vito-lcf-shapefiles-waw4-1
|
|
8
|
+
TERRASCOPE_PATH: /vitodata/vegteam/auxdata/LCFM/shapefiles-lfs/src/lcfm_shapefiles/data
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
sync:
|
|
12
|
+
name: Sync repository on VM cluster and S3
|
|
13
|
+
runs-on: self-hosted # This will use your self-hosted runner
|
|
14
|
+
steps:
|
|
15
|
+
- name: Pull latest changes
|
|
16
|
+
run: |
|
|
17
|
+
eval "$(ssh-agent -s)"
|
|
18
|
+
ssh-add ~/Private/runner/.ssh/github_deploy_key
|
|
19
|
+
|
|
20
|
+
cd /vitodata/vegteam/auxdata/LCFM/shapefiles-lfs
|
|
21
|
+
git checkout ${{ github.ref_name }} # Change this to your default branch if different
|
|
22
|
+
|
|
23
|
+
git remote set-url origin git@github.com:VITO-RS-Vegetation/lcfm-shapefiles.git
|
|
24
|
+
git fetch origin
|
|
25
|
+
git pull origin ${{ github.ref_name }} # Change this to your default branch if different
|
|
26
|
+
|
|
27
|
+
# If using LFS, also pull LFS objects
|
|
28
|
+
if [ -f ".gitattributes" ] && grep -q "filter=lfs" .gitattributes; then
|
|
29
|
+
git lfs pull
|
|
30
|
+
fi
|
|
31
|
+
|
|
32
|
+
echo "Repository successfully synced at $(date)"
|
|
33
|
+
|
|
34
|
+
- name: push_s3
|
|
35
|
+
env:
|
|
36
|
+
AWS_CLI_S3_MULTIPART_THRESHOLD: 5GB
|
|
37
|
+
run: |
|
|
38
|
+
cd ${{ env.TERRASCOPE_PATH }}
|
|
39
|
+
|
|
40
|
+
echo "Starting intelligent S3 sync with ETag comparison..."
|
|
41
|
+
|
|
42
|
+
# Function to calculate MD5 hash of local file
|
|
43
|
+
calculate_local_etag() {
|
|
44
|
+
local file="$1"
|
|
45
|
+
md5sum "$file" | cut -d' ' -f1
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
# Function to get S3 object ETag
|
|
49
|
+
get_s3_etag() {
|
|
50
|
+
local s3_key="$1"
|
|
51
|
+
aws s3api head-object --profile cf --bucket ${{ env.SHAPEFILES_BUCKET }} --key "$s3_key" --query 'ETag' --output text 2>/dev/null | tr -d '"' || echo ""
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
# Counters for statistics
|
|
55
|
+
uploaded_count=0
|
|
56
|
+
skipped_count=0
|
|
57
|
+
total_count=0
|
|
58
|
+
|
|
59
|
+
# Find all files to sync (excluding hidden files and directories)
|
|
60
|
+
find . -type f \( ! -path '*/.*' -o -path './blocks_parquet/*' \) | while read -r file; do
|
|
61
|
+
# Remove leading ./ from file path
|
|
62
|
+
relative_path="${file#./}"
|
|
63
|
+
s3_key="$relative_path"
|
|
64
|
+
|
|
65
|
+
total_count=$((total_count + 1))
|
|
66
|
+
|
|
67
|
+
echo "Processing: $relative_path"
|
|
68
|
+
|
|
69
|
+
# Calculate local file ETag
|
|
70
|
+
local_etag=$(calculate_local_etag "$file")
|
|
71
|
+
|
|
72
|
+
# Get S3 object ETag
|
|
73
|
+
s3_etag=$(get_s3_etag "$s3_key")
|
|
74
|
+
|
|
75
|
+
if [ -z "$s3_etag" ]; then
|
|
76
|
+
echo " → File doesn't exist in S3, uploading..."
|
|
77
|
+
aws s3 cp --profile cf "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
|
|
78
|
+
uploaded_count=$((uploaded_count + 1))
|
|
79
|
+
echo " ✓ Uploaded"
|
|
80
|
+
elif [ "$local_etag" != "$s3_etag" ]; then
|
|
81
|
+
echo " → ETags differ (local: $local_etag, S3: $s3_etag), uploading..."
|
|
82
|
+
aws s3 cp --profile cf "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
|
|
83
|
+
uploaded_count=$((uploaded_count + 1))
|
|
84
|
+
echo " ✓ Updated"
|
|
85
|
+
else
|
|
86
|
+
echo " → ETags match, skipping upload"
|
|
87
|
+
skipped_count=$((skipped_count + 1))
|
|
88
|
+
echo " ⏭ Skipped"
|
|
89
|
+
fi
|
|
90
|
+
done
|
|
91
|
+
|
|
92
|
+
# Also sync the file_timestamps.json specifically
|
|
93
|
+
timestamps_file="file_timestamps.json"
|
|
94
|
+
if [ -f "$timestamps_file" ]; then
|
|
95
|
+
echo "Processing timestamps file: $timestamps_file"
|
|
96
|
+
local_etag=$(calculate_local_etag "$timestamps_file")
|
|
97
|
+
s3_etag=$(get_s3_etag "$timestamps_file")
|
|
98
|
+
|
|
99
|
+
if [ -z "$s3_etag" ] || [ "$local_etag" != "$s3_etag" ]; then
|
|
100
|
+
echo " → Uploading timestamps file..."
|
|
101
|
+
aws s3 cp --profile cf "$timestamps_file" "s3://${{ env.SHAPEFILES_BUCKET }}/$timestamps_file"
|
|
102
|
+
echo " ✓ Timestamps file uploaded"
|
|
103
|
+
else
|
|
104
|
+
echo " → Timestamps file unchanged, skipping"
|
|
105
|
+
fi
|
|
106
|
+
fi
|
|
107
|
+
|
|
108
|
+
echo ""
|
|
109
|
+
echo "=== Sync Summary ==="
|
|
110
|
+
echo "Files uploaded: $uploaded_count"
|
|
111
|
+
echo "Files skipped: $skipped_count"
|
|
112
|
+
echo "Total files processed: $total_count"
|
|
113
|
+
echo "Bucket: ${{ env.SHAPEFILES_BUCKET }}"
|
|
114
|
+
echo "Sync completed at $(date)"
|
|
115
|
+
|
|
116
|
+
- name: Update S3 metadata and commit
|
|
117
|
+
run: |
|
|
118
|
+
eval "$(ssh-agent -s)"
|
|
119
|
+
ssh-add ~/Private/runner/.ssh/github_deploy_key
|
|
120
|
+
|
|
121
|
+
cd /vitodata/vegteam/auxdata/LCFM/shapefiles-lfs
|
|
122
|
+
|
|
123
|
+
echo "Fetching S3 metadata for latest object versions..."
|
|
124
|
+
|
|
125
|
+
# Get all object versions from S3 and filter for latest versions only
|
|
126
|
+
aws s3api list-object-versions \
|
|
127
|
+
--profile cf \
|
|
128
|
+
--endpoint-url https://s3.waw4-1.cloudferro.com \
|
|
129
|
+
--bucket ${{ env.SHAPEFILES_BUCKET }} \
|
|
130
|
+
--query 'Versions[?IsLatest==`true`]' \
|
|
131
|
+
--output json > ${{ env.TERRASCOPE_PATH}}/keys_metadata.json
|
|
132
|
+
|
|
133
|
+
# Configure git identity for the commit
|
|
134
|
+
git config --local user.email "actions@github.com"
|
|
135
|
+
git config --local user.name "GitHub Actions"
|
|
136
|
+
|
|
137
|
+
# Commit and push the updated metadata file if there are changes
|
|
138
|
+
git add src/lcfm_shapefiles/data/keys_metadata.json
|
|
139
|
+
if ! git diff --quiet --staged; then
|
|
140
|
+
git commit -m "Update S3 keys metadata [skip ci]"
|
|
141
|
+
git push origin ${{ github.ref_name }}
|
|
142
|
+
echo "✓ Updated keys_metadata.json with latest S3 object versions"
|
|
143
|
+
else
|
|
144
|
+
echo "→ No changes to keys_metadata.json"
|
|
145
|
+
fi
|
|
146
|
+
|
|
147
|
+
echo "Metadata sync completed at $(date)"
|
|
148
|
+
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: vegorg workflow
|
|
2
|
+
|
|
3
|
+
on: #add your hook triggers in the array [push, pull, merge]
|
|
4
|
+
push: # Trigger only on push events
|
|
5
|
+
branches:
|
|
6
|
+
- dev # Trigger only on pushes to the 'main' branch
|
|
7
|
+
- master
|
|
8
|
+
pull_request: # Trigger only on pull request events
|
|
9
|
+
branches:
|
|
10
|
+
- dev
|
|
11
|
+
- master # Trigger only on pull requests targeting the 'main' branch
|
|
12
|
+
workflow_dispatch: # Allow manual triggering of the workflow
|
|
13
|
+
|
|
14
|
+
env:
|
|
15
|
+
token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GITHUB_TOKEN for authentication
|
|
16
|
+
UV_INDEX_VITO_ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
|
|
17
|
+
UV_INDEX_VITO_ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
format-lint:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
- uses: astral-sh/ruff-action@v3
|
|
25
|
+
with:
|
|
26
|
+
args: "format --check --diff"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
check-uv-lock:
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/checkout@v4
|
|
33
|
+
- uses: astral-sh/setup-uv@v5
|
|
34
|
+
with:
|
|
35
|
+
version: "0.8.x"
|
|
36
|
+
python-version: "3.11"
|
|
37
|
+
- name: Check uv lock
|
|
38
|
+
run: uv lock --check
|
|
39
|
+
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: integration tests
|
|
2
|
+
|
|
3
|
+
on: #add your hook triggers in the array [push, pull, merge]
|
|
4
|
+
pull_request: # Trigger only on pull request events
|
|
5
|
+
branches:
|
|
6
|
+
- master
|
|
7
|
+
workflow_dispatch: # Allow manual triggering of the workflow
|
|
8
|
+
|
|
9
|
+
env:
|
|
10
|
+
token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GITHUB_TOKEN for authentication
|
|
11
|
+
UV_INDEX_VITO_ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
|
|
12
|
+
UV_INDEX_VITO_ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
integration-tests:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- name: Checkout repository
|
|
19
|
+
uses: actions/checkout@v4
|
|
20
|
+
with:
|
|
21
|
+
lfs: true
|
|
22
|
+
|
|
23
|
+
- name: Setup build environment
|
|
24
|
+
uses: VITO-RS-Vegetation/veg-template@v2 # Reuse the composite action
|
|
25
|
+
with:
|
|
26
|
+
uv-args: "--all-groups -vv"
|
|
27
|
+
- name: Run tests
|
|
28
|
+
run: uv run pytest tests -m "not s3"
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# Publishes lcfm-shapefiles to the public PyPI (pypi.org).
|
|
2
|
+
# Separate from publish_wheel.yml (Artifactory) - manual trigger only.
|
|
3
|
+
name: Publish to PyPI
|
|
4
|
+
on:
|
|
5
|
+
workflow_dispatch:
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
release-build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- name: Setup build environment
|
|
18
|
+
uses: VITO-RS-Vegetation/veg-template@v2
|
|
19
|
+
with:
|
|
20
|
+
uv-args: "--all-groups -vv"
|
|
21
|
+
|
|
22
|
+
- name: Build release distributions
|
|
23
|
+
run: uv build --wheel --sdist
|
|
24
|
+
|
|
25
|
+
- name: Upload distributions
|
|
26
|
+
uses: actions/upload-artifact@v4
|
|
27
|
+
with:
|
|
28
|
+
name: pypi-dists
|
|
29
|
+
path: dist/
|
|
30
|
+
|
|
31
|
+
pypi-publish:
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
needs:
|
|
34
|
+
- release-build
|
|
35
|
+
environment:
|
|
36
|
+
name: pypi
|
|
37
|
+
permissions:
|
|
38
|
+
id-token: write # required for trusted publishing (OIDC), no token secrets needed
|
|
39
|
+
|
|
40
|
+
steps:
|
|
41
|
+
- name: Retrieve release distributions
|
|
42
|
+
uses: actions/download-artifact@v4
|
|
43
|
+
with:
|
|
44
|
+
name: pypi-dists
|
|
45
|
+
path: dist/
|
|
46
|
+
|
|
47
|
+
- name: Install uv
|
|
48
|
+
uses: astral-sh/setup-uv@v5
|
|
49
|
+
with:
|
|
50
|
+
version: "0.8.x"
|
|
51
|
+
|
|
52
|
+
- name: Publish to PyPI
|
|
53
|
+
run: uv publish --trusted-publishing automatic
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Uploads veg-workflows wheel to vito public artifactory
|
|
2
|
+
|
|
3
|
+
name: Publish Wheel
|
|
4
|
+
on:
|
|
5
|
+
release:
|
|
6
|
+
types: [published]
|
|
7
|
+
|
|
8
|
+
workflow_dispatch: # Allow manual triggering of the workflow (optional)
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
|
|
13
|
+
env:
|
|
14
|
+
token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GITHUB_TOKEN for authentication
|
|
15
|
+
UV_INDEX_VITO_ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
|
|
16
|
+
UV_INDEX_VITO_ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
release-build:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
|
|
25
|
+
- name: Setup build environment
|
|
26
|
+
uses: VITO-RS-Vegetation/veg-template@v2 # Reuse the composite action
|
|
27
|
+
with:
|
|
28
|
+
uv-args: "--all-groups -vv"
|
|
29
|
+
|
|
30
|
+
- name: Build release distributions
|
|
31
|
+
run: |
|
|
32
|
+
# NOTE: put your own distribution build steps here.
|
|
33
|
+
uv build --wheel
|
|
34
|
+
|
|
35
|
+
- name: Upload distributions
|
|
36
|
+
uses: actions/upload-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: release-dists
|
|
39
|
+
path: dist/
|
|
40
|
+
|
|
41
|
+
public-artifactory-publish:
|
|
42
|
+
env:
|
|
43
|
+
UV_PUBLISH_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
|
|
44
|
+
UV_PUBLISH_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
|
|
45
|
+
UV_PUBLISH_URL: ${{ vars.UV_PUBLISH_URL }}
|
|
46
|
+
permissions:
|
|
47
|
+
contents: read
|
|
48
|
+
id-token: write
|
|
49
|
+
|
|
50
|
+
runs-on: ubuntu-latest
|
|
51
|
+
needs:
|
|
52
|
+
- release-build
|
|
53
|
+
|
|
54
|
+
steps:
|
|
55
|
+
- name: Retrieve release distributions
|
|
56
|
+
uses: actions/download-artifact@v4
|
|
57
|
+
with:
|
|
58
|
+
name: release-dists
|
|
59
|
+
path: dist/
|
|
60
|
+
|
|
61
|
+
- name: Install uv
|
|
62
|
+
uses: astral-sh/setup-uv@v5
|
|
63
|
+
with:
|
|
64
|
+
version: "0.8.x"
|
|
65
|
+
|
|
66
|
+
- name: Publish to vito-artifactory-public
|
|
67
|
+
run: uv publish
|
|
68
|
+
|