lcfm-shapefiles 0.9.33__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. lcfm_shapefiles-0.9.33/.gitattributes +3 -0
  2. lcfm_shapefiles-0.9.33/.github/copilot-instructions.md +114 -0
  3. lcfm_shapefiles-0.9.33/.github/workflows/auto-sync-github.yml +133 -0
  4. lcfm_shapefiles-0.9.33/.github/workflows/auto-sync.yml +148 -0
  5. lcfm_shapefiles-0.9.33/.github/workflows/format_test.yml +39 -0
  6. lcfm_shapefiles-0.9.33/.github/workflows/integration_tests.yml +28 -0
  7. lcfm_shapefiles-0.9.33/.github/workflows/publish_pypi.yml +53 -0
  8. lcfm_shapefiles-0.9.33/.github/workflows/publish_wheel.yml +68 -0
  9. lcfm_shapefiles-0.9.33/.gitignore +180 -0
  10. lcfm_shapefiles-0.9.33/.pre-commit-config.yaml +12 -0
  11. lcfm_shapefiles-0.9.33/.python-version +1 -0
  12. lcfm_shapefiles-0.9.33/Jenkinsfile +14 -0
  13. lcfm_shapefiles-0.9.33/LICENSE +27 -0
  14. lcfm_shapefiles-0.9.33/LICENSE-DATA +28 -0
  15. lcfm_shapefiles-0.9.33/PKG-INFO +145 -0
  16. lcfm_shapefiles-0.9.33/README-auto-sync.md +74 -0
  17. lcfm_shapefiles-0.9.33/README.md +122 -0
  18. lcfm_shapefiles-0.9.33/bucket_policy.json +15 -0
  19. lcfm_shapefiles-0.9.33/ci_cd_policy.json +69 -0
  20. lcfm_shapefiles-0.9.33/figures/grid.png +0 -0
  21. lcfm_shapefiles-0.9.33/makefile +63 -0
  22. lcfm_shapefiles-0.9.33/pyproject.toml +80 -0
  23. lcfm_shapefiles-0.9.33/shapefiles_sync.svg +4 -0
  24. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/__init__.py +61 -0
  25. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/_version.py +1 -0
  26. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/HRVPP_100p_S2-tiles.fgb +3 -0
  27. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_AOI-tropical_v1.fgb +3 -0
  28. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-reduced-tiles.fgb +3 -0
  29. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles-land_buffered-30km.fgb +3 -0
  30. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles.fgb +3 -0
  31. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2018.fgb +3 -0
  32. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2019.fgb +3 -0
  33. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2020.fgb +3 -0
  34. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2021.fgb +3 -0
  35. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2022.fgb +3 -0
  36. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2023.fgb +3 -0
  37. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2024.fgb +3 -0
  38. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_S2-tiles_LSF-S1-2025.fgb +3 -0
  39. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-1deg.fgb +3 -0
  40. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-30deg.fgb +3 -0
  41. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-3deg-TCD.fgb +3 -0
  42. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-3deg-land_buffered-30km.fgb +3 -0
  43. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_100p_latlon-3deg.fgb +3 -0
  44. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_AOI-land_v5.fgb +3 -0
  45. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_AOI-tropical_v3.fgb +3 -0
  46. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_S2-tiles.fgb +3 -0
  47. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_latlon-30deg.fgb +3 -0
  48. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_latlon-3deg-TCD.fgb +3 -0
  49. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_10p_latlon-3deg.fgb +3 -0
  50. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_1p_S2-tiles.fgb +3 -0
  51. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/LCFM_2p_S2-tiles.fgb +3 -0
  52. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_global.fgb +3 -0
  53. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_10p-random.parquet +3 -0
  54. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_10p.parquet +3 -0
  55. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_1p.parquet +3 -0
  56. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_2p.parquet +3 -0
  57. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_global.parquet +3 -0
  58. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/blocks_parquet/blocks_tropics.parquet +3 -0
  59. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/keys_metadata.json +535 -0
  60. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/unclassified_2020/LCFM_100p_S2-tiles-unclassified.fgb +3 -0
  61. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data/unclassified_2020/blocks_global_v12-unclassified.fgb +3 -0
  62. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/data_availability.py +335 -0
  63. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/fix_antimeridian.py +74 -0
  64. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/labels.py +199 -0
  65. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/products.py +2233 -0
  66. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/readme.md +10 -0
  67. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/shapefiles.py +470 -0
  68. lcfm_shapefiles-0.9.33/src/lcfm_shapefiles/tiling.py +132 -0
  69. lcfm_shapefiles-0.9.33/sync-shapefiles-terrascope.sh +40 -0
  70. lcfm_shapefiles-0.9.33/tests/pytest.ini +7 -0
  71. lcfm_shapefiles-0.9.33/tests/test_file.py +50 -0
  72. lcfm_shapefiles-0.9.33/tests/test_labels.py +54 -0
  73. lcfm_shapefiles-0.9.33/tests/test_packaging.py +48 -0
  74. lcfm_shapefiles-0.9.33/tests/test_products.py +637 -0
  75. lcfm_shapefiles-0.9.33/tests/test_s3_upload.py +133 -0
  76. lcfm_shapefiles-0.9.33/tests/test_shapefiles.py +416 -0
  77. lcfm_shapefiles-0.9.33/uv.lock +5492 -0
@@ -0,0 +1,3 @@
1
+ *.fgb filter=lfs diff=lfs merge=lfs -text
2
+ *.parquet filter=lfs diff=lfs merge=lfs -text
3
+ *.ipynb linguist-vendored
@@ -0,0 +1,114 @@
1
+ # LCFM Shapefiles - AI Coding Agent Instructions
2
+
3
+ ## Project Overview
4
+ Python package providing wrapper functionality around FlatGeobuf (`.fgb`) shapefiles for the Land Cover Fraction Map (LCFM) project. The package handles downloading versioned shapefiles from S3 CloudFerro storage with MD5-based caching and provides product specifications in the form of classes. Product classes contain names, layers, data types, etc. and allow generation of metadata for LCFM datasets.
5
+
6
+ **Key pattern**: FlatGeobuf files are NOT stored in the repository due to size. Instead, they are stored in S3 (CloudFerro WAW4-1) with Git LFS pointers in the VM at `/vitodata/vegteam/auxdata/LCFM/shapefiles-lfs`. The package downloads them on-demand with version-based caching.
7
+
8
+ ## Architecture & Core Components
9
+
10
+ ### 1. Shapefile Management (`shapefiles.py`)
11
+ **Singleton pattern**: `ShapefileCacheManager` manages all shapefile downloads and caching
12
+ - **Cache location hierarchy**: `LCFM_SHAPEFILES_CACHE_HOME` → `XDG_CACHE_HOME/lcfm_shapefiles` → `~/.cache/lcfm_shapefiles`
13
+ - **Version-based caching**: Cache organized by package version (`~/.cache/lcfm_shapefiles/0.8.16/`)
14
+ - **Metadata driven**: `keys_metadata.json` contains S3 metadata (ETags for MD5 verification, version IDs, sizes)
15
+ - **S3 URI pattern**: `https://vito-lcf-shapefiles-waw4-1.s3.waw4-1.cloudferro.com/<filename>?versionId=<version_id>`
16
+ - **Integrity checks**: MD5 checksums verify downloaded files match expected ETags
17
+
18
+ **Typical usage pattern**:
19
+ ```python
20
+ from lcfm_shapefiles import get_filepath
21
+ import geopandas as gpd
22
+
23
+ fp = get_filepath("LCFM_1p_S2-tiles.fgb") # Downloads if needed
24
+ gdf = gpd.read_file(fp)
25
+ ```
26
+
27
+ ### 2. Product Abstraction (`products.py`)
28
+ **Registry pattern**: Products register themselves via `@Product.register_product()` decorator
29
+ - **Product hierarchy**: `Product` (abstract) → `LOSProduct`, `LSFProduct`, `LSCProduct`, `LCM10Product`, `LCM100Product`, etc.
30
+ - **Band-based routing**: Products register with band prefixes (e.g., `"LCM10:MAP"` for categorical map data)
31
+ - **Factory method**: `Product.create_product(product_name, version, band, year)` routes to correct subclass
32
+ - **Tile coordinate systems**: UTM zones determined from Sentinel-2 MGRS tile IDs (e.g., `34UCV` → EPSG:32634)
33
+
34
+ **Key functions**:
35
+ - `tile_to_epsg(tile)`: Convert S2 tile ID to EPSG code (row letter determines hemisphere)
36
+ - `utm_to_epsg(utm)`: Convert UTM zone string to EPSG code
37
+
38
+ ### 3. Label Definitions (`labels.py`)
39
+ Enums defining land cover class labels, colors, and names for visualization
40
+ - `LCM10LabelsColors`: 10-meter resolution land cover classes (tree, shrub, grass, crop, wetland, etc.)
41
+ - Color schemes use normalized RGB arrays for matplotlib compatibility
42
+
43
+ ### 4. Antimeridian Handling (`fix_antimeridian.py`)
44
+ Utility to split polygons crossing longitude ±180°
45
+ - `intersects_antimeridian()`: Detects large longitude jumps (>100° threshold)
46
+ - `fix_antimeridian_split()`: Splits into MultiPolygon with normalized coordinates
47
+
48
+ ## Development Workflow
49
+
50
+ ### Environment Setup
51
+ ```bash
52
+ uv sync # Build venv with dependencies
53
+ uv run pytest # Run tests
54
+ uv run ruff format . # Format code (not ruff check!)
55
+ ```
56
+
57
+ **Important**: Use `uv` (not pip/conda) for all dependency management. Dev dependencies include geopandas, pandas[pyarrow], pytest, ruff.
58
+
59
+ ### Testing
60
+ - Tests in `tests/` validate shapefile downloads (`test_shapefiles.py`), product creation (`test_products.py`), and labels (`test_labels.py`)
61
+ - Tests use `tmp_path` fixture and set `LCFM_SHAPEFILES_CACHE_HOME` to isolate cache
62
+ - **Critical**: Tests verify checksum validation and cached file integrity
63
+ - **When adding new Product classes**: Add corresponding tests to `tests/test_file.py`
64
+
65
+ ### Deploying New/Updated Shapefiles
66
+ 1. Add/update `.fgb` files in `src/lcfm_shapefiles/data/`
67
+ 2. Update tests accordingly
68
+ 3. Bump version in `pyproject.toml` → `src/lcfm_shapefiles/_version.py`
69
+ 4. Push to `master` → if build passes, trigger promotion job
70
+ 5. Auto-sync workflow uploads to S3 and updates VM cache
71
+
72
+ ## Project-Specific Conventions
73
+
74
+ ### File Naming
75
+ **Shapefiles**: `LCFM_<percent>_<grid/AOI>-<granularity>_v<version>.fgb`
76
+ - Example: `LCFM_100p_S2-tiles.fgb`, `LCFM_10p_latlon-3deg-TCD.fgb`
77
+ - Version tags removed in package to avoid conflicts with semantic versioning
78
+
79
+ ### Version String Format
80
+ Products expect version as `vXXX` (3 digits), e.g., `v001`, `v127`. Constructor warns if format violated.
81
+
82
+ ### Scripts Directory
83
+ `scripts/` contains data processing utilities (not part of package distribution):
84
+ - Block extraction/manipulation (`add_blocks.py`, `extract_block.py`)
85
+ - Tile selection/comparison (`check_tiles.py`, `select_tiles.py`)
86
+ - File format conversion (`compare_fgb_parquet_columns.py`)
87
+
88
+ These scripts often import from package: `from lcfm_shapefiles import get_filepath`
89
+
90
+ ### Parquet Blocks
91
+ Since `v0.8.7`: Parquet-format grid blocks in `src/lcfm_shapefiles/data/blocks_parquet/`
92
+ - Source: `/vitodata/vegteam/auxdata/grid/blocks_global/` on Terrascope
93
+
94
+ ### Blocks Grid Structure
95
+ `blocks_global.fgb` and `blocks_global.parquet` are **not** one row per tile — they contain one row per **block** (sub-tile subdivision), ~1.4M rows total.
96
+ - The unique `tile` column values in blocks files must match the tile IDs in `LCFM_100p_S2-tiles.fgb` exactly (currently **18,422 tiles** each)
97
+ - Columns: `tile_block_id`, `tile`, `block_id`, `epsg`, `landtype`, `antimeridian`, `xmin`, `ymin`, `xmax`, `ymax`
98
+ - `blocks_global.parquet` has no geometry column; `blocks_global.fgb` has geometry
99
+ - To verify consistency: compare `set(blocks['tile'])` against `set(s2_tiles['tile'])` — row counts will differ by design
100
+
101
+ ## Critical Gotchas
102
+
103
+ 1. **FGB files excluded from wheel**: `pyproject.toml` excludes `*.fgb` from built wheel to reduce package size
104
+ 2. **Git LFS required**: On Terrascope VM, Git LFS pulls actual files; elsewhere package downloads from S3
105
+ 3. **Cache invalidation**: Cache organized by package version; upgrading package creates new cache directory
106
+ 4. **EPSG code logic**: S2 tile rows C-M are southern hemisphere (326XX), N-X are northern (327XX)
107
+ 5. **Product registry**: Subclasses must define `product_name` ClassVar or `@register()` raises ValueError
108
+ 6. **Metadata updates**: Call `update_cache_metadata()` to refresh `keys_metadata.json` from S3
109
+
110
+ ## External Dependencies
111
+
112
+ - **S3 bucket**: `vito-lcf-shapefiles-waw4-1` on CloudFerro WAW4-1 (Warsaw region)
113
+ - **Artifactory**: `https://artifactory.vgt.vito.be/artifactory/api/pypi/python-packages-public/`
114
+ - **Terrascope paths**: `/vitodata/vegteam/auxdata/LCFM/shapefiles-lfs`, `/vitodata/vegteam_vol2/data/lcfm/shapefiles`
@@ -0,0 +1,133 @@
1
+ name: Auto-Sync LFS Files Github
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - master # Change this to your default branch if different
7
+ paths:
8
+ - 'src/lcfm_shapefiles/data/**' # Only trigger when files in this path change
9
+ workflow_dispatch: # Allows manual triggering of the workflow
10
+
11
+ env:
12
+ SHAPEFILES_BUCKET: vito-lcf-shapefiles-waw4-1
13
+
14
+ jobs:
15
+ sync:
16
+ name: Sync repository on VM cluster and S3
17
+ runs-on: ubuntu-latest # This will use your self-hosted runner
18
+ environment: CF_S3_ACCESS
19
+ env:
20
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
21
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
22
+ AWS_ENDPOINT_URL: ${{vars.AWS_ENDPOINT_URL}}
23
+ AWS_REGION: ${{vars.AWS_REGION}}
24
+ permissions:
25
+ contents: write
26
+ steps:
27
+ - name: Checkout repository
28
+ uses: actions/checkout@v4
29
+ with:
30
+ lfs: true
31
+ path: src/lcfm_shapefiles/data
32
+
33
+ - name: Configure AWS CLI for single-part uploads
34
+ run: |
35
+ # Set multipart threshold to 5GB to ensure files are uploaded as single objects
36
+ # This preserves MD5 ETags for integrity validation
37
+ aws configure set default.s3.multipart_threshold 5GB
38
+ aws configure set default.s3.multipart_chunksize 5GB
39
+
40
+ - name: push_s3
41
+ id: push_s3
42
+ working-directory: src/lcfm_shapefiles/data
43
+ run: |
44
+
45
+ echo "Starting intelligent S3 sync with ETag comparison..."
46
+
47
+ # Flag file to track if any uploads occurred (needed because while loop runs in subshell)
48
+ UPLOAD_FLAG=$(mktemp)
49
+
50
+ # Function to calculate MD5 hash of local file
51
+ calculate_local_etag() {
52
+ local file="$1"
53
+ md5sum "$file" | cut -d' ' -f1
54
+ }
55
+
56
+ # Function to get S3 object ETag
57
+ get_s3_etag() {
58
+ local s3_key="$1"
59
+ aws s3api head-object --bucket ${{ env.SHAPEFILES_BUCKET }} --key "$s3_key" --query 'ETag' --output text 2>/dev/null | tr -d '"' || echo ""
60
+ }
61
+
62
+ # Counters for statistics
63
+ uploaded_count=0
64
+ skipped_count=0
65
+ total_count=0
66
+
67
+ # Find all files to sync (excluding hidden files and directories)
68
+ find src/lcfm_shapefiles/data -type f ! -path '*/.*' ! -name 'keys_metadata.json' | while read -r file; do
69
+ # Extract s3_key as path relative to src/lcfm_shapefiles/data/
70
+ s3_key="${file#src/lcfm_shapefiles/data/}"
71
+
72
+ total_count=$((total_count + 1))
73
+
74
+ echo "Processing: $s3_key"
75
+
76
+ # Calculate local file ETag
77
+ local_etag=$(calculate_local_etag "$file")
78
+
79
+ # Get S3 object ETag
80
+ s3_etag=$(get_s3_etag "$s3_key")
81
+
82
+ if [ -z "$s3_etag" ]; then
83
+ echo " → File doesn't exist in S3, uploading..."
84
+ aws s3 cp "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
85
+ echo "uploaded" >> "$UPLOAD_FLAG"
86
+ echo " ✓ Uploaded"
87
+ elif [ "$local_etag" != "$s3_etag" ]; then
88
+ echo " → ETags differ (local: $local_etag, S3: $s3_etag), uploading..."
89
+ aws s3 cp "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
90
+ echo "uploaded" >> "$UPLOAD_FLAG"
91
+ echo " ✓ Updated"
92
+ else
93
+ echo " → ETags match, skipping upload"
94
+ echo " ⏭ Skipped"
95
+ fi
96
+ done
97
+
98
+ # Set output flag based on whether any files were uploaded
99
+ if [ -s "$UPLOAD_FLAG" ]; then
100
+ echo "files_uploaded=true" >> $GITHUB_OUTPUT
101
+ echo "Files were uploaded - metadata update required"
102
+ else
103
+ echo "files_uploaded=false" >> $GITHUB_OUTPUT
104
+ echo "No files uploaded - skipping metadata update"
105
+ fi
106
+ rm -f "$UPLOAD_FLAG"
107
+
108
+ - name: Update S3 metadata and commit
109
+ if: steps.push_s3.outputs.files_uploaded == 'true'
110
+ working-directory: src/lcfm_shapefiles/data
111
+ run: |
112
+ echo "Fetching S3 metadata for latest object versions..."
113
+
114
+ # Get all object versions from S3 and filter for latest versions only
115
+ aws s3api list-object-versions \
116
+ --endpoint-url ${{ vars.AWS_ENDPOINT_URL }} \
117
+ --bucket ${{ env.SHAPEFILES_BUCKET }} \
118
+ --query 'Versions[?IsLatest==`true`]' \
119
+ --output json > src/lcfm_shapefiles/data/keys_metadata.json
120
+
121
+ # Configure git identity for the commit
122
+ git config --local user.email "actions@github.com"
123
+ git config --local user.name "GitHub Actions"
124
+
125
+ # Commit and push the updated metadata file if there are changes
126
+ git add src/lcfm_shapefiles/data/keys_metadata.json
127
+ git commit -m "Update S3 keys metadata [skip ci]"
128
+ git push origin ${{ github.ref_name }}
129
+ echo "✓ Updated keys_metadata.json with latest S3 object versions"
130
+ aws s3 cp src/lcfm_shapefiles/data/keys_metadata.json "s3://${{ env.SHAPEFILES_BUCKET }}/keys_metadata.json"
131
+ echo "✓ Pushed keys_metadata.json to s3://${{ env.SHAPEFILES_BUCKET }}/keys_metadata.json"
132
+
133
+ echo "Metadata sync completed at $(date)"
@@ -0,0 +1,148 @@
1
+ name: Auto-Sync LFS Files Terrascope
2
+
3
+ on:
4
+ workflow_dispatch: # Allows manual triggering of the workflow
5
+
6
+ env:
7
+ SHAPEFILES_BUCKET: vito-lcf-shapefiles-waw4-1
8
+ TERRASCOPE_PATH: /vitodata/vegteam/auxdata/LCFM/shapefiles-lfs/src/lcfm_shapefiles/data
9
+
10
+ jobs:
11
+ sync:
12
+ name: Sync repository on VM cluster and S3
13
+ runs-on: self-hosted # This will use your self-hosted runner
14
+ steps:
15
+ - name: Pull latest changes
16
+ run: |
17
+ eval "$(ssh-agent -s)"
18
+ ssh-add ~/Private/runner/.ssh/github_deploy_key
19
+
20
+ cd /vitodata/vegteam/auxdata/LCFM/shapefiles-lfs
21
+ git checkout ${{ github.ref_name }} # Change this to your default branch if different
22
+
23
+ git remote set-url origin git@github.com:VITO-RS-Vegetation/lcfm-shapefiles.git
24
+ git fetch origin
25
+ git pull origin ${{ github.ref_name }} # Change this to your default branch if different
26
+
27
+ # If using LFS, also pull LFS objects
28
+ if [ -f ".gitattributes" ] && grep -q "filter=lfs" .gitattributes; then
29
+ git lfs pull
30
+ fi
31
+
32
+ echo "Repository successfully synced at $(date)"
33
+
34
+ - name: push_s3
35
+ env:
36
+ AWS_CLI_S3_MULTIPART_THRESHOLD: 5GB
37
+ run: |
38
+ cd ${{ env.TERRASCOPE_PATH }}
39
+
40
+ echo "Starting intelligent S3 sync with ETag comparison..."
41
+
42
+ # Function to calculate MD5 hash of local file
43
+ calculate_local_etag() {
44
+ local file="$1"
45
+ md5sum "$file" | cut -d' ' -f1
46
+ }
47
+
48
+ # Function to get S3 object ETag
49
+ get_s3_etag() {
50
+ local s3_key="$1"
51
+ aws s3api head-object --profile cf --bucket ${{ env.SHAPEFILES_BUCKET }} --key "$s3_key" --query 'ETag' --output text 2>/dev/null | tr -d '"' || echo ""
52
+ }
53
+
54
+ # Counters for statistics
55
+ uploaded_count=0
56
+ skipped_count=0
57
+ total_count=0
58
+
59
+ # Find all files to sync (excluding hidden files and directories)
60
+ find . -type f \( ! -path '*/.*' -o -path './blocks_parquet/*' \) | while read -r file; do
61
+ # Remove leading ./ from file path
62
+ relative_path="${file#./}"
63
+ s3_key="$relative_path"
64
+
65
+ total_count=$((total_count + 1))
66
+
67
+ echo "Processing: $relative_path"
68
+
69
+ # Calculate local file ETag
70
+ local_etag=$(calculate_local_etag "$file")
71
+
72
+ # Get S3 object ETag
73
+ s3_etag=$(get_s3_etag "$s3_key")
74
+
75
+ if [ -z "$s3_etag" ]; then
76
+ echo " → File doesn't exist in S3, uploading..."
77
+ aws s3 cp --profile cf "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
78
+ uploaded_count=$((uploaded_count + 1))
79
+ echo " ✓ Uploaded"
80
+ elif [ "$local_etag" != "$s3_etag" ]; then
81
+ echo " → ETags differ (local: $local_etag, S3: $s3_etag), uploading..."
82
+ aws s3 cp --profile cf "$file" "s3://${{ env.SHAPEFILES_BUCKET }}/$s3_key"
83
+ uploaded_count=$((uploaded_count + 1))
84
+ echo " ✓ Updated"
85
+ else
86
+ echo " → ETags match, skipping upload"
87
+ skipped_count=$((skipped_count + 1))
88
+ echo " ⏭ Skipped"
89
+ fi
90
+ done
91
+
92
+ # Also sync the file_timestamps.json specifically
93
+ timestamps_file="file_timestamps.json"
94
+ if [ -f "$timestamps_file" ]; then
95
+ echo "Processing timestamps file: $timestamps_file"
96
+ local_etag=$(calculate_local_etag "$timestamps_file")
97
+ s3_etag=$(get_s3_etag "$timestamps_file")
98
+
99
+ if [ -z "$s3_etag" ] || [ "$local_etag" != "$s3_etag" ]; then
100
+ echo " → Uploading timestamps file..."
101
+ aws s3 cp --profile cf "$timestamps_file" "s3://${{ env.SHAPEFILES_BUCKET }}/$timestamps_file"
102
+ echo " ✓ Timestamps file uploaded"
103
+ else
104
+ echo " → Timestamps file unchanged, skipping"
105
+ fi
106
+ fi
107
+
108
+ echo ""
109
+ echo "=== Sync Summary ==="
110
+ echo "Files uploaded: $uploaded_count"
111
+ echo "Files skipped: $skipped_count"
112
+ echo "Total files processed: $total_count"
113
+ echo "Bucket: ${{ env.SHAPEFILES_BUCKET }}"
114
+ echo "Sync completed at $(date)"
115
+
116
+ - name: Update S3 metadata and commit
117
+ run: |
118
+ eval "$(ssh-agent -s)"
119
+ ssh-add ~/Private/runner/.ssh/github_deploy_key
120
+
121
+ cd /vitodata/vegteam/auxdata/LCFM/shapefiles-lfs
122
+
123
+ echo "Fetching S3 metadata for latest object versions..."
124
+
125
+ # Get all object versions from S3 and filter for latest versions only
126
+ aws s3api list-object-versions \
127
+ --profile cf \
128
+ --endpoint-url https://s3.waw4-1.cloudferro.com \
129
+ --bucket ${{ env.SHAPEFILES_BUCKET }} \
130
+ --query 'Versions[?IsLatest==`true`]' \
131
+ --output json > ${{ env.TERRASCOPE_PATH}}/keys_metadata.json
132
+
133
+ # Configure git identity for the commit
134
+ git config --local user.email "actions@github.com"
135
+ git config --local user.name "GitHub Actions"
136
+
137
+ # Commit and push the updated metadata file if there are changes
138
+ git add src/lcfm_shapefiles/data/keys_metadata.json
139
+ if ! git diff --quiet --staged; then
140
+ git commit -m "Update S3 keys metadata [skip ci]"
141
+ git push origin ${{ github.ref_name }}
142
+ echo "✓ Updated keys_metadata.json with latest S3 object versions"
143
+ else
144
+ echo "→ No changes to keys_metadata.json"
145
+ fi
146
+
147
+ echo "Metadata sync completed at $(date)"
148
+
@@ -0,0 +1,39 @@
1
+ name: vegorg workflow
2
+
3
+ on: #add your hook triggers in the array [push, pull, merge]
4
+ push: # Trigger only on push events
5
+ branches:
6
+ - dev # Trigger only on pushes to the 'main' branch
7
+ - master
8
+ pull_request: # Trigger only on pull request events
9
+ branches:
10
+ - dev
11
+ - master # Trigger only on pull requests targeting the 'main' branch
12
+ workflow_dispatch: # Allow manual triggering of the workflow
13
+
14
+ env:
15
+ token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GITHUB_TOKEN for authentication
16
+ UV_INDEX_VITO_ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
17
+ UV_INDEX_VITO_ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
18
+
19
+ jobs:
20
+ format-lint:
21
+ runs-on: ubuntu-latest
22
+ steps:
23
+ - uses: actions/checkout@v4
24
+ - uses: astral-sh/ruff-action@v3
25
+ with:
26
+ args: "format --check --diff"
27
+
28
+
29
+ check-uv-lock:
30
+ runs-on: ubuntu-latest
31
+ steps:
32
+ - uses: actions/checkout@v4
33
+ - uses: astral-sh/setup-uv@v5
34
+ with:
35
+ version: "0.8.x"
36
+ python-version: "3.11"
37
+ - name: Check uv lock
38
+ run: uv lock --check
39
+
@@ -0,0 +1,28 @@
1
+ name: integration tests
2
+
3
+ on: #add your hook triggers in the array [push, pull, merge]
4
+ pull_request: # Trigger only on pull request events
5
+ branches:
6
+ - master
7
+ workflow_dispatch: # Allow manual triggering of the workflow
8
+
9
+ env:
10
+ token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GITHUB_TOKEN for authentication
11
+ UV_INDEX_VITO_ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
12
+ UV_INDEX_VITO_ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
13
+
14
+ jobs:
15
+ integration-tests:
16
+ runs-on: ubuntu-latest
17
+ steps:
18
+ - name: Checkout repository
19
+ uses: actions/checkout@v4
20
+ with:
21
+ lfs: true
22
+
23
+ - name: Setup build environment
24
+ uses: VITO-RS-Vegetation/veg-template@v2 # Reuse the composite action
25
+ with:
26
+ uv-args: "--all-groups -vv"
27
+ - name: Run tests
28
+ run: uv run pytest tests -m "not s3"
@@ -0,0 +1,53 @@
1
+ # Publishes lcfm-shapefiles to the public PyPI (pypi.org).
2
+ # Separate from publish_wheel.yml (Artifactory) - manual trigger only.
3
+ name: Publish to PyPI
4
+ on:
5
+ workflow_dispatch:
6
+
7
+ permissions:
8
+ contents: read
9
+
10
+ jobs:
11
+ release-build:
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+
17
+ - name: Setup build environment
18
+ uses: VITO-RS-Vegetation/veg-template@v2
19
+ with:
20
+ uv-args: "--all-groups -vv"
21
+
22
+ - name: Build release distributions
23
+ run: uv build --wheel --sdist
24
+
25
+ - name: Upload distributions
26
+ uses: actions/upload-artifact@v4
27
+ with:
28
+ name: pypi-dists
29
+ path: dist/
30
+
31
+ pypi-publish:
32
+ runs-on: ubuntu-latest
33
+ needs:
34
+ - release-build
35
+ environment:
36
+ name: pypi
37
+ permissions:
38
+ id-token: write # required for trusted publishing (OIDC), no token secrets needed
39
+
40
+ steps:
41
+ - name: Retrieve release distributions
42
+ uses: actions/download-artifact@v4
43
+ with:
44
+ name: pypi-dists
45
+ path: dist/
46
+
47
+ - name: Install uv
48
+ uses: astral-sh/setup-uv@v5
49
+ with:
50
+ version: "0.8.x"
51
+
52
+ - name: Publish to PyPI
53
+ run: uv publish --trusted-publishing automatic
@@ -0,0 +1,68 @@
1
+ # Uploads veg-workflows wheel to vito public artifactory
2
+
3
+ name: Publish Wheel
4
+ on:
5
+ release:
6
+ types: [published]
7
+
8
+ workflow_dispatch: # Allow manual triggering of the workflow (optional)
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ env:
14
+ token: ${{ secrets.GITHUB_TOKEN }} # Use the built-in GITHUB_TOKEN for authentication
15
+ UV_INDEX_VITO_ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
16
+ UV_INDEX_VITO_ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
17
+
18
+ jobs:
19
+ release-build:
20
+ runs-on: ubuntu-latest
21
+
22
+ steps:
23
+ - uses: actions/checkout@v4
24
+
25
+ - name: Setup build environment
26
+ uses: VITO-RS-Vegetation/veg-template@v2 # Reuse the composite action
27
+ with:
28
+ uv-args: "--all-groups -vv"
29
+
30
+ - name: Build release distributions
31
+ run: |
32
+ # NOTE: put your own distribution build steps here.
33
+ uv build --wheel
34
+
35
+ - name: Upload distributions
36
+ uses: actions/upload-artifact@v4
37
+ with:
38
+ name: release-dists
39
+ path: dist/
40
+
41
+ public-artifactory-publish:
42
+ env:
43
+ UV_PUBLISH_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }}
44
+ UV_PUBLISH_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }}
45
+ UV_PUBLISH_URL: ${{ vars.UV_PUBLISH_URL }}
46
+ permissions:
47
+ contents: read
48
+ id-token: write
49
+
50
+ runs-on: ubuntu-latest
51
+ needs:
52
+ - release-build
53
+
54
+ steps:
55
+ - name: Retrieve release distributions
56
+ uses: actions/download-artifact@v4
57
+ with:
58
+ name: release-dists
59
+ path: dist/
60
+
61
+ - name: Install uv
62
+ uses: astral-sh/setup-uv@v5
63
+ with:
64
+ version: "0.8.x"
65
+
66
+ - name: Publish to vito-artifactory-public
67
+ run: uv publish
68
+