pcoras 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pcoras-1.0.0/AUTHORS +13 -0
- pcoras-1.0.0/LICENSE +22 -0
- pcoras-1.0.0/PKG-INFO +162 -0
- pcoras-1.0.0/README.md +135 -0
- pcoras-1.0.0/pyproject.toml +96 -0
- pcoras-1.0.0/setup.cfg +4 -0
- pcoras-1.0.0/setup.py +5 -0
- pcoras-1.0.0/src/_version.py +1 -0
- pcoras-1.0.0/src/cli/__init__.py +5 -0
- pcoras-1.0.0/src/cli/cli.py +157 -0
- pcoras-1.0.0/src/cli/convert.py +162 -0
- pcoras-1.0.0/src/pcoras.egg-info/PKG-INFO +162 -0
- pcoras-1.0.0/src/pcoras.egg-info/SOURCES.txt +20 -0
- pcoras-1.0.0/src/pcoras.egg-info/dependency_links.txt +1 -0
- pcoras-1.0.0/src/pcoras.egg-info/entry_points.txt +2 -0
- pcoras-1.0.0/src/pcoras.egg-info/requires.txt +7 -0
- pcoras-1.0.0/src/pcoras.egg-info/top_level.txt +4 -0
- pcoras-1.0.0/src/pcoras.py +551 -0
- pcoras-1.0.0/src/sub/__init__.py +15 -0
- pcoras-1.0.0/src/sub/data_process.py +493 -0
- pcoras-1.0.0/src/sub/preprocess.py +452 -0
- pcoras-1.0.0/src/sub/utils.py +745 -0
pcoras-1.0.0/AUTHORS
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Choose your fighter!
|
|
2
|
+
|
|
3
|
+
Miguel Angel Martinez Guerrero <miguel.martinezguerrero@eurac.edu>
|
|
4
|
+
- Main developer (and only, as far as we know) and mainteiner of this repository, Junior Researcher at Eurac Research
|
|
5
|
+
|
|
6
|
+
Piero Campalani <piero.campalani@eurac.edu>
|
|
7
|
+
- Senior dev of this repository and senior researcher at Eurac Research
|
|
8
|
+
|
|
9
|
+
Alexander Jacob <alexandre.jacob@eurac.edu>
|
|
10
|
+
- Vice Head of Institute of Earth Observation & Leader of the Advance Computing team at Eurac Research
|
|
11
|
+
|
|
12
|
+
Michele Claus <michele.claus@eurac.edu>
|
|
13
|
+
- Main developer of the repo Raster2STAC, researcher at Eurac Research
|
pcoras-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 Eurac Research - Institute for Earth Observation
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
pcoras-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pcoras
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Package for Cloud-Optimized Raster and Spatio-temporal data
|
|
5
|
+
Author-email: Miguel Martinez <mmartinezguerrero@eurac.edu>, Piero Campalani <pcampalani@eurac.edu>, Alexander Jacob <Alexander.Jacob@eurac.edu>, Michele Claus <michele.claus@eurac.edu>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://gitlab.inf.unibz.it/earth_observation_public/pcoras
|
|
8
|
+
Project-URL: Issues, https://gitlab.inf.unibz.it/earth_observation_public/pcoras/-/issues
|
|
9
|
+
Keywords: STAC,Metadata,Zarr,NetCDF
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Education
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Development Status :: 4 - Beta
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: raster2stac>=2026.2.1
|
|
21
|
+
Requires-Dist: requests
|
|
22
|
+
Requires-Dist: importlib_metadata
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: isort; extra == "dev"
|
|
25
|
+
Requires-Dist: ruff; extra == "dev"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+

|
|
29
|
+
|
|
30
|
+

|
|
31
|
+

|
|
32
|
+

|
|
33
|
+

|
|
34
|
+
|
|
35
|
+
# PCoRaS
|
|
36
|
+
|
|
37
|
+
Python Package for Cloud Optimzed Raster and Spatio-temporal Data, following the STAC guidelines this repository
|
|
38
|
+
is based Cloud optimized type of files and Raster2stac functions to create an STAC collections compatible with
|
|
39
|
+
OpenEO backend, getting an output up to a 5D hypercube dataset for different spatio-temporal data, following a
|
|
40
|
+
collection of local OpenEO processes to test the compability of the item.
|
|
41
|
+
|
|
42
|
+
## Setup
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install pcoras
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Note: Conda support is in development
|
|
49
|
+
|
|
50
|
+
## Getting started
|
|
51
|
+
|
|
52
|
+
### Command-line interface
|
|
53
|
+
|
|
54
|
+
Convert a NetCDF file into a local Zarr data cube:
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
python -m pcoras convert \
|
|
58
|
+
--path ./data/precipitation.nc \
|
|
59
|
+
--output-folder ./output \
|
|
60
|
+
--output-name precipitation \
|
|
61
|
+
--xdim x \
|
|
62
|
+
--ydim y \
|
|
63
|
+
--tdim time
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Replace the input path and dimension names with those in your dataset. For example, a dataset with longitude and latitude dimensions may require `--xdim lon --ydim lat`.
|
|
67
|
+
|
|
68
|
+
These six arguments are required by the command-line parser. NetCDF (`.nc`) is the default input extension. S3 upload and STAC publication are disabled by default; enable them with `--upload-s3-data` and `--publish-stac`, respectively, and supply the corresponding storage or collection settings.
|
|
69
|
+
|
|
70
|
+
The `--path` option accepts one command-line value: a file or directory path. The `co` command is an alias for `convert`.
|
|
71
|
+
|
|
72
|
+
Display the available commands and conversion options:
|
|
73
|
+
|
|
74
|
+
```sh
|
|
75
|
+
python -m pcoras --help
|
|
76
|
+
python -m pcoras convert --help
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Selected conversion options:
|
|
80
|
+
|
|
81
|
+
| Option | Purpose |
|
|
82
|
+
| --- | --- |
|
|
83
|
+
| `--variables pr tas` | Select variables by name. Replace `pr` and `tas` with names from your dataset. |
|
|
84
|
+
| `--file-ext .nc` | Set the input file extension; defaults to `.nc`. |
|
|
85
|
+
| `--crs 32632` | Supply a CRS as an integer EPSG code appropriate for your dataset. |
|
|
86
|
+
| `--chunking '{"time": 31}'` | Supply chunk sizes as a JSON object, using your dataset's dimension names. |
|
|
87
|
+
|
|
88
|
+
Options defined as flags, such as `--upload-s3-data`, are enabled by their presence; do not append `True` or `False`.
|
|
89
|
+
|
|
90
|
+
### Python API
|
|
91
|
+
|
|
92
|
+
Check that the package can be imported and inspect its Python API:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from pcoras import pcoras
|
|
96
|
+
|
|
97
|
+
help(pcoras)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Conversion examples are available in the repository's test folder. They illustrate how to configure input datasets, output data cubes, metadata, and storage options.
|
|
101
|
+
|
|
102
|
+
## Metadata
|
|
103
|
+
|
|
104
|
+
PCoRaS preserves attributes shared by the source datasets. You can supply additional attributes through `extra_attr`, for example:
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
extra_attr = {
|
|
108
|
+
"keywords": ["climate", "precipitation", "Zarr"],
|
|
109
|
+
"License": "MIT",
|
|
110
|
+
"Producer": "Your organisation",
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
These are example dataset attributes. Set them to match the source data and its actual licensing and provenance.
|
|
115
|
+
|
|
116
|
+
STAC metadata describes the collection or item, including its title, description, providers, and licensing. The constructor accepts additional dataset attributes through `extra_attr`; The user must ensure that attibutes contains `title`, `description`, `providers`, `License`, `Producer` or `links` parameters. Inspect the generated STAC metadata to check that it describes your dataset correctly.
|
|
117
|
+
|
|
118
|
+
The dataset license may differ from the software license.
|
|
119
|
+
|
|
120
|
+
## S3-compatible storage
|
|
121
|
+
|
|
122
|
+
For workflows that upload outputs to S3-compatible storage, configure the upload options and credentials. The following snippet defines example configuration values; it does not start an upload:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
upload_s3_data = True
|
|
126
|
+
bucket_name = "pcoras-data"
|
|
127
|
+
bucket_file_prefix = "precipitation/v01"
|
|
128
|
+
aws_region = None
|
|
129
|
+
aws_access_key = ["AWS_ACCESS_KEY_ID"]
|
|
130
|
+
aws_secret_key = ["AWS_SECRET_ACCESS_KEY"]
|
|
131
|
+
s3_endpoint_url = "https://your-s3-endpoint.example.org"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Set the environment variables before running the workflow, and replace the bucket, prefix, endpoint, and region with values for your storage service.
|
|
135
|
+
|
|
136
|
+
For the CLI, enable `--upload-s3-data` and pass the storage settings through `--bucket-name`, `--bucket-file-prefix`, `--aws-region`, `--aws-access-key`, `--aws-secret-key`, and `--s3-endpoint-url`, as required by your storage service. The Python snippet above reads environment variables explicitly; it does not imply that the CLI reads them automatically.
|
|
137
|
+
|
|
138
|
+
Set `stac_version` to the STAC version supported by your workflow and target catalog.
|
|
139
|
+
|
|
140
|
+
### Mindmap and Flowchart
|
|
141
|
+
|
|
142
|
+
Below, is a the current mindmap and flowchart follow by this repo
|
|
143
|
+
|
|
144
|
+
[Mindmap](/docs/Diagrams/mindmap.png)
|
|
145
|
+
|
|
146
|
+
[Flowchart](/docs/Diagrams/PCoRaS_flowchart.png)
|
|
147
|
+
|
|
148
|
+
### Not supported
|
|
149
|
+
|
|
150
|
+
Currently authentification to upload the STAC collection is not supported yet. However the code for request and publication is provided.
|
|
151
|
+
|
|
152
|
+
## License
|
|
153
|
+
|
|
154
|
+
[MIT](https://choosealicense.com/licenses/mit/).
|
|
155
|
+
|
|
156
|
+
## Credits
|
|
157
|
+
|
|
158
|
+
This project was done thanks to the collaboration of the Institute of Earth Observation and the Center for Climate change and Transformation at Eurac Research
|
|
159
|
+
|
|
160
|
+
Thanks To Irma Ruiz Velasco for the great work on graphics and pcoras logo!
|
|
161
|
+
|
|
162
|
+

|
pcoras-1.0.0/README.md
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+

|
|
2
|
+
|
|
3
|
+

|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
# PCoRaS
|
|
9
|
+
|
|
10
|
+
Python Package for Cloud Optimzed Raster and Spatio-temporal Data, following the STAC guidelines this repository
|
|
11
|
+
is based Cloud optimized type of files and Raster2stac functions to create an STAC collections compatible with
|
|
12
|
+
OpenEO backend, getting an output up to a 5D hypercube dataset for different spatio-temporal data, following a
|
|
13
|
+
collection of local OpenEO processes to test the compability of the item.
|
|
14
|
+
|
|
15
|
+
## Setup
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install pcoras
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Note: Conda support is in development
|
|
22
|
+
|
|
23
|
+
## Getting started
|
|
24
|
+
|
|
25
|
+
### Command-line interface
|
|
26
|
+
|
|
27
|
+
Convert a NetCDF file into a local Zarr data cube:
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
python -m pcoras convert \
|
|
31
|
+
--path ./data/precipitation.nc \
|
|
32
|
+
--output-folder ./output \
|
|
33
|
+
--output-name precipitation \
|
|
34
|
+
--xdim x \
|
|
35
|
+
--ydim y \
|
|
36
|
+
--tdim time
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Replace the input path and dimension names with those in your dataset. For example, a dataset with longitude and latitude dimensions may require `--xdim lon --ydim lat`.
|
|
40
|
+
|
|
41
|
+
These six arguments are required by the command-line parser. NetCDF (`.nc`) is the default input extension. S3 upload and STAC publication are disabled by default; enable them with `--upload-s3-data` and `--publish-stac`, respectively, and supply the corresponding storage or collection settings.
|
|
42
|
+
|
|
43
|
+
The `--path` option accepts one command-line value: a file or directory path. The `co` command is an alias for `convert`.
|
|
44
|
+
|
|
45
|
+
Display the available commands and conversion options:
|
|
46
|
+
|
|
47
|
+
```sh
|
|
48
|
+
python -m pcoras --help
|
|
49
|
+
python -m pcoras convert --help
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Selected conversion options:
|
|
53
|
+
|
|
54
|
+
| Option | Purpose |
|
|
55
|
+
| --- | --- |
|
|
56
|
+
| `--variables pr tas` | Select variables by name. Replace `pr` and `tas` with names from your dataset. |
|
|
57
|
+
| `--file-ext .nc` | Set the input file extension; defaults to `.nc`. |
|
|
58
|
+
| `--crs 32632` | Supply a CRS as an integer EPSG code appropriate for your dataset. |
|
|
59
|
+
| `--chunking '{"time": 31}'` | Supply chunk sizes as a JSON object, using your dataset's dimension names. |
|
|
60
|
+
|
|
61
|
+
Options defined as flags, such as `--upload-s3-data`, are enabled by their presence; do not append `True` or `False`.
|
|
62
|
+
|
|
63
|
+
### Python API
|
|
64
|
+
|
|
65
|
+
Check that the package can be imported and inspect its Python API:
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from pcoras import pcoras
|
|
69
|
+
|
|
70
|
+
help(pcoras)
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Conversion examples are available in the repository's test folder. They illustrate how to configure input datasets, output data cubes, metadata, and storage options.
|
|
74
|
+
|
|
75
|
+
## Metadata
|
|
76
|
+
|
|
77
|
+
PCoRaS preserves attributes shared by the source datasets. You can supply additional attributes through `extra_attr`, for example:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
extra_attr = {
|
|
81
|
+
"keywords": ["climate", "precipitation", "Zarr"],
|
|
82
|
+
"License": "MIT",
|
|
83
|
+
"Producer": "Your organisation",
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
These are example dataset attributes. Set them to match the source data and its actual licensing and provenance.
|
|
88
|
+
|
|
89
|
+
STAC metadata describes the collection or item, including its title, description, providers, and licensing. The constructor accepts additional dataset attributes through `extra_attr`; The user must ensure that attibutes contains `title`, `description`, `providers`, `License`, `Producer` or `links` parameters. Inspect the generated STAC metadata to check that it describes your dataset correctly.
|
|
90
|
+
|
|
91
|
+
The dataset license may differ from the software license.
|
|
92
|
+
|
|
93
|
+
## S3-compatible storage
|
|
94
|
+
|
|
95
|
+
For workflows that upload outputs to S3-compatible storage, configure the upload options and credentials. The following snippet defines example configuration values; it does not start an upload:
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
upload_s3_data = True
|
|
99
|
+
bucket_name = "pcoras-data"
|
|
100
|
+
bucket_file_prefix = "precipitation/v01"
|
|
101
|
+
aws_region = None
|
|
102
|
+
aws_access_key = ["AWS_ACCESS_KEY_ID"]
|
|
103
|
+
aws_secret_key = ["AWS_SECRET_ACCESS_KEY"]
|
|
104
|
+
s3_endpoint_url = "https://your-s3-endpoint.example.org"
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Set the environment variables before running the workflow, and replace the bucket, prefix, endpoint, and region with values for your storage service.
|
|
108
|
+
|
|
109
|
+
For the CLI, enable `--upload-s3-data` and pass the storage settings through `--bucket-name`, `--bucket-file-prefix`, `--aws-region`, `--aws-access-key`, `--aws-secret-key`, and `--s3-endpoint-url`, as required by your storage service. The Python snippet above reads environment variables explicitly; it does not imply that the CLI reads them automatically.
|
|
110
|
+
|
|
111
|
+
Set `stac_version` to the STAC version supported by your workflow and target catalog.
|
|
112
|
+
|
|
113
|
+
### Mindmap and Flowchart
|
|
114
|
+
|
|
115
|
+
Below, is a the current mindmap and flowchart follow by this repo
|
|
116
|
+
|
|
117
|
+
[Mindmap](/docs/Diagrams/mindmap.png)
|
|
118
|
+
|
|
119
|
+
[Flowchart](/docs/Diagrams/PCoRaS_flowchart.png)
|
|
120
|
+
|
|
121
|
+
### Not supported
|
|
122
|
+
|
|
123
|
+
Currently authentification to upload the STAC collection is not supported yet. However the code for request and publication is provided.
|
|
124
|
+
|
|
125
|
+
## License
|
|
126
|
+
|
|
127
|
+
[MIT](https://choosealicense.com/licenses/mit/).
|
|
128
|
+
|
|
129
|
+
## Credits
|
|
130
|
+
|
|
131
|
+
This project was done thanks to the collaboration of the Institute of Earth Observation and the Center for Climate change and Transformation at Eurac Research
|
|
132
|
+
|
|
133
|
+
Thanks To Irma Ruiz Velasco for the great work on graphics and pcoras logo!
|
|
134
|
+
|
|
135
|
+

|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pcoras"
|
|
3
|
+
version = "1.0.0"
|
|
4
|
+
|
|
5
|
+
dependencies = [
|
|
6
|
+
"raster2stac>=2026.2.1",
|
|
7
|
+
"requests",
|
|
8
|
+
"importlib_metadata"
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
authors = [
|
|
12
|
+
{ name = "Miguel Martinez", email = "mmartinezguerrero@eurac.edu" },
|
|
13
|
+
{ name = "Piero Campalani", email = "pcampalani@eurac.edu" },
|
|
14
|
+
{ name = "Alexander Jacob", email = "Alexander.Jacob@eurac.edu"},
|
|
15
|
+
{ name = "Michele Claus", email = "michele.claus@eurac.edu" }
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
license = "MIT"
|
|
19
|
+
license-files = ["LICENSE"]
|
|
20
|
+
|
|
21
|
+
description = "Package for Cloud-Optimized Raster and Spatio-temporal data"
|
|
22
|
+
readme = "README.md"
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Programming Language :: Python :: 3",
|
|
25
|
+
"Operating System :: OS Independent",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"Intended Audience :: Education",
|
|
28
|
+
"Intended Audience :: Science/Research",
|
|
29
|
+
"Development Status :: 4 - Beta",
|
|
30
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Information Analysis"
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
keywords = ["STAC", "Metadata", "Zarr", "NetCDF"]
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
dev = ["isort", "ruff"]
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://gitlab.inf.unibz.it/earth_observation_public/pcoras"
|
|
41
|
+
Issues = "https://gitlab.inf.unibz.it/earth_observation_public/pcoras/-/issues"
|
|
42
|
+
|
|
43
|
+
[project.scripts]
|
|
44
|
+
pcoras = "cli.cli:main"
|
|
45
|
+
|
|
46
|
+
[build-system]
|
|
47
|
+
requires = [
|
|
48
|
+
"numpy",
|
|
49
|
+
"raster2stac",
|
|
50
|
+
"pandas",
|
|
51
|
+
"xarray",
|
|
52
|
+
"rioxarray",
|
|
53
|
+
"ujson",
|
|
54
|
+
"setuptools",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
build-backend = "setuptools.build_meta"
|
|
58
|
+
|
|
59
|
+
[tool.setuptools]
|
|
60
|
+
package-dir = {"" = "src"}
|
|
61
|
+
py-modules = ["pcoras", "_version"]
|
|
62
|
+
|
|
63
|
+
[tool.setuptools.packages.find]
|
|
64
|
+
where = ["src"]
|
|
65
|
+
include = ["cli*", "sub*"]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
[tool.isort]
|
|
69
|
+
sections = [
|
|
70
|
+
"FUTURE",
|
|
71
|
+
"STDLIB",
|
|
72
|
+
"THIRDPARTY",
|
|
73
|
+
"FIRSTPARTY",
|
|
74
|
+
"LOCALFOLDER"
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
[tool.black]
|
|
78
|
+
line-length = 79
|
|
79
|
+
include = '\.pyi?$'
|
|
80
|
+
exclude = '''
|
|
81
|
+
/(
|
|
82
|
+
\.git
|
|
83
|
+
| \.hg
|
|
84
|
+
| \.mypy_cache
|
|
85
|
+
| \.tox
|
|
86
|
+
| \.venv
|
|
87
|
+
| _build
|
|
88
|
+
| buck-out
|
|
89
|
+
| build
|
|
90
|
+
| dist
|
|
91
|
+
| \.conda
|
|
92
|
+
| \.cache
|
|
93
|
+
| empty_conda_pkgs_dir
|
|
94
|
+
| site-packages
|
|
95
|
+
)/
|
|
96
|
+
'''
|
pcoras-1.0.0/setup.cfg
ADDED
pcoras-1.0.0/setup.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Command-line parser and entry point for PCoRaS."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
from importlib.metadata import PackageNotFoundError
|
|
6
|
+
from importlib.metadata import version as package_version
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from . import convert
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
try:
|
|
13
|
+
VERSION = package_version("pcoras")
|
|
14
|
+
except PackageNotFoundError:
|
|
15
|
+
VERSION = "unknown"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def parse_json_object(value):
|
|
19
|
+
"""Read a JSON object from an inline value or a JSON file."""
|
|
20
|
+
candidate = Path(value).expanduser()
|
|
21
|
+
try:
|
|
22
|
+
if candidate.is_file():
|
|
23
|
+
with candidate.open(encoding="utf-8") as stream:
|
|
24
|
+
parsed = json.load(stream)
|
|
25
|
+
else:
|
|
26
|
+
parsed = json.loads(value)
|
|
27
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
28
|
+
raise argparse.ArgumentTypeError(
|
|
29
|
+
f"Expected a JSON object or JSON file: {exc}"
|
|
30
|
+
) from exc
|
|
31
|
+
|
|
32
|
+
if not isinstance(parsed, dict):
|
|
33
|
+
raise argparse.ArgumentTypeError("The value must be a JSON object.")
|
|
34
|
+
return parsed
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def parse_extra_attrs(value):
|
|
38
|
+
"""Read extra attributes from JSON, a JSON file, or key=value pairs."""
|
|
39
|
+
try:
|
|
40
|
+
return parse_json_object(value)
|
|
41
|
+
except argparse.ArgumentTypeError:
|
|
42
|
+
parsed = {}
|
|
43
|
+
for item in value.split(","):
|
|
44
|
+
key, separator, raw_value = item.partition("=")
|
|
45
|
+
if not separator or not key.strip():
|
|
46
|
+
raise argparse.ArgumentTypeError(
|
|
47
|
+
"--extra-attrs expects a JSON object, JSON file, "
|
|
48
|
+
"or comma-separated key=value pairs."
|
|
49
|
+
)
|
|
50
|
+
try:
|
|
51
|
+
parsed[key.strip()] = json.loads(raw_value.strip())
|
|
52
|
+
except json.JSONDecodeError:
|
|
53
|
+
parsed[key.strip()] = raw_value.strip()
|
|
54
|
+
return parsed
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def csv_values(value):
|
|
58
|
+
"""Split a comma-separated CLI value."""
|
|
59
|
+
return [item.strip() for item in value.split(",")]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def add_convert_subparser(subparsers):
|
|
63
|
+
"""Add the ``convert``/``co`` command."""
|
|
64
|
+
parser = subparsers.add_parser(
|
|
65
|
+
"convert",
|
|
66
|
+
aliases=["co"],
|
|
67
|
+
help="Convert multidimensional datasets to a Zarr data cube.",
|
|
68
|
+
)
|
|
69
|
+
parser.set_defaults(validate=convert.validate_args, run=convert.run)
|
|
70
|
+
|
|
71
|
+
required = parser.add_argument_group("required arguments")
|
|
72
|
+
required.add_argument("--path", required=True, help="List or String, Path to the input dataset (file or directory).")
|
|
73
|
+
required.add_argument("--output-folder", required=True, help="Path to the output folder.")
|
|
74
|
+
required.add_argument("--output-name", required=True, help="String, Name of the output dataset.")
|
|
75
|
+
required.add_argument("--xdim", required=True, help="String, Name of the x dimension.")
|
|
76
|
+
required.add_argument("--ydim", required=True, help="String, Name of the y dimension.")
|
|
77
|
+
required.add_argument("--tdim", required=True, help="String, Name of the time dimension.")
|
|
78
|
+
|
|
79
|
+
stac = parser.add_argument_group("STAC")
|
|
80
|
+
stac.add_argument("--collection-url", help="URL of the STAC collection to publish to.")
|
|
81
|
+
stac.add_argument("--collection-id", help="ID of the STAC collection to publish to.")
|
|
82
|
+
stac.add_argument("--publish-stac", default=False, action="store_true", help="Boolean, Whether to publish the output dataset to a STAC collection.")
|
|
83
|
+
stac.add_argument("--stac-version", default="1.0.0", help="String, Version of the STAC specification to use for publishing (default is 1.0.0).")
|
|
84
|
+
|
|
85
|
+
storage = parser.add_argument_group("S3 storage")
|
|
86
|
+
storage.add_argument("--bucket-name", help="String, Name of the S3 bucket to upload data to.")
|
|
87
|
+
storage.add_argument("--bucket-file-prefix", help="String, Prefix for the files in the S3 bucket.")
|
|
88
|
+
storage.add_argument("--aws-region", help="String, AWS region for the S3 bucket.")
|
|
89
|
+
storage.add_argument("--aws-access-key", help="String, AWS access key for the S3 bucket.")
|
|
90
|
+
storage.add_argument("--aws-secret-key", help="String, AWS secret key for the S3 bucket.")
|
|
91
|
+
storage.add_argument("--s3-endpoint-url", help="String, Endpoint URL for the S3 service.")
|
|
92
|
+
storage.add_argument("--upload-s3-data", default=False, action="store_true", help="Boolean, Whether to upload the data to S3.")
|
|
93
|
+
|
|
94
|
+
dataset = parser.add_argument_group("dataset")
|
|
95
|
+
dataset.add_argument("--file-ext", default=".nc", help="String, File extension for the input dataset.")
|
|
96
|
+
dataset.add_argument("--sel-ds", default="unmatched", help="String, Dataset selection criteria.")
|
|
97
|
+
dataset.add_argument("--filter-keyword", help="List or String, Keywords to filter the dataset by their filename.")
|
|
98
|
+
dataset.add_argument("--filter-on", default=False, action="store_true", help="Boolean, Toggle for --filter-keyword (True/False to exclude/include files by keyword).")
|
|
99
|
+
dataset.add_argument("--variables", nargs="+", help="List, Variables to include in the output.")
|
|
100
|
+
dataset.add_argument("--odim", nargs="+", type=int, help="list of int, List of dimension indices to be used for the output dataset (default is None, meaning all dimensions will be used)")
|
|
101
|
+
dataset.add_argument("--olabel", help="String, Name of the label dimension to be used for the output dataset (default is None, meaning no label dimension will be added)")
|
|
102
|
+
dataset.add_argument("--bands", nargs="+", help="list of strings, Bands to include.")
|
|
103
|
+
dataset.add_argument("--band-specification", help="String, specification for how to create the band dimension (default is None, meaning no band dimension will be created).")
|
|
104
|
+
dataset.add_argument("--bands-dimensions", type=parse_json_object, help="Dictionary, specifying how to create the band dimension based on the input datasets (default is None, meaning no band dimension will be created).")
|
|
105
|
+
dataset.add_argument("--pattern", help="String, Pattern for matching files with similar names.")
|
|
106
|
+
|
|
107
|
+
temporal = parser.add_argument_group("temporal transformation")
|
|
108
|
+
temporal.add_argument("--time-interval", type=csv_values, help="list of Strings, List specifying the time interval for selecting data from the input datasets (default is None, meaning no time filtering will be applied")
|
|
109
|
+
temporal.add_argument("--new-time-dimension", help="String, Name of a new time dimension to be added to the output dataset (default is None, meaning no new dimension will be added)")
|
|
110
|
+
temporal.add_argument("--new-time-labels", type=parse_json_object, help="Dictionary, Labels for the new time dimension to be added to the output dataset (default is None, meaning no new dimension will be added)")
|
|
111
|
+
|
|
112
|
+
metadata = parser.add_argument_group("metadata and output")
|
|
113
|
+
metadata.add_argument("--concat-attrs", nargs="+", help="list of strings, List of attribute names to concatenate from the input datasets to the output dataset (default is None, meaning no attributes will be concatenated).")
|
|
114
|
+
metadata.add_argument("--extra-attrs", type=parse_extra_attrs, help="Dictionary of extra attributes to be added to the output dataset (default is None, meaning no extra attributes will be added).")
|
|
115
|
+
metadata.add_argument("--crs", type=int, help="Integer, EPSG code for the coordinate reference system to be used for the output dataset (default is None, meaning no CRS will be assigned to the output dataset).")
|
|
116
|
+
metadata.add_argument("--chunking", type=parse_json_object, default={}, help="Dictionary, specifying the chunk sizes for the output dataset (default is an empty dictionary, meaning no chunking will be applied).")
|
|
117
|
+
|
|
118
|
+
parser.add_argument("--dry-run", action="store_true")
|
|
119
|
+
return parser
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def cli_parser():
|
|
123
|
+
"""Create the top-level PCoRaS parser."""
|
|
124
|
+
parser = argparse.ArgumentParser(
|
|
125
|
+
prog="pcoras",
|
|
126
|
+
description="Build Zarr data cubes from raster datasets.",
|
|
127
|
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
128
|
+
)
|
|
129
|
+
parser.add_argument("-d", "--debug", action="store_true")
|
|
130
|
+
parser.add_argument(
|
|
131
|
+
"--version",
|
|
132
|
+
action="version",
|
|
133
|
+
version=f"%(prog)s {VERSION}",
|
|
134
|
+
)
|
|
135
|
+
subparsers = parser.add_subparsers(dest="action_name", title="actions")
|
|
136
|
+
add_convert_subparser(subparsers)
|
|
137
|
+
return parser
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def main():
|
|
141
|
+
"""Run the PCoRaS CLI."""
|
|
142
|
+
parser = cli_parser()
|
|
143
|
+
args = parser.parse_args()
|
|
144
|
+
|
|
145
|
+
if args.action_name is None:
|
|
146
|
+
parser.print_help()
|
|
147
|
+
return 0
|
|
148
|
+
|
|
149
|
+
try:
|
|
150
|
+
args.validate(args)
|
|
151
|
+
return args.run(args)
|
|
152
|
+
except (OSError, RuntimeError, TypeError, ValueError) as exc:
|
|
153
|
+
parser.error(str(exc))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
if __name__ == "__main__":
|
|
157
|
+
raise SystemExit(main())
|