gedidb 2025.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ci/build_gedidb_testsuite_image.sh +51 -0
- ci/context/gedidb_ci.docker +35 -0
- ci/requirements/doc.yml +36 -0
- doc/Makefile +248 -0
- doc/README.rst +6 -0
- doc/_build/html/_downloads/0fa6048a3a12de4be37de324795251bf/authentification.py +34 -0
- doc/_build/html/_downloads/21ca9a06987343a7ec7abae68c32f811/data_provider.py +110 -0
- doc/_build/html/_downloads/6d70e5a3852380bb31dac8aa12f1412d/data_processor.py +75 -0
- doc/_build/html/_downloads/95c319d764d0d1356ad1b764e7bf66b3/get_variable_list.py +39 -0
- doc/_static/gedidb.css +74 -0
- doc/_static/images/GEDIDB_FLOWCHART.png +0 -0
- doc/_static/index-images/api_reference.svg +97 -0
- doc/_static/index-images/contributor.svg +76 -0
- doc/_static/index-images/getting_started.svg +66 -0
- doc/_static/index-images/user_guide.svg +67 -0
- doc/_static/logos/gediDB_logo.png +0 -0
- doc/_static/logos/gediDB_logo.svg +87 -0
- doc/_static/test_files/data_config.yml +732 -0
- doc/_static/test_files/test.geojson +22 -0
- doc/auto_examples/authentification.codeobj.json +20 -0
- doc/auto_examples/authentification.ipynb +43 -0
- doc/auto_examples/authentification.py +34 -0
- doc/auto_examples/authentification.rst +83 -0
- doc/auto_examples/authentification.zip +0 -0
- doc/auto_examples/auto_examples_jupyter.zip +0 -0
- doc/auto_examples/auto_examples_python.zip +0 -0
- doc/auto_examples/data_processor.codeobj.json +38 -0
- doc/auto_examples/data_processor.ipynb +43 -0
- doc/auto_examples/data_processor.py +75 -0
- doc/auto_examples/data_processor.rst +123 -0
- doc/auto_examples/data_processor.zip +0 -0
- doc/auto_examples/data_provider.codeobj.json +20 -0
- doc/auto_examples/data_provider.ipynb +97 -0
- doc/auto_examples/data_provider.py +110 -0
- doc/auto_examples/data_provider.rst +179 -0
- doc/auto_examples/data_provider.zip +0 -0
- doc/auto_examples/get_variable_list.codeobj.json +11 -0
- doc/auto_examples/get_variable_list.ipynb +43 -0
- doc/auto_examples/get_variable_list.py +39 -0
- doc/auto_examples/get_variable_list.rst +88 -0
- doc/auto_examples/get_variable_list.zip +0 -0
- doc/auto_examples/images/thumb/sphx_glr_authentification_thumb.png +0 -0
- doc/auto_examples/images/thumb/sphx_glr_data_processor_thumb.png +0 -0
- doc/auto_examples/images/thumb/sphx_glr_data_provider_thumb.png +0 -0
- doc/auto_examples/images/thumb/sphx_glr_get_variable_list_thumb.png +0 -0
- doc/auto_examples/index.rst +114 -0
- doc/auto_examples/sg_execution_times.rst +37 -0
- doc/conf.py +484 -0
- doc/gallery/README.rst +2 -0
- doc/gallery/authentification.py +34 -0
- doc/gallery/data_processor.py +75 -0
- doc/gallery/data_provider.py +110 -0
- doc/gallery/get_variable_list.py +39 -0
- doc/index.html +3 -0
- doc/index.rst +97 -0
- doc/sg_execution_times.rst +37 -0
- doc/user/api.rst +70 -0
- doc/user/contributing.rst +132 -0
- doc/user/database.rst +104 -0
- doc/user/faq.rst +69 -0
- doc/user/fundamentals.authenticate.rst +92 -0
- doc/user/fundamentals.filters.rst +43 -0
- doc/user/fundamentals.processor.rst +123 -0
- doc/user/fundamentals.provider.rst +177 -0
- doc/user/fundamentals.rst +24 -0
- doc/user/fundamentals.setup.rst +127 -0
- doc/user/generated/gedidb.CMRDataDownloader.rst +23 -0
- doc/user/generated/gedidb.CMRQuery.rst +22 -0
- doc/user/generated/gedidb.EarthDataAuthenticator.rst +23 -0
- doc/user/generated/gedidb.GEDIDatabase.rst +27 -0
- doc/user/generated/gedidb.GEDIDownloader.rst +22 -0
- doc/user/generated/gedidb.GEDIGranule.rst +24 -0
- doc/user/generated/gedidb.GEDIProcessor.rst +25 -0
- doc/user/generated/gedidb.GEDIProvider.rst +28 -0
- doc/user/generated/gedidb.GediNameMetadata.rst +40 -0
- doc/user/generated/gedidb.GranuleParser.rst +24 -0
- doc/user/generated/gedidb.H5FileDownloader.rst +23 -0
- doc/user/generated/gedidb.L2ABeam.rst +61 -0
- doc/user/generated/gedidb.L2AGranule.rst +77 -0
- doc/user/generated/gedidb.L2BBeam.rst +61 -0
- doc/user/generated/gedidb.L2BGranule.rst +77 -0
- doc/user/generated/gedidb.L4ABeam.rst +61 -0
- doc/user/generated/gedidb.L4AGranule.rst +77 -0
- doc/user/generated/gedidb.L4CBeam.rst +61 -0
- doc/user/generated/gedidb.L4CGranule.rst +77 -0
- doc/user/generated/gedidb.SpatialConsolidationPlan.rst +24 -0
- doc/user/generated/gedidb.SpatialConsolidationPlanner.rst +23 -0
- doc/user/generated/gedidb.TileDBProvider.rst +23 -0
- doc/user/generated/gedidb.beam_handler.rst +61 -0
- doc/user/generated/gedidb.granule_handler.rst +76 -0
- doc/user/index.rst +35 -0
- doc/user/installing.rst +94 -0
- doc/user/quick-overview.rst +91 -0
- doc/user/tiledb_database.rst +254 -0
- doc/user/tuning-config.rst +45 -0
- doc/user/why-gedidb.rst +93 -0
- examples/authentification.ipynb +43 -0
- examples/data_processor.ipynb +43 -0
- examples/data_provider.ipynb +97 -0
- gedidb/__init__.py +145 -0
- gedidb/beam/Beam.py +124 -0
- gedidb/beam/__init__.py +6 -0
- gedidb/beam/l2a_beam.py +97 -0
- gedidb/beam/l2b_beam.py +75 -0
- gedidb/beam/l4a_beam.py +67 -0
- gedidb/beam/l4c_beam.py +67 -0
- gedidb/core/__init__.py +7 -0
- gedidb/core/gedidatabase.py +709 -0
- gedidb/core/gedigranule.py +165 -0
- gedidb/core/gediprocessor.py +459 -0
- gedidb/core/gediprovider.py +534 -0
- gedidb/downloader/__init__.py +7 -0
- gedidb/downloader/authentication.py +140 -0
- gedidb/downloader/cmr_query.py +227 -0
- gedidb/downloader/data_downloader.py +260 -0
- gedidb/granule/Granule.py +193 -0
- gedidb/granule/__init__.py +7 -0
- gedidb/granule/granule_name.py +87 -0
- gedidb/granule/granule_parser.py +155 -0
- gedidb/granule/l2a_granule.py +66 -0
- gedidb/granule/l2b_granule.py +66 -0
- gedidb/granule/l4a_granule.py +66 -0
- gedidb/granule/l4c_granule.py +66 -0
- gedidb/providers/__init__.py +7 -0
- gedidb/providers/tiledb_provider.py +168 -0
- gedidb/utils/__init__.py +7 -0
- gedidb/utils/constants.py +43 -0
- gedidb/utils/geo_processing.py +221 -0
- gedidb/utils/print_versions.py +154 -0
- gedidb/utils/tiledb_consolidation.py +179 -0
- gedidb-2025.2.0.dist-info/LICENSE +287 -0
- gedidb-2025.2.0.dist-info/METADATA +426 -0
- gedidb-2025.2.0.dist-info/RECORD +135 -0
- gedidb-2025.2.0.dist-info/WHEEL +5 -0
- gedidb-2025.2.0.dist-info/top_level.txt +4 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
|
|
3
|
+
context_dir="./context"
|
|
4
|
+
dockerfile="gedidb_ci.docker"
|
|
5
|
+
python_script='
|
|
6
|
+
version = {}
|
|
7
|
+
with open("../../gedidb/version.py") as version_file:
|
|
8
|
+
exec(version_file.read(), version)
|
|
9
|
+
print(version["__version__"])
|
|
10
|
+
'
|
|
11
|
+
version=`python -c "$python_script"`
|
|
12
|
+
|
|
13
|
+
runner_version="v$version"
|
|
14
|
+
runner_tag="gedidb_ci:$runner_version"
|
|
15
|
+
gitlab_runner="gedidb_ci_gitlab_ci_runner_$runner_version"
|
|
16
|
+
|
|
17
|
+
echo "#### Build runner docker image"
|
|
18
|
+
docker rmi ${runner_tag}
|
|
19
|
+
docker build --network=host -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir}
|
|
20
|
+
|
|
21
|
+
# create the gitlab-runner docker container for the current project
|
|
22
|
+
# Remove packages
|
|
23
|
+
rm -fr context/gedidb
|
|
24
|
+
|
|
25
|
+
echo "#### Create gitlab-runner (daemon) container"
|
|
26
|
+
docker stop ${gitlab_runner}
|
|
27
|
+
docker rm ${gitlab_runner}
|
|
28
|
+
docker run -d --name ${gitlab_runner} --network host --restart always -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
|
|
29
|
+
|
|
30
|
+
echo "#### Register container at gitlab"
|
|
31
|
+
# register the runner at the corresponding GitLab repository via a registration-token
|
|
32
|
+
# NOTE: In case of locally stored images (like here), the docker pull policy 'never' must be used
|
|
33
|
+
# (see https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work).
|
|
34
|
+
read -p "Please enter gitlab token: " token
|
|
35
|
+
echo ""
|
|
36
|
+
|
|
37
|
+
url='https://git.gfz-potsdam.de'
|
|
38
|
+
|
|
39
|
+
cmd="gitlab-runner --debug register \
|
|
40
|
+
--executor 'docker' \
|
|
41
|
+
--docker-image '${runner_tag}' \
|
|
42
|
+
--url '${url}' \
|
|
43
|
+
--token '${token}' \
|
|
44
|
+
--description '${gitlab_runner}' \
|
|
45
|
+
--docker-pull-policy='never'
|
|
46
|
+
"
|
|
47
|
+
echo "Running the following command:"
|
|
48
|
+
echo "${cmd}"
|
|
49
|
+
docker exec -it ${gitlab_runner} /bin/bash -c "${cmd}"
|
|
50
|
+
echo 'Done'
|
|
51
|
+
echo 'NOTE: If the runner stays inactive, re-create the runner and register it again.'
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# ref https://github.com/tebeka/pythonwise/blob/master/docker-miniconda/Dockerfile
|
|
2
|
+
FROM python:3.11-slim-buster
|
|
3
|
+
|
|
4
|
+
# System packages
|
|
5
|
+
RUN apt-get update -y && \
|
|
6
|
+
echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \
|
|
7
|
+
apt-get install -y -q dialog apt-utils && \
|
|
8
|
+
apt-get install libudunits2-dev build-essential bash-completion automake -y && \
|
|
9
|
+
apt-get install cmake zlib1g-dev libpng-dev libjpeg-dev -y && \
|
|
10
|
+
apt-get install wget vim zip unzip curl git git-lfs jq apt-rdepends -y && \
|
|
11
|
+
apt-get install strace bzip2 cron make gcc -y && git-lfs install
|
|
12
|
+
|
|
13
|
+
# Install mambaforge
|
|
14
|
+
RUN curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh
|
|
15
|
+
RUN bash Mambaforge-$(uname)-$(uname -m).sh -b -p /root/mambaforge
|
|
16
|
+
RUN rm Mambaforge-$(uname)-$(uname -m).sh
|
|
17
|
+
ENV PATH=/root/mambaforge/bin:${PATH}
|
|
18
|
+
|
|
19
|
+
# Python packages from conda
|
|
20
|
+
RUN mamba install -y python=3.12
|
|
21
|
+
RUN mamba install -y pip
|
|
22
|
+
RUN mamba init bash
|
|
23
|
+
|
|
24
|
+
# copy some needed stuff to /root
|
|
25
|
+
COPY *.yml /root/
|
|
26
|
+
RUN mamba env create -n ci_env -f /root/environment.yml
|
|
27
|
+
|
|
28
|
+
# Make RUN commands use the new environment:
|
|
29
|
+
SHELL ["conda", "run", "-n", "ci_env", "/bin/bash", "-c"]
|
|
30
|
+
|
|
31
|
+
# Allow statements and log messages to immediately appear in the Knative logs
|
|
32
|
+
ENV PYTHONUNBUFFERED True
|
|
33
|
+
|
|
34
|
+
# NOTE: The pkgs directory (cache) is deleted because otherwise conda env create takes hours within a docker container.
|
|
35
|
+
RUN /bin/bash -i -c "rm -rf /root/mambaforge/pkgs"
|
ci/requirements/doc.yml
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
name: gedidb-docs
|
|
2
|
+
channels:
|
|
3
|
+
- conda-forge
|
|
4
|
+
- nodefaults
|
|
5
|
+
dependencies:
|
|
6
|
+
- python=3.12
|
|
7
|
+
- bottleneck
|
|
8
|
+
- cfgrib
|
|
9
|
+
- kerchunk
|
|
10
|
+
- hypothesis>=6.75.8
|
|
11
|
+
- matplotlib-base
|
|
12
|
+
- nbsphinx
|
|
13
|
+
- packaging>=21.3
|
|
14
|
+
- pooch
|
|
15
|
+
- pyarrow
|
|
16
|
+
- pyproj
|
|
17
|
+
- psycopg2
|
|
18
|
+
- setuptools
|
|
19
|
+
- sparse
|
|
20
|
+
- breathe
|
|
21
|
+
- sphinx-autosummary-accessors
|
|
22
|
+
- sphinx-book-theme<=1.0.1
|
|
23
|
+
- sphinx-copybutton
|
|
24
|
+
- sphinx-design
|
|
25
|
+
- sphinx-inline-tabs
|
|
26
|
+
- sphinx>=5.0,<7.0
|
|
27
|
+
- sphinxcontrib-srclinks
|
|
28
|
+
- sphinx-remove-toctrees
|
|
29
|
+
- sphinxext-opengraph
|
|
30
|
+
- sphinxext-rediraffe
|
|
31
|
+
- sphinx-gallery
|
|
32
|
+
- pip
|
|
33
|
+
- pip:
|
|
34
|
+
- ipython
|
|
35
|
+
- sphinxcontrib-mermaid
|
|
36
|
+
- -e ../..
|
doc/Makefile
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# Makefile for Sphinx documentation
|
|
2
|
+
#
|
|
3
|
+
|
|
4
|
+
# You can set these variables from the command line.
|
|
5
|
+
SPHINXOPTS =
|
|
6
|
+
SPHINXBUILD = sphinx-build
|
|
7
|
+
SPHINXATUOBUILD = sphinx-autobuild
|
|
8
|
+
PAPER =
|
|
9
|
+
BUILDDIR = _build
|
|
10
|
+
|
|
11
|
+
# Internal variables.
|
|
12
|
+
PAPEROPT_a4 = -D latex_paper_size=a4
|
|
13
|
+
PAPEROPT_letter = -D latex_paper_size=letter
|
|
14
|
+
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
|
|
15
|
+
# the i18n builder cannot share the environment and doctrees with the others
|
|
16
|
+
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
|
|
17
|
+
|
|
18
|
+
.PHONY: help
|
|
19
|
+
help:
|
|
20
|
+
@echo "Please use \`make <target>' where <target> is one of"
|
|
21
|
+
@echo " html to make standalone HTML files"
|
|
22
|
+
@echo " rtdhtml Build html using same settings used on ReadtheDocs"
|
|
23
|
+
@echo " livehtml Make standalone HTML files and rebuild the documentation when a change is detected. Also includes a livereload enabled web server"
|
|
24
|
+
@echo " dirhtml to make HTML files named index.html in directories"
|
|
25
|
+
@echo " singlehtml to make a single large HTML file"
|
|
26
|
+
@echo " pickle to make pickle files"
|
|
27
|
+
@echo " json to make JSON files"
|
|
28
|
+
@echo " htmlhelp to make HTML files and a HTML help project"
|
|
29
|
+
@echo " qthelp to make HTML files and a qthelp project"
|
|
30
|
+
@echo " applehelp to make an Apple Help Book"
|
|
31
|
+
@echo " devhelp to make HTML files and a Devhelp project"
|
|
32
|
+
@echo " epub to make an epub"
|
|
33
|
+
@echo " epub3 to make an epub3"
|
|
34
|
+
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
|
|
35
|
+
@echo " latexpdf to make LaTeX files and run them through pdflatex"
|
|
36
|
+
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
|
|
37
|
+
@echo " text to make text files"
|
|
38
|
+
@echo " man to make manual pages"
|
|
39
|
+
@echo " texinfo to make Texinfo files"
|
|
40
|
+
@echo " info to make Texinfo files and run them through makeinfo"
|
|
41
|
+
@echo " gettext to make PO message catalogs"
|
|
42
|
+
@echo " changes to make an overview of all changed/added/deprecated items"
|
|
43
|
+
@echo " xml to make Docutils-native XML files"
|
|
44
|
+
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
|
|
45
|
+
@echo " linkcheck to check all external links for integrity"
|
|
46
|
+
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
|
|
47
|
+
@echo " coverage to run coverage check of the documentation (if enabled)"
|
|
48
|
+
@echo " dummy to check syntax errors of document sources"
|
|
49
|
+
|
|
50
|
+
.PHONY: clean
|
|
51
|
+
clean:
|
|
52
|
+
rm -rf $(BUILDDIR)/*
|
|
53
|
+
rm -rf generated/*
|
|
54
|
+
rm -rf auto_gallery/
|
|
55
|
+
|
|
56
|
+
.PHONY: html
|
|
57
|
+
html:
|
|
58
|
+
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
|
|
59
|
+
@echo
|
|
60
|
+
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
|
|
61
|
+
|
|
62
|
+
.PHONY: rtdhtml
|
|
63
|
+
rtdhtml:
|
|
64
|
+
$(SPHINXBUILD) -T -j auto -E -W --keep-going -b html -d $(BUILDDIR)/doctrees -D language=en . $(BUILDDIR)/html
|
|
65
|
+
@echo
|
|
66
|
+
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
.PHONY: livehtml
|
|
70
|
+
livehtml:
|
|
71
|
+
# @echo "$(SPHINXATUOBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html"
|
|
72
|
+
$(SPHINXATUOBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
|
|
73
|
+
|
|
74
|
+
.PHONY: dirhtml
|
|
75
|
+
dirhtml:
|
|
76
|
+
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
|
|
77
|
+
@echo
|
|
78
|
+
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
|
|
79
|
+
|
|
80
|
+
.PHONY: singlehtml
|
|
81
|
+
singlehtml:
|
|
82
|
+
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
|
|
83
|
+
@echo
|
|
84
|
+
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
|
|
85
|
+
|
|
86
|
+
.PHONY: html-noplot
|
|
87
|
+
html-noplot:
|
|
88
|
+
$(SPHINXBUILD) -D plot_gallery=0 -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
|
|
89
|
+
@echo
|
|
90
|
+
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
|
|
91
|
+
|
|
92
|
+
.PHONY: pickle
|
|
93
|
+
pickle:
|
|
94
|
+
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
|
|
95
|
+
@echo
|
|
96
|
+
@echo "Build finished; now you can process the pickle files."
|
|
97
|
+
|
|
98
|
+
.PHONY: json
|
|
99
|
+
json:
|
|
100
|
+
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
|
|
101
|
+
@echo
|
|
102
|
+
@echo "Build finished; now you can process the JSON files."
|
|
103
|
+
|
|
104
|
+
.PHONY: htmlhelp
|
|
105
|
+
htmlhelp:
|
|
106
|
+
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
|
|
107
|
+
@echo
|
|
108
|
+
@echo "Build finished; now you can run HTML Help Workshop with the" \
|
|
109
|
+
".hhp project file in $(BUILDDIR)/htmlhelp."
|
|
110
|
+
|
|
111
|
+
.PHONY: qthelp
|
|
112
|
+
qthelp:
|
|
113
|
+
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
|
|
114
|
+
@echo
|
|
115
|
+
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
|
|
116
|
+
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
|
|
117
|
+
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/gedidb.qhcp"
|
|
118
|
+
@echo "To view the help file:"
|
|
119
|
+
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/gedidb.qhc"
|
|
120
|
+
|
|
121
|
+
.PHONY: applehelp
|
|
122
|
+
applehelp:
|
|
123
|
+
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
|
|
124
|
+
@echo
|
|
125
|
+
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
|
|
126
|
+
@echo "N.B. You won't be able to view it unless you put it in" \
|
|
127
|
+
"~/Library/Documentation/Help or install it in your application" \
|
|
128
|
+
"bundle."
|
|
129
|
+
|
|
130
|
+
.PHONY: devhelp
|
|
131
|
+
devhelp:
|
|
132
|
+
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
|
|
133
|
+
@echo
|
|
134
|
+
@echo "Build finished."
|
|
135
|
+
@echo "To view the help file:"
|
|
136
|
+
@echo "# mkdir -p $$HOME/.local/share/devhelp/gedidb"
|
|
137
|
+
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/gedidb"
|
|
138
|
+
@echo "# devhelp"
|
|
139
|
+
|
|
140
|
+
.PHONY: epub
|
|
141
|
+
epub:
|
|
142
|
+
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
|
|
143
|
+
@echo
|
|
144
|
+
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
|
|
145
|
+
|
|
146
|
+
.PHONY: epub3
|
|
147
|
+
epub3:
|
|
148
|
+
$(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3
|
|
149
|
+
@echo
|
|
150
|
+
@echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3."
|
|
151
|
+
|
|
152
|
+
.PHONY: latex
|
|
153
|
+
latex:
|
|
154
|
+
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
|
|
155
|
+
@echo
|
|
156
|
+
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
|
|
157
|
+
@echo "Run \`make' in that directory to run these through (pdf)latex" \
|
|
158
|
+
"(use \`make latexpdf' here to do that automatically)."
|
|
159
|
+
|
|
160
|
+
.PHONY: latexpdf
|
|
161
|
+
latexpdf:
|
|
162
|
+
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
|
|
163
|
+
@echo "Running LaTeX files through pdflatex..."
|
|
164
|
+
$(MAKE) -C $(BUILDDIR)/latex all-pdf
|
|
165
|
+
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
|
|
166
|
+
|
|
167
|
+
.PHONY: latexpdfja
|
|
168
|
+
latexpdfja:
|
|
169
|
+
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
|
|
170
|
+
@echo "Running LaTeX files through platex and dvipdfmx..."
|
|
171
|
+
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
|
|
172
|
+
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
|
|
173
|
+
|
|
174
|
+
.PHONY: text
|
|
175
|
+
text:
|
|
176
|
+
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
|
|
177
|
+
@echo
|
|
178
|
+
@echo "Build finished. The text files are in $(BUILDDIR)/text."
|
|
179
|
+
|
|
180
|
+
.PHONY: man
|
|
181
|
+
man:
|
|
182
|
+
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
|
|
183
|
+
@echo
|
|
184
|
+
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
|
|
185
|
+
|
|
186
|
+
.PHONY: texinfo
|
|
187
|
+
texinfo:
|
|
188
|
+
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
|
|
189
|
+
@echo
|
|
190
|
+
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
|
|
191
|
+
@echo "Run \`make' in that directory to run these through makeinfo" \
|
|
192
|
+
"(use \`make info' here to do that automatically)."
|
|
193
|
+
|
|
194
|
+
.PHONY: info
|
|
195
|
+
info:
|
|
196
|
+
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
|
|
197
|
+
@echo "Running Texinfo files through makeinfo..."
|
|
198
|
+
make -C $(BUILDDIR)/texinfo info
|
|
199
|
+
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
|
|
200
|
+
|
|
201
|
+
.PHONY: gettext
|
|
202
|
+
gettext:
|
|
203
|
+
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
|
|
204
|
+
@echo
|
|
205
|
+
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
|
|
206
|
+
|
|
207
|
+
.PHONY: changes
|
|
208
|
+
changes:
|
|
209
|
+
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
|
|
210
|
+
@echo
|
|
211
|
+
@echo "The overview file is in $(BUILDDIR)/changes."
|
|
212
|
+
|
|
213
|
+
.PHONY: linkcheck
|
|
214
|
+
linkcheck:
|
|
215
|
+
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
|
|
216
|
+
@echo
|
|
217
|
+
@echo "Link check complete; look for any errors in the above output " \
|
|
218
|
+
"or in $(BUILDDIR)/linkcheck/output.txt."
|
|
219
|
+
|
|
220
|
+
.PHONY: doctest
|
|
221
|
+
doctest:
|
|
222
|
+
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
|
|
223
|
+
@echo "Testing of doctests in the sources finished, look at the " \
|
|
224
|
+
"results in $(BUILDDIR)/doctest/output.txt."
|
|
225
|
+
|
|
226
|
+
.PHONY: coverage
|
|
227
|
+
coverage:
|
|
228
|
+
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
|
|
229
|
+
@echo "Testing of coverage in the sources finished, look at the " \
|
|
230
|
+
"results in $(BUILDDIR)/coverage/python.txt."
|
|
231
|
+
|
|
232
|
+
.PHONY: xml
|
|
233
|
+
xml:
|
|
234
|
+
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
|
|
235
|
+
@echo
|
|
236
|
+
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
|
|
237
|
+
|
|
238
|
+
.PHONY: pseudoxml
|
|
239
|
+
pseudoxml:
|
|
240
|
+
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
|
|
241
|
+
@echo
|
|
242
|
+
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
|
|
243
|
+
|
|
244
|
+
.PHONY: dummy
|
|
245
|
+
dummy:
|
|
246
|
+
$(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy
|
|
247
|
+
@echo
|
|
248
|
+
@echo "Build finished. Dummy builder generates no files."
|
doc/README.rst
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Authenticating with NASA EarthData API
|
|
3
|
+
======================================
|
|
4
|
+
|
|
5
|
+
This example demonstrates how to authenticate with the NASA EarthData API using the `EarthDataAuthenticator` class.
|
|
6
|
+
The authentication process involves managing `.netrc` and cookie files to ensure seamless automated login.
|
|
7
|
+
|
|
8
|
+
Before running this example:
|
|
9
|
+
|
|
10
|
+
1. Ensure you have a valid EarthData account. You can create one at https://urs.earthdata.nasa.gov.
|
|
11
|
+
2. Install `wget`, which is used to fetch cookies. You can install it via your system's package manager (e.g., `apt-get install wget` on Debian/Ubuntu).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from gedidb.downloader.authentication import EarthDataAuthenticator
|
|
16
|
+
|
|
17
|
+
# Specify the directory where `.netrc` and cookies will be stored
|
|
18
|
+
earth_data_dir = Path.home()
|
|
19
|
+
|
|
20
|
+
# Initialize the authenticator
|
|
21
|
+
authenticator = EarthDataAuthenticator(earth_data_dir=earth_data_dir, strict=False)
|
|
22
|
+
|
|
23
|
+
# Authenticate and ensure `.netrc` and cookies are valid
|
|
24
|
+
authenticator.authenticate()
|
|
25
|
+
|
|
26
|
+
# Expected Output:
|
|
27
|
+
# ----------------
|
|
28
|
+
# INFO:__main__:EarthData authentication setup incomplete; starting setup.
|
|
29
|
+
# Please enter your Earthdata Login username: <your_username>
|
|
30
|
+
# Please enter your Earthdata Login password:
|
|
31
|
+
# INFO:__main__:Credentials added to .netrc file.
|
|
32
|
+
# INFO:__main__:Attempting to fetch Earthdata cookies and save to /home/username/.earthdata_auth/.cookies
|
|
33
|
+
# INFO:__main__:Earthdata cookies successfully fetched and saved to /home/username/.earthdata_auth/.cookies.
|
|
34
|
+
# Authentication complete.
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Querying Processed GEDI Data from TileDB
|
|
3
|
+
========================================
|
|
4
|
+
|
|
5
|
+
This example demonstrates how to use the `GEDIProvider` class to query and retrieve GEDI data stored in TileDB arrays.
|
|
6
|
+
We will:
|
|
7
|
+
|
|
8
|
+
1. Set up the `GEDIProvider` with a TileDB storage backend.
|
|
9
|
+
2. Query data using spatial and temporal filters.
|
|
10
|
+
3. Retrieve data in both `xarray.Dataset` and `pandas.DataFrame` formats.
|
|
11
|
+
4. Perform nearest-shot queries to retrieve GEDI data closest to a given point.
|
|
12
|
+
|
|
13
|
+
Before running this example:
|
|
14
|
+
|
|
15
|
+
- Ensure that the GEDI data has been processed and stored in TileDB arrays.
|
|
16
|
+
- Configure the TileDB storage settings (local or S3) based on your data location.
|
|
17
|
+
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import geopandas as gpd
|
|
21
|
+
import gedidb as gdb
|
|
22
|
+
|
|
23
|
+
# Configure the TileDB storage backend
|
|
24
|
+
storage_type = "local" # Options: "local" or "s3"
|
|
25
|
+
local_path = "/path/to/processed/gedi/data"
|
|
26
|
+
s3_bucket = None # Only required if using "s3"
|
|
27
|
+
|
|
28
|
+
# Initialize the GEDIProvider
|
|
29
|
+
provider = gdb.GEDIProvider(
|
|
30
|
+
storage_type=storage_type,
|
|
31
|
+
local_path=local_path,
|
|
32
|
+
s3_bucket=s3_bucket,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# Define variables to retrieve
|
|
36
|
+
variables = ["latitude", "longitude", "rh100", "beam"]
|
|
37
|
+
|
|
38
|
+
# Define the spatial region of interest (ROI) as a GeoJSON file or GeoDataFrame
|
|
39
|
+
geojson_path = "/path/to/roi.geojson"
|
|
40
|
+
geometry = gpd.read_file(geojson_path)
|
|
41
|
+
|
|
42
|
+
# Define the temporal range
|
|
43
|
+
start_time = "2020-01-01"
|
|
44
|
+
end_time = "2020-12-31"
|
|
45
|
+
|
|
46
|
+
##############################################################
|
|
47
|
+
# -----------------------------------------------------------------
|
|
48
|
+
# Section 1: Bounding Box Query
|
|
49
|
+
# -----------------------------------------------------------------
|
|
50
|
+
print("=== Bounding Box Query ===")
|
|
51
|
+
print("Querying GEDI data within a specified spatial and temporal range.")
|
|
52
|
+
|
|
53
|
+
# Query data within the bounding box and time range, and retrieve it as an `xarray.Dataset`
|
|
54
|
+
data_xarray = provider.get_data(
|
|
55
|
+
variables=variables,
|
|
56
|
+
geometry=geometry,
|
|
57
|
+
start_time=start_time,
|
|
58
|
+
end_time=end_time,
|
|
59
|
+
return_type="xarray",
|
|
60
|
+
query_type="bounding_box",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Print the retrieved `xarray.Dataset`
|
|
64
|
+
print("Retrieved data as an xarray.Dataset:")
|
|
65
|
+
print(data_xarray)
|
|
66
|
+
|
|
67
|
+
##############################################################
|
|
68
|
+
# -----------------------------------------------------------------
|
|
69
|
+
# Section 2: Nearest Shot Query
|
|
70
|
+
# -----------------------------------------------------------------
|
|
71
|
+
print("\n=== Nearest Shot Query ===")
|
|
72
|
+
print("Querying GEDI data for the nearest shots to a specific point.")
|
|
73
|
+
|
|
74
|
+
# Specify a geographic point (longitude, latitude) and the number of nearest shots
|
|
75
|
+
point = (-55.0, -10.0) # Example longitude and latitude
|
|
76
|
+
num_shots = 5 # Retrieve the 5 nearest shots
|
|
77
|
+
|
|
78
|
+
# Query data for the nearest GEDI shots
|
|
79
|
+
nearest_data = provider.get_data(
|
|
80
|
+
variables=variables,
|
|
81
|
+
point=point,
|
|
82
|
+
num_shots=num_shots,
|
|
83
|
+
query_type="nearest",
|
|
84
|
+
return_type="xarray",
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Print the nearest GEDI shots
|
|
88
|
+
print("Retrieved nearest GEDI shots as an xarray.Dataset:")
|
|
89
|
+
print(nearest_data)
|
|
90
|
+
|
|
91
|
+
##############################################################
|
|
92
|
+
# -----------------------------------------------------------------
|
|
93
|
+
# Section 3: Data Format Options
|
|
94
|
+
# -----------------------------------------------------------------
|
|
95
|
+
print("\n=== Data Format Options ===")
|
|
96
|
+
print("Retrieving GEDI data as a pandas.DataFrame for easier tabular analysis.")
|
|
97
|
+
|
|
98
|
+
# Query the data within the bounding box and retrieve it as a `pandas.DataFrame`
|
|
99
|
+
data_dataframe = provider.get_data(
|
|
100
|
+
variables=variables,
|
|
101
|
+
geometry=geometry,
|
|
102
|
+
start_time=start_time,
|
|
103
|
+
end_time=end_time,
|
|
104
|
+
return_type="dataframe",
|
|
105
|
+
query_type="bounding_box",
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
# Print the retrieved `pandas.DataFrame`
|
|
109
|
+
print("Retrieved data as a pandas.DataFrame:")
|
|
110
|
+
print(data_dataframe)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""
|
|
2
|
+
GEDIProcessor example with different parallel engines
|
|
3
|
+
=======================================================
|
|
4
|
+
|
|
5
|
+
This example demonstrates how to use the `gedidb` library to process GEDI granules
|
|
6
|
+
with different parallel engines, such as `concurrent.futures.ThreadPoolExecutor` and `dask.distributed.Client`.
|
|
7
|
+
|
|
8
|
+
A default data configuration file (`data_config.yml`) can be downloaded here:
|
|
9
|
+
|
|
10
|
+
:download:`Download data_config.yml <../_static/test_files/data_config.yml>`
|
|
11
|
+
|
|
12
|
+
A default geojson file (`test.geojson`) can be downloaded here:
|
|
13
|
+
|
|
14
|
+
:download:`Download test.geojson <../_static/test_files/test.geojson>`
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
We will:
|
|
18
|
+
|
|
19
|
+
1. Set up configuration paths.
|
|
20
|
+
2. Initialize different parallel engines (Dask and concurrent futures).
|
|
21
|
+
3. Run the GEDIProcessor to process granules and consolidate fragments.
|
|
22
|
+
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
# Import required libraries
|
|
26
|
+
import gedidb as gdb
|
|
27
|
+
from dask.distributed import Client, LocalCluster
|
|
28
|
+
import concurrent.futures
|
|
29
|
+
|
|
30
|
+
# Configuration file path
|
|
31
|
+
config_file = '/path/to/data_config.yml'
|
|
32
|
+
|
|
33
|
+
# Paths to GeoJSON region and EarthData credentials
|
|
34
|
+
geojson_path = '/path/to/test.geojson'
|
|
35
|
+
earth_data_dir = '/path/to/EarthData_credentials'
|
|
36
|
+
|
|
37
|
+
# Define the start and end date for processing
|
|
38
|
+
start_date = '2020-01-01'
|
|
39
|
+
end_date = '2020-12-31'
|
|
40
|
+
|
|
41
|
+
# Option 1: Using a ThreadPoolExecutor (concurrent.futures)
|
|
42
|
+
# ---------------------------------------------------------
|
|
43
|
+
# This option uses Python's standard library for parallel execution.
|
|
44
|
+
# Useful for lightweight, multi-threaded tasks on a single machine.
|
|
45
|
+
print("Initializing ThreadPoolExecutor...")
|
|
46
|
+
concurrent_engine = concurrent.futures.ThreadPoolExecutor(max_workers=5)
|
|
47
|
+
|
|
48
|
+
# Option 2: Using Dask for Parallel Execution
|
|
49
|
+
# -------------------------------------------
|
|
50
|
+
# This option is suitable for distributed computing or managing memory more efficiently.
|
|
51
|
+
print("Initializing Dask Client...")
|
|
52
|
+
n_workers = 1 # Number of Dask workers
|
|
53
|
+
cluster = LocalCluster(
|
|
54
|
+
n_workers=5,
|
|
55
|
+
threads_per_worker=1,
|
|
56
|
+
processes=True,
|
|
57
|
+
memory_limit='8GB',
|
|
58
|
+
dashboard_address=None,
|
|
59
|
+
)
|
|
60
|
+
dask_client = Client(cluster)
|
|
61
|
+
|
|
62
|
+
# Initialize the GEDIProcessor with the chosen parallel engine
|
|
63
|
+
# ------------------------------------------------------------
|
|
64
|
+
# Here, we demonstrate usage with `concurrent.futures.ThreadPoolExecutor`.
|
|
65
|
+
# You can replace `parallel_engine=concurrent_engine` with `parallel_engine=dask_client` to use Dask instead.
|
|
66
|
+
with gdb.GEDIProcessor(
|
|
67
|
+
config_file=config_file,
|
|
68
|
+
geometry=geojson_path,
|
|
69
|
+
start_date=start_date,
|
|
70
|
+
end_date=end_date,
|
|
71
|
+
earth_data_dir=earth_data_dir,
|
|
72
|
+
parallel_engine=concurrent_engine, # Change to `parallel_engine=dask_client` for Dask
|
|
73
|
+
) as processor:
|
|
74
|
+
# Run the GEDIProcessor to process granules and consolidate fragments
|
|
75
|
+
processor.compute(consolidate=True)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Retrieve List of Variables in TileDB Array
|
|
3
|
+
==========================================
|
|
4
|
+
|
|
5
|
+
This example demonstrates how to retrieve the list of variables stored in a TileDB array
|
|
6
|
+
using the `GEDIProvider` class from the `gedidb` package. The output is a `pandas.DataFrame`
|
|
7
|
+
containing variable names, descriptions, and metadata.
|
|
8
|
+
|
|
9
|
+
Prerequisites:
|
|
10
|
+
--------------
|
|
11
|
+
1. GEDI data must be processed and stored in TileDB arrays.
|
|
12
|
+
2. Configure the TileDB storage backend (local or S3).
|
|
13
|
+
|
|
14
|
+
Steps:
|
|
15
|
+
------
|
|
16
|
+
1. Configure the TileDB backend (local or S3).
|
|
17
|
+
2. Initialize the `GEDIProvider`.
|
|
18
|
+
3. Retrieve and display the list of available variables.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import gedidb as gdb
|
|
22
|
+
|
|
23
|
+
# Step 1: Configure the TileDB storage backend
|
|
24
|
+
storage_type = "local" # Options: "local" or "s3"
|
|
25
|
+
local_path = "/path/to/processed/gedi/data" # Update with your local TileDB path
|
|
26
|
+
s3_bucket = None # Set the S3 bucket name if using S3 storage
|
|
27
|
+
|
|
28
|
+
# Step 2: Initialize the GEDIProvider
|
|
29
|
+
provider = gdb.GEDIProvider(
|
|
30
|
+
storage_type=storage_type,
|
|
31
|
+
local_path=local_path,
|
|
32
|
+
s3_bucket=s3_bucket,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# Step 3: Retrieve the list of available variables in the TileDB array
|
|
36
|
+
variables_df = provider.get_available_variables()
|
|
37
|
+
|
|
38
|
+
# Display the resulting DataFrame
|
|
39
|
+
print(variables_df)
|