asf_cherry_pick 2.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- asf_cherry_pick-2.0.2/LICENSE +21 -0
- asf_cherry_pick-2.0.2/PKG-INFO +51 -0
- asf_cherry_pick-2.0.2/README.rst +32 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick/__init__.py +4 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick/asf_cherry_pick.py +344 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick/get_missings.py +145 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick.egg-info/PKG-INFO +51 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick.egg-info/SOURCES.txt +13 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick.egg-info/dependency_links.txt +1 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick.egg-info/entry_points.txt +3 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick.egg-info/requires.txt +4 -0
- asf_cherry_pick-2.0.2/asf_cherry_pick.egg-info/top_level.txt +6 -0
- asf_cherry_pick-2.0.2/pyproject.toml +34 -0
- asf_cherry_pick-2.0.2/setup.cfg +4 -0
- asf_cherry_pick-2.0.2/setup.py +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 NSBAS
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: asf_cherry_pick
|
|
3
|
+
Version: 2.0.2
|
|
4
|
+
Summary: Python package for cherry picking Data into asf
|
|
5
|
+
Author-email: Franck Thollard <franck.thollard@univ-grenoble-alpes.fr>
|
|
6
|
+
License: BSD 3-Clause License
|
|
7
|
+
Project-URL: Homepage, https://gricad-gitlab.univ-grenoble-alpes.fr/NSBAS/asf_cherry_pick
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Description-Content-Type: text/x-rst
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: asf_search==9.0.9
|
|
15
|
+
Requires-Dist: fastkml
|
|
16
|
+
Requires-Dist: lxml
|
|
17
|
+
Requires-Dist: remotezip>=0.10.0
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
asf_cherry_pick
|
|
21
|
+
===============
|
|
22
|
+
|
|
23
|
+
The Alaska Sar Fundation mirrors the sentinel1 data from the ESA. For a given acquisition,
|
|
24
|
+
data are provided as a zip file.
|
|
25
|
+
|
|
26
|
+
We provide here a script (and an API) that can fetch data into the zip file, allowing to
|
|
27
|
+
download only the product of interest. This is useflull either for downloading meta data
|
|
28
|
+
or only the tiff file of interest (e.g. only one polarization and one sub swath).
|
|
29
|
+
|
|
30
|
+
Elements to fetch are describe as a pattern (in the sens of regular expression).
|
|
31
|
+
|
|
32
|
+
The set of product are given either
|
|
33
|
+
|
|
34
|
+
- a file in kml format (that contains a polygon which represent the region of interest)
|
|
35
|
+
- a file containing the list of zip files to fetch
|
|
36
|
+
- a file containing the list of urls
|
|
37
|
+
|
|
38
|
+
Installation
|
|
39
|
+
------------
|
|
40
|
+
|
|
41
|
+
- Unzip the file.
|
|
42
|
+
- create a virtual env environment (e.g. python3 -m venv asf_cherry_pick)
|
|
43
|
+
- go to the directory in asf_cherry_pick where the setup.py lies
|
|
44
|
+
- run the command **pip install .**
|
|
45
|
+
|
|
46
|
+
Upon installation, you will have access to the script asf_cherry_pick.
|
|
47
|
+
|
|
48
|
+
Run **asf_cherry_pick -h** and **asf_cherry_pick -H** for a more detail help.
|
|
49
|
+
|
|
50
|
+
Enjoy.
|
|
51
|
+
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
asf_cherry_pick
|
|
2
|
+
===============
|
|
3
|
+
|
|
4
|
+
The Alaska Sar Fundation mirrors the sentinel1 data from the ESA. For a given acquisition,
|
|
5
|
+
data are provided as a zip file.
|
|
6
|
+
|
|
7
|
+
We provide here a script (and an API) that can fetch data into the zip file, allowing to
|
|
8
|
+
download only the product of interest. This is useflull either for downloading meta data
|
|
9
|
+
or only the tiff file of interest (e.g. only one polarization and one sub swath).
|
|
10
|
+
|
|
11
|
+
Elements to fetch are describe as a pattern (in the sens of regular expression).
|
|
12
|
+
|
|
13
|
+
The set of product are given either
|
|
14
|
+
|
|
15
|
+
- a file in kml format (that contains a polygon which represent the region of interest)
|
|
16
|
+
- a file containing the list of zip files to fetch
|
|
17
|
+
- a file containing the list of urls
|
|
18
|
+
|
|
19
|
+
Installation
|
|
20
|
+
------------
|
|
21
|
+
|
|
22
|
+
- Unzip the file.
|
|
23
|
+
- create a virtual env environment (e.g. python3 -m venv asf_cherry_pick)
|
|
24
|
+
- go to the directory in asf_cherry_pick where the setup.py lies
|
|
25
|
+
- run the command **pip install .**
|
|
26
|
+
|
|
27
|
+
Upon installation, you will have access to the script asf_cherry_pick.
|
|
28
|
+
|
|
29
|
+
Run **asf_cherry_pick -h** and **asf_cherry_pick -H** for a more detail help.
|
|
30
|
+
|
|
31
|
+
Enjoy.
|
|
32
|
+
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
Script that can fetch part of zip file of sentinel1 data from ASF.
|
|
6
|
+
|
|
7
|
+
Requirement: have an access (ie login, passwd) at asf.
|
|
8
|
+
|
|
9
|
+
Algorithm:
|
|
10
|
+
=========
|
|
11
|
+
|
|
12
|
+
The algorithm is two folds:
|
|
13
|
+
|
|
14
|
+
i) get the set of urls that matchs a given query. The goegraphical query is
|
|
15
|
+
done through a kml file. A orbit has to be provided that allows to select
|
|
16
|
+
either a ascending or descending track. A time range is also mandatory.
|
|
17
|
+
ii) Downlowd from the url the data in the zip files that match the pattern.
|
|
18
|
+
This can be used either to download only one tiff file (e.g. one polarisation, one subswath)
|
|
19
|
+
or meta data like e.g. xml files.
|
|
20
|
+
|
|
21
|
+
Dependencies
|
|
22
|
+
============
|
|
23
|
+
The script depends upon the asf_search tool that can be installed using pypi.
|
|
24
|
+
The script also uses remotezip package, which has to be modified. We thus embed the modified version.
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
Examples:
|
|
28
|
+
=========
|
|
29
|
+
|
|
30
|
+
url.txt contains urls (one per line) that can be fetched from asf (like e.g. )
|
|
31
|
+
products contains safe names, one per line.
|
|
32
|
+
D041_S1_Mexique.kml is a kml containing a polygon that represent the region of interest
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
asf_cherry_pick --kml D041_S1_Mexique.kml -d 2019-01-30:2020-01-23 -c credentials.txt -v 4 -o D041 --saveproducts to_download.txt
|
|
37
|
+
asf_cherry_pick --urls url -c credentials.txt -o A0034
|
|
38
|
+
asf_cherry_pick --safelist products -c credentials.txt -o A0034
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
API example:
|
|
42
|
+
===========
|
|
43
|
+
|
|
44
|
+
import asf_search as asf
|
|
45
|
+
from fastkml import kml, geometry
|
|
46
|
+
import nsb_remotezip as remotezip
|
|
47
|
+
import flatsim_get_xml_from_kml as fx
|
|
48
|
+
|
|
49
|
+
picker = fx.AsfCherryPick()
|
|
50
|
+
roi = [(-98.82686004479159, 20.4077763126375),
|
|
51
|
+
(-101.1931787671002, 20.81921608915112),
|
|
52
|
+
(-101.8919952521377, 17.07545047337062),
|
|
53
|
+
(-99.55373281701526, 16.60957331722261),
|
|
54
|
+
(-98.82686004479159, 20.4077763126375)]
|
|
55
|
+
picker.search_urls(roi, 'D041', '2019-01-30', '2020-01-23')
|
|
56
|
+
print(f"urls= {picker.urls}")
|
|
57
|
+
picker.connect(user, passwd)
|
|
58
|
+
nb_processes = 4
|
|
59
|
+
picker.cherry_pick_all(target_dir, '(manifest.safe|s1.*xml$)', nb_processes)
|
|
60
|
+
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
# classical imports
|
|
64
|
+
# from IPython import embed; embed()
|
|
65
|
+
|
|
66
|
+
import argparse
|
|
67
|
+
import logging
|
|
68
|
+
import sys
|
|
69
|
+
from pathlib import Path
|
|
70
|
+
import multiprocessing
|
|
71
|
+
from multiprocessing import Pool
|
|
72
|
+
import re
|
|
73
|
+
|
|
74
|
+
import asf_search as asf
|
|
75
|
+
from fastkml import kml, geometry
|
|
76
|
+
import remotezip
|
|
77
|
+
|
|
78
|
+
logger = logging.getLogger(__name__)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class AsfCherryPick(object):
|
|
82
|
+
|
|
83
|
+
"""AsfCherryPick. A class for picking data at asf."""
|
|
84
|
+
|
|
85
|
+
def __init__(self, temp_dir="./asf_tmp"):
|
|
86
|
+
"""Initialize the cherry picker
|
|
87
|
+
:roi: (str) region of interest (format )
|
|
88
|
+
:orbit: (str) orbit in the form D041
|
|
89
|
+
:auth_file: (str) a file containing login passwd information
|
|
90
|
+
:processes: (int) nb of download in //. all the cpus if None,
|
|
91
|
+
default=1
|
|
92
|
+
"""
|
|
93
|
+
self.auth = None
|
|
94
|
+
self.tempdir = Path(temp_dir)
|
|
95
|
+
self.tempdir.mkdir(parents=True, exist_ok=True)
|
|
96
|
+
# self.cookiejar = None
|
|
97
|
+
# self.cookiejar_session = None
|
|
98
|
+
|
|
99
|
+
def search_urls(self, roi, orbit, start_date='2014-09-01', end_date='2022-06-01'):
|
|
100
|
+
""" search the urls that fit the requirements and updates self.urls member
|
|
101
|
+
:roi: list of (lon, lat) tuple
|
|
102
|
+
:orbit: (str) string of format like A001 of D089
|
|
103
|
+
:start_date: (str) start date, format YYYY-MM-DD
|
|
104
|
+
:end_date: (str) start date, format YYYY-MM-DD
|
|
105
|
+
"""
|
|
106
|
+
roi_string = [(str(x[0]), str(x[1])) for x in roi]
|
|
107
|
+
relative_orbit = orbit.lstrip('AD')
|
|
108
|
+
direction = 'ASCENDING' if orbit[0] == 'A' else "DESCENDING"
|
|
109
|
+
aoi = 'polygon((' + \
|
|
110
|
+
','.join([' '.join(x) for x in roi_string]) + \
|
|
111
|
+
'))'
|
|
112
|
+
opts = {
|
|
113
|
+
'platform': asf.PLATFORM.SENTINEL1,
|
|
114
|
+
'relativeOrbit': relative_orbit,
|
|
115
|
+
'processingLevel': 'SLC',
|
|
116
|
+
'beamMode': 'IW',
|
|
117
|
+
'flightDirection': direction,
|
|
118
|
+
'start': start_date+'T00:00:00Z',
|
|
119
|
+
'end': end_date+'T23:59:59Z'
|
|
120
|
+
}
|
|
121
|
+
logger.debug(f'looking for {opts}')
|
|
122
|
+
results = asf.geo_search(intersectsWith=aoi, **opts)
|
|
123
|
+
logger.info(f"found {len(results)} results")
|
|
124
|
+
self.urls = []
|
|
125
|
+
for product in results:
|
|
126
|
+
product_dict = product.geojson()
|
|
127
|
+
self.urls.append(product_dict['properties']['url'])
|
|
128
|
+
logger.info(f"found {self.urls}, {len(self.urls)} products")
|
|
129
|
+
|
|
130
|
+
def connect(self, user, passwd):
|
|
131
|
+
"""Authenticate and update the session.
|
|
132
|
+
|
|
133
|
+
:user: (str) the asf user
|
|
134
|
+
:passwd: (str) the asf passwrd
|
|
135
|
+
"""
|
|
136
|
+
self.auth = asf.ASFSession()
|
|
137
|
+
self.auth.auth_with_creds(user, passwd)
|
|
138
|
+
# self.cookiejar = self.auth.cookies
|
|
139
|
+
# self.cookiejar_session = asf.ASFSession().auth_with_cookiejar(self.cookiejar)
|
|
140
|
+
|
|
141
|
+
def cherry_pick_all(self, target_dir, pattern='(manifest.safe|annotation.*/s1.*.xml)', processes=None):
|
|
142
|
+
"""Fetch from the url the elemtents that matches pattern and send then in target dir.
|
|
143
|
+
connect method should have been called and urls not empty.
|
|
144
|
+
RuntimeError is raised if connect is not done.
|
|
145
|
+
self.missings will be set to missings downloaded
|
|
146
|
+
|
|
147
|
+
:target_dir: (str) target dir
|
|
148
|
+
:pattern: (str) regex that will be used to compile the pattern
|
|
149
|
+
:processes: (str) number of processes to run in parallel. Default use all available
|
|
150
|
+
"""
|
|
151
|
+
if processes is None:
|
|
152
|
+
self.nb_process = multiprocessing.cpu_count()
|
|
153
|
+
else:
|
|
154
|
+
self.nb_process = processes
|
|
155
|
+
self.missings = []
|
|
156
|
+
if self.auth is None:
|
|
157
|
+
raise RuntimeError(("AsfCherryPick:cherry_pick_all called "
|
|
158
|
+
"but auth not set. Call the connect method "
|
|
159
|
+
"before calling the cherry_pick_all one"))
|
|
160
|
+
parameters = []
|
|
161
|
+
for url in self.urls:
|
|
162
|
+
parameters.append((url, target_dir, pattern))
|
|
163
|
+
if processes == 1:
|
|
164
|
+
for parameter in parameters:
|
|
165
|
+
self.cherry_pick(parameter)
|
|
166
|
+
else:
|
|
167
|
+
with Pool(processes=processes) as pool:
|
|
168
|
+
pool.map(self.cherry_pick, parameters)
|
|
169
|
+
|
|
170
|
+
def cherry_pick(self, parameters):
|
|
171
|
+
"""download data from parameters. parameters is a tuple containing
|
|
172
|
+
an url, a taraget_dir and a pattern.
|
|
173
|
+
"""
|
|
174
|
+
url, target_dir, pattern = parameters
|
|
175
|
+
re_pattern = re.compile(pattern)
|
|
176
|
+
my_missings = []
|
|
177
|
+
with remotezip.RemoteZip(url, session=self.auth) as zip:
|
|
178
|
+
for zip_info in zip.infolist():
|
|
179
|
+
if re_pattern.search(str(zip_info.filename)):
|
|
180
|
+
logger.info(f'extracting {zip_info.filename}')
|
|
181
|
+
target = target_dir / Path(zip_info.filename)
|
|
182
|
+
if target.exists():
|
|
183
|
+
logger.info(f"{target} exists, skipping download")
|
|
184
|
+
continue
|
|
185
|
+
try:
|
|
186
|
+
zip.extract(zip_info.filename, path=self.tempdir)
|
|
187
|
+
except Exception as excpt:
|
|
188
|
+
logger.warning(f"Fail to extract {target}: {excpt}")
|
|
189
|
+
my_missings.append((url, target))
|
|
190
|
+
continue
|
|
191
|
+
temp_filename = self.tempdir / zip_info.filename
|
|
192
|
+
if not target.parent.exists():
|
|
193
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
194
|
+
temp_filename.rename(target)
|
|
195
|
+
return my_missings
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def get_roi_from_kml(kml_filename, padding=None):
|
|
199
|
+
"""Extract region of interest given the kml
|
|
200
|
+
:returns:(list) bounding box as a list of (lon, lat) tuples
|
|
201
|
+
"""
|
|
202
|
+
res = []
|
|
203
|
+
with open(kml_filename) as kml_file:
|
|
204
|
+
doc = kml_file.read().encode('utf-8')
|
|
205
|
+
k = kml.KML()
|
|
206
|
+
k.from_string(doc)
|
|
207
|
+
for feature0 in k.features():
|
|
208
|
+
print("{}, {}".format(feature0.name, feature0.description))
|
|
209
|
+
for feature1 in feature0.features():
|
|
210
|
+
logger.debug(f"found feature: {feature1}")
|
|
211
|
+
if isinstance(feature1.geometry, geometry.Polygon):
|
|
212
|
+
polygon = feature1.geometry
|
|
213
|
+
res = [(x[0], x[1]) for x in polygon.exterior.coords]
|
|
214
|
+
logger.info(f"polygon found before padding: {res}")
|
|
215
|
+
if padding is not None:
|
|
216
|
+
min_lat = min([x[1] for x in res])
|
|
217
|
+
max_lat = max([x[1] for x in res])
|
|
218
|
+
for idx, val in enumerate(res):
|
|
219
|
+
if val[1] == min_lat:
|
|
220
|
+
res[idx] = (res[idx][0], res[idx][1] - padding)
|
|
221
|
+
continue
|
|
222
|
+
if val[1] == max_lat:
|
|
223
|
+
res[idx] = (res[idx][0], res[idx][1] + padding)
|
|
224
|
+
logger.info(f"polygon found after padding of {padding} on lat. {res}")
|
|
225
|
+
return res
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def read_auth(filename):
|
|
229
|
+
""" reads filename that contains authentification
|
|
230
|
+
file is a text file with on line, containing login and password
|
|
231
|
+
separated by a space.
|
|
232
|
+
|
|
233
|
+
:param filename: the file that contains the credential
|
|
234
|
+
:type filename: str
|
|
235
|
+
:return: login,password
|
|
236
|
+
:rtype: tuple
|
|
237
|
+
"""
|
|
238
|
+
try:
|
|
239
|
+
with open(filename, "r") as _f:
|
|
240
|
+
(login, password) = _f.readline().split(' ')
|
|
241
|
+
if password.endswith('\n'):
|
|
242
|
+
password = password[:-1]
|
|
243
|
+
except Exception as excpt:
|
|
244
|
+
logger.critical(f"cannot read file {filename}: {excpt}")
|
|
245
|
+
raise Exception(f"read_auth raised {excpt}")
|
|
246
|
+
return (login, password)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def main():
|
|
250
|
+
parser = argparse.ArgumentParser(description="extract coordinate from kml and query asf to get elements matching the pattern")
|
|
251
|
+
group = parser.add_mutually_exclusive_group(required=True)
|
|
252
|
+
|
|
253
|
+
group.add_argument("--kml", type=str, help="input is a kml file. Geographic search")
|
|
254
|
+
group.add_argument("--safelist", type=str,
|
|
255
|
+
help="input is a text file containing safe names")
|
|
256
|
+
group.add_argument("--urls", type=str, help="input is a text file containing urls pointing to zip files")
|
|
257
|
+
group.add_argument("--granules", type=str, help="input is a coma separated list of granule, e.g. safe name without .SAFE extension")
|
|
258
|
+
parser.add_argument("-c", type=str,
|
|
259
|
+
help=("(str) file containing asf user credentials: "
|
|
260
|
+
"format: one line of the form user password"))
|
|
261
|
+
parser.add_argument("-d", type=str, default='2014-09-01:2022-06-01',
|
|
262
|
+
help="(str) date range in the format. Default=%(default)s")
|
|
263
|
+
parser.add_argument("-j", type=int, default=1,
|
|
264
|
+
help="(int) number of download to do in parallel, default=%(default)s")
|
|
265
|
+
parser.add_argument("-m", type=float, default=0.3,
|
|
266
|
+
help="(float) adding a margin on the latitute as found in the xml. default=%(default)s")
|
|
267
|
+
parser.add_argument("-o", type=str, default=None, help="(str) relative orbit number in the form of D041")
|
|
268
|
+
parser.add_argument("-p", type=str, default='(manifest.safe|s1.*xml$)',
|
|
269
|
+
help='(str) regex for patterns. Default=%(default)s')
|
|
270
|
+
parser.add_argument("-t", type=str, default='SAFES', help="(str) target dir. Default=%(default)s")
|
|
271
|
+
parser.add_argument("--saveproducts", type=str, default=None,
|
|
272
|
+
help="(str) if set, filename in which we want to write the set of url found")
|
|
273
|
+
parser.add_argument("-H", action="store_true",
|
|
274
|
+
help="provide more detailed help")
|
|
275
|
+
parser.add_argument("-v", type=int, default=3,
|
|
276
|
+
help=("set logging level:"
|
|
277
|
+
"0 critical, 1 error, 2 warning,"
|
|
278
|
+
"3 info, 4 debug, default=info"))
|
|
279
|
+
if "-H" in sys.argv:
|
|
280
|
+
print(__doc__)
|
|
281
|
+
|
|
282
|
+
sys.exit(0)
|
|
283
|
+
|
|
284
|
+
args = parser.parse_args()
|
|
285
|
+
logging_translate = [logging.CRITICAL, logging.ERROR, logging.WARNING,
|
|
286
|
+
logging.INFO, logging.DEBUG]
|
|
287
|
+
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
288
|
+
level=logging_translate[args.v])
|
|
289
|
+
|
|
290
|
+
script_base_name = Path(__file__).name
|
|
291
|
+
logger = logging.getLogger(script_base_name + ":main")
|
|
292
|
+
cmd_line = ' '.join(sys.argv)
|
|
293
|
+
logger.info(f"called as {cmd_line}")
|
|
294
|
+
|
|
295
|
+
user, passwd = read_auth(args.c)
|
|
296
|
+
|
|
297
|
+
target_dir = Path(args.t)
|
|
298
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
299
|
+
|
|
300
|
+
picker = AsfCherryPick()
|
|
301
|
+
if args.kml is not None:
|
|
302
|
+
orbit = args.o
|
|
303
|
+
if orbit is None:
|
|
304
|
+
logger.error("orbit should be provided when searching using kml")
|
|
305
|
+
sys.exit(1)
|
|
306
|
+
if orbit[0] != 'A' and orbit[0] != 'D':
|
|
307
|
+
logger.error(('wrong format for relative orbit. Should be of the form'
|
|
308
|
+
f'D0041 or A0034, got {args.o}'))
|
|
309
|
+
sys.exit(1)
|
|
310
|
+
roi = get_roi_from_kml(args.kml, padding=args.m)
|
|
311
|
+
if len(roi) == 0:
|
|
312
|
+
logger.error("Cannot extract region interest from kml, aborting")
|
|
313
|
+
sys.exit(1)
|
|
314
|
+
logger.info(f"Extracted region of interest (lon,lat, alt): {roi}")
|
|
315
|
+
picker.search_urls(roi, orbit, args.d.split(":")[0], args.d.split(":")[1])
|
|
316
|
+
if args.urls is not None:
|
|
317
|
+
with open(args.urls, "r") as _url:
|
|
318
|
+
refs_url = [line.strip() for line in _url]
|
|
319
|
+
picker.urls = refs_url
|
|
320
|
+
if args.granules or args.safelist:
|
|
321
|
+
refs_products = []
|
|
322
|
+
if args.granules:
|
|
323
|
+
refs_products = args.granules.split(",")
|
|
324
|
+
if args.safelist:
|
|
325
|
+
with open(args.safelist, "r") as _url:
|
|
326
|
+
refs_products = [line.strip() for line in _url]
|
|
327
|
+
search_results = asf.granule_search(refs_products)
|
|
328
|
+
refs_url = []
|
|
329
|
+
for product in search_results:
|
|
330
|
+
product_dict = product.geojson()
|
|
331
|
+
if product_dict['properties']['url'].endswith(".zip"):
|
|
332
|
+
refs_url.append(product_dict['properties']['url'])
|
|
333
|
+
picker.urls = refs_url
|
|
334
|
+
logger.info(f"fetching urls: {picker.urls}")
|
|
335
|
+
if args.saveproducts is not None:
|
|
336
|
+
with open(args.saveproducts, "w") as _urls:
|
|
337
|
+
for url in picker.urls:
|
|
338
|
+
_urls.write(f"{url}\n")
|
|
339
|
+
picker.connect(user, passwd)
|
|
340
|
+
picker.cherry_pick_all(target_dir, args.p, args.j)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
if __name__ == "__main__":
|
|
344
|
+
sys.exit(main())
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
"""
|
|
4
|
+
Does a diff between two sets to check missing elements.
|
|
5
|
+
Basically makes the same as the comm command.
|
|
6
|
+
|
|
7
|
+
Sets are built either from a filename or as a list of elements in a directory.
|
|
8
|
+
|
|
9
|
+
If input are not formated in the same way, user can provide a regex to select
|
|
10
|
+
the elements that will be compared.
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
Examples:
|
|
14
|
+
|
|
15
|
+
=========
|
|
16
|
+
|
|
17
|
+
Suppose file D041_urls.txt contains lines like
|
|
18
|
+
|
|
19
|
+
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20220523T123454_20220523T123520_043338_052CE2_EDD3.zip
|
|
20
|
+
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20220523T123429_20220523T123456_043338_052CE2_6C68.zip
|
|
21
|
+
https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20220523T123404_20220523T123431_043338_052CE2_5FAF.zip
|
|
22
|
+
|
|
23
|
+
and directory SAFE_D041 contains directories like
|
|
24
|
+
SAFE_D041/S1A_IW_SLC__1SDV_20220523T123454_20220523T123520_043338_052CE2_EDD3.SAFE
|
|
25
|
+
SAFE_D041/S1A_IW_SLC__1SDV_20170218T123330_20170218T123357_015338_01926D_76B4.SAFE
|
|
26
|
+
SAFE_D041/S1A_IW_SLC__1SDV_20170218T123355_20170218T123422_015338_01926D_3DDE.SAFE
|
|
27
|
+
SAFE_D041/S1A_IW_SLC__1SDV_20170218T123420_20170218T123452_015338_01926D_787F.SAFE
|
|
28
|
+
SAFE_D041/S1A_IW_SLC__1SDV_20170302T123330_20170302T123357_015513_0197BE_2F8E.SAFE
|
|
29
|
+
|
|
30
|
+
Looking at the first lines we want to match the
|
|
31
|
+
S1A_IW_SLC__1SDV_20220523T123454_20220523T123520_043338_052CE2_EDD3
|
|
32
|
+
|
|
33
|
+
We provide the script with two inputs (input a and input b). For each input, we need to provide
|
|
34
|
+
if we will list a director or a file. Finally we need to provide a pattern to match. Pattern
|
|
35
|
+
will follow the python regex. The first set of parenthesis define the part that will be
|
|
36
|
+
compared.
|
|
37
|
+
|
|
38
|
+
The script will output all the patterns that are in a and not in b. The last option allows the
|
|
39
|
+
user to add a prefix (option -P) and a suffix (option -S) to the found names
|
|
40
|
+
|
|
41
|
+
Here is a full example:
|
|
42
|
+
|
|
43
|
+
get_missings.py # name of the script
|
|
44
|
+
-a D041_urls.txt # input a
|
|
45
|
+
-ta f # type of a is f, ie a file
|
|
46
|
+
-b SAFE_D041 # input b
|
|
47
|
+
-tb d # type of b is a directory
|
|
48
|
+
-pa '.*/(.*).zip' # pattern of a: skip all everything up to the last /
|
|
49
|
+
# then grab everything up to pattern .zip
|
|
50
|
+
-pb 'SAFE_D041/(.*).SAFE' # grab everything after SAFE_D041/ up to .SAFE
|
|
51
|
+
-P 'https://datapool.asf.alaska.edu/SLC/SA/' -S '.zip'
|
|
52
|
+
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
# classical imports
|
|
56
|
+
import argparse
|
|
57
|
+
import logging
|
|
58
|
+
# import numpy as np
|
|
59
|
+
from pathlib import Path
|
|
60
|
+
import sys
|
|
61
|
+
import re
|
|
62
|
+
|
|
63
|
+
# import matplotlib.pyplot as plt
|
|
64
|
+
# if "GDAL_SYS" in os.environ and os.environ["GDAL_SYS"] == 'True':
|
|
65
|
+
# from osgeo import gdal
|
|
66
|
+
# else:
|
|
67
|
+
# import nsbas.gdal as gdal
|
|
68
|
+
# gdal.UseExceptions()
|
|
69
|
+
|
|
70
|
+
logger = logging.getLogger(__name__)
|
|
71
|
+
|
|
72
|
+
def filter(str_list, pattern):
|
|
73
|
+
for elem in str_list:
|
|
74
|
+
m = pattern.search(elem)
|
|
75
|
+
if m:
|
|
76
|
+
yield m[1]
|
|
77
|
+
|
|
78
|
+
def get_matching_dirs(directory, pattern):
|
|
79
|
+
str_dirs = [str(x) for x in directory.glob("*")]
|
|
80
|
+
res = set(filter(str_dirs, pattern))
|
|
81
|
+
return res
|
|
82
|
+
|
|
83
|
+
def get_matching_lines(filename, pattern):
|
|
84
|
+
res = set()
|
|
85
|
+
with open(filename) as _f:
|
|
86
|
+
res = set(filter(_f.readlines(), pattern))
|
|
87
|
+
return res
|
|
88
|
+
|
|
89
|
+
def main():
|
|
90
|
+
parser = argparse.ArgumentParser(description="extract missing data from a target list, given a directory where fetched data lies")
|
|
91
|
+
parser.add_argument("-a", type=str,
|
|
92
|
+
help="(str) input set a")
|
|
93
|
+
parser.add_argument("-ta", type=str, choices=['f', 'd'],
|
|
94
|
+
help="(str) type of the input 1, f(ile), d(irectory)")
|
|
95
|
+
parser.add_argument("-b", type=str,
|
|
96
|
+
help="(str) input set b")
|
|
97
|
+
parser.add_argument("-tb", type=str, choices=['f', 'd'],
|
|
98
|
+
help="(str) type of the input 2, f(ile), d(irectory)")
|
|
99
|
+
parser.add_argument("-pa", type=str,
|
|
100
|
+
help="(str, regex) pattern for input 1")
|
|
101
|
+
parser.add_argument("-pb", type=str,
|
|
102
|
+
help="(str, regex) pattern for input 2")
|
|
103
|
+
parser.add_argument("-P", type=str, default="",
|
|
104
|
+
help="Prefix to use when printing the missing elems")
|
|
105
|
+
parser.add_argument("-S", type=str, default="",
|
|
106
|
+
help="Suffix to use when printing the missing elems")
|
|
107
|
+
parser.add_argument("-H", action="store_true",
|
|
108
|
+
help="provide more detailed help")
|
|
109
|
+
parser.add_argument("-v", type=int, default=3,
|
|
110
|
+
help=("set logging level:"
|
|
111
|
+
"0 critical, 1 error, 2 warning,"
|
|
112
|
+
"3 info, 4 debug, default=info"))
|
|
113
|
+
if "-H" in sys.argv:
|
|
114
|
+
print(__doc__)
|
|
115
|
+
sys.exit(0)
|
|
116
|
+
args = parser.parse_args()
|
|
117
|
+
logging_translate = [logging.CRITICAL, logging.ERROR, logging.WARNING,
|
|
118
|
+
logging.INFO, logging.DEBUG]
|
|
119
|
+
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
|
120
|
+
level=logging_translate[args.v])
|
|
121
|
+
script_base_name = Path(__file__).name
|
|
122
|
+
logger = logging.getLogger(script_base_name + ":main")
|
|
123
|
+
cmd_line = ' '.join(sys.argv)
|
|
124
|
+
logger.info(f"called as {cmd_line}")
|
|
125
|
+
input_1 = args.a
|
|
126
|
+
input_2 = args.b
|
|
127
|
+
pattern1 = re.compile(args.pa)
|
|
128
|
+
pattern2 = re.compile(args.pb)
|
|
129
|
+
if args.ta == "d":
|
|
130
|
+
path_1 = Path(input_1)
|
|
131
|
+
input_1_set = get_matching_dirs(path_1, pattern1)
|
|
132
|
+
else:
|
|
133
|
+
input_1_set = get_matching_lines(input_1, pattern1)
|
|
134
|
+
if args.tb == "d":
|
|
135
|
+
path_2 = Path(input_2)
|
|
136
|
+
input_2_set = get_matching_dirs(path_2, pattern2)
|
|
137
|
+
else:
|
|
138
|
+
input_2_set = get_matching_lines(input_2, pattern2)
|
|
139
|
+
missings = input_1_set - input_2_set
|
|
140
|
+
for elem in missings:
|
|
141
|
+
print(f"{args.P}{elem}{args.S}")
|
|
142
|
+
sys.exit(0)
|
|
143
|
+
|
|
144
|
+
if __name__ == "__main__":
|
|
145
|
+
main()
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: asf_cherry_pick
|
|
3
|
+
Version: 2.0.2
|
|
4
|
+
Summary: Python package for cherry picking Data into asf
|
|
5
|
+
Author-email: Franck Thollard <franck.thollard@univ-grenoble-alpes.fr>
|
|
6
|
+
License: BSD 3-Clause License
|
|
7
|
+
Project-URL: Homepage, https://gricad-gitlab.univ-grenoble-alpes.fr/NSBAS/asf_cherry_pick
|
|
8
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
9
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Description-Content-Type: text/x-rst
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: asf_search==9.0.9
|
|
15
|
+
Requires-Dist: fastkml
|
|
16
|
+
Requires-Dist: lxml
|
|
17
|
+
Requires-Dist: remotezip>=0.10.0
|
|
18
|
+
Dynamic: license-file
|
|
19
|
+
|
|
20
|
+
asf_cherry_pick
|
|
21
|
+
===============
|
|
22
|
+
|
|
23
|
+
The Alaska Sar Fundation mirrors the sentinel1 data from the ESA. For a given acquisition,
|
|
24
|
+
data are provided as a zip file.
|
|
25
|
+
|
|
26
|
+
We provide here a script (and an API) that can fetch data into the zip file, allowing to
|
|
27
|
+
download only the product of interest. This is useflull either for downloading meta data
|
|
28
|
+
or only the tiff file of interest (e.g. only one polarization and one sub swath).
|
|
29
|
+
|
|
30
|
+
Elements to fetch are describe as a pattern (in the sens of regular expression).
|
|
31
|
+
|
|
32
|
+
The set of product are given either
|
|
33
|
+
|
|
34
|
+
- a file in kml format (that contains a polygon which represent the region of interest)
|
|
35
|
+
- a file containing the list of zip files to fetch
|
|
36
|
+
- a file containing the list of urls
|
|
37
|
+
|
|
38
|
+
Installation
|
|
39
|
+
------------
|
|
40
|
+
|
|
41
|
+
- Unzip the file.
|
|
42
|
+
- create a virtual env environment (e.g. python3 -m venv asf_cherry_pick)
|
|
43
|
+
- go to the directory in asf_cherry_pick where the setup.py lies
|
|
44
|
+
- run the command **pip install .**
|
|
45
|
+
|
|
46
|
+
Upon installation, you will have access to the script asf_cherry_pick.
|
|
47
|
+
|
|
48
|
+
Run **asf_cherry_pick -h** and **asf_cherry_pick -H** for a more detail help.
|
|
49
|
+
|
|
50
|
+
Enjoy.
|
|
51
|
+
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.rst
|
|
3
|
+
pyproject.toml
|
|
4
|
+
setup.py
|
|
5
|
+
asf_cherry_pick/__init__.py
|
|
6
|
+
asf_cherry_pick/asf_cherry_pick.py
|
|
7
|
+
asf_cherry_pick/get_missings.py
|
|
8
|
+
asf_cherry_pick.egg-info/PKG-INFO
|
|
9
|
+
asf_cherry_pick.egg-info/SOURCES.txt
|
|
10
|
+
asf_cherry_pick.egg-info/dependency_links.txt
|
|
11
|
+
asf_cherry_pick.egg-info/entry_points.txt
|
|
12
|
+
asf_cherry_pick.egg-info/requires.txt
|
|
13
|
+
asf_cherry_pick.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "asf_cherry_pick"
|
|
3
|
+
description = "Python package for cherry picking Data into asf"
|
|
4
|
+
version = "2.0.2"
|
|
5
|
+
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "Franck Thollard", email = "franck.thollard@univ-grenoble-alpes.fr" }
|
|
8
|
+
]
|
|
9
|
+
license = {text = "BSD 3-Clause License"}
|
|
10
|
+
readme = "README.rst"
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
classifiers=[
|
|
13
|
+
'Programming Language :: Python :: 3 :: Only',
|
|
14
|
+
'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
|
|
15
|
+
'Intended Audience :: Science/Research',
|
|
16
|
+
]
|
|
17
|
+
dependencies = [
|
|
18
|
+
"asf_search==9.0.9",
|
|
19
|
+
"fastkml",
|
|
20
|
+
"lxml",
|
|
21
|
+
"remotezip>=0.10.0"
|
|
22
|
+
]
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://gricad-gitlab.univ-grenoble-alpes.fr/NSBAS/asf_cherry_pick"
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
asf_cherry_pick = "asf_cherry_pick.asf_cherry_pick:main"
|
|
28
|
+
get_missings = "asf_cherry_pick.get_missings:main"
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.packages.find]
|
|
31
|
+
|
|
32
|
+
[build-system]
|
|
33
|
+
requires = ["setuptools", "wheel"] # PEP 508 specifications.
|
|
34
|
+
build-backend = "setuptools.build_meta"
|