NASADEM 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,443 @@
1
+ import netrc
2
+ import hashlib
3
+ import logging
4
+ import os
5
+ import posixpath
6
+ import re
7
+ import shutil
8
+ import urllib
9
+ from datetime import date
10
+ from fnmatch import fnmatch
11
+ from http.cookiejar import CookieJar
12
+ from os import makedirs, remove
13
+ from os.path import dirname
14
+ from os.path import exists
15
+ from os.path import getsize
16
+ from os.path import isdir
17
+ from os.path import join
18
+ from time import sleep
19
+ from typing import List, OrderedDict
20
+
21
+ import requests
22
+ import xmltodict
23
+ from bs4 import BeautifulSoup
24
+ from dateutil import parser
25
+ from pycksum import cksum
26
+
27
+ import colored_logging as cl
28
+
29
+
30
+ class DownloadFailed(Exception):
31
+ pass
32
+
33
+ CONNECTION_CLOSE = {
34
+ "Connection": "close",
35
+ }
36
+
37
+ DEFAULT_REMOTE = "https://e4ftl01.cr.usgs.gov"
38
+ RETRIES = 6
39
+ WAIT_SECONDS = 360
40
+ XML_RETRIES = RETRIES
41
+ XML_TIMEOUT = WAIT_SECONDS
42
+ DOWNLOAD_RETRIES = RETRIES
43
+ DOWNLOAD_WAIT_SECONDS = WAIT_SECONDS
44
+
45
+ __author__ = "Gregory Halverson"
46
+
47
+ logger = logging.getLogger(__name__)
48
+
49
+
50
+ class LPDAACServerUnreachable(ConnectionError):
51
+ pass
52
+
53
+
54
+ class LPDAACDataPool:
55
+ logger = logging.getLogger(__name__)
56
+ DEFAULT_CHUNK_SIZE = 2 ** 20
57
+ DATE_REGEX = re.compile(r'^(19|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])$')
58
+ DEFAULT_REMOTE = DEFAULT_REMOTE
59
+
60
+ def __init__(self, username: str = None, password: str = None, remote: str = None, offline_ok: bool = False):
61
+ if remote is None:
62
+ remote = DEFAULT_REMOTE
63
+
64
+ if username is None or password is None:
65
+ try:
66
+ netrc_file = netrc.netrc()
67
+ username, _, password = netrc_file.authenticators("urs.earthdata.nasa.gov")
68
+ except Exception as e:
69
+ logger.warning("netrc credentials not found for urs.earthdata.nasa.gov")
70
+
71
+ if username is None or password is None:
72
+ if not "LPDAAC_USERNAME" in os.environ or not "LPDAAC_PASSWORD" in os.environ:
73
+ raise RuntimeError("Missing environment variable 'LPDAAC_USERNAME' or 'LPDAAC_PASSWORD'")
74
+
75
+ username = os.environ["LPDAAC_USERNAME"]
76
+ password = os.environ["LPDAAC_PASSWORD"]
77
+
78
+ self._remote = remote
79
+ self._username = username
80
+ self._password = password
81
+ self.offline_ok = offline_ok
82
+
83
+ # if self.offline_ok:
84
+ # logger.warning("going into offline mode")
85
+
86
+ self._listings = {}
87
+
88
+ try:
89
+ self._authenticate()
90
+ self._check_remote()
91
+ except Exception as e:
92
+ if self.offline_ok:
93
+ logger.warning("unable to connect to LP-DAAC data pool")
94
+ else:
95
+ raise e
96
+
97
+ def _authenticate(self):
98
+ try:
99
+ # https://wiki.earthdata.nasa.gov/display/EL/How+To+Access+Data+With+Python
100
+
101
+ password_manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
102
+
103
+ password_manager.add_password(
104
+ realm=None,
105
+ uri="https://urs.earthdata.nasa.gov",
106
+ user=self._username,
107
+ passwd=self._password
108
+ )
109
+
110
+ cookie_jar = CookieJar()
111
+
112
+ # Install all the handlers.
113
+
114
+ opener = urllib.request.build_opener(
115
+ urllib.request.HTTPBasicAuthHandler(password_manager),
116
+ # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see
117
+ # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses
118
+ urllib.request.HTTPCookieProcessor(cookie_jar)
119
+ )
120
+
121
+ urllib.request.install_opener(opener)
122
+ except Exception as e:
123
+ message = "unable to authenticate with LP-DAAC data pool"
124
+ if self.offline_ok:
125
+ logger.warning(message)
126
+ else:
127
+ raise ConnectionError(message)
128
+
129
+ def _check_remote(self):
130
+ logger.info(f"checking URL: {cl.URL(self.remote)}")
131
+
132
+ try:
133
+ response = requests.head(self.remote, headers=CONNECTION_CLOSE)
134
+ status = response.status_code
135
+ duration = response.elapsed.total_seconds()
136
+ except Exception as e:
137
+ logger.exception(e)
138
+ message = f"unable to connect to URL: {self.remote}"
139
+
140
+ if self.offline_ok:
141
+ logger.warning(message)
142
+ return
143
+ else:
144
+ raise LPDAACServerUnreachable(message)
145
+
146
+ if status == 200:
147
+ logger.info(
148
+ "remote verified with status " + cl.val(200) +
149
+ " in " + cl.time(f"{duration:0.2f}") +
150
+ " seconds: " + cl.URL(self.remote))
151
+ else:
152
+ message = f"status: {status} URL: {self.remote}"
153
+
154
+ if self.offline_ok:
155
+ logger.warning(message)
156
+ else:
157
+ raise ConnectionError(message)
158
+
159
+ @property
160
+ def remote(self):
161
+ return self._remote
162
+
163
+ def get_HTTP_text(self, URL: str) -> str:
164
+ try:
165
+ request = urllib.request.Request(URL)
166
+ response = urllib.request.urlopen(request)
167
+ body = response.read().decode()
168
+ except Exception as e:
169
+ logger.exception(e)
170
+ raise ConnectionError(f"cannot connect to URL: {URL}")
171
+
172
+ return body
173
+
174
+ def get_HTTP_listing(self, URL: str, pattern: str = None) -> List[str]:
175
+ if URL in self._listings:
176
+ listing = self._listings[URL]
177
+ else:
178
+ text = self.get_HTTP_text(URL)
179
+ soup = BeautifulSoup(text, 'html.parser')
180
+ links = list(soup.find_all('a', href=True))
181
+
182
+ # get directory names from links on http site
183
+ listing = sorted([link['href'].replace('/', '') for link in links])
184
+ self._listings[URL] = listing
185
+
186
+ if pattern is not None:
187
+ listing = sorted([
188
+ item
189
+ for item
190
+ in listing
191
+ if fnmatch(item, pattern)
192
+ ])
193
+
194
+ return listing
195
+
196
+ def get_HTTP_date_listing(self, URL: str) -> List[date]:
197
+ return sorted([
198
+ parser.parse(item).date()
199
+ for item
200
+ in self.get_HTTP_listing(URL)
201
+ if self.DATE_REGEX.match(item)
202
+ ])
203
+
204
+ def read_HTTP_XML(self, URL: str) -> OrderedDict:
205
+ return xmltodict.parse(self.get_HTTP_text(URL))
206
+
207
+ def generate_XML_URL(self, URL: str) -> str:
208
+ return f"{URL}.xml"
209
+
210
+ def get_metadata(self, data_URL: str) -> OrderedDict:
211
+ metadata_URL = f"{data_URL}.xml"
212
+ logger.info(f"checking metadata: {cl.URL(metadata_URL)}")
213
+ request = urllib.request.Request(metadata_URL)
214
+ response = urllib.request.urlopen(request)
215
+ duration = response.elapsed.total_seconds()
216
+ body = response.read().decode()
217
+ metadata = xmltodict.parse(body)
218
+ logger.info(f"metadata retrieved in {cl.val(f'{duration:0.2f}')} seconds: {cl.URL(metadata_URL)}")
219
+
220
+ return metadata
221
+
222
+ def get_remote_checksum(self, URL: str) -> int:
223
+ return int(self.get_metadata(URL)["GranuleMetaDataFile"]["GranuleURMetaData"]["DataFiles"]["DataFileContainer"][
224
+ "Checksum"])
225
+
226
+ def get_remote_filesize(self, URL: str) -> int:
227
+ return int(self.get_metadata(URL)["GranuleMetaDataFile"]["GranuleURMetaData"]["DataFiles"]["DataFileContainer"][
228
+ "FileSize"])
229
+
230
+ def get_local_checksum(self, filename: str, checksum_type: str = "CKSUM") -> str:
231
+ with open(filename, "rb") as file:
232
+ if checksum_type == "CKSUM":
233
+ return str(int(cksum(file)))
234
+ elif checksum_type == "MD5":
235
+ return str(hashlib.md5(file.read()).hexdigest())
236
+
237
+ def get_local_filesize(self, filename: str) -> int:
238
+ return getsize(filename)
239
+
240
+ def product_directory(self, platform: str, product: str, build: str = None) -> str:
241
+ if build is None:
242
+ build = "001"
243
+ elif isinstance(build, float):
244
+ build = f"{int(build * 10)}:03d"
245
+ elif isinstance(build, int):
246
+ build = f"{build:03d}"
247
+
248
+ URL = posixpath.join(self._remote, platform, f"{product}.{build}")
249
+
250
+ return URL
251
+
252
+ def dates(self, platform: str, product: str, build: str = None) -> List[date]:
253
+ return self.get_HTTP_date_listing(self.product_directory(platform, product, build))
254
+
255
+ def date_URL(
256
+ self,
257
+ platform: str,
258
+ product: str,
259
+ acquisition_date: date or str,
260
+ build: str = None) -> str:
261
+ if isinstance(acquisition_date, str):
262
+ acquisition_date = parser.parse(acquisition_date).date()
263
+
264
+ URL = posixpath.join(
265
+ self.product_directory(platform, product, build),
266
+ f"{acquisition_date:%Y.%m.%d}"
267
+ )
268
+
269
+ return URL
270
+
271
+ def files(
272
+ self,
273
+ platform: str,
274
+ product: str,
275
+ acquisition_date: date or str,
276
+ build: str = None,
277
+ pattern: str = None) -> List[str]:
278
+ URL = self.date_URL(platform, product, acquisition_date, build)
279
+ listing = self.get_HTTP_listing(URL, pattern)
280
+
281
+ return listing
282
+
283
+ def download_URL(
284
+ self,
285
+ URL: str,
286
+ download_location: str = None,
287
+ XML_retries: int = None,
288
+ XML_timeout_seconds: int = None,
289
+ download_retries: int = None,
290
+ download_wait_seconds: int = None) -> str:
291
+ if isdir(download_location):
292
+ filename = join(download_location, posixpath.basename(URL))
293
+ else:
294
+ filename = download_location
295
+
296
+ if exists(filename):
297
+ logger.info(f"file already retrieved: {cl.file(filename)}")
298
+ return filename
299
+
300
+ # metadata = self.get_metadata(URL)
301
+ metadata_URL = f"{URL}.xml"
302
+ logger.info(f"checking metadata: {cl.URL(metadata_URL)}")
303
+
304
+ if isdir(download_location):
305
+ metadata_filename = join(download_location, posixpath.basename(metadata_URL))
306
+ else:
307
+ metadata_filename = f"{download_location}.xml"
308
+
309
+ makedirs(dirname(metadata_filename), exist_ok=True)
310
+
311
+ if XML_retries is None:
312
+ XML_retries = XML_RETRIES
313
+
314
+ if XML_timeout_seconds is None:
315
+ XML_timeout_seconds = XML_TIMEOUT
316
+
317
+ if download_retries is None:
318
+ download_retries = DOWNLOAD_RETRIES
319
+
320
+ if download_wait_seconds is None:
321
+ download_wait_seconds = DOWNLOAD_WAIT_SECONDS
322
+
323
+ metadata = None
324
+
325
+ while XML_retries > 0:
326
+ XML_retries -= 1
327
+ command = f"wget -nc -c --user {self._username} --password {self._password} -O {metadata_filename} {metadata_URL}"
328
+ logger.info(command)
329
+ os.system(command)
330
+
331
+ if not exists(metadata_filename):
332
+ logger.warning(f"download not found for metadata URL: {metadata_URL}")
333
+ logger.warning(f"waiting {XML_timeout_seconds} for retry")
334
+ sleep(XML_timeout_seconds)
335
+ continue
336
+
337
+ XML_metadata_filesize = self.get_local_filesize(metadata_filename)
338
+
339
+ if XML_metadata_filesize == 0 and exists(metadata_filename):
340
+ logger.warning(f"removing corrupted zero-size metadata file: {metadata_filename}")
341
+
342
+ try:
343
+ os.remove(metadata_filename)
344
+ except:
345
+ logger.warning(f"unable to remove zero-size metadata file: {metadata_filename}")
346
+
347
+ logger.warning(f"waiting {XML_timeout_seconds} for retry")
348
+ sleep(XML_timeout_seconds)
349
+ continue
350
+
351
+ try:
352
+ with open(metadata_filename, "r") as file:
353
+ metadata = xmltodict.parse(file.read())
354
+ except Exception as e:
355
+ logger.warning(e)
356
+ logger.warning(f"unable to parse metadata file: {metadata_filename}")
357
+ os.remove(metadata_filename)
358
+ logger.warning(f"waiting {XML_timeout_seconds} for retry")
359
+ sleep(XML_timeout_seconds)
360
+ continue
361
+
362
+ if metadata is None:
363
+ raise DownloadFailed(f"unable to retrieve metadata URL: {metadata_URL}") # exit code 16
364
+
365
+ remote_checksum = str(
366
+ metadata["GranuleMetaDataFile"]["GranuleURMetaData"]["DataFiles"]["DataFileContainer"]["Checksum"])
367
+ checksum_type = str(
368
+ metadata["GranuleMetaDataFile"]["GranuleURMetaData"]["DataFiles"]["DataFileContainer"]["ChecksumType"])
369
+ remote_filesize = int(
370
+ metadata["GranuleMetaDataFile"]["GranuleURMetaData"]["DataFiles"]["DataFileContainer"]["FileSize"])
371
+
372
+ logger.info(
373
+ f"metadata retrieved {checksum_type} checksum: {cl.val(remote_checksum)} size: {cl.val(remote_filesize)} URL: {cl.URL(metadata_URL)}")
374
+ makedirs(dirname(filename), exist_ok=True)
375
+ logger.info(f"downloading {cl.URL(URL)} -> {cl.file(filename)}")
376
+
377
+ # Use a temporary file for downloading
378
+ temporary_filename = f"{filename}.download"
379
+
380
+ while download_retries > 0:
381
+ download_retries -=1
382
+
383
+ try:
384
+ if exists(temporary_filename):
385
+ temporary_filesize = self.get_local_filesize(temporary_filename)
386
+
387
+ if temporary_filesize > remote_filesize:
388
+ logger.warning(
389
+ f"removing corrupted file with size {temporary_filesize} greater than remote size {remote_filesize}: {temporary_filename}")
390
+ remove(temporary_filename)
391
+
392
+ elif temporary_filesize == remote_filesize:
393
+ local_checksum = self.get_local_checksum(temporary_filename, checksum_type=checksum_type)
394
+
395
+ if local_checksum == remote_checksum:
396
+ try:
397
+ shutil.move(temporary_filename, filename)
398
+ except Exception as e:
399
+ if exists(filename):
400
+ logger.warning(f"unable to move temporary file: {temporary_filename}")
401
+ return filename
402
+
403
+ logger.exception(e)
404
+ raise DownloadFailed(f"unable to move temporary file: {temporary_filename}")
405
+
406
+ return filename
407
+ else:
408
+ logger.warning(
409
+ f"removing corrupted file with local checksum {local_checksum} and remote checksum {remote_checksum}: {temporary_filename}")
410
+ remove(temporary_filename)
411
+ else:
412
+ logger.info(f"resuming incomplete download: {cl.file(temporary_filename)}")
413
+
414
+ command = f"wget -nc -c --user {self._username} --password {self._password} -O {temporary_filename} {URL}"
415
+ logger.info(command)
416
+ os.system(command)
417
+
418
+ if not exists(temporary_filename):
419
+ raise ConnectionError(f"unable to download URL: {URL}")
420
+
421
+ local_filesize = self.get_local_filesize(temporary_filename)
422
+ local_checksum = self.get_local_checksum(temporary_filename, checksum_type=checksum_type)
423
+
424
+ if local_filesize != remote_filesize or local_checksum != remote_checksum:
425
+ os.remove(temporary_filename)
426
+ raise ConnectionError(
427
+ f"removing corrupted file with local filesize {local_filesize} remote filesize {remote_filesize} local checksum {local_checksum} remote checksum {remote_checksum}: {temporary_filename}")
428
+
429
+ # Download successful, rename the temporary file to its proper name
430
+ shutil.move(temporary_filename, filename)
431
+
432
+ logger.info(
433
+ f"successful download with filesize {cl.val(local_filesize)} checksum {cl.val(local_checksum)}: {cl.file(filename)}")
434
+
435
+ return filename
436
+ except Exception as e:
437
+ if download_retries == 0:
438
+ raise e
439
+ else:
440
+ logger.warning(e)
441
+ logger.warning(f"waiting {download_wait_seconds} seconds to retry download")
442
+ sleep(download_wait_seconds)
443
+ continue
@@ -0,0 +1,9 @@
1
+ from os.path import join, dirname, abspath
2
+
3
+ from .LPDAACDataPool import LPDAACDataPool
4
+
5
+ with open(join(abspath(dirname(__file__)), "version.txt")) as f:
6
+ version = f.read()
7
+
8
+ __version__ = version
9
+ __author__ = "Gregory H. Halverson"
@@ -0,0 +1 @@
1
+ 0.1.3