gpload-py3 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
gpload.py ADDED
@@ -0,0 +1,3206 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ # gpload - load file(s) into Greenplum Database
4
+ # Copyright Greenplum 2008
5
+ #
6
+ # Python 3 port: Copyright 2026 Vitchenko Kirill
7
+ # This file is a Python 3 port of the original gpload utility.
8
+ # See NOTICE and LICENSE for attribution and license terms.
9
+
10
+ '''gpload [options] -f configuration file
11
+
12
+ Options:
13
+ -h hostname: host to connect to
14
+ -p port: port to connect to
15
+ -U username: user to connect as
16
+ -d database: database to connect to
17
+ -W: force password authentication
18
+ -q: quiet mode
19
+ -D: do not actually load data
20
+ -v: verbose
21
+ -V: very verbose
22
+ -l logfile: log output to logfile
23
+ --no_auto_trans: do not wrap gpload in transaction
24
+ --gpfdist_timeout timeout: gpfdist timeout value
25
+ --version: print version number and exit
26
+ -?: help
27
+ '''
28
+
29
+ import re
30
+ import sys
31
+
32
+ try:
33
+ import yaml
34
+ except ImportError:
35
+ sys.stderr.write("gpload needs pyyaml. You can get it from http://pyyaml.org.\n")
36
+ sys.exit(2)
37
+
38
+ import platform
39
+
40
+ psycopg2 = None
41
+ pg = None
42
+
43
+
44
+ def load_db_driver():
45
+ """Import psycopg2 or PyGreSQL, exiting with code 2 if neither works.
46
+
47
+ The import is deferred so that ``import gpload`` can be used for the
48
+ programmatic interface without a database driver in the calling process.
49
+ """
50
+ global psycopg2, pg
51
+ if psycopg2 is not None or pg is not None:
52
+ return
53
+
54
+ try:
55
+ import psycopg2 as _psycopg2
56
+ except ImportError:
57
+ _psycopg2 = None
58
+ if _psycopg2 is not None:
59
+ psycopg2 = _psycopg2
60
+ return
61
+
62
+ try:
63
+ from pygresql import pg as _pg
64
+ except Exception:
65
+ try:
66
+ import pg as _pg
67
+ except Exception as e:
68
+ from struct import calcsize
69
+ sysWordSize = calcsize("P") * 8
70
+ if (platform.system()) in ['Windows', 'Microsoft'] and (sysWordSize == 64):
71
+ errorMsg = "gpload appears to be running in 64-bit Python under Windows.\n"
72
+ errorMsg = errorMsg + "Currently only 32-bit Python is supported. Please \n"
73
+ errorMsg = errorMsg + "reinstall a 32-bit Python interpreter.\n"
74
+ else:
75
+ errorMsg = "gpload was unable to import psycopg2 or PyGreSQL (pg.py) - %s\n" % str(e)
76
+ sys.stderr.write(str(errorMsg))
77
+ sys.exit(2)
78
+
79
+ pg = _pg
80
+
81
+ import hashlib
82
+ import codecs
83
+ import csv
84
+ import fnmatch
85
+ import site
86
+ import sysconfig
87
+ import tempfile
88
+ import datetime,getpass,os,signal,socket,subprocess,threading,time,traceback,re
89
+ import uuid
90
+ import socket
91
+
92
+ thePlatform = platform.system()
93
+ if thePlatform in ['Windows', 'Microsoft']:
94
+ windowsPlatform = True
95
+ else:
96
+ windowsPlatform = False
97
+
98
+ if windowsPlatform == False:
99
+ import select
100
+
101
+
102
+ EXECNAME = 'gpload'
103
+
104
+ NUM_WARN_ROWS = 0
105
+
106
+ # PyGreSQL versions differ: the Greenplum build exposes DB.notices(),
107
+ # while the upstream package only exposes a notice receiver callback.
108
+ PG_NOTICES = []
109
+
110
+
111
+ def _notice_text(notice):
112
+ message = getattr(notice, 'message', notice)
113
+ if isinstance(message, bytes):
114
+ message = message.decode('utf-8', 'replace')
115
+ return str(message)
116
+
117
+
118
+ def _pg_notice_receiver(notice):
119
+ try:
120
+ PG_NOTICES.append(_notice_text(notice))
121
+ except Exception:
122
+ pass
123
+
124
+
125
+ def _get_notices(db):
126
+ if hasattr(db, 'notices'):
127
+ try:
128
+ return list(db.notices())
129
+ except Exception:
130
+ return []
131
+ return list(PG_NOTICES)
132
+
133
+
134
+ def _install_notice_receiver(db):
135
+ if not hasattr(db, 'set_notice_receiver'):
136
+ return
137
+ try:
138
+ del PG_NOTICES[:]
139
+ db.set_notice_receiver(_pg_notice_receiver)
140
+ except Exception:
141
+ pass
142
+
143
+
144
+ class QueryResult:
145
+ def __init__(self, rows, columns, rowcount):
146
+ self.rows = rows
147
+ self.columns = columns
148
+ self.rowcount = rowcount
149
+
150
+ def getresult(self):
151
+ return self.rows
152
+
153
+ def dictresult(self):
154
+ return [dict(zip(self.columns, row)) for row in self.rows]
155
+
156
+ def __iter__(self):
157
+ return iter(self.rows)
158
+
159
+ def __str__(self):
160
+ return str(self.rowcount)
161
+
162
+
163
+ class Psycopg2DB:
164
+ def __init__(self, dbname, host, port, user, passwd):
165
+ params = {'dbname': dbname}
166
+ if host:
167
+ params['host'] = host
168
+ if port:
169
+ params['port'] = port
170
+ if user:
171
+ params['user'] = user
172
+ if passwd is not None:
173
+ params['password'] = passwd
174
+ self._conn = psycopg2.connect(**params)
175
+ self._conn.autocommit = True
176
+
177
+ def close(self):
178
+ self._conn.close()
179
+
180
+ def query(self, sql):
181
+ cur = self._conn.cursor()
182
+ try:
183
+ cur.execute(sql)
184
+ columns = [d[0] for d in cur.description] if cur.description else []
185
+ rows = cur.fetchall() if cur.description else []
186
+ return QueryResult(rows, columns, cur.rowcount)
187
+ finally:
188
+ cur.close()
189
+
190
+ def notices(self):
191
+ return list(getattr(self._conn, 'notices', []))
192
+
193
+
194
+ def db_connect(dbname, host, port, user, passwd):
195
+ load_db_driver()
196
+ if psycopg2 is not None:
197
+ return Psycopg2DB(dbname, host, port, user, passwd)
198
+ return pg.DB(dbname=dbname, host=host, port=port, user=user, passwd=passwd)
199
+
200
+
201
+ # Mapping for validing our configuration file. We're only concerned with
202
+ # keys -- stuff left of ':'. It gets complex in two cases: firstly when
203
+ # we handle blocks which have keys which are not keywords -- such as under
204
+ # COLUMNS:. Secondly, we want to detect when users put keywords in the wrong
205
+ # place. To that end, the mapping is structured such that:
206
+ #
207
+ # key -> { 'parse_children' -> [ True | False ],
208
+ # 'parent' -> <parent name> }
209
+ #
210
+ # Each key is a keyword in the configuration file. parse_children tells us
211
+ # whether children are expected to be keywords. parent tells us the parent
212
+ # keyword or None
213
+ valid_tokens = {
214
+ "version": {'parse_children': True, 'parent': None},
215
+ "database": {'parse_children': True, 'parent': None},
216
+ "user": {'parse_children': True, 'parent': None},
217
+ "host": {'parse_children': True, 'parent': None},
218
+ "port": {'parse_children': True, 'parent': [None, "source"]},
219
+ "password": {'parse_children': True, 'parent': None},
220
+ "gpload": {'parse_children': True, 'parent': None},
221
+ "input": {'parse_children': True, 'parent': "gpload"},
222
+ "source": {'parse_children': True, 'parent': "input"},
223
+ "local_hostname": {'parse_children': False, 'parent': "source"},
224
+ "port_range": {'parse_children': False, 'parent': "source"},
225
+ "file": {'parse_children': False, 'parent': "source"},
226
+ "ssl": {'parse_children': False, 'parent': "source"},
227
+ "certificates_path": {'parse_children': False, 'parent': "source"},
228
+ "columns": {'parse_children': False, 'parent': "input"},
229
+ "transform": {'parse_children': True, 'parent': "input"},
230
+ "transform_config": {'parse_children': True, 'parent': "input"},
231
+ "max_line_length": {'parse_children': True, 'parent': "input"},
232
+ "format": {'parse_children': True, 'parent': "input"},
233
+ "delimiter": {'parse_children': True, 'parent': "input"},
234
+ "escape": {'parse_children': True, 'parent': "input"},
235
+ "null_as": {'parse_children': True, 'parent': "input"},
236
+ "quote": {'parse_children': True, 'parent': "input"},
237
+ "encoding": {'parse_children': True, 'parent': "input"},
238
+ "force_not_null": {'parse_children': False, 'parent': "input"},
239
+ "error_limit": {'parse_children': True, 'parent': "input"},
240
+ "error_percent": {'parse_children': True, 'parent': "input"},
241
+ "error_table": {'parse_children': True, 'parent': "input"},
242
+ "log_errors": {'parse_children': False, 'parent': "input"},
243
+ "header": {'parse_children': True, 'parent': "input"},
244
+ "fully_qualified_domain_name": {'parse_children': False, 'parent': 'input'},
245
+ "output": {'parse_children': True, 'parent': "gpload"},
246
+ "table": {'parse_children': True, 'parent': "output"},
247
+ "mode": {'parse_children': True, 'parent': "output"},
248
+ "match_columns": {'parse_children': False, 'parent': "output"},
249
+ "update_columns": {'parse_children': False, 'parent': "output"},
250
+ "update_condition": {'parse_children': True, 'parent': "output"},
251
+ "mapping": {'parse_children': False, 'parent': "output"},
252
+ "including_defaults": {'parse_children': False, 'parent': 'output'},
253
+ "preload": {'parse_children': True, 'parent': 'gpload'},
254
+ "truncate": {'parse_children': False, 'parent': 'preload'},
255
+ "reuse_tables": {'parse_children': False, 'parent': 'preload'},
256
+ "sql": {'parse_children': True, 'parent': 'gpload'},
257
+ "before": {'parse_children': False, 'parent': 'sql'},
258
+ "after": {'parse_children': False, 'parent': 'sql'},
259
+ "external": {'parse_children': True, 'parent': 'gpload'},
260
+ "schema": {'parse_children': False, 'parent': 'external'}}
261
+
262
+ _abbrevs = [
263
+ (1<<50, ' PB'),
264
+ (1<<40, ' TB'),
265
+ (1<<30, ' GB'),
266
+ (1<<20, ' MB'),
267
+ (1<<10, ' kB'),
268
+ (1, ' bytes')
269
+ ]
270
+
271
+ received_kill = False
272
+
273
+
274
+ def caseInsensitiveDictLookup(key, dictionary):
275
+ """
276
+ Do a case insensitive dictionary lookup. Return the dictionary value if found,
277
+ or None if not found.
278
+ """
279
+ for entry in dictionary:
280
+ if entry.lower() == key.lower():
281
+ return dictionary[entry]
282
+ return None
283
+
284
+
285
+
286
+ def sqlIdentifierCompare(x, y):
287
+ """
288
+ Compare x and y as SQL identifiers. Use SQL rules for comparing delimited
289
+ and non-delimited identifiers. Return True if they are equivalent or False
290
+ if they are not equivalent.
291
+ """
292
+ if x == None or y == None:
293
+ return False
294
+
295
+ if isDelimited(x):
296
+ x = quote_unident(x)
297
+ else:
298
+ x = x.lower()
299
+ if isDelimited(y):
300
+ y = quote_unident(y)
301
+ else:
302
+ y = y.lower()
303
+
304
+ if x == y:
305
+ return True
306
+ else:
307
+ return False
308
+
309
+
310
+ def isDelimited(value):
311
+ """
312
+ This method simply checks to see if the user supplied value has delimiters.
313
+ That is, if it starts and ends with double-quotes, then it is delimited.
314
+ """
315
+ if len(value) < 2:
316
+ return False
317
+ if value[0] == '"' and value[-1] == '"':
318
+ return True
319
+ else:
320
+ return False
321
+
322
+
323
+ def convertListToDelimited(identifiers):
324
+ """
325
+ This method will convert a list of identifiers, which may be a mix of
326
+ delimited and non-delimited identifiers, and return a list of
327
+ delimited identifiers.
328
+ """
329
+ returnList = []
330
+
331
+ for id in identifiers:
332
+ if isDelimited(id) == False:
333
+ id = id.lower()
334
+ returnList.append(quote_ident(id))
335
+ else:
336
+ returnList.append(id)
337
+ return returnList
338
+
339
+
340
+
341
+ def splitUpMultipartIdentifier(id):
342
+ """
343
+ Given a sql identifer like sch.tab, return a list of its
344
+ individual elements (e.g. sch.tab would return ['sch','tab']
345
+ """
346
+ returnList = []
347
+
348
+ elementList = splitIntoLiteralsAndNonLiterals(id, quoteValue='"')
349
+ # If there is a leading empty string, remove it.
350
+ if elementList[0] == ' ':
351
+ elementList.pop(0)
352
+
353
+ # Remove the dots, and split up undelimited multipart names
354
+ for e in elementList:
355
+ if e != '.':
356
+ if e[0] != '"':
357
+ subElementList = e.split('.')
358
+ else:
359
+ subElementList = [e]
360
+ for se in subElementList:
361
+ # remove any empty elements
362
+ if se != '':
363
+ returnList.append(se)
364
+
365
+ return returnList
366
+
367
+
368
+ def splitIntoLiteralsAndNonLiterals(str1, quoteValue="'"):
369
+ """
370
+ Break the string (str1) into a list of literals and non-literals where every
371
+ even number element is a non-literal and every odd number element is a literal.
372
+ The delimiter between literals and non-literals is the quoteValue, so this
373
+ function will not take into account any modifiers on a literal (e.g. E'adf').
374
+ """
375
+ returnList = []
376
+
377
+ if len(str1) > 1 and str1[0] == quoteValue:
378
+ # Always start with a non-literal
379
+ str1 = ' ' + str1
380
+
381
+ inLiteral = False
382
+ i = 0
383
+ tokenStart = 0
384
+ while i < len(str1):
385
+ if str1[i] == quoteValue:
386
+ if inLiteral == False:
387
+ # We are at start of literal
388
+ inLiteral = True
389
+ returnList.append(str1[tokenStart:i])
390
+ tokenStart = i
391
+ elif i + 1 < len(str1) and str1[i+1] == quoteValue:
392
+ # We are in a literal and found quote quote, so skip over it
393
+ i = i + 1
394
+ else:
395
+ # We are at the end of a literal or end of str1
396
+ returnList.append(str1[tokenStart:i+1])
397
+ tokenStart = i + 1
398
+ inLiteral = False
399
+ i = i + 1
400
+ if tokenStart < len(str1):
401
+ returnList.append(str1[tokenStart:])
402
+ return returnList
403
+
404
+
405
+ def quote_ident(val):
406
+ """
407
+ This method returns a new string replacing " with "",
408
+ and adding a " at the start and end of the string.
409
+ """
410
+ return '"' + val.replace('"', '""') + '"'
411
+
412
+
413
+ def quote_unident(val):
414
+ """
415
+ This method returns a new string replacing "" with ",
416
+ and removing the " at the start and end of the string.
417
+ """
418
+ if val != None and len(val) > 0:
419
+ val = val.replace('""', '"')
420
+ if val != None and len(val) > 1 and val[0] == '"' and val[-1] == '"':
421
+ val = val[1:-1]
422
+
423
+ return val
424
+
425
+
426
+ def notice_processor(self):
427
+ if windowsPlatform == True:
428
+ # We don't have a pygresql with our notice fix, so skip for windows.
429
+ # This means we will not get any warnings on windows (MPP10989).
430
+ return
431
+
432
+ theNotices = _get_notices(self.db)
433
+ # Greenplum 6 lowercases the first word: "NOTICE: found N data
434
+ # formatting errors (N or more input rows), rejected related input data".
435
+ r = re.compile(r"^NOTICE:\s+found (\d+) data formatting errors", re.IGNORECASE)
436
+ messageNumber = 0
437
+ m = None
438
+ while messageNumber < len(theNotices) and m == None:
439
+ aNotice = _notice_text(theNotices[messageNumber])
440
+ m = r.match(aNotice)
441
+ messageNumber = messageNumber + 1
442
+ if m:
443
+ global NUM_WARN_ROWS
444
+ NUM_WARN_ROWS = int(m.group(1))
445
+
446
+ def handle_kill(signum, frame):
447
+ # already dying?
448
+ global received_kill
449
+ if received_kill:
450
+ return
451
+
452
+ received_kill = True
453
+
454
+ g.log(g.INFO, "received signal %d" % signum)
455
+ g.exitValue = 2
456
+ sys.exit(2)
457
+
458
+
459
+ def bytestr(size, precision=1):
460
+ """Return a string representing the greek/metric suffix of a size"""
461
+ if size==1:
462
+ return '1 byte'
463
+ for factor, suffix in _abbrevs:
464
+ if size >= factor:
465
+ break
466
+
467
+ float_string_split = repr(size/float(factor)).split('.')
468
+ integer_part = float_string_split[0]
469
+ decimal_part = float_string_split[1]
470
+ if int(decimal_part[0:precision]):
471
+ float_string = '.'.join([integer_part, decimal_part[0:precision]])
472
+ else:
473
+ float_string = integer_part
474
+ return float_string + suffix
475
+
476
+ class CatThread(threading.Thread):
477
+ """
478
+ Simple threading wrapper to read a file descriptor and put the contents
479
+ in the log file.
480
+
481
+ The fd is assumed to be stdout and stderr from gpfdist. We must use select.select
482
+ and locks to ensure both threads are not read at the same time. A dead lock
483
+ situation could happen if they did. communicate() is not used since it blocks.
484
+ We will wait 1 second between read attempts.
485
+
486
+ """
487
+ def __init__(self,gpload,fd, sharedLock = None):
488
+ threading.Thread.__init__(self)
489
+ self.gpload = gpload
490
+ self.fd = fd
491
+ self.theLock = sharedLock
492
+
493
+ def run(self):
494
+ if windowsPlatform == True:
495
+ while 1:
496
+ # Windows select does not support select on non-file fd's, so we can use the lock fix. Deadlock is possible here.
497
+ # We need to look into the Python windows module to see if there is another way to do this in Windows.
498
+ line = self.fd.readline()
499
+ if line=='':
500
+ break
501
+ self.gpload.log(self.gpload.DEBUG, 'gpfdist: ' + line.strip('\n'))
502
+ else:
503
+ while 1:
504
+ retList = select.select( [self.fd]
505
+ , []
506
+ , []
507
+ , 1
508
+ )
509
+ if retList[0] == [self.fd]:
510
+ self.theLock.acquire()
511
+ line = self.fd.readline()
512
+ self.theLock.release()
513
+ else:
514
+ continue
515
+ if line=='':
516
+ break
517
+ self.gpload.log(self.gpload.DEBUG, 'gpfdist: ' + line.strip('\n'))
518
+
519
+
520
+ def cli_help():
521
+ help_path = os.path.join(sys.path[0], '..', 'docs', 'cli_help', EXECNAME +
522
+ '_help');
523
+ f = None
524
+ try:
525
+ try:
526
+ f = open(help_path);
527
+ return f.read(-1)
528
+ except:
529
+ return ''
530
+ finally:
531
+ if f: f.close()
532
+
533
+ #============================================================
534
+ def usage(error = None):
535
+ print(cli_help() or __doc__)
536
+ sys.stdout.flush()
537
+ if error:
538
+ sys.stderr.write('ERROR: ' + error + '\n')
539
+ sys.stderr.write('\n')
540
+ sys.stderr.flush()
541
+
542
+ sys.exit(2)
543
+
544
+ def quote(a):
545
+ """
546
+ SQLify a string
547
+ """
548
+ return "'"+a.replace("'","''").replace('\\','\\\\')+"'"
549
+
550
+
551
+ def quote_copy_value(a):
552
+ """Quote a COPY option value (delimiter, null, escape, quote).
553
+
554
+ quote() doubles backslashes for ordinary SQL literals, but COPY options
555
+ compare against the raw input bytes and ESCAPE requires exactly one
556
+ character. With standard_conforming_strings=on, '\\\\N' therefore became
557
+ a three-character NULL string instead of the usual two-character \\N.
558
+ E'' keeps the value unambiguous on both settings.
559
+ """
560
+ if '\\' in a:
561
+ return "E'" + a.replace('\\', '\\\\').replace("'", "''") + "'"
562
+ return quote(a)
563
+
564
+
565
+ def decode_unicode_escape(a):
566
+ return codecs.decode(a, 'unicode_escape')
567
+
568
+
569
+ def to_db_text(value):
570
+ if isinstance(value, bytes):
571
+ return value.decode('utf-8')
572
+ return value
573
+
574
+
575
+ def to_text(value):
576
+ if isinstance(value, bytes):
577
+ return value.decode('utf-8')
578
+ return value
579
+
580
+
581
+ def escape_string(value):
582
+ value = to_text(value)
583
+ return value.replace('\\', '\\\\').replace("'", "''")
584
+
585
+
586
+ def to_bool(value):
587
+ value = to_text(value)
588
+ if isinstance(value, str):
589
+ return value.lower() not in ('', 'f', 'false', '0')
590
+ return bool(value)
591
+
592
+
593
+ def error_log_filename(value):
594
+ """Return the source file name from gp_read_error_log(filename).
595
+
596
+ Older Greenplum releases returned values like
597
+ ``gpfdist://host:port/dir/ [1.csv]``; Greenplum 6 returns
598
+ ``gpfdist://host:port/dir/file.csv [dir/file.csv]``. The caller only
599
+ needs the file name because load.sh resolves it against data_path.
600
+ """
601
+ value = to_text(value)
602
+ if not value:
603
+ return value
604
+
605
+ # GP6+ keeps the original path in square brackets.
606
+ match = re.search(r"\[([^\]]+)\]", value)
607
+ candidate = match.group(1) if match else value
608
+
609
+ # Drop gpfdist://host:port/ and gpfdists://... prefixes when present.
610
+ candidate = re.sub(r"^[A-Za-z][A-Za-z0-9+.-]*://[^/]*/", "", candidate)
611
+ candidate = candidate.replace('\\', '/').strip()
612
+
613
+ if not candidate:
614
+ return value
615
+ return os.path.basename(candidate)
616
+
617
+
618
+ def splitPgpassLine(a):
619
+ """
620
+ If the user has specified a .pgpass file, we'll have to parse it. We simply
621
+ split the string into arrays at :. We could just use a native python
622
+ function but we need to escape the ':' character.
623
+ """
624
+ b = []
625
+ escape = False
626
+ d = ''
627
+ for c in a:
628
+ if not escape and c=='\\':
629
+ escape = True
630
+ elif not escape and c==':':
631
+ b.append(d)
632
+ d = ''
633
+ else:
634
+ d += c
635
+ escape = False
636
+ if escape:
637
+ d += '\\'
638
+ b.append(d)
639
+ return b
640
+
641
+ def test_key(gp, key, crumb):
642
+ """
643
+ Make sure that a key is a valid keyword in the configuration grammar and
644
+ that it appears in the configuration file where we expect -- that is, where
645
+ it has the parent we expect
646
+ """
647
+ val = valid_tokens.get(key)
648
+ if val == None:
649
+ gp.log(gp.ERROR, 'unrecognized key: "%s"' % key)
650
+
651
+ p = val['parent']
652
+
653
+ # simplify for when the same keyword can appear in multiple places
654
+ if type(p) != list:
655
+ p = [p]
656
+
657
+ c = None
658
+ if len(crumb):
659
+ c = crumb[-1]
660
+
661
+ found = False
662
+ for m in p:
663
+ if m == c:
664
+ found = True
665
+ break
666
+
667
+ if not found:
668
+ gp.log(gp.ERROR, 'unexpected key: "%s"' % key)
669
+
670
+ return val
671
+
672
+ def yaml_walk(gp, node, crumb):
673
+ if type(node) == list:
674
+ for a in node:
675
+ if type(a) == tuple:
676
+ key = a[0].value.lower()
677
+
678
+ val = test_key(gp, key, crumb)
679
+
680
+ if (len(a) > 1 and val['parse_children'] and
681
+ (isinstance(a[1], yaml.nodes.MappingNode) or
682
+ isinstance(a[1], yaml.nodes.SequenceNode))):
683
+ crumb.append(key)
684
+ yaml_walk(gp, a[1], crumb)
685
+ crumb.pop()
686
+ elif isinstance(a, yaml.nodes.ScalarNode):
687
+ test_key(gp, a.value, crumb)
688
+ else:
689
+ yaml_walk(gp, a, crumb)
690
+ elif isinstance(node, yaml.nodes.MappingNode):
691
+ yaml_walk(gp, node.value, crumb)
692
+
693
+ elif isinstance(node, yaml.nodes.ScalarNode):
694
+ pass
695
+
696
+ elif isinstance(node, yaml.nodes.SequenceNode):
697
+ yaml_walk(gp, node.value, crumb)
698
+
699
+ elif isinstance(node, yaml.nodes.CollectionNode):
700
+ pass
701
+
702
+
703
+ def changeToUnicode(a):
704
+ """
705
+ Recursively normalize the containers in a YAML document
706
+ """
707
+ if isinstance(a, list):
708
+ return [changeToUnicode(item) for item in a]
709
+ if isinstance(a, dict):
710
+ return {key: changeToUnicode(value) for key, value in a.items()}
711
+ return a
712
+
713
+
714
+
715
+ def dictKeyToLower(a):
716
+ """
717
+ down case all entries in a list or dict
718
+ """
719
+ if type(a) == list:
720
+ return list(map(dictKeyToLower,a))
721
+ if type(a) == dict:
722
+ b = dict()
723
+ for key,value in a.items():
724
+ if type(key) == str:
725
+ key = str(key.lower())
726
+ b[key] = dictKeyToLower(value)
727
+ return b
728
+ if type(a) == str:
729
+ a = str(a)
730
+ return a
731
+
732
+ #
733
+ # MPP-13348
734
+ #
735
+
736
+ '''Jenkins hash - http://burtleburtle.net/bob/hash/doobs.html'''
737
+
738
+ def jenkinsmix(a, b, c):
739
+ a &= 0xffffffff; b &= 0xffffffff; c &= 0xffffffff
740
+ a -= b; a -= c; a ^= (c>>13); a &= 0xffffffff
741
+ b -= c; b -= a; b ^= (a<<8); b &= 0xffffffff
742
+ c -= a; c -= b; c ^= (b>>13); c &= 0xffffffff
743
+ a -= b; a -= c; a ^= (c>>12); a &= 0xffffffff
744
+ b -= c; b -= a; b ^= (a<<16); b &= 0xffffffff
745
+ c -= a; c -= b; c ^= (b>>5); c &= 0xffffffff
746
+ a -= b; a -= c; a ^= (c>>3); a &= 0xffffffff
747
+ b -= c; b -= a; b ^= (a<<10); b &= 0xffffffff
748
+ c -= a; c -= b; c ^= (b>>15); c &= 0xffffffff
749
+ return a, b, c
750
+
751
+
752
+ def jenkins(data, initval = 0):
753
+ length = lenpos = len(data)
754
+ if length == 0:
755
+ return 0
756
+ a = b = 0x9e3779b9
757
+ c = initval
758
+ p = 0
759
+ while lenpos >= 12:
760
+ a += (ord(data[p+0]) + (ord(data[p+1])<<8) + (ord(data[p+2])<<16) + (ord(data[p+3])<<24))
761
+ b += (ord(data[p+4]) + (ord(data[p+5])<<8) + (ord(data[p+6])<<16) + (ord(data[p+7])<<24))
762
+ c += (ord(data[p+8]) + (ord(data[p+9])<<8) + (ord(data[p+10])<<16) + (ord(data[p+11])<<24))
763
+ a, b, c = jenkinsmix(a, b, c)
764
+ p += 12
765
+ lenpos -= 12
766
+ c += length
767
+ if lenpos >= 11: c += ord(data[p+10])<<24
768
+ if lenpos >= 10: c += ord(data[p+9])<<16
769
+ if lenpos >= 9: c += ord(data[p+8])<<8
770
+ if lenpos >= 8: b += ord(data[p+7])<<24
771
+ if lenpos >= 7: b += ord(data[p+6])<<16
772
+ if lenpos >= 6: b += ord(data[p+5])<<8
773
+ if lenpos >= 5: b += ord(data[p+4])
774
+ if lenpos >= 4: a += ord(data[p+3])<<24
775
+ if lenpos >= 3: a += ord(data[p+2])<<16
776
+ if lenpos >= 2: a += ord(data[p+1])<<8
777
+ if lenpos >= 1: a += ord(data[p+0])
778
+ a, b, c = jenkinsmix(a, b, c)
779
+ return c
780
+
781
+ # MPP-20927 Citibank: gpload external table name problem
782
+ # Not sure if it is used by other components, just leave it here.
783
+ def shortname(name):
784
+ """
785
+ Returns a 10 character string formed by concatenating the first two characters
786
+ of the name with another 8 character string computed using the Jenkins hash
787
+ function of the table name. When the original name has only a single non-space
788
+ ascii character, we return '00' followed by 8 char hash.
789
+
790
+ For example:
791
+
792
+ >>> shortname('mytable')
793
+ 'my3cbb7ba8'
794
+ >>> shortname('some_pretty_long_test_table_name')
795
+ 'so9068664a'
796
+ >>> shortname('t')
797
+ '006742be70'
798
+
799
+ @param name: the input tablename
800
+ @returns: a string 10 characters or less built from the table name
801
+ """
802
+
803
+ # Remove spaces from original name
804
+ name = re.sub(r' ', '', name)
805
+
806
+ # Run the hash function
807
+ j = jenkins(name)
808
+
809
+ # Now also remove non ascii chars from original name.
810
+ # We do this after jenkins so that we exclude the
811
+ # (very rare) case of passing an empty string to jenkins
812
+ name = "".join(i for i in name if ord(i) < 128)
813
+
814
+ if len(name) > 1:
815
+ return '%2s%08x' % (name[0:2], j)
816
+ else:
817
+ return '00%08x' % (j) # could be len 0 or 1
818
+
819
+
820
+ def staging_conditions_hash(conditions_str):
821
+ return hashlib.md5(conditions_str.encode('utf-8')).hexdigest()
822
+
823
+
824
+ class options:
825
+ pass
826
+
827
+ class gpload:
828
+ """
829
+ Main class wrapper
830
+ """
831
+
832
+ def __init__(self,argv):
833
+ # The database driver is imported by db_connect(), so --version, -?
834
+ # and control-file validation work without psycopg2/PyGreSQL.
835
+ self.threads = [] # remember threads so that we can join() against them
836
+ self.exitValue = 0
837
+ self.options = options()
838
+ self.options.h = None
839
+ self.options.gpfdist_timeout = None
840
+ self.options.p = None
841
+ self.options.U = None
842
+ self.options.W = False
843
+ self.options.D = False
844
+ self.options.no_auto_trans = False
845
+ self.options.password = None
846
+ self.options.d = None
847
+ self.DEBUG = 5
848
+ self.LOG = 4
849
+ self.INFO = 3
850
+ self.WARN = 2
851
+ self.ERROR = 1
852
+ self.options.qv = self.INFO
853
+ self.options.l = None
854
+ self.lastcmdtime = ''
855
+
856
+ seenv = False
857
+ seenq = False
858
+
859
+ # Create Temp and External table names. However external table name could
860
+ # get overwritten with another name later on (see create_external_table_name).
861
+ # MPP-20927 Citibank: gpload external table name problem. We use uuid to avoid
862
+ # external table name confliction.
863
+ self.unique_suffix = str(uuid.uuid1()).replace('-', '_')
864
+ self.staging_table_name = 'temp_staging_gpload_' + self.unique_suffix
865
+ self.extTableName = 'ext_gpload_' + self.unique_suffix
866
+
867
+ # SQL to run in order to undo our temporary work
868
+ self.cleanupSql = []
869
+ self.distkey = None
870
+ configFilename = None
871
+ while argv:
872
+ try:
873
+ try:
874
+ if argv[0]=='-h':
875
+ self.options.h = argv[1]
876
+ argv = argv[2:]
877
+ if argv[0]=='--gpfdist_timeout':
878
+ self.options.gpfdist_timeout = argv[1]
879
+ argv = argv[2:]
880
+ elif argv[0]=='-p':
881
+ self.options.p = int(argv[1])
882
+ argv = argv[2:]
883
+ elif argv[0]=='-l':
884
+ self.options.l = argv[1]
885
+ argv = argv[2:]
886
+ elif argv[0]=='-q':
887
+ self.options.qv -= 1
888
+ argv = argv[1:]
889
+ seenq = True
890
+ elif argv[0]=='--version':
891
+ sys.stderr.write("gpload version $Revision$\n")
892
+ sys.exit(0)
893
+ elif argv[0]=='-v':
894
+ self.options.qv = self.LOG
895
+ argv = argv[1:]
896
+ seenv = True
897
+ elif argv[0]=='-V':
898
+ self.options.qv = self.DEBUG
899
+ argv = argv[1:]
900
+ seenv = True
901
+ elif argv[0]=='-W':
902
+ self.options.W = True
903
+ argv = argv[1:]
904
+ elif argv[0]=='-D':
905
+ self.options.D = True
906
+ argv = argv[1:]
907
+ elif argv[0]=='-U':
908
+ self.options.U = argv[1]
909
+ argv = argv[2:]
910
+ elif argv[0]=='-d':
911
+ self.options.d = argv[1]
912
+ argv = argv[2:]
913
+ elif argv[0]=='-f':
914
+ configFilename = argv[1]
915
+ argv = argv[2:]
916
+ elif argv[0]=='--no_auto_trans':
917
+ self.options.no_auto_trans = True
918
+ argv = argv[1:]
919
+ elif argv[0]=='-?':
920
+ usage()
921
+ else:
922
+ break
923
+ except IndexError:
924
+ sys.stderr.write("Option %s needs a parameter.\n"%argv[0])
925
+ sys.exit(2)
926
+ except ValueError:
927
+ sys.stderr.write("Parameter for option %s must be an integer.\n"%argv[0])
928
+ sys.exit(2)
929
+
930
+ if configFilename==None:
931
+ usage('configuration file required')
932
+ elif argv:
933
+ a = ""
934
+ if len(argv) > 1:
935
+ a = "s"
936
+ usage('unrecognized argument%s: %s' % (a, ' '.join(argv)))
937
+
938
+ # default to gpAdminLogs for a log file, may be overwritten
939
+ if self.options.l is None:
940
+ self.options.l = os.path.join(os.environ.get('HOME', '.'),'gpAdminLogs')
941
+ if not os.path.isdir(self.options.l):
942
+ os.mkdir(self.options.l)
943
+
944
+ self.options.l = os.path.join(self.options.l, 'gpload_' + \
945
+ datetime.date.today().strftime('%Y%m%d') + '.log')
946
+
947
+ try:
948
+ self.logfile = open(self.options.l,'a', encoding='utf-8')
949
+ except Exception as e:
950
+ self.log(self.ERROR, "could not open logfile %s: %s" % \
951
+ (self.options.l, e))
952
+
953
+ if seenv and seenq:
954
+ self.log(self.ERROR, "-q conflicts with -v and -V")
955
+
956
+ if self.options.D:
957
+ self.log(self.INFO, 'gpload has the -D option, so it does not actually load any data')
958
+
959
+ try:
960
+ f = open(configFilename,'rb')
961
+ except IOError as e:
962
+ self.log(self.ERROR, "could not open configuration file: %s" % e)
963
+
964
+ # pull in the config file, which should be in valid YAML
965
+ try:
966
+ # do an initial parse, validating the config file
967
+ doc = f.read()
968
+ self.config = yaml.safe_load(doc)
969
+
970
+ self.configOriginal = changeToUnicode(self.config)
971
+ self.config = dictKeyToLower(self.config)
972
+ ver = self.getconfig('version', str, extraStuff = ' tag')
973
+ if ver != '1.0.0.1':
974
+ self.control_file_error("gpload configuration schema version must be 1.0.0.1")
975
+ # second parse, to check that the keywords are sensible
976
+ y = yaml.compose(doc, Loader=yaml.SafeLoader)
977
+ # first should be MappingNode
978
+ if not isinstance(y, yaml.MappingNode):
979
+ self.control_file_error("configuration file must begin with a mapping")
980
+
981
+ yaml_walk(self, y.value, [])
982
+ except yaml.scanner.ScannerError as e:
983
+ self.log(self.ERROR, "configuration file error: %s, line %s" % \
984
+ (e.problem, e.problem_mark.line))
985
+ except yaml.reader.ReaderError as e:
986
+ es = "unacceptable character #x%04x at byte %d: %s" \
987
+ % (e.character, e.position, e.reason)
988
+ self.log(self.ERROR, es)
989
+ except yaml.error.MarkedYAMLError as e:
990
+ self.log(self.ERROR, "configuration file error: %s, line %s" % \
991
+ (e.problem, e.problem_mark.line))
992
+
993
+ f.close()
994
+ self.subprocesses = []
995
+ self.log(self.INFO,'gpload session started ' + \
996
+ datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
997
+
998
+ def control_file_warning(self, msg):
999
+ self.log(self.WARN, "A gpload control file processing warning occurred. %s" % msg)
1000
+
1001
+ def control_file_error(self, msg):
1002
+ self.log(self.ERROR, "A gpload control file processing error occurred. %s" % msg)
1003
+
1004
+ def elevel2str(self, level):
1005
+ if level == self.DEBUG:
1006
+ return "DEBUG"
1007
+ elif level == self.LOG:
1008
+ return "LOG"
1009
+ elif level == self.INFO:
1010
+ return "INFO"
1011
+ elif level == self.ERROR:
1012
+ return "ERROR"
1013
+ elif level == self.WARN:
1014
+ return "WARN"
1015
+ else:
1016
+ self.log(self.ERROR, "unknown log type %i" % level)
1017
+
1018
+ def log(self, level, a):
1019
+ """
1020
+ Level is either DEBUG, LOG, INFO, ERROR. a is the message
1021
+ """
1022
+ t = time.localtime()
1023
+ message = '|'.join(
1024
+ [datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S'),
1025
+ self.elevel2str(level), a]) + '\n'
1026
+
1027
+ if level <= self.options.qv:
1028
+ sys.stdout.write(message)
1029
+
1030
+ if level <= self.options.qv or level <= self.INFO:
1031
+ try:
1032
+ self.logfile.write(message)
1033
+ self.logfile.flush()
1034
+ except AttributeError as e:
1035
+ pass
1036
+
1037
+ if level == self.ERROR:
1038
+ self.exitValue = 2;
1039
+ sys.exit(self.exitValue)
1040
+
1041
+ def getconfig(self, a, typ=None, default='error', extraStuff='', returnOriginal=False):
1042
+ """
1043
+ Look for a config entry, via a column delimited string. a:b:c points to
1044
+
1045
+ a:
1046
+ b:
1047
+ c
1048
+
1049
+ Make sure that end point is of type 'typ' when not set to None.
1050
+
1051
+ If returnOriginal is False, the return value will be in lower case,
1052
+ else the return value will be in its original form (i.e. the case that
1053
+ the user specified in their yaml file).
1054
+ """
1055
+ self.log(self.DEBUG, "getting config for " + a)
1056
+ if returnOriginal == True:
1057
+ config = self.configOriginal
1058
+ else:
1059
+ config = self.config
1060
+ for s in a.split(':'):
1061
+ self.log(self.DEBUG, "trying " + s)
1062
+ index = 1
1063
+
1064
+ if s[-1:]==')':
1065
+ j = s.index('(')
1066
+ index = int(s[j+1:-1])
1067
+ s = s[:j]
1068
+
1069
+ if type(config)!=list:
1070
+ config = [config]
1071
+
1072
+ for c in config:
1073
+ if type(c)==dict:
1074
+ temp = caseInsensitiveDictLookup(s, c)
1075
+ if temp != None:
1076
+ index -= 1
1077
+ if not index:
1078
+ self.log(self.DEBUG, "found " + s)
1079
+ config = temp
1080
+ break
1081
+ else:
1082
+ if default=='error':
1083
+ self.control_file_error("The configuration must contain %s%s"%(a,extraStuff))
1084
+ sys.exit(2)
1085
+ return default
1086
+
1087
+ if typ != None and type(config) != typ:
1088
+ if typ == list:
1089
+ self.control_file_error("The %s entry must be a YAML sequence %s"% (a ,extraStuff))
1090
+ elif typ == dict:
1091
+ self.control_file_error("The %s entry must be a YAML mapping %s"% (a, extraStuff))
1092
+ elif typ == str:
1093
+ self.control_file_error("%s must be a string %s" % (a, extraStuff))
1094
+ elif typ == int:
1095
+ self.control_file_error("The %s entry must be a YAML integer %s" % (a, extraStuff))
1096
+ else:
1097
+ assert 0
1098
+
1099
+ self.control_file_error("Encountered unknown configuration type %s"% type(config))
1100
+ sys.exit(2)
1101
+ return config
1102
+
1103
+ def read_config(self):
1104
+ """
1105
+ Configure ourselves
1106
+ """
1107
+
1108
+ # ensure output is of type list
1109
+ self.getconfig('gpload:output', list)
1110
+
1111
+ # The user supplied table name can be completely or partially delimited,
1112
+ # and it can be a one or two part name. Get the originally supplied name
1113
+ # and parse it into its delimited one or two part name.
1114
+ self.schemaTable = self.getconfig('gpload:output:table', str, returnOriginal=True)
1115
+ schemaTableList = splitUpMultipartIdentifier(self.schemaTable)
1116
+ schemaTableList = convertListToDelimited(schemaTableList)
1117
+ if len(schemaTableList) == 2:
1118
+ self.schema = schemaTableList[0]
1119
+ self.table = schemaTableList[1]
1120
+ else:
1121
+ self.schema = None
1122
+ self.table = schemaTableList[0]
1123
+
1124
+ # Precendence for configuration: command line > config file > env
1125
+ # variable
1126
+
1127
+ # host to connect to
1128
+ if not self.options.h:
1129
+ self.options.h = self.getconfig('host', str, None)
1130
+ if self.options.h:
1131
+ self.options.h = str(self.options.h)
1132
+ if not self.options.h:
1133
+ self.options.h = os.environ.get('PGHOST')
1134
+ if not self.options.h or len(self.options.h) == 0:
1135
+ self.log(self.INFO, "no host supplied, defaulting to localhost")
1136
+ self.options.h = "localhost"
1137
+
1138
+ # Port to connect to
1139
+ if not self.options.p:
1140
+ self.options.p = self.getconfig('port',int,None)
1141
+ if not self.options.p:
1142
+ try:
1143
+ self.options.p = int(os.environ.get('PGPORT'))
1144
+ except (ValueError, TypeError):
1145
+ pass
1146
+ if not self.options.p:
1147
+ self.options.p = 5432
1148
+
1149
+ # User to connect as
1150
+ if not self.options.U:
1151
+ self.options.U = self.getconfig('user', str, None)
1152
+ if not self.options.U:
1153
+ self.options.U = os.environ.get('PGUSER')
1154
+ if not self.options.U:
1155
+ self.options.U = os.environ.get('USER') or \
1156
+ os.environ.get('LOGNAME') or \
1157
+ os.environ.get('USERNAME')
1158
+
1159
+ if not self.options.U or len(self.options.U) == 0:
1160
+ self.log(self.ERROR,
1161
+ "You need to specify your username with the -U " +
1162
+ "option or in your configuration or in your " +
1163
+ "environment as PGUSER")
1164
+
1165
+ # database to connect to
1166
+ if not self.options.d:
1167
+ self.options.d = self.getconfig('database', str, None)
1168
+ if not self.options.d:
1169
+ self.options.d = os.environ.get('PGDATABASE')
1170
+ if not self.options.d:
1171
+ # like libpq, just inherit USER
1172
+ self.options.d = self.options.U
1173
+
1174
+ if self.getconfig('gpload:input:error_table', str, None):
1175
+ self.control_file_error("ERROR_TABLE is not supported. Please use LOG_ERRORS instead.")
1176
+
1177
+ def gpfdist_port_options(self, name, availablePorts, popenList):
1178
+ """
1179
+ Adds gpfdist -p / -P port options to popenList based on port and port_range in YAML file.
1180
+ Raises errors if options are invalid or ports are unavailable.
1181
+
1182
+ @param name: input source name from YAML file.
1183
+ @param availablePorts: current set of available ports
1184
+ @param popenList: gpfdist options (updated)
1185
+ """
1186
+ port = self.getconfig(name + ':port', int, None)
1187
+ port_range = self.getconfig(name+':port_range', list, None)
1188
+
1189
+ if port:
1190
+ startPort = endPort = port
1191
+ endPort += 1
1192
+ elif port_range:
1193
+ try:
1194
+ startPort = int(port_range[0])
1195
+ endPort = int(port_range[1])
1196
+ except (IndexError,ValueError):
1197
+ self.control_file_error(name + ":port_range must be a YAML sequence of two integers")
1198
+ else:
1199
+ startPort = self.getconfig(name+':port',int,8000)
1200
+ endPort = self.getconfig(name+':port',int,9000)
1201
+
1202
+ if (startPort > 65535 or endPort > 65535):
1203
+ # Do not allow invalid ports
1204
+ self.control_file_error("Invalid port. Port values must be less than or equal to 65535.")
1205
+ elif not (set(range(startPort,endPort+1)) & availablePorts):
1206
+ self.log(self.ERROR, "no more ports available for gpfdist")
1207
+
1208
+ popenList.append('-p')
1209
+ popenList.append(str(startPort))
1210
+
1211
+ popenList.append('-P')
1212
+ popenList.append(str(endPort))
1213
+
1214
+
1215
+ def gpfdist_filenames(self, name, popenList):
1216
+ """
1217
+ Adds gpfdist -f filenames to popenList.
1218
+ Raises errors if YAML file option is invalid.
1219
+
1220
+ @param name: input source name from YAML file.
1221
+ @param popenList: gpfdist options (updated)
1222
+ @return: list of files names
1223
+ """
1224
+ file = self.getconfig(name+':file',list)
1225
+ for i in file:
1226
+ if not isinstance(i, str):
1227
+ self.control_file_error(name + ":file must be a YAML sequence of strings")
1228
+ popenList.append('-f')
1229
+ popenList.append('"'+' '.join(file)+'"')
1230
+ return file
1231
+
1232
+
1233
+ def gpfdist_timeout_options(self, popenList):
1234
+ """
1235
+ Adds gpfdist -t timeout option to popenList.
1236
+
1237
+ @param popenList: gpfdist options (updated)
1238
+ """
1239
+ if self.options.gpfdist_timeout != None:
1240
+ gpfdistTimeout = self.options.gpfdist_timeout
1241
+ else:
1242
+ gpfdistTimeout = 30
1243
+ popenList.append('-t')
1244
+ popenList.append(str(gpfdistTimeout))
1245
+
1246
+
1247
+ def gpfdist_verbose_options(self, popenList):
1248
+ """
1249
+ Adds gpfdist -v / -V options to popenList depending on logging level
1250
+
1251
+ @param popenList: gpfdist options (updated)
1252
+ """
1253
+ if self.options.qv == self.LOG:
1254
+ popenList.append('-v')
1255
+ elif self.options.qv > self.LOG:
1256
+ popenList.append('-V')
1257
+
1258
+
1259
+ def gpfdist_max_line_length(self, popenList):
1260
+ """
1261
+ Adds gpfdist -m option to popenList when max_line_length option specified in YAML file.
1262
+
1263
+ @param popenList: gpfdist options (updated)
1264
+ """
1265
+ max_line_length = self.getconfig('gpload:input:max_line_length',int,None)
1266
+ if max_line_length is not None:
1267
+ popenList.append('-m')
1268
+ popenList.append(str(max_line_length))
1269
+
1270
+
1271
+ def gpfdist_transform(self, popenList):
1272
+ """
1273
+ Compute and return url fragment if transform option specified in YAML file.
1274
+ Checks for readable transform config file if transform_config option is specified.
1275
+ Adds gpfdist -c option to popenList if transform_config is specified.
1276
+ Validates that transform_config is present when transform option is specified.
1277
+
1278
+ @param popenList: gpfdist options (updated)
1279
+ @returns: uri fragment for transform or "" if not appropriate.
1280
+ """
1281
+ transform = self.getconfig('gpload:input:transform', str, None)
1282
+ transform_config = self.getconfig('gpload:input:transform_config', str, None)
1283
+ if transform_config:
1284
+ try:
1285
+ f = open(transform_config,'r')
1286
+ except IOError as e:
1287
+ self.log(self.ERROR, "could not open transform_config file: %s" % e)
1288
+ f.close()
1289
+ popenList.append('-c')
1290
+ popenList.append(transform_config)
1291
+ else:
1292
+ if transform:
1293
+ self.control_file_error("transform_config is required when transform is specified")
1294
+
1295
+ fragment = ""
1296
+ if transform is not None:
1297
+ fragment = "#transform=" + transform
1298
+ return fragment
1299
+
1300
+
1301
+ def gpfdist_ssl(self, popenList):
1302
+ """
1303
+ Adds gpfdist --ssl option to popenList when ssl option specified as true in YAML file.
1304
+
1305
+ @param popenList: gpfdist options (updated)
1306
+ """
1307
+ ssl = self.getconfig('gpload:input:source:ssl',bool, False)
1308
+ certificates_path = self.getconfig('gpload:input:source:certificates_path', str, None)
1309
+
1310
+ if ssl and certificates_path:
1311
+ dir_exists = os.path.isdir(certificates_path)
1312
+ if dir_exists == False:
1313
+ self.log(self.ERROR, "could not access CERTIFICATES_PATH directory: %s" % certificates_path)
1314
+
1315
+ popenList.append('--ssl')
1316
+ popenList.append(certificates_path)
1317
+
1318
+ else:
1319
+ if ssl:
1320
+ self.control_file_error("CERTIFICATES_PATH is required when SSL is specified as true")
1321
+ elif certificates_path: # ssl=false (or not specified) and certificates_path is specified
1322
+ self.control_file_error("CERTIFICATES_PATH is specified while SSL is not specified as true")
1323
+
1324
+
1325
+ def start_gpfdists(self):
1326
+ """
1327
+ Start gpfdist daemon(s)
1328
+ """
1329
+ self.locations = []
1330
+ self.ports = []
1331
+ sourceIndex = 0
1332
+ availablePorts = set(range(1,65535))
1333
+ found_source = False
1334
+
1335
+ self.getconfig('gpload:input', list)
1336
+
1337
+ while 1:
1338
+ sourceIndex += 1
1339
+ name = 'gpload:input:source(%d)'%sourceIndex
1340
+ a = self.getconfig(name,None,None)
1341
+ if not a:
1342
+ break
1343
+ found_source = True
1344
+ local_hostname = self.getconfig(name+':local_hostname', list, False)
1345
+
1346
+ # do default host, the current one
1347
+ if not local_hostname:
1348
+ # if fully_qualified_domain_name is defined and set to true we want to
1349
+ # resolve the fqdn rather than just grabbing the hostname.
1350
+ fqdn = self.getconfig('gpload:input:fully_qualified_domain_name', bool, False)
1351
+ if fqdn:
1352
+ local_hostname = socket.getfqdn()
1353
+ else:
1354
+ local_hostname = socket.gethostname()
1355
+
1356
+ # build gpfdist parameters
1357
+ popenList = ['gpfdist']
1358
+ self.gpfdist_ssl(popenList)
1359
+ self.gpfdist_port_options(name, availablePorts, popenList)
1360
+ file = self.gpfdist_filenames(name, popenList)
1361
+ self.gpfdist_timeout_options(popenList)
1362
+ self.gpfdist_verbose_options(popenList)
1363
+ self.gpfdist_max_line_length(popenList)
1364
+ fragment = self.gpfdist_transform(popenList)
1365
+
1366
+ try:
1367
+ self.log(self.LOG, 'trying to run %s' % ' '.join(popenList))
1368
+ cfds = True
1369
+ if platform.system() in ['Windows', 'Microsoft']: # not supported on win32
1370
+ cfds = False
1371
+ cmd = ' '.join(popenList)
1372
+ needshell = False
1373
+ else:
1374
+ srcfile = None
1375
+ if os.environ.get('GPHOME_LOADERS'):
1376
+ srcfile = os.path.join(os.environ.get('GPHOME_LOADERS'),
1377
+ 'greenplum_loaders_path.sh')
1378
+ elif os.environ.get('GPHOME'):
1379
+ srcfile = os.path.join(os.environ.get('GPHOME'),
1380
+ 'greenplum_path.sh')
1381
+
1382
+ if (not (srcfile and os.path.exists(srcfile))):
1383
+ self.log(self.ERROR, 'cannot find greenplum environment ' +
1384
+ 'file: environment misconfigured')
1385
+
1386
+ cmd = '. %s ; exec ' % srcfile
1387
+ cmd += ' '.join(popenList)
1388
+ needshell = True
1389
+
1390
+ a = subprocess.Popen(cmd, stdout=subprocess.PIPE,
1391
+ stderr=subprocess.PIPE,
1392
+ close_fds=cfds, shell=needshell,
1393
+ text=True, encoding='utf-8',
1394
+ errors='replace')
1395
+ self.subprocesses.append(a)
1396
+ except Exception as e:
1397
+ self.log(self.ERROR, "could not run %s: %s" % \
1398
+ (' '.join(popenList), str(e)))
1399
+
1400
+ """
1401
+ Reading from stderr and stdout on a Popen object can result in a dead lock if done at the same time.
1402
+ Create a lock to share when reading stderr and stdout from gpfdist.
1403
+ """
1404
+ readLock = threading.Lock()
1405
+
1406
+ # get all the output from the daemon(s)
1407
+ t = CatThread(self,a.stderr, readLock)
1408
+ t.start()
1409
+ self.threads.append(t)
1410
+
1411
+ while 1:
1412
+ readLock.acquire()
1413
+ line = a.stdout.readline()
1414
+ readLock.release()
1415
+ if line=='':
1416
+ self.log(self.ERROR,'failed to start gpfdist: ' +
1417
+ 'gpfdist command line: ' + ' '.join(popenList))
1418
+
1419
+ line = line.strip('\n')
1420
+ self.log(self.LOG,'gpfdist says: ' + line)
1421
+ if (line.startswith('Serving HTTP on port ') or line.startswith('Serving HTTPS on port ')):
1422
+ port = int(line[21:line.index(',')])
1423
+ break
1424
+
1425
+ self.log(self.INFO, 'started %s' % ' '.join(popenList))
1426
+ self.log(self.LOG,'gpfdist is running on port %d'%port)
1427
+ if port in availablePorts:
1428
+ availablePorts.remove(port)
1429
+ self.ports.append(port)
1430
+ t = CatThread(self,a.stdout,readLock)
1431
+ t.start()
1432
+ self.threads.append(t)
1433
+
1434
+ ssl = self.getconfig('gpload:input:source:ssl', bool, False)
1435
+ if ssl:
1436
+ protocol = 'gpfdists'
1437
+ else:
1438
+ protocol = 'gpfdist'
1439
+
1440
+ for l in local_hostname:
1441
+ if not isinstance(l, str):
1442
+ self.control_file_error(name + ":local_hostname must be a YAML sequence of strings")
1443
+ l = str(l)
1444
+ sep = ''
1445
+ if file[0] != '/':
1446
+ sep = '/'
1447
+ # MPP-13617
1448
+ if ':' in l:
1449
+ l = '[' + l + ']'
1450
+ self.locations.append('%s://%s:%d%s%s%s' % (protocol, l, port, sep, '%20'.join(file), fragment))
1451
+ if not found_source:
1452
+ self.control_file_error("configuration file must contain source definition")
1453
+
1454
+ def readPgpass(self,pgpassname):
1455
+ """
1456
+ Get password form .pgpass file
1457
+ """
1458
+ try:
1459
+ f = open(pgpassname,'r')
1460
+ except IOError:
1461
+ return
1462
+ for row in f:
1463
+ try:
1464
+ row = row.rstrip("\n")
1465
+ line = splitPgpassLine(row)
1466
+ if line[0]!='*' and line[0].lower()!=self.options.h.lower():
1467
+ continue
1468
+ if line[1]!='*' and int(line[1])!=self.options.p:
1469
+ continue
1470
+ if line[2]!='*' and line[2]!=self.options.d:
1471
+ continue
1472
+ if line[3]!='*' and line[3]!=self.options.U:
1473
+ continue
1474
+ self.options.password = line[4]
1475
+ break
1476
+ except (ValueError,IndexError):
1477
+ pass
1478
+ f.close()
1479
+
1480
+
1481
+ def setup_connection(self, recurse = 0):
1482
+ """
1483
+ Connect to the backend
1484
+ """
1485
+ if self.db != None:
1486
+ self.db.close()
1487
+ self.db = None
1488
+ if self.options.W:
1489
+ if self.options.password==None:
1490
+ self.options.password = getpass.getpass()
1491
+ else:
1492
+ if self.options.password==None:
1493
+ self.options.password = self.getconfig('password', str,
1494
+ None)
1495
+ if self.options.password==None:
1496
+ self.options.password = os.environ.get('PGPASSWORD')
1497
+ if self.options.password==None:
1498
+ self.readPgpass(os.environ.get('PGPASSFILE',
1499
+ os.environ.get('HOME','.')+'/.pgpass'))
1500
+ try:
1501
+ self.log(self.DEBUG, "connection string:" +
1502
+ " user=" + str(self.options.U) +
1503
+ " host=" + str(self.options.h) +
1504
+ " port=" + str(self.options.p) +
1505
+ " database=" + str(self.options.d))
1506
+ self.db = db_connect( dbname=self.options.d
1507
+ , host=self.options.h
1508
+ , port=self.options.p
1509
+ , user=self.options.U
1510
+ , passwd=self.options.password
1511
+ )
1512
+ _install_notice_receiver(self.db)
1513
+ self.log(self.DEBUG, "Successfully connected to database")
1514
+ except Exception as e:
1515
+ errorMessage = str(e)
1516
+ if errorMessage.find("no password supplied") != -1:
1517
+ self.options.password = getpass.getpass()
1518
+ recurse += 1
1519
+ if recurse > 10:
1520
+ self.log(self.ERROR, "too many login attempt failures")
1521
+ self.setup_connection(recurse)
1522
+ else:
1523
+ self.log(self.ERROR, "could not connect to database: %s. Is " \
1524
+ "the Greenplum Database running on port %i?" % (errorMessage,
1525
+ self.options.p))
1526
+
1527
+ def read_columns(self):
1528
+ columns = self.getconfig('gpload:input:columns',list,None, returnOriginal=True)
1529
+ if columns != None:
1530
+ self.from_cols_from_user = True # user specified from columns
1531
+ self.from_columns = []
1532
+ for d in columns:
1533
+ if type(d)!=dict:
1534
+ self.control_file_error("gpload:input:columns must be a sequence of YAML mappings")
1535
+ tempkey = next(iter(d))
1536
+ value = d[tempkey]
1537
+ """ remove leading or trailing spaces """
1538
+ d = { tempkey.strip() : value }
1539
+ key = next(iter(d))
1540
+ if d[key] == None:
1541
+ self.log(self.DEBUG,
1542
+ 'getting source column data type from target')
1543
+ for name, typ, mapto, hasseq in self.into_columns:
1544
+ if sqlIdentifierCompare(name, key):
1545
+ d[key] = typ
1546
+ break
1547
+
1548
+ # perform the same kind of magic type change that postgres does
1549
+ if d[key] == 'bigserial':
1550
+ d[key] = 'bigint'
1551
+ elif d[key] == 'serial':
1552
+ d[key] = 'int4'
1553
+
1554
+ # Mark this column as having no mapping, which is important
1555
+ # for do_insert()
1556
+ self.from_columns.append([key,d[key],None, False])
1557
+ else:
1558
+ self.from_columns = self.into_columns
1559
+ self.from_cols_from_user = False
1560
+
1561
+ # make sure that all columns have a type
1562
+ for name, typ, map, hasseq in self.from_columns:
1563
+ if typ == None:
1564
+ self.log(self.ERROR, 'column "%s" has no type ' % name +
1565
+ 'and does not appear in target table "%s"' % self.schemaTable)
1566
+ self.log(self.DEBUG, 'from columns are:')
1567
+ for c in self.from_columns:
1568
+ name = c[0]
1569
+ typ = c[1]
1570
+ self.log(self.DEBUG, '%s: %s'%(name,typ))
1571
+
1572
+
1573
+
1574
+ def read_table_metadata(self):
1575
+ # KAS Note to self. If schema is specified, then probably should use PostgreSQL rules for defining it.
1576
+
1577
+ # find the shema name for this table (according to search_path)
1578
+ # if it was not explicitly specified in the configuration file.
1579
+ if self.schema == None:
1580
+ queryString = """SELECT n.nspname
1581
+ FROM pg_catalog.pg_class c
1582
+ LEFT JOIN pg_catalog.pg_namespace n
1583
+ ON n.oid = c.relnamespace
1584
+ WHERE c.relname = '%s'
1585
+ AND pg_catalog.pg_table_is_visible(c.oid);""" % quote_unident(self.table)
1586
+
1587
+ resultList = self.db.query(to_db_text(queryString)).getresult()
1588
+
1589
+ if len(resultList) > 0:
1590
+ self.schema = to_text((resultList[0])[0])
1591
+ self.log(self.INFO, "setting schema '%s' for table '%s'" % (self.schema, quote_unident(self.table)))
1592
+ else:
1593
+ self.log(self.ERROR, "table %s not found in any database schema" % self.table)
1594
+
1595
+
1596
+ queryString = """select nt.nspname as table_schema,
1597
+ c.relname as table_name,
1598
+ a.attname as column_name,
1599
+ a.attnum as ordinal_position,
1600
+ format_type(a.atttypid, a.atttypmod) as data_type,
1601
+ c.relkind = 'r' AS is_updatable,
1602
+ a.atttypid in (23, 20) and a.atthasdef and
1603
+ (select position ( 'nextval(' in pg_catalog.pg_get_expr(adbin,adrelid) ) > 0 and
1604
+ position ( '::regclass)' in pg_catalog.pg_get_expr(adbin,adrelid) ) > 0
1605
+ FROM pg_catalog.pg_attrdef d
1606
+ WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as has_sequence
1607
+ from pg_catalog.pg_class c join pg_catalog.pg_namespace nt on (c.relnamespace = nt.oid)
1608
+ join pg_attribute a on (a.attrelid = c.oid)
1609
+ where c.relname = '%s' and nt.nspname = '%s'
1610
+ and a.attnum > 0 and a.attisdropped = 'f'
1611
+ order by a.attnum """ % (quote_unident(self.table), quote_unident(self.schema))
1612
+
1613
+ count = 0
1614
+ self.into_columns = []
1615
+ self.into_columns_dict = dict()
1616
+ resultList = self.db.query(to_db_text(queryString)).dictresult()
1617
+ while count < len(resultList):
1618
+ row = resultList[count]
1619
+ count += 1
1620
+ ct = to_text(row['data_type'])
1621
+ if ct == 'bigserial':
1622
+ ct = 'bigint'
1623
+ elif ct == 'serial':
1624
+ ct = 'int4'
1625
+ name = to_text(row['column_name'])
1626
+ name = quote_ident(name)
1627
+ has_seq = to_bool(row['has_sequence'])
1628
+ i = [name,ct,None, has_seq]
1629
+ self.into_columns.append(i)
1630
+ self.into_columns_dict[name] = i
1631
+ self.log(self.DEBUG, "found input column: " + str(i))
1632
+ if count == 0:
1633
+ # see if it's a permissions issue or it actually doesn't exist
1634
+ tableName = quote_unident(self.table)
1635
+ tableSchema = quote_unident(self.schema)
1636
+ sql = """select 1 from pg_class c, pg_namespace n
1637
+ where c.relname = '%s' and
1638
+ n.nspname = '%s' and
1639
+ n.oid = c.relnamespace""" % (tableName, tableSchema)
1640
+ resultList = self.db.query(to_db_text(sql)).getresult()
1641
+ if len(resultList) > 0:
1642
+ self.log(self.ERROR, "permission denied for table %s.%s" % \
1643
+ (tableSchema, tableName))
1644
+ else:
1645
+ self.log(self.ERROR, 'table %s.%s does not exist in database %s'% (tableSchema, tableName, self.options.d))
1646
+
1647
+ def read_mapping(self):
1648
+ mapping = self.getconfig('gpload:output:mapping',dict,None, returnOriginal=True)
1649
+
1650
+ if mapping:
1651
+ for key,value in mapping.items():
1652
+ if not isinstance(key, str) or not isinstance(value, str):
1653
+ self.control_file_error("gpload:output:mapping must be a YAML type mapping from strings to strings")
1654
+ found = False
1655
+ for a in self.into_columns:
1656
+ if sqlIdentifierCompare(a[0], key) == True:
1657
+ a[2] = value
1658
+ found = True
1659
+ break
1660
+ if found == False:
1661
+ self.log(self.ERROR,'%s in mapping is not in table %s'% \
1662
+ (key, self.schemaTable))
1663
+ else:
1664
+ # Now, map anything yet to be mapped to itself, picking up on those
1665
+ # columns which are not found in the table.
1666
+ for x in self.from_columns:
1667
+ # Check to see if it already has a mapping value
1668
+ i = list(filter(lambda a:a[2] == x[0], self.into_columns))
1669
+ if not i:
1670
+ # Check to see if the target column names match the input column names.
1671
+ for a in self.into_columns:
1672
+ if sqlIdentifierCompare(a[0], x[0]) == True:
1673
+ i = a
1674
+ found = True
1675
+ break
1676
+ if i:
1677
+ if i[2] == None: i[2] = i[0]
1678
+ else:
1679
+ self.log(self.ERROR, 'no mapping for input column ' +
1680
+ '"%s" to output table' % x[0])
1681
+ for name,typ,mapto,seq in self.into_columns:
1682
+ self.log(self.DEBUG,'%s: %s = %s'%(name,typ,mapto))
1683
+
1684
+ # In order to find out whether we have an existing external table in the
1685
+ # catalog which could be reused for this operation we need to make sure
1686
+ # that it has the same column names and types, the same data format, and
1687
+ # location specification, and single row error handling specs.
1688
+ #
1689
+ # This function will return the SQL to run in order to find out whether
1690
+ # such a table exists.
1691
+ #
1692
+ def get_reuse_exttable_query(self, formatType, formatOpts, limitStr, from_cols, schemaName, log_errors):
1693
+ # Greenplum 6 changed pg_exttable: location was renamed to
1694
+ # urilocation and the fmterrtbl error-table oid was replaced by the
1695
+ # logerrors boolean. Detect the available columns first so the same
1696
+ # code works on both old and new catalog layouts.
1697
+ ext_col_sql = "select attname from pg_catalog.pg_attribute " + \
1698
+ "where attrelid = 'pg_catalog.pg_exttable'::regclass " + \
1699
+ "and attname in ('urilocation', 'location', 'logerrors', 'fmterrtbl'); "
1700
+ ext_cols = set(to_text((row)[0]) for row in self.db.query(to_db_text(ext_col_sql)).getresult())
1701
+ location_col = 'urilocation' if 'urilocation' in ext_cols else 'location'
1702
+ error_col = 'logerrors' if 'logerrors' in ext_cols else 'fmterrtbl'
1703
+
1704
+ sqlFormat = """select attrelid::regclass
1705
+ from (
1706
+ select
1707
+ attrelid,
1708
+ row_number() over (partition by attrelid order by attnum) as attord,
1709
+ attnum,
1710
+ attname,
1711
+ atttypid::regtype
1712
+ from
1713
+ pg_attribute
1714
+ join
1715
+ pg_class
1716
+ on (pg_class.oid = attrelid)
1717
+ %s
1718
+ where
1719
+ relstorage = 'x' and
1720
+ relname like 'ext_gpload_reusable_%%' and
1721
+ attnum > 0 and
1722
+ not attisdropped and %s
1723
+ ) pgattr
1724
+ join
1725
+ pg_exttable pgext
1726
+ on(pgattr.attrelid = pgext.reloid)
1727
+ """
1728
+ joinStr = ""
1729
+ conditionStr = ""
1730
+
1731
+ # if schemaName is None, find the resuable ext table which is visible to
1732
+ # current search path. Else find the resuable ext table under the specific
1733
+ # schema, and this needs to join pg_namespace.
1734
+ if schemaName is None:
1735
+ joinStr = ""
1736
+ conditionStr = "pg_table_is_visible(pg_class.oid)"
1737
+ else:
1738
+ joinStr = """join
1739
+ pg_namespace pgns
1740
+ on(pg_class.relnamespace = pgns.oid)
1741
+ """
1742
+ conditionStr = "pgns.nspname = '%s'" % schemaName
1743
+
1744
+ sql = sqlFormat % (joinStr, conditionStr)
1745
+
1746
+ if log_errors:
1747
+ if error_col == 'logerrors':
1748
+ sql += " WHERE pgext.logerrors = true "
1749
+ else:
1750
+ sql += " WHERE pgext.fmterrtbl = pgext.reloid "
1751
+ else:
1752
+ if error_col == 'logerrors':
1753
+ sql += " WHERE pgext.logerrors = false "
1754
+ else:
1755
+ sql += " WHERE pgext.fmterrtbl IS NULL "
1756
+
1757
+ for i, l in enumerate(self.locations):
1758
+ sql += " and pgext.%s[%s] = %s\n" % (location_col, i + 1, quote(l))
1759
+
1760
+ sql+= """and pgext.fmttype = %s
1761
+ and pgext.writable = false
1762
+ and pgext.fmtopts like %s """ % (quote(formatType[0]),quote("%" + quote_unident(formatOpts.rstrip()) +"%"))
1763
+
1764
+ if limitStr:
1765
+ sql += "and pgext.rejectlimit = %s " % limitStr
1766
+ else:
1767
+ sql += "and pgext.rejectlimit IS NULL "
1768
+
1769
+ sql+= "group by attrelid "
1770
+
1771
+ sql+= """having
1772
+ count(*) = %s and
1773
+ bool_and(case """ % len(from_cols)
1774
+
1775
+ for i, c in enumerate(from_cols):
1776
+ name = c[0]
1777
+ typ = c[1]
1778
+ sql+= "when attord = %s then atttypid = %s::regtype and attname = %s\n" % (i+1, quote(typ), quote(quote_unident(name)))
1779
+
1780
+ sql+= """else true
1781
+ end)
1782
+ limit 1;"""
1783
+
1784
+ self.log(self.DEBUG, "query used to identify reusable external relations: %s" % sql)
1785
+ return sql
1786
+
1787
+ #
1788
+ # Create a string from the following conditions to reuse staging table:
1789
+ # 1. same target table
1790
+ # 2. same number of columns
1791
+ # 3. same names and types, in the same order
1792
+ # 4. same distribution key (according to columns' names and thier order)
1793
+ #
1794
+ def get_staging_conditions_string(self, target_table_name, staging_cols, distribution_cols):
1795
+
1796
+ columns_num = len(staging_cols)
1797
+
1798
+ staging_cols_str = '-'.join(list(map(lambda col:'%s-%s' % (quote(quote_unident(col[0])), quote(col[1])), staging_cols)))
1799
+
1800
+ distribution_cols_str = '-'.join([quote(quote_unident(col)) for col in distribution_cols])
1801
+
1802
+ return '%s:%s:%s:%s' % (target_table_name, columns_num, staging_cols_str, distribution_cols_str)
1803
+
1804
+
1805
+ #
1806
+ # This function will return the SQL to run in order to find out whether
1807
+ # we have an existing staging table in the catalog which could be reused for this
1808
+ # operation, according to the mathod and the encoding conditions.
1809
+ #
1810
+ def get_reuse_staging_table_query(self, encoding_conditions):
1811
+
1812
+ sql = """SELECT oid::regclass
1813
+ FROM pg_class
1814
+ WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions)
1815
+
1816
+ self.log(self.DEBUG, "query used to identify reusable temporary relations: %s" % sql)
1817
+ return sql
1818
+
1819
+ #
1820
+ # get oid for table from pg_class, None if not exist
1821
+ #
1822
+ def get_table_oid(self, tableName):
1823
+ if tableName:
1824
+ sql = "select %s::regclass::oid" % quote(quote_unident(tableName))
1825
+ try:
1826
+ resultList = self.db.query(to_db_text(sql)).getresult()
1827
+ return resultList[0][0]
1828
+ except Exception as e:
1829
+ pass
1830
+ return None
1831
+
1832
+ def get_ext_schematable(self, schemaName, tableName):
1833
+ if schemaName == None:
1834
+ return tableName
1835
+ else:
1836
+ schemaTable = "%s.%s" % (schemaName, tableName)
1837
+ return schemaTable
1838
+
1839
+ #
1840
+ # Create a new external table or find a reusable external table to use for this operation
1841
+ #
1842
+ def create_external_table(self):
1843
+
1844
+ # extract all control file information and transform it accordingly
1845
+ # in order to construct a CREATE EXTERNAL TABLE statement if will be
1846
+ # needed later on
1847
+
1848
+ formatType = self.getconfig('gpload:input:format', str, 'text').lower()
1849
+ formatOpts = ""
1850
+ locationStr = ','.join(list(map(quote,self.locations)))
1851
+
1852
+ delimiterValue = self.getconfig('gpload:input:delimiter', str, False)
1853
+ if isinstance(delimiterValue, bool) and delimiterValue == False:
1854
+ """ implies the DELIMITER option has no value in the yaml file, so use default """
1855
+ if formatType=='csv':
1856
+ formatOpts += "delimiter ',' "
1857
+ elif len(delimiterValue) != 1:
1858
+ # is a escape sequence character like '\x1B' or '\u001B'?
1859
+ if len(decode_unicode_escape(delimiterValue)) == 1:
1860
+ formatOpts += "delimiter '%s' " % \
1861
+ self.getconfig('gpload:input:delimiter', str)
1862
+ # is a escape string syntax support by gpdb like E'\x1B' or E'\x\u001B'
1863
+ elif len(decode_unicode_escape(delimiterValue.lstrip("E'").rstrip("'"))) ==1:
1864
+ formatOpts += "delimiter %s " % self.getconfig('gpload:input:delimiter', str)
1865
+ else:
1866
+ self.control_file_warning('''A delimiter must be single ASCII charactor, you can also use unprintable characters(for example: '\\x1c' / E'\\x1c' or '\\u001c' / E'\\u001c' ''')
1867
+ self.control_file_error("Invalid delimiter, gpload quit immediately")
1868
+ sys.exit(2);
1869
+ else:
1870
+ formatOpts += "delimiter %s " % \
1871
+ quote_copy_value(self.getconfig('gpload:input:delimiter', str))
1872
+
1873
+ nullas = self.getconfig('gpload:input:null_as', str, False)
1874
+ self.log(self.DEBUG, "null " + str(nullas))
1875
+ if nullas != False: # could be empty string
1876
+ formatOpts += "null %s " % quote_copy_value(nullas)
1877
+ elif formatType=='csv':
1878
+ formatOpts += "null '' "
1879
+ else:
1880
+ formatOpts += "null %s " % quote_copy_value(r"\N")
1881
+
1882
+ esc = self.getconfig('gpload:input:escape', None, None)
1883
+ if esc:
1884
+ if not isinstance(esc, str):
1885
+ self.control_file_error("gpload:input:escape must be a string")
1886
+ if esc.lower() == 'off':
1887
+ if formatType == 'csv':
1888
+ self.control_file_error("ESCAPE cannot be set to OFF in CSV mode")
1889
+ formatOpts += "escape 'off' "
1890
+ else:
1891
+ formatOpts += "escape %s " % quote_copy_value(esc)
1892
+ else:
1893
+ if formatType=='csv':
1894
+ formatOpts += "escape %s " % quote_copy_value(self.getconfig('gpload:input:quote',
1895
+ str, extraStuff='for csv formatted data'))
1896
+ else:
1897
+ formatOpts += "escape %s " % quote_copy_value("\\")
1898
+
1899
+ if formatType=='csv':
1900
+ formatOpts += "quote %s " % quote_copy_value(self.getconfig('gpload:input:quote',
1901
+ str, extraStuff='for csv formatted data'))
1902
+
1903
+ if self.getconfig('gpload:input:header',bool,False):
1904
+ formatOpts += "header "
1905
+
1906
+ force_not_null_columns = self.getconfig('gpload:input:force_not_null',list,[])
1907
+ if force_not_null_columns:
1908
+ for i in force_not_null_columns:
1909
+ if not isinstance(i, str):
1910
+ self.control_file_error("gpload:input:force_not_null must be a YAML sequence of strings")
1911
+ formatOpts += "force not null %s " % ','.join(force_not_null_columns)
1912
+
1913
+ encodingStr = self.getconfig('gpload:input:encoding', str, None)
1914
+
1915
+ limitStr = self.getconfig('gpload:input:error_limit',int, None)
1916
+ if self.log_errors and not limitStr:
1917
+ self.control_file_error("gpload:input:log_errors requires " +
1918
+ "gpload:input:error_limit to be specified")
1919
+
1920
+ self.extSchemaName = self.getconfig('gpload:external:schema', str, None)
1921
+ if self.extSchemaName == '%':
1922
+ self.extSchemaName = self.schema
1923
+
1924
+ # get the list of columns to use in the extnernal table
1925
+ if not self.from_cols_from_user:
1926
+ # don't put values serial columns
1927
+ from_cols = list(filter(lambda a: a[3] != True,
1928
+ self.from_columns))
1929
+ else:
1930
+ from_cols = self.from_columns
1931
+
1932
+ # If the 'reuse tables' option was specified we now try to find an
1933
+ # already existing external table in the catalog which will match
1934
+ # the one that we need to use. It must have identical attributes,
1935
+ # external location, format, and encoding specifications.
1936
+ if self.reuse_tables == True:
1937
+ sql = self.get_reuse_exttable_query(formatType, formatOpts,
1938
+ limitStr, from_cols, self.extSchemaName, self.log_errors)
1939
+ resultList = self.db.query(to_db_text(sql)).getresult()
1940
+ if len(resultList) > 0:
1941
+ # found an external table to reuse. no need to create one. we're done here.
1942
+ self.extTableName = to_text((resultList[0])[0])
1943
+ self.extSchemaTable = self.extTableName
1944
+ self.log(self.INFO, "reusing external table %s" % self.extSchemaTable)
1945
+ return
1946
+
1947
+ # didn't find an existing external table suitable for reuse. Format a reusable
1948
+ # name and issue a CREATE EXTERNAL TABLE on it. Hopefully we can use it next time
1949
+ # around
1950
+
1951
+ self.extTableName = "ext_gpload_reusable_%s" % self.unique_suffix
1952
+ self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.extTableName)
1953
+
1954
+ # construct a CREATE EXTERNAL TABLE statement and execute it
1955
+ self.extSchemaTable = self.get_ext_schematable(self.extSchemaName, self.extTableName)
1956
+ sql = "create external table %s" % self.extSchemaTable
1957
+ sql += "(%s)" % ','.join(list(map(lambda a:'%s %s' % (a[0], a[1]), from_cols)))
1958
+
1959
+ sql += "location(%s) "%locationStr
1960
+ sql += "format%s "% quote(formatType)
1961
+ if len(formatOpts) > 0:
1962
+ sql += "(%s) "% formatOpts
1963
+ if encodingStr:
1964
+ sql += "encoding%s "%quote(encodingStr)
1965
+ if self.log_errors:
1966
+ sql += "log errors "
1967
+
1968
+ if limitStr:
1969
+ if limitStr < 2:
1970
+ self.control_file_error("error_limit must be 2 or higher")
1971
+ sql += "segment reject limit %s "%limitStr
1972
+
1973
+ try:
1974
+ self.db.query(to_db_text(sql))
1975
+ except Exception as e:
1976
+ self.log(self.ERROR, 'could not run SQL "%s": %s' % (sql, str(e)))
1977
+
1978
+ # set up to drop the external table at the end of operation, unless user
1979
+ # specified the 'reuse_tables' option, in which case we don't drop
1980
+ if self.reuse_tables == False:
1981
+ self.cleanupSql.append('drop external table if exists %s'%self.extSchemaTable)
1982
+
1983
+
1984
+ #
1985
+ # Create a new staging table or find a reusable staging table to use for this operation
1986
+ # (only valid for update/merge operations).
1987
+ #
1988
+ def create_staging_table(self):
1989
+
1990
+ # Do some initial work to extract the update_columns and metadata
1991
+ # that may be needed in order to create or reuse a temp table
1992
+ if not self.from_cols_from_user:
1993
+ # don't put values serial columns
1994
+ from_cols = list(filter(lambda a: a[3] != True, self.from_columns))
1995
+ else:
1996
+ from_cols = self.from_columns
1997
+
1998
+ # make sure we set the correct distribution policy
1999
+ distcols = self.getconfig('gpload:output:match_columns', list)
2000
+
2001
+ # MPP-13399, CR-2227
2002
+ including_defaults = ""
2003
+ if self.getconfig('gpload:output:including_defaults',bool,True):
2004
+ including_defaults = " including defaults"
2005
+
2006
+ sql = "SELECT * FROM pg_class WHERE relname LIKE 'temp_gpload_reusable_%%';"
2007
+ resultList = self.db.query(to_db_text(sql)).getresult()
2008
+ if len(resultList) > 0:
2009
+ self.log(self.WARN, """Old style, reusable tables named "temp_gpload_reusable_*" from a previous versions were found.
2010
+ Greenplum recommends running "DROP TABLE temp_gpload_reusable_..." on each table. This only needs to be done once.""")
2011
+
2012
+ # If the 'reuse tables' option was specified we now try to find an
2013
+ # already existing staging table in the catalog which will match
2014
+ # the one that we need to use. It must meet the reuse conditions
2015
+ is_temp_table = 'TEMP '
2016
+ target_columns = []
2017
+ for column in self.into_columns:
2018
+ if column[2]:
2019
+ target_columns.append([quote_unident(column[0]), column[1]])
2020
+
2021
+ if self.reuse_tables == True:
2022
+ is_temp_table = ''
2023
+ target_table_name = quote_unident(self.table)
2024
+
2025
+ # create a string from all reuse conditions for staging tables and ancode it
2026
+ conditions_str = self.get_staging_conditions_string(target_table_name, target_columns, distcols)
2027
+ encoding_conditions = staging_conditions_hash(conditions_str)
2028
+
2029
+ sql = self.get_reuse_staging_table_query(encoding_conditions)
2030
+ resultList = self.db.query(to_db_text(sql)).getresult()
2031
+
2032
+ if len(resultList) > 0:
2033
+
2034
+ # found a temp table to reuse. no need to create one. we're done here.
2035
+ self.staging_table_name = to_text((resultList[0])[0])
2036
+ self.log(self.INFO, "reusing staging table %s" % self.staging_table_name)
2037
+
2038
+ # truncate it so we don't use old data
2039
+ self.do_truncate(self.staging_table_name)
2040
+
2041
+ return
2042
+
2043
+ # didn't find an existing staging table suitable for reuse. Format a reusable
2044
+ # name and issue a CREATE TABLE on it (without TEMP!). Hopefully we can use it
2045
+ # next time around
2046
+ # we no longer need the timestamp, since we will never want to create few
2047
+ # tables with same encoding_conditions
2048
+ self.staging_table_name = "staging_gpload_reusable_%s" % (encoding_conditions)
2049
+ self.log(self.INFO, "did not find a staging table to reuse. creating %s" % self.staging_table_name)
2050
+
2051
+ # MPP-14667 - self.reuse_tables should change one, and only one, aspect of how we build the following table,
2052
+ # and that is, whether it's a temp table or not. In other words, is_temp_table = '' iff self.reuse_tables == True.
2053
+ sql = 'CREATE %sTABLE %s ' % (is_temp_table, self.staging_table_name)
2054
+ cols = list(map(lambda a:'%s %s' % (a[0], a[1]), target_columns))
2055
+ sql += "(%s)" % ','.join(cols)
2056
+ sql += " DISTRIBUTED BY (%s)" % ', '.join(distcols)
2057
+ self.log(self.LOG, sql)
2058
+
2059
+ if not self.options.D:
2060
+ self.db.query(to_db_text(sql))
2061
+ if not self.reuse_tables:
2062
+ self.cleanupSql.append('DROP TABLE IF EXISTS %s' % self.staging_table_name)
2063
+
2064
+
2065
+ def count_errors(self):
2066
+ notice_processor(self)
2067
+ if self.log_errors and not self.options.D:
2068
+ # make sure we only get errors for our own instance
2069
+ if not self.reuse_tables:
2070
+ queryStr = "select count(*) from gp_read_error_log('%s')" % escape_string(self.extTableName)
2071
+ results = self.db.query(to_db_text(queryStr)).getresult()
2072
+ return (results[0])[0]
2073
+ else: # reuse_tables
2074
+ queryStr = "select cmdtime, count(*) from gp_read_error_log('%s') group by cmdtime order by cmdtime desc limit 1" % escape_string(self.extTableName)
2075
+ results = self.db.query(to_db_text(queryStr)).getresult()
2076
+ if not results:
2077
+ # No formatting errors were recorded for this external
2078
+ # table, so there is nothing to report.
2079
+ return 0
2080
+ self.lastcmdtime = to_text((results[0])[0])
2081
+ global NUM_WARN_ROWS
2082
+ NUM_WARN_ROWS = (results[0])[1]
2083
+ return (results[0])[1];
2084
+ return 0
2085
+
2086
+ def report_errors(self):
2087
+ errors = self.count_errors()
2088
+ global NUM_WARN_ROWS
2089
+ # Without LOG ERRORS gp_read_error_log stays empty; the
2090
+ # "found N data formatting errors" NOTICE is the only signal that
2091
+ # rows were rejected. Use whichever source reports more so rejected
2092
+ # rows are never treated as a clean success.
2093
+ bad_rows = errors if errors else NUM_WARN_ROWS
2094
+ if self.log_errors and errors:
2095
+ NUM_WARN_ROWS = errors
2096
+ if bad_rows == 1:
2097
+ self.log(self.WARN, '1 bad row')
2098
+ elif bad_rows:
2099
+ self.log(self.WARN, '%d bad rows' % bad_rows)
2100
+ if bad_rows:
2101
+ NUM_WARN_ROWS = bad_rows
2102
+
2103
+ # error message is also deleted if external table is dropped.
2104
+ # if reuse_table is set, error message is not deleted.
2105
+ if errors and self.log_errors and self.reuse_tables:
2106
+ self.log(self.WARN, "Please use following query to access the detailed error")
2107
+ self.log(self.WARN, "select * from gp_read_error_log('{0}') where cmdtime = '{1}'".format(escape_string(self.extTableName), self.lastcmdtime))
2108
+ self.exitValue = 1 if bad_rows else 0
2109
+
2110
+
2111
+ def do_insert(self, dest):
2112
+ """
2113
+ Handle the INSERT case
2114
+ """
2115
+ self.log(self.DEBUG, "into columns " + str(self.into_columns))
2116
+ cols = list(filter(lambda a:a[2]!=None, self.into_columns))
2117
+
2118
+ # only insert non-serial columns, unless the user told us to
2119
+ # insert the serials explicitly
2120
+ if not self.from_cols_from_user:
2121
+ cols = list(filter(lambda a:a[3] == False, cols))
2122
+
2123
+ sql = 'INSERT INTO %s' % dest
2124
+ sql += ' (%s)' % ','.join(list(map(lambda a:a[0], cols)))
2125
+ sql += ' SELECT %s' % ','.join(list(map(lambda a:a[2], cols)))
2126
+ sql += ' FROM %s' % self.extSchemaTable
2127
+
2128
+ self.log(self.LOG, sql)
2129
+ if not self.options.D:
2130
+ try:
2131
+ self.rowsInserted = self.db.query(to_db_text(sql))
2132
+ except Exception as e:
2133
+ # We need to be a bit careful about the error since it may contain non-unicode characters
2134
+ global NUM_WARN_ROWS
2135
+ strE = str(e)
2136
+ strF = str(sql)
2137
+
2138
+ # report error file and linenum
2139
+ queryStr = "select filename, linenum from gp_read_error_log('%s')" % escape_string(self.extTableName)
2140
+ results = self.db.query(to_db_text(queryStr)).getresult()
2141
+ if results:
2142
+ NUM_WARN_ROWS = len(results)
2143
+ errfile = error_log_filename((results[0])[0])
2144
+ errlinenum = str((results[0])[1])
2145
+ self.log(self.WARN, "error file: " + errfile)
2146
+ self.log(self.WARN, "error line: " + errlinenum)
2147
+ else:
2148
+ self.log(self.WARN, "no rows found in gp_read_error_log for " + self.extTableName)
2149
+
2150
+ self.log(self.ERROR, strE + ' encountered while running ' + strF)
2151
+
2152
+ self.report_errors()
2153
+
2154
+ def do_method_insert(self):
2155
+ self.create_external_table()
2156
+ self.do_insert(self.get_qualified_tablename())
2157
+
2158
+ def map_stuff(self,config,format,index):
2159
+ lis = []
2160
+ theList = self.getconfig(config,list)
2161
+ theList = convertListToDelimited(theList)
2162
+ for i in theList:
2163
+ if not isinstance(i, str):
2164
+ self.control_file_error("%s must be a YAML sequence of strings"%config)
2165
+ j = self.into_columns_dict.get(i)
2166
+ if not j:
2167
+ self.log(self.ERROR,'column %s in %s does not exist'%(i,config))
2168
+ if not j[index]:
2169
+ self.log(self.ERROR,'there is no mapping from the column %s in %s'%(i,config))
2170
+ lis.append(format(j[0],j[index]))
2171
+ return lis
2172
+
2173
+ def fix_update_cond(self, match):
2174
+ self.log(self.DEBUG, match.group(0))
2175
+ return 'into_table.' + match.group(0)
2176
+
2177
+ def do_update(self,fromname,index):
2178
+ """
2179
+ UPDATE case
2180
+ """
2181
+ sql = 'update %s into_table ' % self.get_qualified_tablename()
2182
+ sql += 'set %s '%','.join(self.map_stuff('gpload:output:update_columns',(lambda x,y:'%s=from_table.%s' % (x, y)),index))
2183
+ sql += 'from %s from_table' % fromname
2184
+
2185
+ match = self.map_stuff('gpload:output:match_columns'
2186
+ , lambda x,y:'into_table.%s=from_table.%s' % (x, y)
2187
+ , index)
2188
+
2189
+ update_condition = self.getconfig('gpload:output:update_condition',
2190
+ str, None)
2191
+ if update_condition:
2192
+ #
2193
+ # Place the table alias infront of column references.
2194
+ #
2195
+ # The following logic is not bullet proof. It may not work
2196
+ # correctly if the user uses an identifier in both its
2197
+ # delimited and un-delimited format (e.g. where c1 < 7 and "c1" > 2)
2198
+ # Better lexing and parsing needs to be done here to fix all cases.
2199
+ #
2200
+ update_condition = ' ' + update_condition + ' '
2201
+ for name, type, mapto, seq in self.into_columns:
2202
+ regexp = r'(?<=[^\w])%s(?=[^\w])' % name
2203
+ self.log(self.DEBUG, 'update_condition re: ' + regexp)
2204
+ temp_update_condition = update_condition
2205
+ updateConditionList = splitIntoLiteralsAndNonLiterals(update_condition)
2206
+ skip = False
2207
+ newUpdateConditionList = []
2208
+ update_condition = ''
2209
+ for uc in updateConditionList:
2210
+ if skip == False:
2211
+ uc = re.sub(regexp, self.fix_update_cond, uc)
2212
+ skip = True
2213
+ update_condition = update_condition + uc
2214
+ if update_condition == temp_update_condition:
2215
+ # see if column can be undelimited, and try again.
2216
+ if len(name) > 2 and name[1:-1] == name[1:-1].lower():
2217
+ regexp = r'(?<=[^\w])%s(?=[^\w])' % name[1:-1]
2218
+ self.log(self.DEBUG, 'update_condition undelimited re: ' + regexp)
2219
+ update_condition = re.sub( regexp
2220
+ , self.fix_update_cond
2221
+ , update_condition
2222
+ )
2223
+ self.log(self.DEBUG, "updated update_condition to %s" %
2224
+ update_condition)
2225
+ match.append(update_condition)
2226
+ sql += ' where %s' % ' and '.join(match)
2227
+ self.log(self.LOG, sql)
2228
+ if not self.options.D:
2229
+ try:
2230
+ self.rowsUpdated = self.db.query(to_db_text(sql))
2231
+ except Exception as e:
2232
+ # We need to be a bit careful about the error since it may contain non-unicode characters
2233
+ strE = str(e)
2234
+ strF = str(sql)
2235
+ self.log(self.ERROR, strE + ' encountered while running ' + strF)
2236
+
2237
+ def get_qualified_tablename(self):
2238
+
2239
+ tblname = "%s.%s" % (self.schema, self.table)
2240
+ return tblname
2241
+
2242
+ def get_table_dist_key(self):
2243
+
2244
+ # NOTE: this query should be re-written better. the problem is that it is
2245
+ # not possible to perform a cast on a table name with spaces...
2246
+ # Older Greenplum versions expose the distribution key as
2247
+ # gp_distribution_policy.attrnums, while GP 6.25+ renamed it to
2248
+ # gp_distribution_policy.distkey. Detect the column name first so the
2249
+ # transaction is never aborted by a failed catalog query.
2250
+ distkey_col_sql = "select attname from pg_catalog.pg_attribute " + \
2251
+ "where attrelid = 'pg_catalog.gp_distribution_policy'::regclass " + \
2252
+ "and attname in ('distkey', 'attrnums') " + \
2253
+ "order by attname desc limit 1; "
2254
+ distkey_cols = self.db.query(to_db_text(distkey_col_sql)).getresult()
2255
+ distkey_col = 'attrnums'
2256
+ if len(distkey_cols) > 0:
2257
+ distkey_col = to_text((distkey_cols[0])[0])
2258
+ if distkey_col not in ('distkey', 'attrnums'):
2259
+ distkey_col = 'attrnums'
2260
+
2261
+ sql = "select attname from pg_attribute a, gp_distribution_policy p , pg_class c, pg_namespace n "+\
2262
+ "where a.attrelid = c.oid and " + \
2263
+ "a.attrelid = p.localoid and " + \
2264
+ "a.attnum = any (p.%s) and " % distkey_col + \
2265
+ "c.relnamespace = n.oid and " + \
2266
+ "n.nspname = '%s' and c.relname = '%s'; " % (quote_unident(self.schema), quote_unident(self.table))
2267
+
2268
+
2269
+ resultList = self.db.query(to_db_text(sql)).getresult()
2270
+ attrs = []
2271
+ count = 0
2272
+ while count < len(resultList):
2273
+ attrs.append(to_text((resultList[count])[0]))
2274
+ count = count + 1
2275
+
2276
+ return attrs
2277
+
2278
+ def table_supports_update(self):
2279
+ """Columns being updated cannot appear in the distribution key."""
2280
+ distKeyList = self.get_table_dist_key()
2281
+ distkey = set()
2282
+ for dk in distKeyList:
2283
+ distkey.add(quote_ident(dk))
2284
+
2285
+ self.distkey = distkey
2286
+ if len(distkey) != 0:
2287
+ # not randomly distributed - check that UPDATE_COLUMNS isn't part of the distribution key
2288
+ updateColumnList = self.getconfig('gpload:output:update_columns',
2289
+ list,
2290
+ returnOriginal=True)
2291
+ update_columns = convertListToDelimited(updateColumnList)
2292
+ update_columns = set(update_columns)
2293
+ a = distkey.intersection(update_columns)
2294
+ if len(a):
2295
+ self.control_file_error('update_columns cannot reference column(s) in distribution key (%s)' % ', '.join(list(distkey)))
2296
+
2297
+ def do_method_update(self):
2298
+ """Load the data in and update an existing table based upon it"""
2299
+
2300
+ self.table_supports_update()
2301
+ self.create_staging_table()
2302
+
2303
+ self.create_external_table()
2304
+ self.do_insert(self.staging_table_name)
2305
+ # These rows are inserted temporarily for processing, so set inserted rows back to zero.
2306
+ self.rowsInserted = 0
2307
+ self.do_update(self.staging_table_name, 0)
2308
+
2309
+ def do_method_merge(self):
2310
+ """insert data not already in the table, update remaining items"""
2311
+
2312
+ self.table_supports_update()
2313
+ self.create_staging_table()
2314
+ self.create_external_table()
2315
+ self.do_insert(self.staging_table_name)
2316
+ self.rowsInserted = 0 # MPP-13024. No rows inserted yet (only to temp table).
2317
+ self.do_update(self.staging_table_name, 0)
2318
+
2319
+ # insert new rows to the target table
2320
+ match = self.map_stuff('gpload:output:match_columns',lambda x,y:'into_table.%s=from_table.%s'%(x,y),0)
2321
+ matchColumns = self.getconfig('gpload:output:match_columns',list)
2322
+
2323
+ cols = list(filter(lambda a:a[2] != None, self.into_columns))
2324
+ sql = 'INSERT INTO %s ' % self.get_qualified_tablename()
2325
+ sql += '(%s) ' % ','.join(list(map(lambda a:a[0], cols)))
2326
+ sql += '(SELECT %s ' % ','.join(list(map(lambda a:'from_table.%s' % a[0], cols)))
2327
+ sql += 'FROM (SELECT *, row_number() OVER (PARTITION BY %s) AS gpload_row_number ' % ','.join(matchColumns)
2328
+ sql += 'FROM %s) AS from_table ' % self.staging_table_name
2329
+ sql += 'LEFT OUTER JOIN %s into_table ' % self.get_qualified_tablename()
2330
+ sql += 'ON %s '%' AND '.join(match)
2331
+ where = self.map_stuff('gpload:output:match_columns',lambda x,y:'into_table.%s IS NULL'%x,0)
2332
+ sql += 'WHERE %s ' % ' AND '.join(where)
2333
+ sql += 'AND gpload_row_number=1)'
2334
+
2335
+ self.log(self.LOG, sql)
2336
+ if not self.options.D:
2337
+ try:
2338
+ self.rowsInserted = self.db.query(to_db_text(sql))
2339
+ except Exception as e:
2340
+ # We need to be a bit careful about the error since it may contain non-unicode characters
2341
+ strE = str(e)
2342
+ strF = str(sql)
2343
+ self.log(self.ERROR, strE + ' encountered while running ' + strF)
2344
+
2345
+
2346
+ def do_truncate(self, tblname):
2347
+ self.log(self.LOG, "Truncate table %s" %(tblname))
2348
+ if not self.options.D:
2349
+ try:
2350
+ truncateSQLtext = "truncate %s" % tblname
2351
+ self.db.query(to_db_text(truncateSQLtext))
2352
+ except Exception as e:
2353
+ self.log(self.ERROR, 'could not execute truncate target %s: %s' % (tblname, str(e)))
2354
+
2355
+ def do_method(self):
2356
+ # Is the table to be truncated before the load?
2357
+ preload = self.getconfig('gpload:preload', list, default=None)
2358
+ method = self.getconfig('gpload:output:mode', str, 'insert').lower()
2359
+ self.log_errors = self.getconfig('gpload:input:log_errors', bool, False)
2360
+ truncate = False
2361
+ self.reuse_tables = False
2362
+
2363
+ if not self.options.no_auto_trans and not method=='insert':
2364
+ self.db.query("BEGIN")
2365
+
2366
+ if preload:
2367
+ truncate = self.getconfig('gpload:preload:truncate',bool,False)
2368
+ self.reuse_tables = self.getconfig('gpload:preload:reuse_tables',bool,False)
2369
+ if truncate == True:
2370
+ if method=='insert':
2371
+ self.do_truncate(self.schemaTable)
2372
+ else:
2373
+ self.log(self.ERROR, 'preload truncate operation should be used with insert ' +
2374
+ 'operation only. used with %s' % method)
2375
+
2376
+ # sql pre or post processing?
2377
+ sql = self.getconfig('gpload:sql', list, default=None)
2378
+ before = None
2379
+ after = None
2380
+ if sql:
2381
+ before = self.getconfig('gpload:sql:before', str, default=None)
2382
+ after = self.getconfig('gpload:sql:after', str, default=None)
2383
+ if before:
2384
+ self.log(self.LOG, "Pre-SQL from user: %s" % before)
2385
+ if not self.options.D:
2386
+ try:
2387
+ self.db.query(to_db_text(before))
2388
+ except Exception as e:
2389
+ self.log(self.ERROR, 'could not execute SQL in sql:before "%s": %s' %
2390
+ (before, str(e)))
2391
+
2392
+
2393
+ if method=='insert':
2394
+ self.do_method_insert()
2395
+ elif method=='update':
2396
+ self.do_method_update()
2397
+ elif method=='merge':
2398
+ self.do_method_merge()
2399
+ else:
2400
+ self.control_file_error('unsupported method %s' % method)
2401
+
2402
+ # truncate the staging table to avoid dumping it's content - see MPP-15474
2403
+ if method=='merge' or method=='update':
2404
+ self.do_truncate(self.staging_table_name)
2405
+
2406
+ if after:
2407
+ self.log(self.LOG, "Post-SQL from user: %s" % after)
2408
+ if not self.options.D:
2409
+ try:
2410
+ self.db.query(to_db_text(after))
2411
+ except Exception as e:
2412
+ self.log(self.ERROR, 'could not execute SQL in sql:after "%s": %s' %
2413
+ (after, str(e)))
2414
+
2415
+ if not self.options.no_auto_trans and not method=='insert':
2416
+ self.db.query("COMMIT")
2417
+
2418
+
2419
+ def run2(self):
2420
+ self.log(self.DEBUG, 'config ' + str(self.config))
2421
+ start = time.time()
2422
+ self.read_config()
2423
+ self.setup_connection()
2424
+ self.read_table_metadata()
2425
+ self.read_columns()
2426
+ self.read_mapping()
2427
+ self.start_gpfdists()
2428
+ self.do_method()
2429
+ self.log(self.INFO, 'running time: %.2f seconds'%(time.time()-start))
2430
+
2431
+ def run(self):
2432
+ self.db = None
2433
+ self.rowsInserted = 0
2434
+ self.rowsUpdated = 0
2435
+ signal.signal(signal.SIGINT, handle_kill)
2436
+ signal.signal(signal.SIGTERM, handle_kill)
2437
+ # win32 doesn't do SIGQUIT
2438
+ if not platform.system() in ['Windows', 'Microsoft']:
2439
+ signal.signal(signal.SIGQUIT, handle_kill)
2440
+ signal.signal(signal.SIGHUP, signal.SIG_IGN)
2441
+
2442
+ try:
2443
+ try:
2444
+ self.run2()
2445
+ except Exception:
2446
+ traceback.print_exc(file=self.logfile)
2447
+ self.logfile.flush()
2448
+ self.exitValue = 2
2449
+ if (self.options.qv > self.INFO):
2450
+ traceback.print_exc()
2451
+ else:
2452
+ self.log(self.ERROR, "unexpected error -- backtrace " +
2453
+ "written to log file")
2454
+ finally:
2455
+ if self.cleanupSql:
2456
+ self.log(self.LOG, 'removing temporary data')
2457
+ self.setup_connection()
2458
+ for a in self.cleanupSql:
2459
+ try:
2460
+ self.log(self.DEBUG, a)
2461
+ self.db.query(to_db_text(a))
2462
+ except Exception:
2463
+ traceback.print_exc(file=self.logfile)
2464
+ self.logfile.flush()
2465
+ traceback.print_exc()
2466
+ if self.subprocesses:
2467
+ self.log(self.LOG, 'killing gpfdist')
2468
+ for a in self.subprocesses:
2469
+ try:
2470
+ if platform.system() in ['Windows', 'Microsoft']:
2471
+ # win32 API is better but hard for us
2472
+ # to install, so we use the crude method
2473
+ subprocess.Popen("taskkill /F /T /PID %i" % a.pid,
2474
+ shell=True, stdout=subprocess.PIPE,
2475
+ stderr=subprocess.PIPE)
2476
+
2477
+ else:
2478
+ os.kill(a.pid, signal.SIGTERM)
2479
+ except OSError:
2480
+ pass
2481
+ for t in self.threads:
2482
+ t.join()
2483
+
2484
+ self.log(self.INFO, 'rows Inserted = ' + str(self.rowsInserted))
2485
+ self.log(self.INFO, 'rows Updated = ' + str(self.rowsUpdated))
2486
+ self.log(self.INFO, 'data formatting errors = ' + str(NUM_WARN_ROWS))
2487
+ if self.exitValue==0:
2488
+ self.log(self.INFO, 'gpload succeeded')
2489
+ elif self.exitValue==1:
2490
+ self.log(self.INFO, 'gpload succeeded with warnings')
2491
+ else:
2492
+ self.log(self.INFO, 'gpload failed')
2493
+
2494
+ ## MPP-19015 - Extra python thread shutdown time is needed on HP-UX
2495
+ if platform.uname()[0] == 'HP-UX':
2496
+ time.sleep(1)
2497
+
2498
+
2499
+ # ---------------------------------------------------------------------------
2500
+ # Programmatic interface
2501
+ # ---------------------------------------------------------------------------
2502
+
2503
+ class GploadError(RuntimeError):
2504
+ """Raised by :meth:`GploadResult.raise_for_status` on a failed load."""
2505
+
2506
+ def __init__(self, result):
2507
+ super().__init__('gpload failed with exit code %d' % result.returncode)
2508
+ self.result = result
2509
+
2510
+
2511
+ class GploadResult:
2512
+ """Outcome of one gpload run.
2513
+
2514
+ ``returncode`` follows the gpload/load.sh convention: 0 - success,
2515
+ 1 - loaded with rejected rows, 2 - failure, 124 - API timeout.
2516
+ """
2517
+
2518
+ def __init__(self, returncode, stdout='', stderr='', log_file=None,
2519
+ command=None):
2520
+ self.returncode = returncode
2521
+ self.stdout = stdout
2522
+ self.stderr = stderr
2523
+ self.log_file = log_file
2524
+ self.command = list(command) if command else []
2525
+
2526
+ @property
2527
+ def ok(self):
2528
+ return self.returncode == 0
2529
+
2530
+ @property
2531
+ def has_warnings(self):
2532
+ return self.returncode == 1
2533
+
2534
+ @property
2535
+ def failed(self):
2536
+ return self.returncode >= 2
2537
+
2538
+ def raise_for_status(self):
2539
+ """Raise :class:`GploadError` unless the load succeeded."""
2540
+ if self.returncode != 0:
2541
+ raise GploadError(self)
2542
+ return self
2543
+
2544
+ def __repr__(self):
2545
+ return 'GploadResult(returncode=%d, log_file=%r)' % (
2546
+ self.returncode, self.log_file)
2547
+
2548
+
2549
+ class Gpload:
2550
+ """Programmatic interface to gpload.
2551
+
2552
+ The load runs in a child process: gpload calls ``sys.exit()`` on errors
2553
+ and replaces the process signal handlers, so calling it in-process would
2554
+ affect the host application. A load that exceeds ``timeout`` is killed
2555
+ together with its gpfdist children (exit code 124).
2556
+
2557
+ Example::
2558
+
2559
+ gpload = Gpload()
2560
+ result = gpload.load('/path/config.yaml', log='/var/log/gpload.log')
2561
+ if result.has_warnings:
2562
+ ...
2563
+ result.raise_for_status()
2564
+ """
2565
+
2566
+ def __init__(self, script=None, python=None, timeout=600):
2567
+ script = script or globals().get('__file__')
2568
+ if not script:
2569
+ raise ValueError('cannot determine gpload.py path; pass script=')
2570
+ self.script = os.path.abspath(script)
2571
+ self.python = python or sys.executable
2572
+ self.timeout = timeout
2573
+
2574
+ def build_command(self, config, log=None, host=None, port=None,
2575
+ user=None, database=None, dry_run=False, quiet=False,
2576
+ verbose=False, very_verbose=False, no_auto_trans=False,
2577
+ gpfdist_timeout=None, extra_args=None):
2578
+ """Build the gpload command line without running it."""
2579
+ cmd = [self.python, self.script, '-f', os.fspath(config)]
2580
+ if host is not None:
2581
+ cmd += ['-h', str(host)]
2582
+ if port is not None:
2583
+ cmd += ['-p', str(port)]
2584
+ if user is not None:
2585
+ cmd += ['-U', str(user)]
2586
+ if database is not None:
2587
+ cmd += ['-d', str(database)]
2588
+ if log is not None:
2589
+ cmd += ['-l', os.fspath(log)]
2590
+ if dry_run:
2591
+ cmd.append('-D')
2592
+ if quiet:
2593
+ cmd.append('-q')
2594
+ if verbose:
2595
+ cmd.append('-v')
2596
+ if very_verbose:
2597
+ cmd.append('-V')
2598
+ if no_auto_trans:
2599
+ cmd.append('--no_auto_trans')
2600
+ if gpfdist_timeout is not None:
2601
+ cmd += ['--gpfdist_timeout', str(gpfdist_timeout)]
2602
+ if extra_args:
2603
+ cmd += [str(a) for a in extra_args]
2604
+ return cmd
2605
+
2606
+ @staticmethod
2607
+ def _kill(proc):
2608
+ """Kill the child and its gpfdist children."""
2609
+ if os.name == 'posix':
2610
+ try:
2611
+ os.killpg(proc.pid, signal.SIGKILL)
2612
+ except OSError:
2613
+ pass
2614
+ else:
2615
+ proc.kill()
2616
+
2617
+ def load(self, config, timeout=None, env=None, cwd=None, **options):
2618
+ """Run gpload for ``config`` and return a :class:`GploadResult`.
2619
+
2620
+ ``options`` are the keyword arguments of :meth:`build_command`.
2621
+ ``env`` entries are merged into the inherited environment.
2622
+ """
2623
+ cmd = self.build_command(config, **options)
2624
+ log_file = options.get('log')
2625
+ run_env = os.environ.copy()
2626
+ if env:
2627
+ run_env.update(env)
2628
+ run_timeout = self.timeout if timeout is None else timeout
2629
+
2630
+ popen_kwargs = {}
2631
+ if os.name == 'posix':
2632
+ # Own process group: on timeout gpload and its gpfdist children
2633
+ # are killed together.
2634
+ popen_kwargs['start_new_session'] = True
2635
+
2636
+ proc = subprocess.Popen(
2637
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
2638
+ text=True, encoding='utf-8', errors='replace',
2639
+ env=run_env, cwd=os.fspath(cwd) if cwd else None,
2640
+ **popen_kwargs)
2641
+ try:
2642
+ stdout, stderr = proc.communicate(timeout=run_timeout)
2643
+ except subprocess.TimeoutExpired:
2644
+ self._kill(proc)
2645
+ stdout, stderr = proc.communicate()
2646
+ return GploadResult(124, stdout or '', stderr or '', log_file, cmd)
2647
+ except BaseException:
2648
+ # Ctrl-C in the caller must not leave gpload/gpfdist running:
2649
+ # the child is in its own process group and does not get SIGINT.
2650
+ self._kill(proc)
2651
+ proc.communicate()
2652
+ raise
2653
+
2654
+ return GploadResult(proc.returncode, stdout or '', stderr or '',
2655
+ log_file, cmd)
2656
+
2657
+
2658
+ def run_gpload(config, **kwargs):
2659
+ """Shortcut for ``Gpload().load(config, **kwargs)``."""
2660
+ return Gpload().load(config, **kwargs)
2661
+
2662
+
2663
+ class GploadTableResult:
2664
+ """Outcome of loading one ``<data_dir>/<database>/<table>`` directory.
2665
+
2666
+ ``returncode`` uses the load.sh mapping: 0 - success, 1 - loaded with
2667
+ rejected rows, 2 - failure (including a timeout).
2668
+ """
2669
+
2670
+ def __init__(self, database, table, files=(), returncode=0,
2671
+ gpload_result=None, loaded_files=(), failed_files=(),
2672
+ error_file=None, error_line=None, message=None,
2673
+ config_path=None, config_text=None):
2674
+ self.database = database
2675
+ self.table = table
2676
+ self.files = [os.fspath(path) for path in files]
2677
+ self.returncode = returncode
2678
+ self.gpload_result = gpload_result
2679
+ self.loaded_files = [os.fspath(path) for path in loaded_files]
2680
+ self.failed_files = [os.fspath(path) for path in failed_files]
2681
+ self.error_file = error_file
2682
+ self.error_line = error_line
2683
+ self.message = message
2684
+ self.config_path = config_path
2685
+ self.config_text = config_text
2686
+
2687
+ @property
2688
+ def source_dir(self):
2689
+ if not self.files:
2690
+ return None
2691
+ return os.path.dirname(self.files[0])
2692
+
2693
+ @property
2694
+ def ok(self):
2695
+ return self.returncode == 0
2696
+
2697
+ @property
2698
+ def has_warnings(self):
2699
+ return self.returncode == 1
2700
+
2701
+ @property
2702
+ def failed(self):
2703
+ return self.returncode >= 2
2704
+
2705
+ def __repr__(self):
2706
+ return 'GploadTableResult(database=%r, table=%r, returncode=%d)' % (
2707
+ self.database, self.table, self.returncode)
2708
+
2709
+
2710
+ class GploadBatchResult:
2711
+ """Aggregated outcome of :meth:`GploadBatch.load`."""
2712
+
2713
+ def __init__(self, tables=()):
2714
+ self.tables = list(tables)
2715
+
2716
+ @property
2717
+ def returncode(self):
2718
+ code = 0
2719
+ for table in self.tables:
2720
+ if table.returncode >= 2:
2721
+ return 2
2722
+ if table.returncode == 1:
2723
+ code = 1
2724
+ return code
2725
+
2726
+ @property
2727
+ def ok(self):
2728
+ return self.returncode == 0
2729
+
2730
+ @property
2731
+ def has_warnings(self):
2732
+ return self.returncode == 1
2733
+
2734
+ @property
2735
+ def failed(self):
2736
+ return self.returncode >= 2
2737
+
2738
+ @property
2739
+ def loaded_tables(self):
2740
+ return [table for table in self.tables if table.ok]
2741
+
2742
+ @property
2743
+ def warning_tables(self):
2744
+ return [table for table in self.tables if table.has_warnings]
2745
+
2746
+ @property
2747
+ def failed_tables(self):
2748
+ return [table for table in self.tables if table.failed]
2749
+
2750
+ def raise_for_status(self):
2751
+ """Raise :class:`GploadError` unless every table loaded cleanly."""
2752
+ if self.returncode != 0:
2753
+ raise GploadError(self)
2754
+ return self
2755
+
2756
+ def __iter__(self):
2757
+ return iter(self.tables)
2758
+
2759
+ def __len__(self):
2760
+ return len(self.tables)
2761
+
2762
+ def __repr__(self):
2763
+ return 'GploadBatchResult(tables=%d, returncode=%d)' % (
2764
+ len(self.tables), self.returncode)
2765
+
2766
+
2767
+ def default_config_template(script=None):
2768
+ """Return the default ``gpload.yaml.ori`` path.
2769
+
2770
+ The template is looked up next to ``gpload.py`` (source checkout) and in
2771
+ the ``share/gpload-py3`` directory used by an installed package.
2772
+ """
2773
+ script = script or globals().get('__file__')
2774
+ candidates = []
2775
+ if script:
2776
+ candidates.append(os.path.join(
2777
+ os.path.dirname(os.path.abspath(script)), 'gpload.yaml.ori'))
2778
+
2779
+ try:
2780
+ candidates.append(os.path.join(
2781
+ sysconfig.get_path('data'), 'share', 'gpload-py3',
2782
+ 'gpload.yaml.ori'))
2783
+ except (KeyError, TypeError):
2784
+ pass
2785
+
2786
+ try:
2787
+ candidates.append(os.path.join(
2788
+ site.USER_BASE, 'share', 'gpload-py3', 'gpload.yaml.ori'))
2789
+ except (AttributeError, TypeError):
2790
+ pass
2791
+
2792
+ for candidate in candidates:
2793
+ if os.path.isfile(candidate):
2794
+ return candidate
2795
+ return candidates[0] if candidates else 'gpload.yaml.ori'
2796
+
2797
+
2798
+ class GploadBatch:
2799
+ """Batch loader for a ``<data_dir>/<database>/<table>/*.csv`` tree.
2800
+
2801
+ This is the programmatic equivalent of ``script/load.sh``: for every
2802
+ table directory it renders a control file from ``gpload.yaml.ori`` (using
2803
+ the CSV header for ``HEADER``/``COLUMNS``), loads all CSV files of that
2804
+ directory in one gpload run, removes successfully loaded files and moves
2805
+ the files of a failed load to ``failed/``.
2806
+
2807
+ Each table is loaded in a separate child process, so no database driver
2808
+ is needed in the calling process.
2809
+
2810
+ Example::
2811
+
2812
+ from gpload import GploadBatch
2813
+
2814
+ batch = GploadBatch(
2815
+ '/opt/gpstall/data', # <db>/<table>/*.csv
2816
+ log_dir='/var/log/gpstall', # default: <data_dir>/../log
2817
+ user='gpadmin',
2818
+ password='secret',
2819
+ host='127.0.0.1',
2820
+ port=5432,
2821
+ )
2822
+ result = batch.load()
2823
+ print(result.returncode, len(result.tables))
2824
+ """
2825
+
2826
+ def __init__(self, data_dir, log_dir=None, config_template=None, *,
2827
+ user=None, password=None, host=None, port=None,
2828
+ gpfdist_host=None, port_range='[8081,8082]',
2829
+ error_limit=50000, script=None, python=None, timeout=600,
2830
+ file_pattern='*.csv', keep_temp_configs=False):
2831
+ self.data_dir = os.path.abspath(os.fspath(data_dir))
2832
+ if log_dir is None:
2833
+ log_dir = os.path.join(os.path.dirname(self.data_dir), 'log')
2834
+ self.log_dir = os.path.abspath(os.fspath(log_dir))
2835
+
2836
+ script = script or globals().get('__file__')
2837
+ if not script:
2838
+ raise ValueError('cannot determine gpload.py path; pass script=')
2839
+ self.script = os.path.abspath(script)
2840
+ self.python = python or sys.executable
2841
+
2842
+ if config_template is None:
2843
+ config_template = default_config_template(self.script)
2844
+ self.config_template = os.path.abspath(os.fspath(config_template))
2845
+
2846
+ self.user = user if user is not None else (
2847
+ os.environ.get('PGUSER') or getpass.getuser())
2848
+ self.password = password if password is not None else (
2849
+ os.environ.get('PGPASSWORD'))
2850
+ self.host = host if host is not None else (
2851
+ os.environ.get('PGHOST') or '127.0.0.1')
2852
+ self.port = port if port is not None else (
2853
+ os.environ.get('PGPORT') or 5432)
2854
+ self.gpfdist_host = gpfdist_host if gpfdist_host is not None else (
2855
+ self.host)
2856
+ self.port_range = port_range
2857
+ self.error_limit = error_limit
2858
+
2859
+ self.timeout = timeout
2860
+ self.file_pattern = file_pattern
2861
+ self.keep_temp_configs = keep_temp_configs
2862
+
2863
+ self.log_file = os.path.join(self.log_dir, 'gpload.log')
2864
+ self.latest_failed_file = os.path.join(
2865
+ self.log_dir, 'latest_failed_file')
2866
+
2867
+ self.loader = Gpload(
2868
+ script=self.script, python=self.python, timeout=timeout)
2869
+
2870
+ # -- discovery ---------------------------------------------------------
2871
+
2872
+ @staticmethod
2873
+ def _subdirectories(path):
2874
+ try:
2875
+ with os.scandir(path) as entries:
2876
+ entries = sorted(entries, key=lambda entry: entry.name)
2877
+ except OSError:
2878
+ return []
2879
+
2880
+ names = []
2881
+ for entry in entries:
2882
+ if entry.name.startswith('.'):
2883
+ continue
2884
+ try:
2885
+ if entry.is_dir():
2886
+ names.append(entry.name)
2887
+ except OSError:
2888
+ pass
2889
+ return names
2890
+
2891
+ def table_files(self, table_dir):
2892
+ """Return the CSV files of one table directory, sorted."""
2893
+ files = []
2894
+ try:
2895
+ with os.scandir(table_dir) as entries:
2896
+ entries = list(entries)
2897
+ except OSError:
2898
+ return files
2899
+
2900
+ for entry in entries:
2901
+ try:
2902
+ is_file = entry.is_file()
2903
+ except OSError:
2904
+ is_file = False
2905
+ if is_file and fnmatch.fnmatch(entry.name, self.file_pattern):
2906
+ files.append(os.path.join(table_dir, entry.name))
2907
+ return sorted(files)
2908
+
2909
+ def scan(self):
2910
+ """Return ``[(database, table, [csv_file, ...]), ...]``.
2911
+
2912
+ Only tables that contain at least one matching file are returned.
2913
+ """
2914
+ if not os.path.isdir(self.data_dir):
2915
+ raise FileNotFoundError(
2916
+ 'data directory not found: %s' % self.data_dir)
2917
+
2918
+ tables = []
2919
+ for database in self._subdirectories(self.data_dir):
2920
+ database_dir = os.path.join(self.data_dir, database)
2921
+ for table in self._subdirectories(database_dir):
2922
+ table_dir = os.path.join(database_dir, table)
2923
+ files = self.table_files(table_dir)
2924
+ if files:
2925
+ tables.append((database, table, files))
2926
+ return tables
2927
+
2928
+ # -- control file rendering -------------------------------------------
2929
+
2930
+ @staticmethod
2931
+ def _read_header(path):
2932
+ with open(path, 'r', encoding='utf-8-sig', newline='') as handle:
2933
+ try:
2934
+ row = next(csv.reader(handle))
2935
+ except (StopIteration, csv.Error):
2936
+ return []
2937
+ return [column.strip() for column in row if column.strip()]
2938
+
2939
+ def render_config(self, database, table, files):
2940
+ """Render the control file template for one table."""
2941
+ files = list(files)
2942
+ if not files:
2943
+ raise ValueError('no CSV files for %s.%s' % (database, table))
2944
+
2945
+ with open(self.config_template, 'r', encoding='utf-8') as handle:
2946
+ template = handle.read()
2947
+
2948
+ columns = self._read_header(files[0])
2949
+ if not columns:
2950
+ raise ValueError(
2951
+ 'empty CSV header in %s' % files[0])
2952
+
2953
+ table_dir = os.path.dirname(os.path.abspath(files[0]))
2954
+ file_block = '\n - %s' % os.path.join(
2955
+ table_dir, self.file_pattern)
2956
+
2957
+ header_block = '- HEADER: true\n - COLUMNS:'
2958
+ for column in columns:
2959
+ header_block += '\n - %s:' % column
2960
+
2961
+ values = {
2962
+ 'DATABASE': database,
2963
+ 'TABLE': table,
2964
+ 'FILE': file_block,
2965
+ 'HEADER': header_block,
2966
+ 'GPUSER': self.user,
2967
+ 'GPHOST': self.host,
2968
+ 'GPPORT': self.port,
2969
+ 'GPDHOST': self.gpfdist_host,
2970
+ 'GPDPORT_RANGE': self.port_range,
2971
+ 'ERROR_LIMIT': self.error_limit,
2972
+ }
2973
+ pattern = re.compile(r'\{(' + '|'.join(values) + r')\}')
2974
+ return pattern.sub(
2975
+ lambda match: str(values[match.group(1)]), template)
2976
+
2977
+ # -- loading -----------------------------------------------------------
2978
+
2979
+ @staticmethod
2980
+ def _map_returncode(returncode):
2981
+ if returncode == 0:
2982
+ return 0
2983
+ if returncode == 1:
2984
+ return 1
2985
+ return 2
2986
+
2987
+ def _environment(self):
2988
+ env = os.environ.copy()
2989
+ if self.password is not None:
2990
+ env['PGPASSWORD'] = self.password
2991
+
2992
+ # load.sh removes $GPHOME/* from LD_LIBRARY_PATH: the system psycopg2
2993
+ # is linked against the OS libpq and cannot load the older Greenplum
2994
+ # one. gpfdist is started by gpload.py in its own shell, which
2995
+ # sources the Greenplum environment again.
2996
+ gphome = env.get('GPHOME')
2997
+ ld_path = env.get('LD_LIBRARY_PATH')
2998
+ if gphome and ld_path:
2999
+ prefix = gphome.rstrip(os.sep) + os.sep
3000
+ kept = [path for path in ld_path.split(os.pathsep)
3001
+ if path and not path.startswith(prefix)]
3002
+ if kept:
3003
+ env['LD_LIBRARY_PATH'] = os.pathsep.join(kept)
3004
+ else:
3005
+ env.pop('LD_LIBRARY_PATH', None)
3006
+ return env
3007
+
3008
+ def _append_log_separator(self):
3009
+ with open(self.log_file, 'a', encoding='utf-8') as handle:
3010
+ handle.write('------------------------------\n')
3011
+ handle.flush()
3012
+ return handle.tell()
3013
+
3014
+ def _read_log_since(self, offset):
3015
+ try:
3016
+ with open(self.log_file, 'r', encoding='utf-8',
3017
+ errors='replace') as handle:
3018
+ handle.seek(offset)
3019
+ return handle.read()
3020
+ except OSError:
3021
+ return ''
3022
+
3023
+ @staticmethod
3024
+ def _parse_error_location(text):
3025
+ error_file = None
3026
+ error_line = None
3027
+ for line in (text or '').splitlines()[-50:]:
3028
+ if 'error file:' in line:
3029
+ error_file = line.split('error file:', 1)[1].strip()
3030
+ if 'error line:' in line:
3031
+ error_line = line.split('error line:', 1)[1].strip()
3032
+ return error_file or None, error_line or None
3033
+
3034
+ @staticmethod
3035
+ def _remove_loaded(files):
3036
+ removed = []
3037
+ errors = []
3038
+ for path in files:
3039
+ try:
3040
+ os.unlink(path)
3041
+ removed.append(path)
3042
+ except OSError as exc:
3043
+ errors.append('%s: %s' % (path, exc))
3044
+ message = None
3045
+ if errors:
3046
+ message = 'could not remove loaded files: ' + '; '.join(errors)
3047
+ return removed, message
3048
+
3049
+ @staticmethod
3050
+ def _move_to_failed(files, failed_dir):
3051
+ os.makedirs(failed_dir, exist_ok=True)
3052
+ moved = []
3053
+ for path in files:
3054
+ if not os.path.exists(path):
3055
+ continue
3056
+ destination = os.path.join(
3057
+ failed_dir, os.path.basename(path))
3058
+ try:
3059
+ os.replace(path, destination)
3060
+ moved.append(destination)
3061
+ except OSError:
3062
+ pass
3063
+ return moved
3064
+
3065
+ def _quarantine(self, table_dir, files, error_file, error_line):
3066
+ failed_dir = os.path.join(table_dir, 'failed')
3067
+ os.makedirs(failed_dir, exist_ok=True)
3068
+
3069
+ try:
3070
+ os.unlink(self.latest_failed_file)
3071
+ except OSError:
3072
+ pass
3073
+
3074
+ target = None
3075
+ if error_file:
3076
+ candidate = error_file if os.path.isabs(error_file) else \
3077
+ os.path.join(table_dir, error_file)
3078
+ candidate = os.path.abspath(candidate)
3079
+ # Only quarantine a file that really belongs to this table.
3080
+ if os.path.dirname(candidate) == os.path.abspath(table_dir):
3081
+ target = candidate
3082
+
3083
+ if error_line and target and os.path.isfile(target):
3084
+ destination = os.path.join(
3085
+ failed_dir,
3086
+ os.path.basename(target) + '_' + error_line)
3087
+ try:
3088
+ os.replace(target, destination)
3089
+ moved = [destination]
3090
+ entries = [destination]
3091
+ except OSError:
3092
+ moved = self._move_to_failed(files, failed_dir)
3093
+ entries = moved
3094
+ else:
3095
+ moved = self._move_to_failed(files, failed_dir)
3096
+ try:
3097
+ entries = sorted(
3098
+ os.path.join(failed_dir, name)
3099
+ for name in os.listdir(failed_dir))
3100
+ except OSError:
3101
+ entries = []
3102
+
3103
+ with open(self.latest_failed_file, 'w', encoding='utf-8') as handle:
3104
+ for entry in entries:
3105
+ handle.write(entry + '\n')
3106
+ handle.flush()
3107
+ return moved
3108
+
3109
+ def load_table(self, database, table, files=None):
3110
+ """Load all CSV files of one ``<database>/<table>`` directory."""
3111
+ table_dir = os.path.join(self.data_dir, database, table)
3112
+ if files is None:
3113
+ files = self.table_files(table_dir)
3114
+ files = [os.path.abspath(os.fspath(path)) for path in files]
3115
+ if not files:
3116
+ return GploadTableResult(database, table)
3117
+ os.makedirs(self.log_dir, exist_ok=True)
3118
+
3119
+ try:
3120
+ config_text = self.render_config(database, table, files)
3121
+ except (OSError, UnicodeError, ValueError, csv.Error) as exc:
3122
+ return GploadTableResult(
3123
+ database, table, files=files, returncode=2,
3124
+ message='cannot render control file: %s' % exc)
3125
+
3126
+ descriptor, config_path = tempfile.mkstemp(
3127
+ prefix='gpload-batch-', suffix='.yaml')
3128
+ try:
3129
+ with os.fdopen(descriptor, 'w', encoding='utf-8') as handle:
3130
+ handle.write(config_text)
3131
+
3132
+ log_offset = self._append_log_separator()
3133
+ run = self.loader.load(
3134
+ config_path, log=self.log_file, env=self._environment(),
3135
+ timeout=self.timeout)
3136
+ output = self._read_log_since(log_offset) + run.stdout + \
3137
+ run.stderr
3138
+ finally:
3139
+ if not self.keep_temp_configs:
3140
+ try:
3141
+ os.unlink(config_path)
3142
+ except OSError:
3143
+ pass
3144
+
3145
+ returncode = self._map_returncode(run.returncode)
3146
+ if returncode == 0:
3147
+ loaded, message = self._remove_loaded(files)
3148
+ return GploadTableResult(
3149
+ database, table, files=files, returncode=0,
3150
+ gpload_result=run, loaded_files=loaded, message=message,
3151
+ config_path=config_path, config_text=config_text)
3152
+
3153
+ error_file, error_line = self._parse_error_location(output)
3154
+ failed = self._quarantine(
3155
+ table_dir, files, error_file, error_line)
3156
+ message = 'gpload exited with code %d' % run.returncode
3157
+ if run.returncode == 124:
3158
+ message = 'gpload timed out after %s seconds' % self.timeout
3159
+ return GploadTableResult(
3160
+ database, table, files=files, returncode=returncode,
3161
+ gpload_result=run, failed_files=failed, error_file=error_file,
3162
+ error_line=error_line, message=message,
3163
+ config_path=config_path, config_text=config_text)
3164
+
3165
+ def load_dir(self, on_table=None):
3166
+ """Alias for :meth:`load`."""
3167
+ return self.load(on_table=on_table)
3168
+
3169
+ def load(self, on_table=None):
3170
+ """Load every table directory and return a :class:`GploadBatchResult`.
3171
+
3172
+ ``on_table`` is an optional callback invoked with each
3173
+ :class:`GploadTableResult` as soon as that table is processed.
3174
+ """
3175
+ if not os.path.isdir(self.data_dir):
3176
+ raise FileNotFoundError(
3177
+ 'data directory not found: %s' % self.data_dir)
3178
+ if not os.path.isfile(self.config_template):
3179
+ raise FileNotFoundError(
3180
+ 'control file template not found: %s' % self.config_template)
3181
+
3182
+ os.makedirs(self.log_dir, exist_ok=True)
3183
+
3184
+ results = []
3185
+ for database, table, files in self.scan():
3186
+ result = self.load_table(database, table, files)
3187
+ results.append(result)
3188
+ if on_table is not None:
3189
+ on_table(result)
3190
+ return GploadBatchResult(results)
3191
+
3192
+
3193
+ def run_gpload_batch(data_dir, **kwargs):
3194
+ """Shortcut for ``GploadBatch(data_dir, **kwargs).load()``."""
3195
+ return GploadBatch(data_dir, **kwargs).load()
3196
+
3197
+
3198
+ def main(argv=None):
3199
+ """Command-line entry point used by the ``gpload-py3`` console script."""
3200
+ instance = gpload(sys.argv[1:] if argv is None else argv)
3201
+ instance.run()
3202
+ return instance.exitValue
3203
+
3204
+
3205
+ if __name__ == '__main__':
3206
+ sys.exit(main())