gpload-py3 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gpload.py +3206 -0
- gpload_py3-0.1.1.data/data/share/gpload-py3/NOTICE +15 -0
- gpload_py3-0.1.1.data/data/share/gpload-py3/gpload.yaml.ori +29 -0
- gpload_py3-0.1.1.dist-info/METADATA +644 -0
- gpload_py3-0.1.1.dist-info/RECORD +10 -0
- gpload_py3-0.1.1.dist-info/WHEEL +5 -0
- gpload_py3-0.1.1.dist-info/entry_points.txt +2 -0
- gpload_py3-0.1.1.dist-info/licenses/LICENSE +202 -0
- gpload_py3-0.1.1.dist-info/licenses/NOTICE +15 -0
- gpload_py3-0.1.1.dist-info/top_level.txt +1 -0
gpload.py
ADDED
|
@@ -0,0 +1,3206 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
# gpload - load file(s) into Greenplum Database
|
|
4
|
+
# Copyright Greenplum 2008
|
|
5
|
+
#
|
|
6
|
+
# Python 3 port: Copyright 2026 Vitchenko Kirill
|
|
7
|
+
# This file is a Python 3 port of the original gpload utility.
|
|
8
|
+
# See NOTICE and LICENSE for attribution and license terms.
|
|
9
|
+
|
|
10
|
+
'''gpload [options] -f configuration file
|
|
11
|
+
|
|
12
|
+
Options:
|
|
13
|
+
-h hostname: host to connect to
|
|
14
|
+
-p port: port to connect to
|
|
15
|
+
-U username: user to connect as
|
|
16
|
+
-d database: database to connect to
|
|
17
|
+
-W: force password authentication
|
|
18
|
+
-q: quiet mode
|
|
19
|
+
-D: do not actually load data
|
|
20
|
+
-v: verbose
|
|
21
|
+
-V: very verbose
|
|
22
|
+
-l logfile: log output to logfile
|
|
23
|
+
--no_auto_trans: do not wrap gpload in transaction
|
|
24
|
+
--gpfdist_timeout timeout: gpfdist timeout value
|
|
25
|
+
--version: print version number and exit
|
|
26
|
+
-?: help
|
|
27
|
+
'''
|
|
28
|
+
|
|
29
|
+
import re
|
|
30
|
+
import sys
|
|
31
|
+
|
|
32
|
+
try:
|
|
33
|
+
import yaml
|
|
34
|
+
except ImportError:
|
|
35
|
+
sys.stderr.write("gpload needs pyyaml. You can get it from http://pyyaml.org.\n")
|
|
36
|
+
sys.exit(2)
|
|
37
|
+
|
|
38
|
+
import platform
|
|
39
|
+
|
|
40
|
+
psycopg2 = None
|
|
41
|
+
pg = None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def load_db_driver():
|
|
45
|
+
"""Import psycopg2 or PyGreSQL, exiting with code 2 if neither works.
|
|
46
|
+
|
|
47
|
+
The import is deferred so that ``import gpload`` can be used for the
|
|
48
|
+
programmatic interface without a database driver in the calling process.
|
|
49
|
+
"""
|
|
50
|
+
global psycopg2, pg
|
|
51
|
+
if psycopg2 is not None or pg is not None:
|
|
52
|
+
return
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
import psycopg2 as _psycopg2
|
|
56
|
+
except ImportError:
|
|
57
|
+
_psycopg2 = None
|
|
58
|
+
if _psycopg2 is not None:
|
|
59
|
+
psycopg2 = _psycopg2
|
|
60
|
+
return
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
from pygresql import pg as _pg
|
|
64
|
+
except Exception:
|
|
65
|
+
try:
|
|
66
|
+
import pg as _pg
|
|
67
|
+
except Exception as e:
|
|
68
|
+
from struct import calcsize
|
|
69
|
+
sysWordSize = calcsize("P") * 8
|
|
70
|
+
if (platform.system()) in ['Windows', 'Microsoft'] and (sysWordSize == 64):
|
|
71
|
+
errorMsg = "gpload appears to be running in 64-bit Python under Windows.\n"
|
|
72
|
+
errorMsg = errorMsg + "Currently only 32-bit Python is supported. Please \n"
|
|
73
|
+
errorMsg = errorMsg + "reinstall a 32-bit Python interpreter.\n"
|
|
74
|
+
else:
|
|
75
|
+
errorMsg = "gpload was unable to import psycopg2 or PyGreSQL (pg.py) - %s\n" % str(e)
|
|
76
|
+
sys.stderr.write(str(errorMsg))
|
|
77
|
+
sys.exit(2)
|
|
78
|
+
|
|
79
|
+
pg = _pg
|
|
80
|
+
|
|
81
|
+
import hashlib
|
|
82
|
+
import codecs
|
|
83
|
+
import csv
|
|
84
|
+
import fnmatch
|
|
85
|
+
import site
|
|
86
|
+
import sysconfig
|
|
87
|
+
import tempfile
|
|
88
|
+
import datetime,getpass,os,signal,socket,subprocess,threading,time,traceback,re
|
|
89
|
+
import uuid
|
|
90
|
+
import socket
|
|
91
|
+
|
|
92
|
+
thePlatform = platform.system()
|
|
93
|
+
if thePlatform in ['Windows', 'Microsoft']:
|
|
94
|
+
windowsPlatform = True
|
|
95
|
+
else:
|
|
96
|
+
windowsPlatform = False
|
|
97
|
+
|
|
98
|
+
if windowsPlatform == False:
|
|
99
|
+
import select
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
EXECNAME = 'gpload'
|
|
103
|
+
|
|
104
|
+
NUM_WARN_ROWS = 0
|
|
105
|
+
|
|
106
|
+
# PyGreSQL versions differ: the Greenplum build exposes DB.notices(),
|
|
107
|
+
# while the upstream package only exposes a notice receiver callback.
|
|
108
|
+
PG_NOTICES = []
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _notice_text(notice):
|
|
112
|
+
message = getattr(notice, 'message', notice)
|
|
113
|
+
if isinstance(message, bytes):
|
|
114
|
+
message = message.decode('utf-8', 'replace')
|
|
115
|
+
return str(message)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _pg_notice_receiver(notice):
|
|
119
|
+
try:
|
|
120
|
+
PG_NOTICES.append(_notice_text(notice))
|
|
121
|
+
except Exception:
|
|
122
|
+
pass
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _get_notices(db):
|
|
126
|
+
if hasattr(db, 'notices'):
|
|
127
|
+
try:
|
|
128
|
+
return list(db.notices())
|
|
129
|
+
except Exception:
|
|
130
|
+
return []
|
|
131
|
+
return list(PG_NOTICES)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _install_notice_receiver(db):
|
|
135
|
+
if not hasattr(db, 'set_notice_receiver'):
|
|
136
|
+
return
|
|
137
|
+
try:
|
|
138
|
+
del PG_NOTICES[:]
|
|
139
|
+
db.set_notice_receiver(_pg_notice_receiver)
|
|
140
|
+
except Exception:
|
|
141
|
+
pass
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class QueryResult:
|
|
145
|
+
def __init__(self, rows, columns, rowcount):
|
|
146
|
+
self.rows = rows
|
|
147
|
+
self.columns = columns
|
|
148
|
+
self.rowcount = rowcount
|
|
149
|
+
|
|
150
|
+
def getresult(self):
|
|
151
|
+
return self.rows
|
|
152
|
+
|
|
153
|
+
def dictresult(self):
|
|
154
|
+
return [dict(zip(self.columns, row)) for row in self.rows]
|
|
155
|
+
|
|
156
|
+
def __iter__(self):
|
|
157
|
+
return iter(self.rows)
|
|
158
|
+
|
|
159
|
+
def __str__(self):
|
|
160
|
+
return str(self.rowcount)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class Psycopg2DB:
|
|
164
|
+
def __init__(self, dbname, host, port, user, passwd):
|
|
165
|
+
params = {'dbname': dbname}
|
|
166
|
+
if host:
|
|
167
|
+
params['host'] = host
|
|
168
|
+
if port:
|
|
169
|
+
params['port'] = port
|
|
170
|
+
if user:
|
|
171
|
+
params['user'] = user
|
|
172
|
+
if passwd is not None:
|
|
173
|
+
params['password'] = passwd
|
|
174
|
+
self._conn = psycopg2.connect(**params)
|
|
175
|
+
self._conn.autocommit = True
|
|
176
|
+
|
|
177
|
+
def close(self):
|
|
178
|
+
self._conn.close()
|
|
179
|
+
|
|
180
|
+
def query(self, sql):
|
|
181
|
+
cur = self._conn.cursor()
|
|
182
|
+
try:
|
|
183
|
+
cur.execute(sql)
|
|
184
|
+
columns = [d[0] for d in cur.description] if cur.description else []
|
|
185
|
+
rows = cur.fetchall() if cur.description else []
|
|
186
|
+
return QueryResult(rows, columns, cur.rowcount)
|
|
187
|
+
finally:
|
|
188
|
+
cur.close()
|
|
189
|
+
|
|
190
|
+
def notices(self):
|
|
191
|
+
return list(getattr(self._conn, 'notices', []))
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def db_connect(dbname, host, port, user, passwd):
|
|
195
|
+
load_db_driver()
|
|
196
|
+
if psycopg2 is not None:
|
|
197
|
+
return Psycopg2DB(dbname, host, port, user, passwd)
|
|
198
|
+
return pg.DB(dbname=dbname, host=host, port=port, user=user, passwd=passwd)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# Mapping for validing our configuration file. We're only concerned with
|
|
202
|
+
# keys -- stuff left of ':'. It gets complex in two cases: firstly when
|
|
203
|
+
# we handle blocks which have keys which are not keywords -- such as under
|
|
204
|
+
# COLUMNS:. Secondly, we want to detect when users put keywords in the wrong
|
|
205
|
+
# place. To that end, the mapping is structured such that:
|
|
206
|
+
#
|
|
207
|
+
# key -> { 'parse_children' -> [ True | False ],
|
|
208
|
+
# 'parent' -> <parent name> }
|
|
209
|
+
#
|
|
210
|
+
# Each key is a keyword in the configuration file. parse_children tells us
|
|
211
|
+
# whether children are expected to be keywords. parent tells us the parent
|
|
212
|
+
# keyword or None
|
|
213
|
+
valid_tokens = {
|
|
214
|
+
"version": {'parse_children': True, 'parent': None},
|
|
215
|
+
"database": {'parse_children': True, 'parent': None},
|
|
216
|
+
"user": {'parse_children': True, 'parent': None},
|
|
217
|
+
"host": {'parse_children': True, 'parent': None},
|
|
218
|
+
"port": {'parse_children': True, 'parent': [None, "source"]},
|
|
219
|
+
"password": {'parse_children': True, 'parent': None},
|
|
220
|
+
"gpload": {'parse_children': True, 'parent': None},
|
|
221
|
+
"input": {'parse_children': True, 'parent': "gpload"},
|
|
222
|
+
"source": {'parse_children': True, 'parent': "input"},
|
|
223
|
+
"local_hostname": {'parse_children': False, 'parent': "source"},
|
|
224
|
+
"port_range": {'parse_children': False, 'parent': "source"},
|
|
225
|
+
"file": {'parse_children': False, 'parent': "source"},
|
|
226
|
+
"ssl": {'parse_children': False, 'parent': "source"},
|
|
227
|
+
"certificates_path": {'parse_children': False, 'parent': "source"},
|
|
228
|
+
"columns": {'parse_children': False, 'parent': "input"},
|
|
229
|
+
"transform": {'parse_children': True, 'parent': "input"},
|
|
230
|
+
"transform_config": {'parse_children': True, 'parent': "input"},
|
|
231
|
+
"max_line_length": {'parse_children': True, 'parent': "input"},
|
|
232
|
+
"format": {'parse_children': True, 'parent': "input"},
|
|
233
|
+
"delimiter": {'parse_children': True, 'parent': "input"},
|
|
234
|
+
"escape": {'parse_children': True, 'parent': "input"},
|
|
235
|
+
"null_as": {'parse_children': True, 'parent': "input"},
|
|
236
|
+
"quote": {'parse_children': True, 'parent': "input"},
|
|
237
|
+
"encoding": {'parse_children': True, 'parent': "input"},
|
|
238
|
+
"force_not_null": {'parse_children': False, 'parent': "input"},
|
|
239
|
+
"error_limit": {'parse_children': True, 'parent': "input"},
|
|
240
|
+
"error_percent": {'parse_children': True, 'parent': "input"},
|
|
241
|
+
"error_table": {'parse_children': True, 'parent': "input"},
|
|
242
|
+
"log_errors": {'parse_children': False, 'parent': "input"},
|
|
243
|
+
"header": {'parse_children': True, 'parent': "input"},
|
|
244
|
+
"fully_qualified_domain_name": {'parse_children': False, 'parent': 'input'},
|
|
245
|
+
"output": {'parse_children': True, 'parent': "gpload"},
|
|
246
|
+
"table": {'parse_children': True, 'parent': "output"},
|
|
247
|
+
"mode": {'parse_children': True, 'parent': "output"},
|
|
248
|
+
"match_columns": {'parse_children': False, 'parent': "output"},
|
|
249
|
+
"update_columns": {'parse_children': False, 'parent': "output"},
|
|
250
|
+
"update_condition": {'parse_children': True, 'parent': "output"},
|
|
251
|
+
"mapping": {'parse_children': False, 'parent': "output"},
|
|
252
|
+
"including_defaults": {'parse_children': False, 'parent': 'output'},
|
|
253
|
+
"preload": {'parse_children': True, 'parent': 'gpload'},
|
|
254
|
+
"truncate": {'parse_children': False, 'parent': 'preload'},
|
|
255
|
+
"reuse_tables": {'parse_children': False, 'parent': 'preload'},
|
|
256
|
+
"sql": {'parse_children': True, 'parent': 'gpload'},
|
|
257
|
+
"before": {'parse_children': False, 'parent': 'sql'},
|
|
258
|
+
"after": {'parse_children': False, 'parent': 'sql'},
|
|
259
|
+
"external": {'parse_children': True, 'parent': 'gpload'},
|
|
260
|
+
"schema": {'parse_children': False, 'parent': 'external'}}
|
|
261
|
+
|
|
262
|
+
_abbrevs = [
|
|
263
|
+
(1<<50, ' PB'),
|
|
264
|
+
(1<<40, ' TB'),
|
|
265
|
+
(1<<30, ' GB'),
|
|
266
|
+
(1<<20, ' MB'),
|
|
267
|
+
(1<<10, ' kB'),
|
|
268
|
+
(1, ' bytes')
|
|
269
|
+
]
|
|
270
|
+
|
|
271
|
+
received_kill = False
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def caseInsensitiveDictLookup(key, dictionary):
|
|
275
|
+
"""
|
|
276
|
+
Do a case insensitive dictionary lookup. Return the dictionary value if found,
|
|
277
|
+
or None if not found.
|
|
278
|
+
"""
|
|
279
|
+
for entry in dictionary:
|
|
280
|
+
if entry.lower() == key.lower():
|
|
281
|
+
return dictionary[entry]
|
|
282
|
+
return None
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def sqlIdentifierCompare(x, y):
|
|
287
|
+
"""
|
|
288
|
+
Compare x and y as SQL identifiers. Use SQL rules for comparing delimited
|
|
289
|
+
and non-delimited identifiers. Return True if they are equivalent or False
|
|
290
|
+
if they are not equivalent.
|
|
291
|
+
"""
|
|
292
|
+
if x == None or y == None:
|
|
293
|
+
return False
|
|
294
|
+
|
|
295
|
+
if isDelimited(x):
|
|
296
|
+
x = quote_unident(x)
|
|
297
|
+
else:
|
|
298
|
+
x = x.lower()
|
|
299
|
+
if isDelimited(y):
|
|
300
|
+
y = quote_unident(y)
|
|
301
|
+
else:
|
|
302
|
+
y = y.lower()
|
|
303
|
+
|
|
304
|
+
if x == y:
|
|
305
|
+
return True
|
|
306
|
+
else:
|
|
307
|
+
return False
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def isDelimited(value):
|
|
311
|
+
"""
|
|
312
|
+
This method simply checks to see if the user supplied value has delimiters.
|
|
313
|
+
That is, if it starts and ends with double-quotes, then it is delimited.
|
|
314
|
+
"""
|
|
315
|
+
if len(value) < 2:
|
|
316
|
+
return False
|
|
317
|
+
if value[0] == '"' and value[-1] == '"':
|
|
318
|
+
return True
|
|
319
|
+
else:
|
|
320
|
+
return False
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def convertListToDelimited(identifiers):
|
|
324
|
+
"""
|
|
325
|
+
This method will convert a list of identifiers, which may be a mix of
|
|
326
|
+
delimited and non-delimited identifiers, and return a list of
|
|
327
|
+
delimited identifiers.
|
|
328
|
+
"""
|
|
329
|
+
returnList = []
|
|
330
|
+
|
|
331
|
+
for id in identifiers:
|
|
332
|
+
if isDelimited(id) == False:
|
|
333
|
+
id = id.lower()
|
|
334
|
+
returnList.append(quote_ident(id))
|
|
335
|
+
else:
|
|
336
|
+
returnList.append(id)
|
|
337
|
+
return returnList
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def splitUpMultipartIdentifier(id):
|
|
342
|
+
"""
|
|
343
|
+
Given a sql identifer like sch.tab, return a list of its
|
|
344
|
+
individual elements (e.g. sch.tab would return ['sch','tab']
|
|
345
|
+
"""
|
|
346
|
+
returnList = []
|
|
347
|
+
|
|
348
|
+
elementList = splitIntoLiteralsAndNonLiterals(id, quoteValue='"')
|
|
349
|
+
# If there is a leading empty string, remove it.
|
|
350
|
+
if elementList[0] == ' ':
|
|
351
|
+
elementList.pop(0)
|
|
352
|
+
|
|
353
|
+
# Remove the dots, and split up undelimited multipart names
|
|
354
|
+
for e in elementList:
|
|
355
|
+
if e != '.':
|
|
356
|
+
if e[0] != '"':
|
|
357
|
+
subElementList = e.split('.')
|
|
358
|
+
else:
|
|
359
|
+
subElementList = [e]
|
|
360
|
+
for se in subElementList:
|
|
361
|
+
# remove any empty elements
|
|
362
|
+
if se != '':
|
|
363
|
+
returnList.append(se)
|
|
364
|
+
|
|
365
|
+
return returnList
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def splitIntoLiteralsAndNonLiterals(str1, quoteValue="'"):
|
|
369
|
+
"""
|
|
370
|
+
Break the string (str1) into a list of literals and non-literals where every
|
|
371
|
+
even number element is a non-literal and every odd number element is a literal.
|
|
372
|
+
The delimiter between literals and non-literals is the quoteValue, so this
|
|
373
|
+
function will not take into account any modifiers on a literal (e.g. E'adf').
|
|
374
|
+
"""
|
|
375
|
+
returnList = []
|
|
376
|
+
|
|
377
|
+
if len(str1) > 1 and str1[0] == quoteValue:
|
|
378
|
+
# Always start with a non-literal
|
|
379
|
+
str1 = ' ' + str1
|
|
380
|
+
|
|
381
|
+
inLiteral = False
|
|
382
|
+
i = 0
|
|
383
|
+
tokenStart = 0
|
|
384
|
+
while i < len(str1):
|
|
385
|
+
if str1[i] == quoteValue:
|
|
386
|
+
if inLiteral == False:
|
|
387
|
+
# We are at start of literal
|
|
388
|
+
inLiteral = True
|
|
389
|
+
returnList.append(str1[tokenStart:i])
|
|
390
|
+
tokenStart = i
|
|
391
|
+
elif i + 1 < len(str1) and str1[i+1] == quoteValue:
|
|
392
|
+
# We are in a literal and found quote quote, so skip over it
|
|
393
|
+
i = i + 1
|
|
394
|
+
else:
|
|
395
|
+
# We are at the end of a literal or end of str1
|
|
396
|
+
returnList.append(str1[tokenStart:i+1])
|
|
397
|
+
tokenStart = i + 1
|
|
398
|
+
inLiteral = False
|
|
399
|
+
i = i + 1
|
|
400
|
+
if tokenStart < len(str1):
|
|
401
|
+
returnList.append(str1[tokenStart:])
|
|
402
|
+
return returnList
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def quote_ident(val):
|
|
406
|
+
"""
|
|
407
|
+
This method returns a new string replacing " with "",
|
|
408
|
+
and adding a " at the start and end of the string.
|
|
409
|
+
"""
|
|
410
|
+
return '"' + val.replace('"', '""') + '"'
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def quote_unident(val):
|
|
414
|
+
"""
|
|
415
|
+
This method returns a new string replacing "" with ",
|
|
416
|
+
and removing the " at the start and end of the string.
|
|
417
|
+
"""
|
|
418
|
+
if val != None and len(val) > 0:
|
|
419
|
+
val = val.replace('""', '"')
|
|
420
|
+
if val != None and len(val) > 1 and val[0] == '"' and val[-1] == '"':
|
|
421
|
+
val = val[1:-1]
|
|
422
|
+
|
|
423
|
+
return val
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def notice_processor(self):
|
|
427
|
+
if windowsPlatform == True:
|
|
428
|
+
# We don't have a pygresql with our notice fix, so skip for windows.
|
|
429
|
+
# This means we will not get any warnings on windows (MPP10989).
|
|
430
|
+
return
|
|
431
|
+
|
|
432
|
+
theNotices = _get_notices(self.db)
|
|
433
|
+
# Greenplum 6 lowercases the first word: "NOTICE: found N data
|
|
434
|
+
# formatting errors (N or more input rows), rejected related input data".
|
|
435
|
+
r = re.compile(r"^NOTICE:\s+found (\d+) data formatting errors", re.IGNORECASE)
|
|
436
|
+
messageNumber = 0
|
|
437
|
+
m = None
|
|
438
|
+
while messageNumber < len(theNotices) and m == None:
|
|
439
|
+
aNotice = _notice_text(theNotices[messageNumber])
|
|
440
|
+
m = r.match(aNotice)
|
|
441
|
+
messageNumber = messageNumber + 1
|
|
442
|
+
if m:
|
|
443
|
+
global NUM_WARN_ROWS
|
|
444
|
+
NUM_WARN_ROWS = int(m.group(1))
|
|
445
|
+
|
|
446
|
+
def handle_kill(signum, frame):
|
|
447
|
+
# already dying?
|
|
448
|
+
global received_kill
|
|
449
|
+
if received_kill:
|
|
450
|
+
return
|
|
451
|
+
|
|
452
|
+
received_kill = True
|
|
453
|
+
|
|
454
|
+
g.log(g.INFO, "received signal %d" % signum)
|
|
455
|
+
g.exitValue = 2
|
|
456
|
+
sys.exit(2)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def bytestr(size, precision=1):
|
|
460
|
+
"""Return a string representing the greek/metric suffix of a size"""
|
|
461
|
+
if size==1:
|
|
462
|
+
return '1 byte'
|
|
463
|
+
for factor, suffix in _abbrevs:
|
|
464
|
+
if size >= factor:
|
|
465
|
+
break
|
|
466
|
+
|
|
467
|
+
float_string_split = repr(size/float(factor)).split('.')
|
|
468
|
+
integer_part = float_string_split[0]
|
|
469
|
+
decimal_part = float_string_split[1]
|
|
470
|
+
if int(decimal_part[0:precision]):
|
|
471
|
+
float_string = '.'.join([integer_part, decimal_part[0:precision]])
|
|
472
|
+
else:
|
|
473
|
+
float_string = integer_part
|
|
474
|
+
return float_string + suffix
|
|
475
|
+
|
|
476
|
+
class CatThread(threading.Thread):
|
|
477
|
+
"""
|
|
478
|
+
Simple threading wrapper to read a file descriptor and put the contents
|
|
479
|
+
in the log file.
|
|
480
|
+
|
|
481
|
+
The fd is assumed to be stdout and stderr from gpfdist. We must use select.select
|
|
482
|
+
and locks to ensure both threads are not read at the same time. A dead lock
|
|
483
|
+
situation could happen if they did. communicate() is not used since it blocks.
|
|
484
|
+
We will wait 1 second between read attempts.
|
|
485
|
+
|
|
486
|
+
"""
|
|
487
|
+
def __init__(self,gpload,fd, sharedLock = None):
|
|
488
|
+
threading.Thread.__init__(self)
|
|
489
|
+
self.gpload = gpload
|
|
490
|
+
self.fd = fd
|
|
491
|
+
self.theLock = sharedLock
|
|
492
|
+
|
|
493
|
+
def run(self):
|
|
494
|
+
if windowsPlatform == True:
|
|
495
|
+
while 1:
|
|
496
|
+
# Windows select does not support select on non-file fd's, so we can use the lock fix. Deadlock is possible here.
|
|
497
|
+
# We need to look into the Python windows module to see if there is another way to do this in Windows.
|
|
498
|
+
line = self.fd.readline()
|
|
499
|
+
if line=='':
|
|
500
|
+
break
|
|
501
|
+
self.gpload.log(self.gpload.DEBUG, 'gpfdist: ' + line.strip('\n'))
|
|
502
|
+
else:
|
|
503
|
+
while 1:
|
|
504
|
+
retList = select.select( [self.fd]
|
|
505
|
+
, []
|
|
506
|
+
, []
|
|
507
|
+
, 1
|
|
508
|
+
)
|
|
509
|
+
if retList[0] == [self.fd]:
|
|
510
|
+
self.theLock.acquire()
|
|
511
|
+
line = self.fd.readline()
|
|
512
|
+
self.theLock.release()
|
|
513
|
+
else:
|
|
514
|
+
continue
|
|
515
|
+
if line=='':
|
|
516
|
+
break
|
|
517
|
+
self.gpload.log(self.gpload.DEBUG, 'gpfdist: ' + line.strip('\n'))
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
def cli_help():
|
|
521
|
+
help_path = os.path.join(sys.path[0], '..', 'docs', 'cli_help', EXECNAME +
|
|
522
|
+
'_help');
|
|
523
|
+
f = None
|
|
524
|
+
try:
|
|
525
|
+
try:
|
|
526
|
+
f = open(help_path);
|
|
527
|
+
return f.read(-1)
|
|
528
|
+
except:
|
|
529
|
+
return ''
|
|
530
|
+
finally:
|
|
531
|
+
if f: f.close()
|
|
532
|
+
|
|
533
|
+
#============================================================
|
|
534
|
+
def usage(error = None):
|
|
535
|
+
print(cli_help() or __doc__)
|
|
536
|
+
sys.stdout.flush()
|
|
537
|
+
if error:
|
|
538
|
+
sys.stderr.write('ERROR: ' + error + '\n')
|
|
539
|
+
sys.stderr.write('\n')
|
|
540
|
+
sys.stderr.flush()
|
|
541
|
+
|
|
542
|
+
sys.exit(2)
|
|
543
|
+
|
|
544
|
+
def quote(a):
|
|
545
|
+
"""
|
|
546
|
+
SQLify a string
|
|
547
|
+
"""
|
|
548
|
+
return "'"+a.replace("'","''").replace('\\','\\\\')+"'"
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def quote_copy_value(a):
|
|
552
|
+
"""Quote a COPY option value (delimiter, null, escape, quote).
|
|
553
|
+
|
|
554
|
+
quote() doubles backslashes for ordinary SQL literals, but COPY options
|
|
555
|
+
compare against the raw input bytes and ESCAPE requires exactly one
|
|
556
|
+
character. With standard_conforming_strings=on, '\\\\N' therefore became
|
|
557
|
+
a three-character NULL string instead of the usual two-character \\N.
|
|
558
|
+
E'' keeps the value unambiguous on both settings.
|
|
559
|
+
"""
|
|
560
|
+
if '\\' in a:
|
|
561
|
+
return "E'" + a.replace('\\', '\\\\').replace("'", "''") + "'"
|
|
562
|
+
return quote(a)
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def decode_unicode_escape(a):
|
|
566
|
+
return codecs.decode(a, 'unicode_escape')
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def to_db_text(value):
|
|
570
|
+
if isinstance(value, bytes):
|
|
571
|
+
return value.decode('utf-8')
|
|
572
|
+
return value
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def to_text(value):
|
|
576
|
+
if isinstance(value, bytes):
|
|
577
|
+
return value.decode('utf-8')
|
|
578
|
+
return value
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def escape_string(value):
|
|
582
|
+
value = to_text(value)
|
|
583
|
+
return value.replace('\\', '\\\\').replace("'", "''")
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def to_bool(value):
|
|
587
|
+
value = to_text(value)
|
|
588
|
+
if isinstance(value, str):
|
|
589
|
+
return value.lower() not in ('', 'f', 'false', '0')
|
|
590
|
+
return bool(value)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def error_log_filename(value):
|
|
594
|
+
"""Return the source file name from gp_read_error_log(filename).
|
|
595
|
+
|
|
596
|
+
Older Greenplum releases returned values like
|
|
597
|
+
``gpfdist://host:port/dir/ [1.csv]``; Greenplum 6 returns
|
|
598
|
+
``gpfdist://host:port/dir/file.csv [dir/file.csv]``. The caller only
|
|
599
|
+
needs the file name because load.sh resolves it against data_path.
|
|
600
|
+
"""
|
|
601
|
+
value = to_text(value)
|
|
602
|
+
if not value:
|
|
603
|
+
return value
|
|
604
|
+
|
|
605
|
+
# GP6+ keeps the original path in square brackets.
|
|
606
|
+
match = re.search(r"\[([^\]]+)\]", value)
|
|
607
|
+
candidate = match.group(1) if match else value
|
|
608
|
+
|
|
609
|
+
# Drop gpfdist://host:port/ and gpfdists://... prefixes when present.
|
|
610
|
+
candidate = re.sub(r"^[A-Za-z][A-Za-z0-9+.-]*://[^/]*/", "", candidate)
|
|
611
|
+
candidate = candidate.replace('\\', '/').strip()
|
|
612
|
+
|
|
613
|
+
if not candidate:
|
|
614
|
+
return value
|
|
615
|
+
return os.path.basename(candidate)
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def splitPgpassLine(a):
|
|
619
|
+
"""
|
|
620
|
+
If the user has specified a .pgpass file, we'll have to parse it. We simply
|
|
621
|
+
split the string into arrays at :. We could just use a native python
|
|
622
|
+
function but we need to escape the ':' character.
|
|
623
|
+
"""
|
|
624
|
+
b = []
|
|
625
|
+
escape = False
|
|
626
|
+
d = ''
|
|
627
|
+
for c in a:
|
|
628
|
+
if not escape and c=='\\':
|
|
629
|
+
escape = True
|
|
630
|
+
elif not escape and c==':':
|
|
631
|
+
b.append(d)
|
|
632
|
+
d = ''
|
|
633
|
+
else:
|
|
634
|
+
d += c
|
|
635
|
+
escape = False
|
|
636
|
+
if escape:
|
|
637
|
+
d += '\\'
|
|
638
|
+
b.append(d)
|
|
639
|
+
return b
|
|
640
|
+
|
|
641
|
+
def test_key(gp, key, crumb):
|
|
642
|
+
"""
|
|
643
|
+
Make sure that a key is a valid keyword in the configuration grammar and
|
|
644
|
+
that it appears in the configuration file where we expect -- that is, where
|
|
645
|
+
it has the parent we expect
|
|
646
|
+
"""
|
|
647
|
+
val = valid_tokens.get(key)
|
|
648
|
+
if val == None:
|
|
649
|
+
gp.log(gp.ERROR, 'unrecognized key: "%s"' % key)
|
|
650
|
+
|
|
651
|
+
p = val['parent']
|
|
652
|
+
|
|
653
|
+
# simplify for when the same keyword can appear in multiple places
|
|
654
|
+
if type(p) != list:
|
|
655
|
+
p = [p]
|
|
656
|
+
|
|
657
|
+
c = None
|
|
658
|
+
if len(crumb):
|
|
659
|
+
c = crumb[-1]
|
|
660
|
+
|
|
661
|
+
found = False
|
|
662
|
+
for m in p:
|
|
663
|
+
if m == c:
|
|
664
|
+
found = True
|
|
665
|
+
break
|
|
666
|
+
|
|
667
|
+
if not found:
|
|
668
|
+
gp.log(gp.ERROR, 'unexpected key: "%s"' % key)
|
|
669
|
+
|
|
670
|
+
return val
|
|
671
|
+
|
|
672
|
+
def yaml_walk(gp, node, crumb):
|
|
673
|
+
if type(node) == list:
|
|
674
|
+
for a in node:
|
|
675
|
+
if type(a) == tuple:
|
|
676
|
+
key = a[0].value.lower()
|
|
677
|
+
|
|
678
|
+
val = test_key(gp, key, crumb)
|
|
679
|
+
|
|
680
|
+
if (len(a) > 1 and val['parse_children'] and
|
|
681
|
+
(isinstance(a[1], yaml.nodes.MappingNode) or
|
|
682
|
+
isinstance(a[1], yaml.nodes.SequenceNode))):
|
|
683
|
+
crumb.append(key)
|
|
684
|
+
yaml_walk(gp, a[1], crumb)
|
|
685
|
+
crumb.pop()
|
|
686
|
+
elif isinstance(a, yaml.nodes.ScalarNode):
|
|
687
|
+
test_key(gp, a.value, crumb)
|
|
688
|
+
else:
|
|
689
|
+
yaml_walk(gp, a, crumb)
|
|
690
|
+
elif isinstance(node, yaml.nodes.MappingNode):
|
|
691
|
+
yaml_walk(gp, node.value, crumb)
|
|
692
|
+
|
|
693
|
+
elif isinstance(node, yaml.nodes.ScalarNode):
|
|
694
|
+
pass
|
|
695
|
+
|
|
696
|
+
elif isinstance(node, yaml.nodes.SequenceNode):
|
|
697
|
+
yaml_walk(gp, node.value, crumb)
|
|
698
|
+
|
|
699
|
+
elif isinstance(node, yaml.nodes.CollectionNode):
|
|
700
|
+
pass
|
|
701
|
+
|
|
702
|
+
|
|
703
|
+
def changeToUnicode(a):
|
|
704
|
+
"""
|
|
705
|
+
Recursively normalize the containers in a YAML document
|
|
706
|
+
"""
|
|
707
|
+
if isinstance(a, list):
|
|
708
|
+
return [changeToUnicode(item) for item in a]
|
|
709
|
+
if isinstance(a, dict):
|
|
710
|
+
return {key: changeToUnicode(value) for key, value in a.items()}
|
|
711
|
+
return a
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def dictKeyToLower(a):
|
|
716
|
+
"""
|
|
717
|
+
down case all entries in a list or dict
|
|
718
|
+
"""
|
|
719
|
+
if type(a) == list:
|
|
720
|
+
return list(map(dictKeyToLower,a))
|
|
721
|
+
if type(a) == dict:
|
|
722
|
+
b = dict()
|
|
723
|
+
for key,value in a.items():
|
|
724
|
+
if type(key) == str:
|
|
725
|
+
key = str(key.lower())
|
|
726
|
+
b[key] = dictKeyToLower(value)
|
|
727
|
+
return b
|
|
728
|
+
if type(a) == str:
|
|
729
|
+
a = str(a)
|
|
730
|
+
return a
|
|
731
|
+
|
|
732
|
+
#
|
|
733
|
+
# MPP-13348
|
|
734
|
+
#
|
|
735
|
+
|
|
736
|
+
'''Jenkins hash - http://burtleburtle.net/bob/hash/doobs.html'''
|
|
737
|
+
|
|
738
|
+
def jenkinsmix(a, b, c):
|
|
739
|
+
a &= 0xffffffff; b &= 0xffffffff; c &= 0xffffffff
|
|
740
|
+
a -= b; a -= c; a ^= (c>>13); a &= 0xffffffff
|
|
741
|
+
b -= c; b -= a; b ^= (a<<8); b &= 0xffffffff
|
|
742
|
+
c -= a; c -= b; c ^= (b>>13); c &= 0xffffffff
|
|
743
|
+
a -= b; a -= c; a ^= (c>>12); a &= 0xffffffff
|
|
744
|
+
b -= c; b -= a; b ^= (a<<16); b &= 0xffffffff
|
|
745
|
+
c -= a; c -= b; c ^= (b>>5); c &= 0xffffffff
|
|
746
|
+
a -= b; a -= c; a ^= (c>>3); a &= 0xffffffff
|
|
747
|
+
b -= c; b -= a; b ^= (a<<10); b &= 0xffffffff
|
|
748
|
+
c -= a; c -= b; c ^= (b>>15); c &= 0xffffffff
|
|
749
|
+
return a, b, c
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def jenkins(data, initval = 0):
|
|
753
|
+
length = lenpos = len(data)
|
|
754
|
+
if length == 0:
|
|
755
|
+
return 0
|
|
756
|
+
a = b = 0x9e3779b9
|
|
757
|
+
c = initval
|
|
758
|
+
p = 0
|
|
759
|
+
while lenpos >= 12:
|
|
760
|
+
a += (ord(data[p+0]) + (ord(data[p+1])<<8) + (ord(data[p+2])<<16) + (ord(data[p+3])<<24))
|
|
761
|
+
b += (ord(data[p+4]) + (ord(data[p+5])<<8) + (ord(data[p+6])<<16) + (ord(data[p+7])<<24))
|
|
762
|
+
c += (ord(data[p+8]) + (ord(data[p+9])<<8) + (ord(data[p+10])<<16) + (ord(data[p+11])<<24))
|
|
763
|
+
a, b, c = jenkinsmix(a, b, c)
|
|
764
|
+
p += 12
|
|
765
|
+
lenpos -= 12
|
|
766
|
+
c += length
|
|
767
|
+
if lenpos >= 11: c += ord(data[p+10])<<24
|
|
768
|
+
if lenpos >= 10: c += ord(data[p+9])<<16
|
|
769
|
+
if lenpos >= 9: c += ord(data[p+8])<<8
|
|
770
|
+
if lenpos >= 8: b += ord(data[p+7])<<24
|
|
771
|
+
if lenpos >= 7: b += ord(data[p+6])<<16
|
|
772
|
+
if lenpos >= 6: b += ord(data[p+5])<<8
|
|
773
|
+
if lenpos >= 5: b += ord(data[p+4])
|
|
774
|
+
if lenpos >= 4: a += ord(data[p+3])<<24
|
|
775
|
+
if lenpos >= 3: a += ord(data[p+2])<<16
|
|
776
|
+
if lenpos >= 2: a += ord(data[p+1])<<8
|
|
777
|
+
if lenpos >= 1: a += ord(data[p+0])
|
|
778
|
+
a, b, c = jenkinsmix(a, b, c)
|
|
779
|
+
return c
|
|
780
|
+
|
|
781
|
+
# MPP-20927 Citibank: gpload external table name problem
|
|
782
|
+
# Not sure if it is used by other components, just leave it here.
|
|
783
|
+
def shortname(name):
|
|
784
|
+
"""
|
|
785
|
+
Returns a 10 character string formed by concatenating the first two characters
|
|
786
|
+
of the name with another 8 character string computed using the Jenkins hash
|
|
787
|
+
function of the table name. When the original name has only a single non-space
|
|
788
|
+
ascii character, we return '00' followed by 8 char hash.
|
|
789
|
+
|
|
790
|
+
For example:
|
|
791
|
+
|
|
792
|
+
>>> shortname('mytable')
|
|
793
|
+
'my3cbb7ba8'
|
|
794
|
+
>>> shortname('some_pretty_long_test_table_name')
|
|
795
|
+
'so9068664a'
|
|
796
|
+
>>> shortname('t')
|
|
797
|
+
'006742be70'
|
|
798
|
+
|
|
799
|
+
@param name: the input tablename
|
|
800
|
+
@returns: a string 10 characters or less built from the table name
|
|
801
|
+
"""
|
|
802
|
+
|
|
803
|
+
# Remove spaces from original name
|
|
804
|
+
name = re.sub(r' ', '', name)
|
|
805
|
+
|
|
806
|
+
# Run the hash function
|
|
807
|
+
j = jenkins(name)
|
|
808
|
+
|
|
809
|
+
# Now also remove non ascii chars from original name.
|
|
810
|
+
# We do this after jenkins so that we exclude the
|
|
811
|
+
# (very rare) case of passing an empty string to jenkins
|
|
812
|
+
name = "".join(i for i in name if ord(i) < 128)
|
|
813
|
+
|
|
814
|
+
if len(name) > 1:
|
|
815
|
+
return '%2s%08x' % (name[0:2], j)
|
|
816
|
+
else:
|
|
817
|
+
return '00%08x' % (j) # could be len 0 or 1
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def staging_conditions_hash(conditions_str):
|
|
821
|
+
return hashlib.md5(conditions_str.encode('utf-8')).hexdigest()
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
class options:
|
|
825
|
+
pass
|
|
826
|
+
|
|
827
|
+
class gpload:
|
|
828
|
+
"""
|
|
829
|
+
Main class wrapper
|
|
830
|
+
"""
|
|
831
|
+
|
|
832
|
+
def __init__(self,argv):
|
|
833
|
+
# The database driver is imported by db_connect(), so --version, -?
|
|
834
|
+
# and control-file validation work without psycopg2/PyGreSQL.
|
|
835
|
+
self.threads = [] # remember threads so that we can join() against them
|
|
836
|
+
self.exitValue = 0
|
|
837
|
+
self.options = options()
|
|
838
|
+
self.options.h = None
|
|
839
|
+
self.options.gpfdist_timeout = None
|
|
840
|
+
self.options.p = None
|
|
841
|
+
self.options.U = None
|
|
842
|
+
self.options.W = False
|
|
843
|
+
self.options.D = False
|
|
844
|
+
self.options.no_auto_trans = False
|
|
845
|
+
self.options.password = None
|
|
846
|
+
self.options.d = None
|
|
847
|
+
self.DEBUG = 5
|
|
848
|
+
self.LOG = 4
|
|
849
|
+
self.INFO = 3
|
|
850
|
+
self.WARN = 2
|
|
851
|
+
self.ERROR = 1
|
|
852
|
+
self.options.qv = self.INFO
|
|
853
|
+
self.options.l = None
|
|
854
|
+
self.lastcmdtime = ''
|
|
855
|
+
|
|
856
|
+
seenv = False
|
|
857
|
+
seenq = False
|
|
858
|
+
|
|
859
|
+
# Create Temp and External table names. However external table name could
|
|
860
|
+
# get overwritten with another name later on (see create_external_table_name).
|
|
861
|
+
# MPP-20927 Citibank: gpload external table name problem. We use uuid to avoid
|
|
862
|
+
# external table name confliction.
|
|
863
|
+
self.unique_suffix = str(uuid.uuid1()).replace('-', '_')
|
|
864
|
+
self.staging_table_name = 'temp_staging_gpload_' + self.unique_suffix
|
|
865
|
+
self.extTableName = 'ext_gpload_' + self.unique_suffix
|
|
866
|
+
|
|
867
|
+
# SQL to run in order to undo our temporary work
|
|
868
|
+
self.cleanupSql = []
|
|
869
|
+
self.distkey = None
|
|
870
|
+
configFilename = None
|
|
871
|
+
while argv:
|
|
872
|
+
try:
|
|
873
|
+
try:
|
|
874
|
+
if argv[0]=='-h':
|
|
875
|
+
self.options.h = argv[1]
|
|
876
|
+
argv = argv[2:]
|
|
877
|
+
if argv[0]=='--gpfdist_timeout':
|
|
878
|
+
self.options.gpfdist_timeout = argv[1]
|
|
879
|
+
argv = argv[2:]
|
|
880
|
+
elif argv[0]=='-p':
|
|
881
|
+
self.options.p = int(argv[1])
|
|
882
|
+
argv = argv[2:]
|
|
883
|
+
elif argv[0]=='-l':
|
|
884
|
+
self.options.l = argv[1]
|
|
885
|
+
argv = argv[2:]
|
|
886
|
+
elif argv[0]=='-q':
|
|
887
|
+
self.options.qv -= 1
|
|
888
|
+
argv = argv[1:]
|
|
889
|
+
seenq = True
|
|
890
|
+
elif argv[0]=='--version':
|
|
891
|
+
sys.stderr.write("gpload version $Revision$\n")
|
|
892
|
+
sys.exit(0)
|
|
893
|
+
elif argv[0]=='-v':
|
|
894
|
+
self.options.qv = self.LOG
|
|
895
|
+
argv = argv[1:]
|
|
896
|
+
seenv = True
|
|
897
|
+
elif argv[0]=='-V':
|
|
898
|
+
self.options.qv = self.DEBUG
|
|
899
|
+
argv = argv[1:]
|
|
900
|
+
seenv = True
|
|
901
|
+
elif argv[0]=='-W':
|
|
902
|
+
self.options.W = True
|
|
903
|
+
argv = argv[1:]
|
|
904
|
+
elif argv[0]=='-D':
|
|
905
|
+
self.options.D = True
|
|
906
|
+
argv = argv[1:]
|
|
907
|
+
elif argv[0]=='-U':
|
|
908
|
+
self.options.U = argv[1]
|
|
909
|
+
argv = argv[2:]
|
|
910
|
+
elif argv[0]=='-d':
|
|
911
|
+
self.options.d = argv[1]
|
|
912
|
+
argv = argv[2:]
|
|
913
|
+
elif argv[0]=='-f':
|
|
914
|
+
configFilename = argv[1]
|
|
915
|
+
argv = argv[2:]
|
|
916
|
+
elif argv[0]=='--no_auto_trans':
|
|
917
|
+
self.options.no_auto_trans = True
|
|
918
|
+
argv = argv[1:]
|
|
919
|
+
elif argv[0]=='-?':
|
|
920
|
+
usage()
|
|
921
|
+
else:
|
|
922
|
+
break
|
|
923
|
+
except IndexError:
|
|
924
|
+
sys.stderr.write("Option %s needs a parameter.\n"%argv[0])
|
|
925
|
+
sys.exit(2)
|
|
926
|
+
except ValueError:
|
|
927
|
+
sys.stderr.write("Parameter for option %s must be an integer.\n"%argv[0])
|
|
928
|
+
sys.exit(2)
|
|
929
|
+
|
|
930
|
+
if configFilename==None:
|
|
931
|
+
usage('configuration file required')
|
|
932
|
+
elif argv:
|
|
933
|
+
a = ""
|
|
934
|
+
if len(argv) > 1:
|
|
935
|
+
a = "s"
|
|
936
|
+
usage('unrecognized argument%s: %s' % (a, ' '.join(argv)))
|
|
937
|
+
|
|
938
|
+
# default to gpAdminLogs for a log file, may be overwritten
|
|
939
|
+
if self.options.l is None:
|
|
940
|
+
self.options.l = os.path.join(os.environ.get('HOME', '.'),'gpAdminLogs')
|
|
941
|
+
if not os.path.isdir(self.options.l):
|
|
942
|
+
os.mkdir(self.options.l)
|
|
943
|
+
|
|
944
|
+
self.options.l = os.path.join(self.options.l, 'gpload_' + \
|
|
945
|
+
datetime.date.today().strftime('%Y%m%d') + '.log')
|
|
946
|
+
|
|
947
|
+
try:
|
|
948
|
+
self.logfile = open(self.options.l,'a', encoding='utf-8')
|
|
949
|
+
except Exception as e:
|
|
950
|
+
self.log(self.ERROR, "could not open logfile %s: %s" % \
|
|
951
|
+
(self.options.l, e))
|
|
952
|
+
|
|
953
|
+
if seenv and seenq:
|
|
954
|
+
self.log(self.ERROR, "-q conflicts with -v and -V")
|
|
955
|
+
|
|
956
|
+
if self.options.D:
|
|
957
|
+
self.log(self.INFO, 'gpload has the -D option, so it does not actually load any data')
|
|
958
|
+
|
|
959
|
+
try:
|
|
960
|
+
f = open(configFilename,'rb')
|
|
961
|
+
except IOError as e:
|
|
962
|
+
self.log(self.ERROR, "could not open configuration file: %s" % e)
|
|
963
|
+
|
|
964
|
+
# pull in the config file, which should be in valid YAML
|
|
965
|
+
try:
|
|
966
|
+
# do an initial parse, validating the config file
|
|
967
|
+
doc = f.read()
|
|
968
|
+
self.config = yaml.safe_load(doc)
|
|
969
|
+
|
|
970
|
+
self.configOriginal = changeToUnicode(self.config)
|
|
971
|
+
self.config = dictKeyToLower(self.config)
|
|
972
|
+
ver = self.getconfig('version', str, extraStuff = ' tag')
|
|
973
|
+
if ver != '1.0.0.1':
|
|
974
|
+
self.control_file_error("gpload configuration schema version must be 1.0.0.1")
|
|
975
|
+
# second parse, to check that the keywords are sensible
|
|
976
|
+
y = yaml.compose(doc, Loader=yaml.SafeLoader)
|
|
977
|
+
# first should be MappingNode
|
|
978
|
+
if not isinstance(y, yaml.MappingNode):
|
|
979
|
+
self.control_file_error("configuration file must begin with a mapping")
|
|
980
|
+
|
|
981
|
+
yaml_walk(self, y.value, [])
|
|
982
|
+
except yaml.scanner.ScannerError as e:
|
|
983
|
+
self.log(self.ERROR, "configuration file error: %s, line %s" % \
|
|
984
|
+
(e.problem, e.problem_mark.line))
|
|
985
|
+
except yaml.reader.ReaderError as e:
|
|
986
|
+
es = "unacceptable character #x%04x at byte %d: %s" \
|
|
987
|
+
% (e.character, e.position, e.reason)
|
|
988
|
+
self.log(self.ERROR, es)
|
|
989
|
+
except yaml.error.MarkedYAMLError as e:
|
|
990
|
+
self.log(self.ERROR, "configuration file error: %s, line %s" % \
|
|
991
|
+
(e.problem, e.problem_mark.line))
|
|
992
|
+
|
|
993
|
+
f.close()
|
|
994
|
+
self.subprocesses = []
|
|
995
|
+
self.log(self.INFO,'gpload session started ' + \
|
|
996
|
+
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
|
|
997
|
+
|
|
998
|
+
def control_file_warning(self, msg):
|
|
999
|
+
self.log(self.WARN, "A gpload control file processing warning occurred. %s" % msg)
|
|
1000
|
+
|
|
1001
|
+
def control_file_error(self, msg):
|
|
1002
|
+
self.log(self.ERROR, "A gpload control file processing error occurred. %s" % msg)
|
|
1003
|
+
|
|
1004
|
+
def elevel2str(self, level):
|
|
1005
|
+
if level == self.DEBUG:
|
|
1006
|
+
return "DEBUG"
|
|
1007
|
+
elif level == self.LOG:
|
|
1008
|
+
return "LOG"
|
|
1009
|
+
elif level == self.INFO:
|
|
1010
|
+
return "INFO"
|
|
1011
|
+
elif level == self.ERROR:
|
|
1012
|
+
return "ERROR"
|
|
1013
|
+
elif level == self.WARN:
|
|
1014
|
+
return "WARN"
|
|
1015
|
+
else:
|
|
1016
|
+
self.log(self.ERROR, "unknown log type %i" % level)
|
|
1017
|
+
|
|
1018
|
+
def log(self, level, a):
|
|
1019
|
+
"""
|
|
1020
|
+
Level is either DEBUG, LOG, INFO, ERROR. a is the message
|
|
1021
|
+
"""
|
|
1022
|
+
t = time.localtime()
|
|
1023
|
+
message = '|'.join(
|
|
1024
|
+
[datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S'),
|
|
1025
|
+
self.elevel2str(level), a]) + '\n'
|
|
1026
|
+
|
|
1027
|
+
if level <= self.options.qv:
|
|
1028
|
+
sys.stdout.write(message)
|
|
1029
|
+
|
|
1030
|
+
if level <= self.options.qv or level <= self.INFO:
|
|
1031
|
+
try:
|
|
1032
|
+
self.logfile.write(message)
|
|
1033
|
+
self.logfile.flush()
|
|
1034
|
+
except AttributeError as e:
|
|
1035
|
+
pass
|
|
1036
|
+
|
|
1037
|
+
if level == self.ERROR:
|
|
1038
|
+
self.exitValue = 2;
|
|
1039
|
+
sys.exit(self.exitValue)
|
|
1040
|
+
|
|
1041
|
+
def getconfig(self, a, typ=None, default='error', extraStuff='', returnOriginal=False):
|
|
1042
|
+
"""
|
|
1043
|
+
Look for a config entry, via a column delimited string. a:b:c points to
|
|
1044
|
+
|
|
1045
|
+
a:
|
|
1046
|
+
b:
|
|
1047
|
+
c
|
|
1048
|
+
|
|
1049
|
+
Make sure that end point is of type 'typ' when not set to None.
|
|
1050
|
+
|
|
1051
|
+
If returnOriginal is False, the return value will be in lower case,
|
|
1052
|
+
else the return value will be in its original form (i.e. the case that
|
|
1053
|
+
the user specified in their yaml file).
|
|
1054
|
+
"""
|
|
1055
|
+
self.log(self.DEBUG, "getting config for " + a)
|
|
1056
|
+
if returnOriginal == True:
|
|
1057
|
+
config = self.configOriginal
|
|
1058
|
+
else:
|
|
1059
|
+
config = self.config
|
|
1060
|
+
for s in a.split(':'):
|
|
1061
|
+
self.log(self.DEBUG, "trying " + s)
|
|
1062
|
+
index = 1
|
|
1063
|
+
|
|
1064
|
+
if s[-1:]==')':
|
|
1065
|
+
j = s.index('(')
|
|
1066
|
+
index = int(s[j+1:-1])
|
|
1067
|
+
s = s[:j]
|
|
1068
|
+
|
|
1069
|
+
if type(config)!=list:
|
|
1070
|
+
config = [config]
|
|
1071
|
+
|
|
1072
|
+
for c in config:
|
|
1073
|
+
if type(c)==dict:
|
|
1074
|
+
temp = caseInsensitiveDictLookup(s, c)
|
|
1075
|
+
if temp != None:
|
|
1076
|
+
index -= 1
|
|
1077
|
+
if not index:
|
|
1078
|
+
self.log(self.DEBUG, "found " + s)
|
|
1079
|
+
config = temp
|
|
1080
|
+
break
|
|
1081
|
+
else:
|
|
1082
|
+
if default=='error':
|
|
1083
|
+
self.control_file_error("The configuration must contain %s%s"%(a,extraStuff))
|
|
1084
|
+
sys.exit(2)
|
|
1085
|
+
return default
|
|
1086
|
+
|
|
1087
|
+
if typ != None and type(config) != typ:
|
|
1088
|
+
if typ == list:
|
|
1089
|
+
self.control_file_error("The %s entry must be a YAML sequence %s"% (a ,extraStuff))
|
|
1090
|
+
elif typ == dict:
|
|
1091
|
+
self.control_file_error("The %s entry must be a YAML mapping %s"% (a, extraStuff))
|
|
1092
|
+
elif typ == str:
|
|
1093
|
+
self.control_file_error("%s must be a string %s" % (a, extraStuff))
|
|
1094
|
+
elif typ == int:
|
|
1095
|
+
self.control_file_error("The %s entry must be a YAML integer %s" % (a, extraStuff))
|
|
1096
|
+
else:
|
|
1097
|
+
assert 0
|
|
1098
|
+
|
|
1099
|
+
self.control_file_error("Encountered unknown configuration type %s"% type(config))
|
|
1100
|
+
sys.exit(2)
|
|
1101
|
+
return config
|
|
1102
|
+
|
|
1103
|
+
def read_config(self):
|
|
1104
|
+
"""
|
|
1105
|
+
Configure ourselves
|
|
1106
|
+
"""
|
|
1107
|
+
|
|
1108
|
+
# ensure output is of type list
|
|
1109
|
+
self.getconfig('gpload:output', list)
|
|
1110
|
+
|
|
1111
|
+
# The user supplied table name can be completely or partially delimited,
|
|
1112
|
+
# and it can be a one or two part name. Get the originally supplied name
|
|
1113
|
+
# and parse it into its delimited one or two part name.
|
|
1114
|
+
self.schemaTable = self.getconfig('gpload:output:table', str, returnOriginal=True)
|
|
1115
|
+
schemaTableList = splitUpMultipartIdentifier(self.schemaTable)
|
|
1116
|
+
schemaTableList = convertListToDelimited(schemaTableList)
|
|
1117
|
+
if len(schemaTableList) == 2:
|
|
1118
|
+
self.schema = schemaTableList[0]
|
|
1119
|
+
self.table = schemaTableList[1]
|
|
1120
|
+
else:
|
|
1121
|
+
self.schema = None
|
|
1122
|
+
self.table = schemaTableList[0]
|
|
1123
|
+
|
|
1124
|
+
# Precendence for configuration: command line > config file > env
|
|
1125
|
+
# variable
|
|
1126
|
+
|
|
1127
|
+
# host to connect to
|
|
1128
|
+
if not self.options.h:
|
|
1129
|
+
self.options.h = self.getconfig('host', str, None)
|
|
1130
|
+
if self.options.h:
|
|
1131
|
+
self.options.h = str(self.options.h)
|
|
1132
|
+
if not self.options.h:
|
|
1133
|
+
self.options.h = os.environ.get('PGHOST')
|
|
1134
|
+
if not self.options.h or len(self.options.h) == 0:
|
|
1135
|
+
self.log(self.INFO, "no host supplied, defaulting to localhost")
|
|
1136
|
+
self.options.h = "localhost"
|
|
1137
|
+
|
|
1138
|
+
# Port to connect to
|
|
1139
|
+
if not self.options.p:
|
|
1140
|
+
self.options.p = self.getconfig('port',int,None)
|
|
1141
|
+
if not self.options.p:
|
|
1142
|
+
try:
|
|
1143
|
+
self.options.p = int(os.environ.get('PGPORT'))
|
|
1144
|
+
except (ValueError, TypeError):
|
|
1145
|
+
pass
|
|
1146
|
+
if not self.options.p:
|
|
1147
|
+
self.options.p = 5432
|
|
1148
|
+
|
|
1149
|
+
# User to connect as
|
|
1150
|
+
if not self.options.U:
|
|
1151
|
+
self.options.U = self.getconfig('user', str, None)
|
|
1152
|
+
if not self.options.U:
|
|
1153
|
+
self.options.U = os.environ.get('PGUSER')
|
|
1154
|
+
if not self.options.U:
|
|
1155
|
+
self.options.U = os.environ.get('USER') or \
|
|
1156
|
+
os.environ.get('LOGNAME') or \
|
|
1157
|
+
os.environ.get('USERNAME')
|
|
1158
|
+
|
|
1159
|
+
if not self.options.U or len(self.options.U) == 0:
|
|
1160
|
+
self.log(self.ERROR,
|
|
1161
|
+
"You need to specify your username with the -U " +
|
|
1162
|
+
"option or in your configuration or in your " +
|
|
1163
|
+
"environment as PGUSER")
|
|
1164
|
+
|
|
1165
|
+
# database to connect to
|
|
1166
|
+
if not self.options.d:
|
|
1167
|
+
self.options.d = self.getconfig('database', str, None)
|
|
1168
|
+
if not self.options.d:
|
|
1169
|
+
self.options.d = os.environ.get('PGDATABASE')
|
|
1170
|
+
if not self.options.d:
|
|
1171
|
+
# like libpq, just inherit USER
|
|
1172
|
+
self.options.d = self.options.U
|
|
1173
|
+
|
|
1174
|
+
if self.getconfig('gpload:input:error_table', str, None):
|
|
1175
|
+
self.control_file_error("ERROR_TABLE is not supported. Please use LOG_ERRORS instead.")
|
|
1176
|
+
|
|
1177
|
+
def gpfdist_port_options(self, name, availablePorts, popenList):
|
|
1178
|
+
"""
|
|
1179
|
+
Adds gpfdist -p / -P port options to popenList based on port and port_range in YAML file.
|
|
1180
|
+
Raises errors if options are invalid or ports are unavailable.
|
|
1181
|
+
|
|
1182
|
+
@param name: input source name from YAML file.
|
|
1183
|
+
@param availablePorts: current set of available ports
|
|
1184
|
+
@param popenList: gpfdist options (updated)
|
|
1185
|
+
"""
|
|
1186
|
+
port = self.getconfig(name + ':port', int, None)
|
|
1187
|
+
port_range = self.getconfig(name+':port_range', list, None)
|
|
1188
|
+
|
|
1189
|
+
if port:
|
|
1190
|
+
startPort = endPort = port
|
|
1191
|
+
endPort += 1
|
|
1192
|
+
elif port_range:
|
|
1193
|
+
try:
|
|
1194
|
+
startPort = int(port_range[0])
|
|
1195
|
+
endPort = int(port_range[1])
|
|
1196
|
+
except (IndexError,ValueError):
|
|
1197
|
+
self.control_file_error(name + ":port_range must be a YAML sequence of two integers")
|
|
1198
|
+
else:
|
|
1199
|
+
startPort = self.getconfig(name+':port',int,8000)
|
|
1200
|
+
endPort = self.getconfig(name+':port',int,9000)
|
|
1201
|
+
|
|
1202
|
+
if (startPort > 65535 or endPort > 65535):
|
|
1203
|
+
# Do not allow invalid ports
|
|
1204
|
+
self.control_file_error("Invalid port. Port values must be less than or equal to 65535.")
|
|
1205
|
+
elif not (set(range(startPort,endPort+1)) & availablePorts):
|
|
1206
|
+
self.log(self.ERROR, "no more ports available for gpfdist")
|
|
1207
|
+
|
|
1208
|
+
popenList.append('-p')
|
|
1209
|
+
popenList.append(str(startPort))
|
|
1210
|
+
|
|
1211
|
+
popenList.append('-P')
|
|
1212
|
+
popenList.append(str(endPort))
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
def gpfdist_filenames(self, name, popenList):
|
|
1216
|
+
"""
|
|
1217
|
+
Adds gpfdist -f filenames to popenList.
|
|
1218
|
+
Raises errors if YAML file option is invalid.
|
|
1219
|
+
|
|
1220
|
+
@param name: input source name from YAML file.
|
|
1221
|
+
@param popenList: gpfdist options (updated)
|
|
1222
|
+
@return: list of files names
|
|
1223
|
+
"""
|
|
1224
|
+
file = self.getconfig(name+':file',list)
|
|
1225
|
+
for i in file:
|
|
1226
|
+
if not isinstance(i, str):
|
|
1227
|
+
self.control_file_error(name + ":file must be a YAML sequence of strings")
|
|
1228
|
+
popenList.append('-f')
|
|
1229
|
+
popenList.append('"'+' '.join(file)+'"')
|
|
1230
|
+
return file
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
def gpfdist_timeout_options(self, popenList):
|
|
1234
|
+
"""
|
|
1235
|
+
Adds gpfdist -t timeout option to popenList.
|
|
1236
|
+
|
|
1237
|
+
@param popenList: gpfdist options (updated)
|
|
1238
|
+
"""
|
|
1239
|
+
if self.options.gpfdist_timeout != None:
|
|
1240
|
+
gpfdistTimeout = self.options.gpfdist_timeout
|
|
1241
|
+
else:
|
|
1242
|
+
gpfdistTimeout = 30
|
|
1243
|
+
popenList.append('-t')
|
|
1244
|
+
popenList.append(str(gpfdistTimeout))
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
def gpfdist_verbose_options(self, popenList):
|
|
1248
|
+
"""
|
|
1249
|
+
Adds gpfdist -v / -V options to popenList depending on logging level
|
|
1250
|
+
|
|
1251
|
+
@param popenList: gpfdist options (updated)
|
|
1252
|
+
"""
|
|
1253
|
+
if self.options.qv == self.LOG:
|
|
1254
|
+
popenList.append('-v')
|
|
1255
|
+
elif self.options.qv > self.LOG:
|
|
1256
|
+
popenList.append('-V')
|
|
1257
|
+
|
|
1258
|
+
|
|
1259
|
+
def gpfdist_max_line_length(self, popenList):
|
|
1260
|
+
"""
|
|
1261
|
+
Adds gpfdist -m option to popenList when max_line_length option specified in YAML file.
|
|
1262
|
+
|
|
1263
|
+
@param popenList: gpfdist options (updated)
|
|
1264
|
+
"""
|
|
1265
|
+
max_line_length = self.getconfig('gpload:input:max_line_length',int,None)
|
|
1266
|
+
if max_line_length is not None:
|
|
1267
|
+
popenList.append('-m')
|
|
1268
|
+
popenList.append(str(max_line_length))
|
|
1269
|
+
|
|
1270
|
+
|
|
1271
|
+
def gpfdist_transform(self, popenList):
|
|
1272
|
+
"""
|
|
1273
|
+
Compute and return url fragment if transform option specified in YAML file.
|
|
1274
|
+
Checks for readable transform config file if transform_config option is specified.
|
|
1275
|
+
Adds gpfdist -c option to popenList if transform_config is specified.
|
|
1276
|
+
Validates that transform_config is present when transform option is specified.
|
|
1277
|
+
|
|
1278
|
+
@param popenList: gpfdist options (updated)
|
|
1279
|
+
@returns: uri fragment for transform or "" if not appropriate.
|
|
1280
|
+
"""
|
|
1281
|
+
transform = self.getconfig('gpload:input:transform', str, None)
|
|
1282
|
+
transform_config = self.getconfig('gpload:input:transform_config', str, None)
|
|
1283
|
+
if transform_config:
|
|
1284
|
+
try:
|
|
1285
|
+
f = open(transform_config,'r')
|
|
1286
|
+
except IOError as e:
|
|
1287
|
+
self.log(self.ERROR, "could not open transform_config file: %s" % e)
|
|
1288
|
+
f.close()
|
|
1289
|
+
popenList.append('-c')
|
|
1290
|
+
popenList.append(transform_config)
|
|
1291
|
+
else:
|
|
1292
|
+
if transform:
|
|
1293
|
+
self.control_file_error("transform_config is required when transform is specified")
|
|
1294
|
+
|
|
1295
|
+
fragment = ""
|
|
1296
|
+
if transform is not None:
|
|
1297
|
+
fragment = "#transform=" + transform
|
|
1298
|
+
return fragment
|
|
1299
|
+
|
|
1300
|
+
|
|
1301
|
+
def gpfdist_ssl(self, popenList):
|
|
1302
|
+
"""
|
|
1303
|
+
Adds gpfdist --ssl option to popenList when ssl option specified as true in YAML file.
|
|
1304
|
+
|
|
1305
|
+
@param popenList: gpfdist options (updated)
|
|
1306
|
+
"""
|
|
1307
|
+
ssl = self.getconfig('gpload:input:source:ssl',bool, False)
|
|
1308
|
+
certificates_path = self.getconfig('gpload:input:source:certificates_path', str, None)
|
|
1309
|
+
|
|
1310
|
+
if ssl and certificates_path:
|
|
1311
|
+
dir_exists = os.path.isdir(certificates_path)
|
|
1312
|
+
if dir_exists == False:
|
|
1313
|
+
self.log(self.ERROR, "could not access CERTIFICATES_PATH directory: %s" % certificates_path)
|
|
1314
|
+
|
|
1315
|
+
popenList.append('--ssl')
|
|
1316
|
+
popenList.append(certificates_path)
|
|
1317
|
+
|
|
1318
|
+
else:
|
|
1319
|
+
if ssl:
|
|
1320
|
+
self.control_file_error("CERTIFICATES_PATH is required when SSL is specified as true")
|
|
1321
|
+
elif certificates_path: # ssl=false (or not specified) and certificates_path is specified
|
|
1322
|
+
self.control_file_error("CERTIFICATES_PATH is specified while SSL is not specified as true")
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def start_gpfdists(self):
|
|
1326
|
+
"""
|
|
1327
|
+
Start gpfdist daemon(s)
|
|
1328
|
+
"""
|
|
1329
|
+
self.locations = []
|
|
1330
|
+
self.ports = []
|
|
1331
|
+
sourceIndex = 0
|
|
1332
|
+
availablePorts = set(range(1,65535))
|
|
1333
|
+
found_source = False
|
|
1334
|
+
|
|
1335
|
+
self.getconfig('gpload:input', list)
|
|
1336
|
+
|
|
1337
|
+
while 1:
|
|
1338
|
+
sourceIndex += 1
|
|
1339
|
+
name = 'gpload:input:source(%d)'%sourceIndex
|
|
1340
|
+
a = self.getconfig(name,None,None)
|
|
1341
|
+
if not a:
|
|
1342
|
+
break
|
|
1343
|
+
found_source = True
|
|
1344
|
+
local_hostname = self.getconfig(name+':local_hostname', list, False)
|
|
1345
|
+
|
|
1346
|
+
# do default host, the current one
|
|
1347
|
+
if not local_hostname:
|
|
1348
|
+
# if fully_qualified_domain_name is defined and set to true we want to
|
|
1349
|
+
# resolve the fqdn rather than just grabbing the hostname.
|
|
1350
|
+
fqdn = self.getconfig('gpload:input:fully_qualified_domain_name', bool, False)
|
|
1351
|
+
if fqdn:
|
|
1352
|
+
local_hostname = socket.getfqdn()
|
|
1353
|
+
else:
|
|
1354
|
+
local_hostname = socket.gethostname()
|
|
1355
|
+
|
|
1356
|
+
# build gpfdist parameters
|
|
1357
|
+
popenList = ['gpfdist']
|
|
1358
|
+
self.gpfdist_ssl(popenList)
|
|
1359
|
+
self.gpfdist_port_options(name, availablePorts, popenList)
|
|
1360
|
+
file = self.gpfdist_filenames(name, popenList)
|
|
1361
|
+
self.gpfdist_timeout_options(popenList)
|
|
1362
|
+
self.gpfdist_verbose_options(popenList)
|
|
1363
|
+
self.gpfdist_max_line_length(popenList)
|
|
1364
|
+
fragment = self.gpfdist_transform(popenList)
|
|
1365
|
+
|
|
1366
|
+
try:
|
|
1367
|
+
self.log(self.LOG, 'trying to run %s' % ' '.join(popenList))
|
|
1368
|
+
cfds = True
|
|
1369
|
+
if platform.system() in ['Windows', 'Microsoft']: # not supported on win32
|
|
1370
|
+
cfds = False
|
|
1371
|
+
cmd = ' '.join(popenList)
|
|
1372
|
+
needshell = False
|
|
1373
|
+
else:
|
|
1374
|
+
srcfile = None
|
|
1375
|
+
if os.environ.get('GPHOME_LOADERS'):
|
|
1376
|
+
srcfile = os.path.join(os.environ.get('GPHOME_LOADERS'),
|
|
1377
|
+
'greenplum_loaders_path.sh')
|
|
1378
|
+
elif os.environ.get('GPHOME'):
|
|
1379
|
+
srcfile = os.path.join(os.environ.get('GPHOME'),
|
|
1380
|
+
'greenplum_path.sh')
|
|
1381
|
+
|
|
1382
|
+
if (not (srcfile and os.path.exists(srcfile))):
|
|
1383
|
+
self.log(self.ERROR, 'cannot find greenplum environment ' +
|
|
1384
|
+
'file: environment misconfigured')
|
|
1385
|
+
|
|
1386
|
+
cmd = '. %s ; exec ' % srcfile
|
|
1387
|
+
cmd += ' '.join(popenList)
|
|
1388
|
+
needshell = True
|
|
1389
|
+
|
|
1390
|
+
a = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
|
1391
|
+
stderr=subprocess.PIPE,
|
|
1392
|
+
close_fds=cfds, shell=needshell,
|
|
1393
|
+
text=True, encoding='utf-8',
|
|
1394
|
+
errors='replace')
|
|
1395
|
+
self.subprocesses.append(a)
|
|
1396
|
+
except Exception as e:
|
|
1397
|
+
self.log(self.ERROR, "could not run %s: %s" % \
|
|
1398
|
+
(' '.join(popenList), str(e)))
|
|
1399
|
+
|
|
1400
|
+
"""
|
|
1401
|
+
Reading from stderr and stdout on a Popen object can result in a dead lock if done at the same time.
|
|
1402
|
+
Create a lock to share when reading stderr and stdout from gpfdist.
|
|
1403
|
+
"""
|
|
1404
|
+
readLock = threading.Lock()
|
|
1405
|
+
|
|
1406
|
+
# get all the output from the daemon(s)
|
|
1407
|
+
t = CatThread(self,a.stderr, readLock)
|
|
1408
|
+
t.start()
|
|
1409
|
+
self.threads.append(t)
|
|
1410
|
+
|
|
1411
|
+
while 1:
|
|
1412
|
+
readLock.acquire()
|
|
1413
|
+
line = a.stdout.readline()
|
|
1414
|
+
readLock.release()
|
|
1415
|
+
if line=='':
|
|
1416
|
+
self.log(self.ERROR,'failed to start gpfdist: ' +
|
|
1417
|
+
'gpfdist command line: ' + ' '.join(popenList))
|
|
1418
|
+
|
|
1419
|
+
line = line.strip('\n')
|
|
1420
|
+
self.log(self.LOG,'gpfdist says: ' + line)
|
|
1421
|
+
if (line.startswith('Serving HTTP on port ') or line.startswith('Serving HTTPS on port ')):
|
|
1422
|
+
port = int(line[21:line.index(',')])
|
|
1423
|
+
break
|
|
1424
|
+
|
|
1425
|
+
self.log(self.INFO, 'started %s' % ' '.join(popenList))
|
|
1426
|
+
self.log(self.LOG,'gpfdist is running on port %d'%port)
|
|
1427
|
+
if port in availablePorts:
|
|
1428
|
+
availablePorts.remove(port)
|
|
1429
|
+
self.ports.append(port)
|
|
1430
|
+
t = CatThread(self,a.stdout,readLock)
|
|
1431
|
+
t.start()
|
|
1432
|
+
self.threads.append(t)
|
|
1433
|
+
|
|
1434
|
+
ssl = self.getconfig('gpload:input:source:ssl', bool, False)
|
|
1435
|
+
if ssl:
|
|
1436
|
+
protocol = 'gpfdists'
|
|
1437
|
+
else:
|
|
1438
|
+
protocol = 'gpfdist'
|
|
1439
|
+
|
|
1440
|
+
for l in local_hostname:
|
|
1441
|
+
if not isinstance(l, str):
|
|
1442
|
+
self.control_file_error(name + ":local_hostname must be a YAML sequence of strings")
|
|
1443
|
+
l = str(l)
|
|
1444
|
+
sep = ''
|
|
1445
|
+
if file[0] != '/':
|
|
1446
|
+
sep = '/'
|
|
1447
|
+
# MPP-13617
|
|
1448
|
+
if ':' in l:
|
|
1449
|
+
l = '[' + l + ']'
|
|
1450
|
+
self.locations.append('%s://%s:%d%s%s%s' % (protocol, l, port, sep, '%20'.join(file), fragment))
|
|
1451
|
+
if not found_source:
|
|
1452
|
+
self.control_file_error("configuration file must contain source definition")
|
|
1453
|
+
|
|
1454
|
+
def readPgpass(self,pgpassname):
|
|
1455
|
+
"""
|
|
1456
|
+
Get password form .pgpass file
|
|
1457
|
+
"""
|
|
1458
|
+
try:
|
|
1459
|
+
f = open(pgpassname,'r')
|
|
1460
|
+
except IOError:
|
|
1461
|
+
return
|
|
1462
|
+
for row in f:
|
|
1463
|
+
try:
|
|
1464
|
+
row = row.rstrip("\n")
|
|
1465
|
+
line = splitPgpassLine(row)
|
|
1466
|
+
if line[0]!='*' and line[0].lower()!=self.options.h.lower():
|
|
1467
|
+
continue
|
|
1468
|
+
if line[1]!='*' and int(line[1])!=self.options.p:
|
|
1469
|
+
continue
|
|
1470
|
+
if line[2]!='*' and line[2]!=self.options.d:
|
|
1471
|
+
continue
|
|
1472
|
+
if line[3]!='*' and line[3]!=self.options.U:
|
|
1473
|
+
continue
|
|
1474
|
+
self.options.password = line[4]
|
|
1475
|
+
break
|
|
1476
|
+
except (ValueError,IndexError):
|
|
1477
|
+
pass
|
|
1478
|
+
f.close()
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
def setup_connection(self, recurse = 0):
|
|
1482
|
+
"""
|
|
1483
|
+
Connect to the backend
|
|
1484
|
+
"""
|
|
1485
|
+
if self.db != None:
|
|
1486
|
+
self.db.close()
|
|
1487
|
+
self.db = None
|
|
1488
|
+
if self.options.W:
|
|
1489
|
+
if self.options.password==None:
|
|
1490
|
+
self.options.password = getpass.getpass()
|
|
1491
|
+
else:
|
|
1492
|
+
if self.options.password==None:
|
|
1493
|
+
self.options.password = self.getconfig('password', str,
|
|
1494
|
+
None)
|
|
1495
|
+
if self.options.password==None:
|
|
1496
|
+
self.options.password = os.environ.get('PGPASSWORD')
|
|
1497
|
+
if self.options.password==None:
|
|
1498
|
+
self.readPgpass(os.environ.get('PGPASSFILE',
|
|
1499
|
+
os.environ.get('HOME','.')+'/.pgpass'))
|
|
1500
|
+
try:
|
|
1501
|
+
self.log(self.DEBUG, "connection string:" +
|
|
1502
|
+
" user=" + str(self.options.U) +
|
|
1503
|
+
" host=" + str(self.options.h) +
|
|
1504
|
+
" port=" + str(self.options.p) +
|
|
1505
|
+
" database=" + str(self.options.d))
|
|
1506
|
+
self.db = db_connect( dbname=self.options.d
|
|
1507
|
+
, host=self.options.h
|
|
1508
|
+
, port=self.options.p
|
|
1509
|
+
, user=self.options.U
|
|
1510
|
+
, passwd=self.options.password
|
|
1511
|
+
)
|
|
1512
|
+
_install_notice_receiver(self.db)
|
|
1513
|
+
self.log(self.DEBUG, "Successfully connected to database")
|
|
1514
|
+
except Exception as e:
|
|
1515
|
+
errorMessage = str(e)
|
|
1516
|
+
if errorMessage.find("no password supplied") != -1:
|
|
1517
|
+
self.options.password = getpass.getpass()
|
|
1518
|
+
recurse += 1
|
|
1519
|
+
if recurse > 10:
|
|
1520
|
+
self.log(self.ERROR, "too many login attempt failures")
|
|
1521
|
+
self.setup_connection(recurse)
|
|
1522
|
+
else:
|
|
1523
|
+
self.log(self.ERROR, "could not connect to database: %s. Is " \
|
|
1524
|
+
"the Greenplum Database running on port %i?" % (errorMessage,
|
|
1525
|
+
self.options.p))
|
|
1526
|
+
|
|
1527
|
+
def read_columns(self):
|
|
1528
|
+
columns = self.getconfig('gpload:input:columns',list,None, returnOriginal=True)
|
|
1529
|
+
if columns != None:
|
|
1530
|
+
self.from_cols_from_user = True # user specified from columns
|
|
1531
|
+
self.from_columns = []
|
|
1532
|
+
for d in columns:
|
|
1533
|
+
if type(d)!=dict:
|
|
1534
|
+
self.control_file_error("gpload:input:columns must be a sequence of YAML mappings")
|
|
1535
|
+
tempkey = next(iter(d))
|
|
1536
|
+
value = d[tempkey]
|
|
1537
|
+
""" remove leading or trailing spaces """
|
|
1538
|
+
d = { tempkey.strip() : value }
|
|
1539
|
+
key = next(iter(d))
|
|
1540
|
+
if d[key] == None:
|
|
1541
|
+
self.log(self.DEBUG,
|
|
1542
|
+
'getting source column data type from target')
|
|
1543
|
+
for name, typ, mapto, hasseq in self.into_columns:
|
|
1544
|
+
if sqlIdentifierCompare(name, key):
|
|
1545
|
+
d[key] = typ
|
|
1546
|
+
break
|
|
1547
|
+
|
|
1548
|
+
# perform the same kind of magic type change that postgres does
|
|
1549
|
+
if d[key] == 'bigserial':
|
|
1550
|
+
d[key] = 'bigint'
|
|
1551
|
+
elif d[key] == 'serial':
|
|
1552
|
+
d[key] = 'int4'
|
|
1553
|
+
|
|
1554
|
+
# Mark this column as having no mapping, which is important
|
|
1555
|
+
# for do_insert()
|
|
1556
|
+
self.from_columns.append([key,d[key],None, False])
|
|
1557
|
+
else:
|
|
1558
|
+
self.from_columns = self.into_columns
|
|
1559
|
+
self.from_cols_from_user = False
|
|
1560
|
+
|
|
1561
|
+
# make sure that all columns have a type
|
|
1562
|
+
for name, typ, map, hasseq in self.from_columns:
|
|
1563
|
+
if typ == None:
|
|
1564
|
+
self.log(self.ERROR, 'column "%s" has no type ' % name +
|
|
1565
|
+
'and does not appear in target table "%s"' % self.schemaTable)
|
|
1566
|
+
self.log(self.DEBUG, 'from columns are:')
|
|
1567
|
+
for c in self.from_columns:
|
|
1568
|
+
name = c[0]
|
|
1569
|
+
typ = c[1]
|
|
1570
|
+
self.log(self.DEBUG, '%s: %s'%(name,typ))
|
|
1571
|
+
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
def read_table_metadata(self):
|
|
1575
|
+
# KAS Note to self. If schema is specified, then probably should use PostgreSQL rules for defining it.
|
|
1576
|
+
|
|
1577
|
+
# find the shema name for this table (according to search_path)
|
|
1578
|
+
# if it was not explicitly specified in the configuration file.
|
|
1579
|
+
if self.schema == None:
|
|
1580
|
+
queryString = """SELECT n.nspname
|
|
1581
|
+
FROM pg_catalog.pg_class c
|
|
1582
|
+
LEFT JOIN pg_catalog.pg_namespace n
|
|
1583
|
+
ON n.oid = c.relnamespace
|
|
1584
|
+
WHERE c.relname = '%s'
|
|
1585
|
+
AND pg_catalog.pg_table_is_visible(c.oid);""" % quote_unident(self.table)
|
|
1586
|
+
|
|
1587
|
+
resultList = self.db.query(to_db_text(queryString)).getresult()
|
|
1588
|
+
|
|
1589
|
+
if len(resultList) > 0:
|
|
1590
|
+
self.schema = to_text((resultList[0])[0])
|
|
1591
|
+
self.log(self.INFO, "setting schema '%s' for table '%s'" % (self.schema, quote_unident(self.table)))
|
|
1592
|
+
else:
|
|
1593
|
+
self.log(self.ERROR, "table %s not found in any database schema" % self.table)
|
|
1594
|
+
|
|
1595
|
+
|
|
1596
|
+
queryString = """select nt.nspname as table_schema,
|
|
1597
|
+
c.relname as table_name,
|
|
1598
|
+
a.attname as column_name,
|
|
1599
|
+
a.attnum as ordinal_position,
|
|
1600
|
+
format_type(a.atttypid, a.atttypmod) as data_type,
|
|
1601
|
+
c.relkind = 'r' AS is_updatable,
|
|
1602
|
+
a.atttypid in (23, 20) and a.atthasdef and
|
|
1603
|
+
(select position ( 'nextval(' in pg_catalog.pg_get_expr(adbin,adrelid) ) > 0 and
|
|
1604
|
+
position ( '::regclass)' in pg_catalog.pg_get_expr(adbin,adrelid) ) > 0
|
|
1605
|
+
FROM pg_catalog.pg_attrdef d
|
|
1606
|
+
WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as has_sequence
|
|
1607
|
+
from pg_catalog.pg_class c join pg_catalog.pg_namespace nt on (c.relnamespace = nt.oid)
|
|
1608
|
+
join pg_attribute a on (a.attrelid = c.oid)
|
|
1609
|
+
where c.relname = '%s' and nt.nspname = '%s'
|
|
1610
|
+
and a.attnum > 0 and a.attisdropped = 'f'
|
|
1611
|
+
order by a.attnum """ % (quote_unident(self.table), quote_unident(self.schema))
|
|
1612
|
+
|
|
1613
|
+
count = 0
|
|
1614
|
+
self.into_columns = []
|
|
1615
|
+
self.into_columns_dict = dict()
|
|
1616
|
+
resultList = self.db.query(to_db_text(queryString)).dictresult()
|
|
1617
|
+
while count < len(resultList):
|
|
1618
|
+
row = resultList[count]
|
|
1619
|
+
count += 1
|
|
1620
|
+
ct = to_text(row['data_type'])
|
|
1621
|
+
if ct == 'bigserial':
|
|
1622
|
+
ct = 'bigint'
|
|
1623
|
+
elif ct == 'serial':
|
|
1624
|
+
ct = 'int4'
|
|
1625
|
+
name = to_text(row['column_name'])
|
|
1626
|
+
name = quote_ident(name)
|
|
1627
|
+
has_seq = to_bool(row['has_sequence'])
|
|
1628
|
+
i = [name,ct,None, has_seq]
|
|
1629
|
+
self.into_columns.append(i)
|
|
1630
|
+
self.into_columns_dict[name] = i
|
|
1631
|
+
self.log(self.DEBUG, "found input column: " + str(i))
|
|
1632
|
+
if count == 0:
|
|
1633
|
+
# see if it's a permissions issue or it actually doesn't exist
|
|
1634
|
+
tableName = quote_unident(self.table)
|
|
1635
|
+
tableSchema = quote_unident(self.schema)
|
|
1636
|
+
sql = """select 1 from pg_class c, pg_namespace n
|
|
1637
|
+
where c.relname = '%s' and
|
|
1638
|
+
n.nspname = '%s' and
|
|
1639
|
+
n.oid = c.relnamespace""" % (tableName, tableSchema)
|
|
1640
|
+
resultList = self.db.query(to_db_text(sql)).getresult()
|
|
1641
|
+
if len(resultList) > 0:
|
|
1642
|
+
self.log(self.ERROR, "permission denied for table %s.%s" % \
|
|
1643
|
+
(tableSchema, tableName))
|
|
1644
|
+
else:
|
|
1645
|
+
self.log(self.ERROR, 'table %s.%s does not exist in database %s'% (tableSchema, tableName, self.options.d))
|
|
1646
|
+
|
|
1647
|
+
def read_mapping(self):
|
|
1648
|
+
mapping = self.getconfig('gpload:output:mapping',dict,None, returnOriginal=True)
|
|
1649
|
+
|
|
1650
|
+
if mapping:
|
|
1651
|
+
for key,value in mapping.items():
|
|
1652
|
+
if not isinstance(key, str) or not isinstance(value, str):
|
|
1653
|
+
self.control_file_error("gpload:output:mapping must be a YAML type mapping from strings to strings")
|
|
1654
|
+
found = False
|
|
1655
|
+
for a in self.into_columns:
|
|
1656
|
+
if sqlIdentifierCompare(a[0], key) == True:
|
|
1657
|
+
a[2] = value
|
|
1658
|
+
found = True
|
|
1659
|
+
break
|
|
1660
|
+
if found == False:
|
|
1661
|
+
self.log(self.ERROR,'%s in mapping is not in table %s'% \
|
|
1662
|
+
(key, self.schemaTable))
|
|
1663
|
+
else:
|
|
1664
|
+
# Now, map anything yet to be mapped to itself, picking up on those
|
|
1665
|
+
# columns which are not found in the table.
|
|
1666
|
+
for x in self.from_columns:
|
|
1667
|
+
# Check to see if it already has a mapping value
|
|
1668
|
+
i = list(filter(lambda a:a[2] == x[0], self.into_columns))
|
|
1669
|
+
if not i:
|
|
1670
|
+
# Check to see if the target column names match the input column names.
|
|
1671
|
+
for a in self.into_columns:
|
|
1672
|
+
if sqlIdentifierCompare(a[0], x[0]) == True:
|
|
1673
|
+
i = a
|
|
1674
|
+
found = True
|
|
1675
|
+
break
|
|
1676
|
+
if i:
|
|
1677
|
+
if i[2] == None: i[2] = i[0]
|
|
1678
|
+
else:
|
|
1679
|
+
self.log(self.ERROR, 'no mapping for input column ' +
|
|
1680
|
+
'"%s" to output table' % x[0])
|
|
1681
|
+
for name,typ,mapto,seq in self.into_columns:
|
|
1682
|
+
self.log(self.DEBUG,'%s: %s = %s'%(name,typ,mapto))
|
|
1683
|
+
|
|
1684
|
+
# In order to find out whether we have an existing external table in the
|
|
1685
|
+
# catalog which could be reused for this operation we need to make sure
|
|
1686
|
+
# that it has the same column names and types, the same data format, and
|
|
1687
|
+
# location specification, and single row error handling specs.
|
|
1688
|
+
#
|
|
1689
|
+
# This function will return the SQL to run in order to find out whether
|
|
1690
|
+
# such a table exists.
|
|
1691
|
+
#
|
|
1692
|
+
def get_reuse_exttable_query(self, formatType, formatOpts, limitStr, from_cols, schemaName, log_errors):
|
|
1693
|
+
# Greenplum 6 changed pg_exttable: location was renamed to
|
|
1694
|
+
# urilocation and the fmterrtbl error-table oid was replaced by the
|
|
1695
|
+
# logerrors boolean. Detect the available columns first so the same
|
|
1696
|
+
# code works on both old and new catalog layouts.
|
|
1697
|
+
ext_col_sql = "select attname from pg_catalog.pg_attribute " + \
|
|
1698
|
+
"where attrelid = 'pg_catalog.pg_exttable'::regclass " + \
|
|
1699
|
+
"and attname in ('urilocation', 'location', 'logerrors', 'fmterrtbl'); "
|
|
1700
|
+
ext_cols = set(to_text((row)[0]) for row in self.db.query(to_db_text(ext_col_sql)).getresult())
|
|
1701
|
+
location_col = 'urilocation' if 'urilocation' in ext_cols else 'location'
|
|
1702
|
+
error_col = 'logerrors' if 'logerrors' in ext_cols else 'fmterrtbl'
|
|
1703
|
+
|
|
1704
|
+
sqlFormat = """select attrelid::regclass
|
|
1705
|
+
from (
|
|
1706
|
+
select
|
|
1707
|
+
attrelid,
|
|
1708
|
+
row_number() over (partition by attrelid order by attnum) as attord,
|
|
1709
|
+
attnum,
|
|
1710
|
+
attname,
|
|
1711
|
+
atttypid::regtype
|
|
1712
|
+
from
|
|
1713
|
+
pg_attribute
|
|
1714
|
+
join
|
|
1715
|
+
pg_class
|
|
1716
|
+
on (pg_class.oid = attrelid)
|
|
1717
|
+
%s
|
|
1718
|
+
where
|
|
1719
|
+
relstorage = 'x' and
|
|
1720
|
+
relname like 'ext_gpload_reusable_%%' and
|
|
1721
|
+
attnum > 0 and
|
|
1722
|
+
not attisdropped and %s
|
|
1723
|
+
) pgattr
|
|
1724
|
+
join
|
|
1725
|
+
pg_exttable pgext
|
|
1726
|
+
on(pgattr.attrelid = pgext.reloid)
|
|
1727
|
+
"""
|
|
1728
|
+
joinStr = ""
|
|
1729
|
+
conditionStr = ""
|
|
1730
|
+
|
|
1731
|
+
# if schemaName is None, find the resuable ext table which is visible to
|
|
1732
|
+
# current search path. Else find the resuable ext table under the specific
|
|
1733
|
+
# schema, and this needs to join pg_namespace.
|
|
1734
|
+
if schemaName is None:
|
|
1735
|
+
joinStr = ""
|
|
1736
|
+
conditionStr = "pg_table_is_visible(pg_class.oid)"
|
|
1737
|
+
else:
|
|
1738
|
+
joinStr = """join
|
|
1739
|
+
pg_namespace pgns
|
|
1740
|
+
on(pg_class.relnamespace = pgns.oid)
|
|
1741
|
+
"""
|
|
1742
|
+
conditionStr = "pgns.nspname = '%s'" % schemaName
|
|
1743
|
+
|
|
1744
|
+
sql = sqlFormat % (joinStr, conditionStr)
|
|
1745
|
+
|
|
1746
|
+
if log_errors:
|
|
1747
|
+
if error_col == 'logerrors':
|
|
1748
|
+
sql += " WHERE pgext.logerrors = true "
|
|
1749
|
+
else:
|
|
1750
|
+
sql += " WHERE pgext.fmterrtbl = pgext.reloid "
|
|
1751
|
+
else:
|
|
1752
|
+
if error_col == 'logerrors':
|
|
1753
|
+
sql += " WHERE pgext.logerrors = false "
|
|
1754
|
+
else:
|
|
1755
|
+
sql += " WHERE pgext.fmterrtbl IS NULL "
|
|
1756
|
+
|
|
1757
|
+
for i, l in enumerate(self.locations):
|
|
1758
|
+
sql += " and pgext.%s[%s] = %s\n" % (location_col, i + 1, quote(l))
|
|
1759
|
+
|
|
1760
|
+
sql+= """and pgext.fmttype = %s
|
|
1761
|
+
and pgext.writable = false
|
|
1762
|
+
and pgext.fmtopts like %s """ % (quote(formatType[0]),quote("%" + quote_unident(formatOpts.rstrip()) +"%"))
|
|
1763
|
+
|
|
1764
|
+
if limitStr:
|
|
1765
|
+
sql += "and pgext.rejectlimit = %s " % limitStr
|
|
1766
|
+
else:
|
|
1767
|
+
sql += "and pgext.rejectlimit IS NULL "
|
|
1768
|
+
|
|
1769
|
+
sql+= "group by attrelid "
|
|
1770
|
+
|
|
1771
|
+
sql+= """having
|
|
1772
|
+
count(*) = %s and
|
|
1773
|
+
bool_and(case """ % len(from_cols)
|
|
1774
|
+
|
|
1775
|
+
for i, c in enumerate(from_cols):
|
|
1776
|
+
name = c[0]
|
|
1777
|
+
typ = c[1]
|
|
1778
|
+
sql+= "when attord = %s then atttypid = %s::regtype and attname = %s\n" % (i+1, quote(typ), quote(quote_unident(name)))
|
|
1779
|
+
|
|
1780
|
+
sql+= """else true
|
|
1781
|
+
end)
|
|
1782
|
+
limit 1;"""
|
|
1783
|
+
|
|
1784
|
+
self.log(self.DEBUG, "query used to identify reusable external relations: %s" % sql)
|
|
1785
|
+
return sql
|
|
1786
|
+
|
|
1787
|
+
#
|
|
1788
|
+
# Create a string from the following conditions to reuse staging table:
|
|
1789
|
+
# 1. same target table
|
|
1790
|
+
# 2. same number of columns
|
|
1791
|
+
# 3. same names and types, in the same order
|
|
1792
|
+
# 4. same distribution key (according to columns' names and thier order)
|
|
1793
|
+
#
|
|
1794
|
+
def get_staging_conditions_string(self, target_table_name, staging_cols, distribution_cols):
|
|
1795
|
+
|
|
1796
|
+
columns_num = len(staging_cols)
|
|
1797
|
+
|
|
1798
|
+
staging_cols_str = '-'.join(list(map(lambda col:'%s-%s' % (quote(quote_unident(col[0])), quote(col[1])), staging_cols)))
|
|
1799
|
+
|
|
1800
|
+
distribution_cols_str = '-'.join([quote(quote_unident(col)) for col in distribution_cols])
|
|
1801
|
+
|
|
1802
|
+
return '%s:%s:%s:%s' % (target_table_name, columns_num, staging_cols_str, distribution_cols_str)
|
|
1803
|
+
|
|
1804
|
+
|
|
1805
|
+
#
|
|
1806
|
+
# This function will return the SQL to run in order to find out whether
|
|
1807
|
+
# we have an existing staging table in the catalog which could be reused for this
|
|
1808
|
+
# operation, according to the mathod and the encoding conditions.
|
|
1809
|
+
#
|
|
1810
|
+
def get_reuse_staging_table_query(self, encoding_conditions):
|
|
1811
|
+
|
|
1812
|
+
sql = """SELECT oid::regclass
|
|
1813
|
+
FROM pg_class
|
|
1814
|
+
WHERE relname = 'staging_gpload_reusable_%s';""" % (encoding_conditions)
|
|
1815
|
+
|
|
1816
|
+
self.log(self.DEBUG, "query used to identify reusable temporary relations: %s" % sql)
|
|
1817
|
+
return sql
|
|
1818
|
+
|
|
1819
|
+
#
|
|
1820
|
+
# get oid for table from pg_class, None if not exist
|
|
1821
|
+
#
|
|
1822
|
+
def get_table_oid(self, tableName):
|
|
1823
|
+
if tableName:
|
|
1824
|
+
sql = "select %s::regclass::oid" % quote(quote_unident(tableName))
|
|
1825
|
+
try:
|
|
1826
|
+
resultList = self.db.query(to_db_text(sql)).getresult()
|
|
1827
|
+
return resultList[0][0]
|
|
1828
|
+
except Exception as e:
|
|
1829
|
+
pass
|
|
1830
|
+
return None
|
|
1831
|
+
|
|
1832
|
+
def get_ext_schematable(self, schemaName, tableName):
|
|
1833
|
+
if schemaName == None:
|
|
1834
|
+
return tableName
|
|
1835
|
+
else:
|
|
1836
|
+
schemaTable = "%s.%s" % (schemaName, tableName)
|
|
1837
|
+
return schemaTable
|
|
1838
|
+
|
|
1839
|
+
#
|
|
1840
|
+
# Create a new external table or find a reusable external table to use for this operation
|
|
1841
|
+
#
|
|
1842
|
+
def create_external_table(self):
|
|
1843
|
+
|
|
1844
|
+
# extract all control file information and transform it accordingly
|
|
1845
|
+
# in order to construct a CREATE EXTERNAL TABLE statement if will be
|
|
1846
|
+
# needed later on
|
|
1847
|
+
|
|
1848
|
+
formatType = self.getconfig('gpload:input:format', str, 'text').lower()
|
|
1849
|
+
formatOpts = ""
|
|
1850
|
+
locationStr = ','.join(list(map(quote,self.locations)))
|
|
1851
|
+
|
|
1852
|
+
delimiterValue = self.getconfig('gpload:input:delimiter', str, False)
|
|
1853
|
+
if isinstance(delimiterValue, bool) and delimiterValue == False:
|
|
1854
|
+
""" implies the DELIMITER option has no value in the yaml file, so use default """
|
|
1855
|
+
if formatType=='csv':
|
|
1856
|
+
formatOpts += "delimiter ',' "
|
|
1857
|
+
elif len(delimiterValue) != 1:
|
|
1858
|
+
# is a escape sequence character like '\x1B' or '\u001B'?
|
|
1859
|
+
if len(decode_unicode_escape(delimiterValue)) == 1:
|
|
1860
|
+
formatOpts += "delimiter '%s' " % \
|
|
1861
|
+
self.getconfig('gpload:input:delimiter', str)
|
|
1862
|
+
# is a escape string syntax support by gpdb like E'\x1B' or E'\x\u001B'
|
|
1863
|
+
elif len(decode_unicode_escape(delimiterValue.lstrip("E'").rstrip("'"))) ==1:
|
|
1864
|
+
formatOpts += "delimiter %s " % self.getconfig('gpload:input:delimiter', str)
|
|
1865
|
+
else:
|
|
1866
|
+
self.control_file_warning('''A delimiter must be single ASCII charactor, you can also use unprintable characters(for example: '\\x1c' / E'\\x1c' or '\\u001c' / E'\\u001c' ''')
|
|
1867
|
+
self.control_file_error("Invalid delimiter, gpload quit immediately")
|
|
1868
|
+
sys.exit(2);
|
|
1869
|
+
else:
|
|
1870
|
+
formatOpts += "delimiter %s " % \
|
|
1871
|
+
quote_copy_value(self.getconfig('gpload:input:delimiter', str))
|
|
1872
|
+
|
|
1873
|
+
nullas = self.getconfig('gpload:input:null_as', str, False)
|
|
1874
|
+
self.log(self.DEBUG, "null " + str(nullas))
|
|
1875
|
+
if nullas != False: # could be empty string
|
|
1876
|
+
formatOpts += "null %s " % quote_copy_value(nullas)
|
|
1877
|
+
elif formatType=='csv':
|
|
1878
|
+
formatOpts += "null '' "
|
|
1879
|
+
else:
|
|
1880
|
+
formatOpts += "null %s " % quote_copy_value(r"\N")
|
|
1881
|
+
|
|
1882
|
+
esc = self.getconfig('gpload:input:escape', None, None)
|
|
1883
|
+
if esc:
|
|
1884
|
+
if not isinstance(esc, str):
|
|
1885
|
+
self.control_file_error("gpload:input:escape must be a string")
|
|
1886
|
+
if esc.lower() == 'off':
|
|
1887
|
+
if formatType == 'csv':
|
|
1888
|
+
self.control_file_error("ESCAPE cannot be set to OFF in CSV mode")
|
|
1889
|
+
formatOpts += "escape 'off' "
|
|
1890
|
+
else:
|
|
1891
|
+
formatOpts += "escape %s " % quote_copy_value(esc)
|
|
1892
|
+
else:
|
|
1893
|
+
if formatType=='csv':
|
|
1894
|
+
formatOpts += "escape %s " % quote_copy_value(self.getconfig('gpload:input:quote',
|
|
1895
|
+
str, extraStuff='for csv formatted data'))
|
|
1896
|
+
else:
|
|
1897
|
+
formatOpts += "escape %s " % quote_copy_value("\\")
|
|
1898
|
+
|
|
1899
|
+
if formatType=='csv':
|
|
1900
|
+
formatOpts += "quote %s " % quote_copy_value(self.getconfig('gpload:input:quote',
|
|
1901
|
+
str, extraStuff='for csv formatted data'))
|
|
1902
|
+
|
|
1903
|
+
if self.getconfig('gpload:input:header',bool,False):
|
|
1904
|
+
formatOpts += "header "
|
|
1905
|
+
|
|
1906
|
+
force_not_null_columns = self.getconfig('gpload:input:force_not_null',list,[])
|
|
1907
|
+
if force_not_null_columns:
|
|
1908
|
+
for i in force_not_null_columns:
|
|
1909
|
+
if not isinstance(i, str):
|
|
1910
|
+
self.control_file_error("gpload:input:force_not_null must be a YAML sequence of strings")
|
|
1911
|
+
formatOpts += "force not null %s " % ','.join(force_not_null_columns)
|
|
1912
|
+
|
|
1913
|
+
encodingStr = self.getconfig('gpload:input:encoding', str, None)
|
|
1914
|
+
|
|
1915
|
+
limitStr = self.getconfig('gpload:input:error_limit',int, None)
|
|
1916
|
+
if self.log_errors and not limitStr:
|
|
1917
|
+
self.control_file_error("gpload:input:log_errors requires " +
|
|
1918
|
+
"gpload:input:error_limit to be specified")
|
|
1919
|
+
|
|
1920
|
+
self.extSchemaName = self.getconfig('gpload:external:schema', str, None)
|
|
1921
|
+
if self.extSchemaName == '%':
|
|
1922
|
+
self.extSchemaName = self.schema
|
|
1923
|
+
|
|
1924
|
+
# get the list of columns to use in the extnernal table
|
|
1925
|
+
if not self.from_cols_from_user:
|
|
1926
|
+
# don't put values serial columns
|
|
1927
|
+
from_cols = list(filter(lambda a: a[3] != True,
|
|
1928
|
+
self.from_columns))
|
|
1929
|
+
else:
|
|
1930
|
+
from_cols = self.from_columns
|
|
1931
|
+
|
|
1932
|
+
# If the 'reuse tables' option was specified we now try to find an
|
|
1933
|
+
# already existing external table in the catalog which will match
|
|
1934
|
+
# the one that we need to use. It must have identical attributes,
|
|
1935
|
+
# external location, format, and encoding specifications.
|
|
1936
|
+
if self.reuse_tables == True:
|
|
1937
|
+
sql = self.get_reuse_exttable_query(formatType, formatOpts,
|
|
1938
|
+
limitStr, from_cols, self.extSchemaName, self.log_errors)
|
|
1939
|
+
resultList = self.db.query(to_db_text(sql)).getresult()
|
|
1940
|
+
if len(resultList) > 0:
|
|
1941
|
+
# found an external table to reuse. no need to create one. we're done here.
|
|
1942
|
+
self.extTableName = to_text((resultList[0])[0])
|
|
1943
|
+
self.extSchemaTable = self.extTableName
|
|
1944
|
+
self.log(self.INFO, "reusing external table %s" % self.extSchemaTable)
|
|
1945
|
+
return
|
|
1946
|
+
|
|
1947
|
+
# didn't find an existing external table suitable for reuse. Format a reusable
|
|
1948
|
+
# name and issue a CREATE EXTERNAL TABLE on it. Hopefully we can use it next time
|
|
1949
|
+
# around
|
|
1950
|
+
|
|
1951
|
+
self.extTableName = "ext_gpload_reusable_%s" % self.unique_suffix
|
|
1952
|
+
self.log(self.INFO, "did not find an external table to reuse. creating %s" % self.extTableName)
|
|
1953
|
+
|
|
1954
|
+
# construct a CREATE EXTERNAL TABLE statement and execute it
|
|
1955
|
+
self.extSchemaTable = self.get_ext_schematable(self.extSchemaName, self.extTableName)
|
|
1956
|
+
sql = "create external table %s" % self.extSchemaTable
|
|
1957
|
+
sql += "(%s)" % ','.join(list(map(lambda a:'%s %s' % (a[0], a[1]), from_cols)))
|
|
1958
|
+
|
|
1959
|
+
sql += "location(%s) "%locationStr
|
|
1960
|
+
sql += "format%s "% quote(formatType)
|
|
1961
|
+
if len(formatOpts) > 0:
|
|
1962
|
+
sql += "(%s) "% formatOpts
|
|
1963
|
+
if encodingStr:
|
|
1964
|
+
sql += "encoding%s "%quote(encodingStr)
|
|
1965
|
+
if self.log_errors:
|
|
1966
|
+
sql += "log errors "
|
|
1967
|
+
|
|
1968
|
+
if limitStr:
|
|
1969
|
+
if limitStr < 2:
|
|
1970
|
+
self.control_file_error("error_limit must be 2 or higher")
|
|
1971
|
+
sql += "segment reject limit %s "%limitStr
|
|
1972
|
+
|
|
1973
|
+
try:
|
|
1974
|
+
self.db.query(to_db_text(sql))
|
|
1975
|
+
except Exception as e:
|
|
1976
|
+
self.log(self.ERROR, 'could not run SQL "%s": %s' % (sql, str(e)))
|
|
1977
|
+
|
|
1978
|
+
# set up to drop the external table at the end of operation, unless user
|
|
1979
|
+
# specified the 'reuse_tables' option, in which case we don't drop
|
|
1980
|
+
if self.reuse_tables == False:
|
|
1981
|
+
self.cleanupSql.append('drop external table if exists %s'%self.extSchemaTable)
|
|
1982
|
+
|
|
1983
|
+
|
|
1984
|
+
#
|
|
1985
|
+
# Create a new staging table or find a reusable staging table to use for this operation
|
|
1986
|
+
# (only valid for update/merge operations).
|
|
1987
|
+
#
|
|
1988
|
+
def create_staging_table(self):
|
|
1989
|
+
|
|
1990
|
+
# Do some initial work to extract the update_columns and metadata
|
|
1991
|
+
# that may be needed in order to create or reuse a temp table
|
|
1992
|
+
if not self.from_cols_from_user:
|
|
1993
|
+
# don't put values serial columns
|
|
1994
|
+
from_cols = list(filter(lambda a: a[3] != True, self.from_columns))
|
|
1995
|
+
else:
|
|
1996
|
+
from_cols = self.from_columns
|
|
1997
|
+
|
|
1998
|
+
# make sure we set the correct distribution policy
|
|
1999
|
+
distcols = self.getconfig('gpload:output:match_columns', list)
|
|
2000
|
+
|
|
2001
|
+
# MPP-13399, CR-2227
|
|
2002
|
+
including_defaults = ""
|
|
2003
|
+
if self.getconfig('gpload:output:including_defaults',bool,True):
|
|
2004
|
+
including_defaults = " including defaults"
|
|
2005
|
+
|
|
2006
|
+
sql = "SELECT * FROM pg_class WHERE relname LIKE 'temp_gpload_reusable_%%';"
|
|
2007
|
+
resultList = self.db.query(to_db_text(sql)).getresult()
|
|
2008
|
+
if len(resultList) > 0:
|
|
2009
|
+
self.log(self.WARN, """Old style, reusable tables named "temp_gpload_reusable_*" from a previous versions were found.
|
|
2010
|
+
Greenplum recommends running "DROP TABLE temp_gpload_reusable_..." on each table. This only needs to be done once.""")
|
|
2011
|
+
|
|
2012
|
+
# If the 'reuse tables' option was specified we now try to find an
|
|
2013
|
+
# already existing staging table in the catalog which will match
|
|
2014
|
+
# the one that we need to use. It must meet the reuse conditions
|
|
2015
|
+
is_temp_table = 'TEMP '
|
|
2016
|
+
target_columns = []
|
|
2017
|
+
for column in self.into_columns:
|
|
2018
|
+
if column[2]:
|
|
2019
|
+
target_columns.append([quote_unident(column[0]), column[1]])
|
|
2020
|
+
|
|
2021
|
+
if self.reuse_tables == True:
|
|
2022
|
+
is_temp_table = ''
|
|
2023
|
+
target_table_name = quote_unident(self.table)
|
|
2024
|
+
|
|
2025
|
+
# create a string from all reuse conditions for staging tables and ancode it
|
|
2026
|
+
conditions_str = self.get_staging_conditions_string(target_table_name, target_columns, distcols)
|
|
2027
|
+
encoding_conditions = staging_conditions_hash(conditions_str)
|
|
2028
|
+
|
|
2029
|
+
sql = self.get_reuse_staging_table_query(encoding_conditions)
|
|
2030
|
+
resultList = self.db.query(to_db_text(sql)).getresult()
|
|
2031
|
+
|
|
2032
|
+
if len(resultList) > 0:
|
|
2033
|
+
|
|
2034
|
+
# found a temp table to reuse. no need to create one. we're done here.
|
|
2035
|
+
self.staging_table_name = to_text((resultList[0])[0])
|
|
2036
|
+
self.log(self.INFO, "reusing staging table %s" % self.staging_table_name)
|
|
2037
|
+
|
|
2038
|
+
# truncate it so we don't use old data
|
|
2039
|
+
self.do_truncate(self.staging_table_name)
|
|
2040
|
+
|
|
2041
|
+
return
|
|
2042
|
+
|
|
2043
|
+
# didn't find an existing staging table suitable for reuse. Format a reusable
|
|
2044
|
+
# name and issue a CREATE TABLE on it (without TEMP!). Hopefully we can use it
|
|
2045
|
+
# next time around
|
|
2046
|
+
# we no longer need the timestamp, since we will never want to create few
|
|
2047
|
+
# tables with same encoding_conditions
|
|
2048
|
+
self.staging_table_name = "staging_gpload_reusable_%s" % (encoding_conditions)
|
|
2049
|
+
self.log(self.INFO, "did not find a staging table to reuse. creating %s" % self.staging_table_name)
|
|
2050
|
+
|
|
2051
|
+
# MPP-14667 - self.reuse_tables should change one, and only one, aspect of how we build the following table,
|
|
2052
|
+
# and that is, whether it's a temp table or not. In other words, is_temp_table = '' iff self.reuse_tables == True.
|
|
2053
|
+
sql = 'CREATE %sTABLE %s ' % (is_temp_table, self.staging_table_name)
|
|
2054
|
+
cols = list(map(lambda a:'%s %s' % (a[0], a[1]), target_columns))
|
|
2055
|
+
sql += "(%s)" % ','.join(cols)
|
|
2056
|
+
sql += " DISTRIBUTED BY (%s)" % ', '.join(distcols)
|
|
2057
|
+
self.log(self.LOG, sql)
|
|
2058
|
+
|
|
2059
|
+
if not self.options.D:
|
|
2060
|
+
self.db.query(to_db_text(sql))
|
|
2061
|
+
if not self.reuse_tables:
|
|
2062
|
+
self.cleanupSql.append('DROP TABLE IF EXISTS %s' % self.staging_table_name)
|
|
2063
|
+
|
|
2064
|
+
|
|
2065
|
+
def count_errors(self):
|
|
2066
|
+
notice_processor(self)
|
|
2067
|
+
if self.log_errors and not self.options.D:
|
|
2068
|
+
# make sure we only get errors for our own instance
|
|
2069
|
+
if not self.reuse_tables:
|
|
2070
|
+
queryStr = "select count(*) from gp_read_error_log('%s')" % escape_string(self.extTableName)
|
|
2071
|
+
results = self.db.query(to_db_text(queryStr)).getresult()
|
|
2072
|
+
return (results[0])[0]
|
|
2073
|
+
else: # reuse_tables
|
|
2074
|
+
queryStr = "select cmdtime, count(*) from gp_read_error_log('%s') group by cmdtime order by cmdtime desc limit 1" % escape_string(self.extTableName)
|
|
2075
|
+
results = self.db.query(to_db_text(queryStr)).getresult()
|
|
2076
|
+
if not results:
|
|
2077
|
+
# No formatting errors were recorded for this external
|
|
2078
|
+
# table, so there is nothing to report.
|
|
2079
|
+
return 0
|
|
2080
|
+
self.lastcmdtime = to_text((results[0])[0])
|
|
2081
|
+
global NUM_WARN_ROWS
|
|
2082
|
+
NUM_WARN_ROWS = (results[0])[1]
|
|
2083
|
+
return (results[0])[1];
|
|
2084
|
+
return 0
|
|
2085
|
+
|
|
2086
|
+
def report_errors(self):
|
|
2087
|
+
errors = self.count_errors()
|
|
2088
|
+
global NUM_WARN_ROWS
|
|
2089
|
+
# Without LOG ERRORS gp_read_error_log stays empty; the
|
|
2090
|
+
# "found N data formatting errors" NOTICE is the only signal that
|
|
2091
|
+
# rows were rejected. Use whichever source reports more so rejected
|
|
2092
|
+
# rows are never treated as a clean success.
|
|
2093
|
+
bad_rows = errors if errors else NUM_WARN_ROWS
|
|
2094
|
+
if self.log_errors and errors:
|
|
2095
|
+
NUM_WARN_ROWS = errors
|
|
2096
|
+
if bad_rows == 1:
|
|
2097
|
+
self.log(self.WARN, '1 bad row')
|
|
2098
|
+
elif bad_rows:
|
|
2099
|
+
self.log(self.WARN, '%d bad rows' % bad_rows)
|
|
2100
|
+
if bad_rows:
|
|
2101
|
+
NUM_WARN_ROWS = bad_rows
|
|
2102
|
+
|
|
2103
|
+
# error message is also deleted if external table is dropped.
|
|
2104
|
+
# if reuse_table is set, error message is not deleted.
|
|
2105
|
+
if errors and self.log_errors and self.reuse_tables:
|
|
2106
|
+
self.log(self.WARN, "Please use following query to access the detailed error")
|
|
2107
|
+
self.log(self.WARN, "select * from gp_read_error_log('{0}') where cmdtime = '{1}'".format(escape_string(self.extTableName), self.lastcmdtime))
|
|
2108
|
+
self.exitValue = 1 if bad_rows else 0
|
|
2109
|
+
|
|
2110
|
+
|
|
2111
|
+
def do_insert(self, dest):
|
|
2112
|
+
"""
|
|
2113
|
+
Handle the INSERT case
|
|
2114
|
+
"""
|
|
2115
|
+
self.log(self.DEBUG, "into columns " + str(self.into_columns))
|
|
2116
|
+
cols = list(filter(lambda a:a[2]!=None, self.into_columns))
|
|
2117
|
+
|
|
2118
|
+
# only insert non-serial columns, unless the user told us to
|
|
2119
|
+
# insert the serials explicitly
|
|
2120
|
+
if not self.from_cols_from_user:
|
|
2121
|
+
cols = list(filter(lambda a:a[3] == False, cols))
|
|
2122
|
+
|
|
2123
|
+
sql = 'INSERT INTO %s' % dest
|
|
2124
|
+
sql += ' (%s)' % ','.join(list(map(lambda a:a[0], cols)))
|
|
2125
|
+
sql += ' SELECT %s' % ','.join(list(map(lambda a:a[2], cols)))
|
|
2126
|
+
sql += ' FROM %s' % self.extSchemaTable
|
|
2127
|
+
|
|
2128
|
+
self.log(self.LOG, sql)
|
|
2129
|
+
if not self.options.D:
|
|
2130
|
+
try:
|
|
2131
|
+
self.rowsInserted = self.db.query(to_db_text(sql))
|
|
2132
|
+
except Exception as e:
|
|
2133
|
+
# We need to be a bit careful about the error since it may contain non-unicode characters
|
|
2134
|
+
global NUM_WARN_ROWS
|
|
2135
|
+
strE = str(e)
|
|
2136
|
+
strF = str(sql)
|
|
2137
|
+
|
|
2138
|
+
# report error file and linenum
|
|
2139
|
+
queryStr = "select filename, linenum from gp_read_error_log('%s')" % escape_string(self.extTableName)
|
|
2140
|
+
results = self.db.query(to_db_text(queryStr)).getresult()
|
|
2141
|
+
if results:
|
|
2142
|
+
NUM_WARN_ROWS = len(results)
|
|
2143
|
+
errfile = error_log_filename((results[0])[0])
|
|
2144
|
+
errlinenum = str((results[0])[1])
|
|
2145
|
+
self.log(self.WARN, "error file: " + errfile)
|
|
2146
|
+
self.log(self.WARN, "error line: " + errlinenum)
|
|
2147
|
+
else:
|
|
2148
|
+
self.log(self.WARN, "no rows found in gp_read_error_log for " + self.extTableName)
|
|
2149
|
+
|
|
2150
|
+
self.log(self.ERROR, strE + ' encountered while running ' + strF)
|
|
2151
|
+
|
|
2152
|
+
self.report_errors()
|
|
2153
|
+
|
|
2154
|
+
def do_method_insert(self):
|
|
2155
|
+
self.create_external_table()
|
|
2156
|
+
self.do_insert(self.get_qualified_tablename())
|
|
2157
|
+
|
|
2158
|
+
def map_stuff(self,config,format,index):
|
|
2159
|
+
lis = []
|
|
2160
|
+
theList = self.getconfig(config,list)
|
|
2161
|
+
theList = convertListToDelimited(theList)
|
|
2162
|
+
for i in theList:
|
|
2163
|
+
if not isinstance(i, str):
|
|
2164
|
+
self.control_file_error("%s must be a YAML sequence of strings"%config)
|
|
2165
|
+
j = self.into_columns_dict.get(i)
|
|
2166
|
+
if not j:
|
|
2167
|
+
self.log(self.ERROR,'column %s in %s does not exist'%(i,config))
|
|
2168
|
+
if not j[index]:
|
|
2169
|
+
self.log(self.ERROR,'there is no mapping from the column %s in %s'%(i,config))
|
|
2170
|
+
lis.append(format(j[0],j[index]))
|
|
2171
|
+
return lis
|
|
2172
|
+
|
|
2173
|
+
def fix_update_cond(self, match):
|
|
2174
|
+
self.log(self.DEBUG, match.group(0))
|
|
2175
|
+
return 'into_table.' + match.group(0)
|
|
2176
|
+
|
|
2177
|
+
def do_update(self,fromname,index):
|
|
2178
|
+
"""
|
|
2179
|
+
UPDATE case
|
|
2180
|
+
"""
|
|
2181
|
+
sql = 'update %s into_table ' % self.get_qualified_tablename()
|
|
2182
|
+
sql += 'set %s '%','.join(self.map_stuff('gpload:output:update_columns',(lambda x,y:'%s=from_table.%s' % (x, y)),index))
|
|
2183
|
+
sql += 'from %s from_table' % fromname
|
|
2184
|
+
|
|
2185
|
+
match = self.map_stuff('gpload:output:match_columns'
|
|
2186
|
+
, lambda x,y:'into_table.%s=from_table.%s' % (x, y)
|
|
2187
|
+
, index)
|
|
2188
|
+
|
|
2189
|
+
update_condition = self.getconfig('gpload:output:update_condition',
|
|
2190
|
+
str, None)
|
|
2191
|
+
if update_condition:
|
|
2192
|
+
#
|
|
2193
|
+
# Place the table alias infront of column references.
|
|
2194
|
+
#
|
|
2195
|
+
# The following logic is not bullet proof. It may not work
|
|
2196
|
+
# correctly if the user uses an identifier in both its
|
|
2197
|
+
# delimited and un-delimited format (e.g. where c1 < 7 and "c1" > 2)
|
|
2198
|
+
# Better lexing and parsing needs to be done here to fix all cases.
|
|
2199
|
+
#
|
|
2200
|
+
update_condition = ' ' + update_condition + ' '
|
|
2201
|
+
for name, type, mapto, seq in self.into_columns:
|
|
2202
|
+
regexp = r'(?<=[^\w])%s(?=[^\w])' % name
|
|
2203
|
+
self.log(self.DEBUG, 'update_condition re: ' + regexp)
|
|
2204
|
+
temp_update_condition = update_condition
|
|
2205
|
+
updateConditionList = splitIntoLiteralsAndNonLiterals(update_condition)
|
|
2206
|
+
skip = False
|
|
2207
|
+
newUpdateConditionList = []
|
|
2208
|
+
update_condition = ''
|
|
2209
|
+
for uc in updateConditionList:
|
|
2210
|
+
if skip == False:
|
|
2211
|
+
uc = re.sub(regexp, self.fix_update_cond, uc)
|
|
2212
|
+
skip = True
|
|
2213
|
+
update_condition = update_condition + uc
|
|
2214
|
+
if update_condition == temp_update_condition:
|
|
2215
|
+
# see if column can be undelimited, and try again.
|
|
2216
|
+
if len(name) > 2 and name[1:-1] == name[1:-1].lower():
|
|
2217
|
+
regexp = r'(?<=[^\w])%s(?=[^\w])' % name[1:-1]
|
|
2218
|
+
self.log(self.DEBUG, 'update_condition undelimited re: ' + regexp)
|
|
2219
|
+
update_condition = re.sub( regexp
|
|
2220
|
+
, self.fix_update_cond
|
|
2221
|
+
, update_condition
|
|
2222
|
+
)
|
|
2223
|
+
self.log(self.DEBUG, "updated update_condition to %s" %
|
|
2224
|
+
update_condition)
|
|
2225
|
+
match.append(update_condition)
|
|
2226
|
+
sql += ' where %s' % ' and '.join(match)
|
|
2227
|
+
self.log(self.LOG, sql)
|
|
2228
|
+
if not self.options.D:
|
|
2229
|
+
try:
|
|
2230
|
+
self.rowsUpdated = self.db.query(to_db_text(sql))
|
|
2231
|
+
except Exception as e:
|
|
2232
|
+
# We need to be a bit careful about the error since it may contain non-unicode characters
|
|
2233
|
+
strE = str(e)
|
|
2234
|
+
strF = str(sql)
|
|
2235
|
+
self.log(self.ERROR, strE + ' encountered while running ' + strF)
|
|
2236
|
+
|
|
2237
|
+
def get_qualified_tablename(self):
|
|
2238
|
+
|
|
2239
|
+
tblname = "%s.%s" % (self.schema, self.table)
|
|
2240
|
+
return tblname
|
|
2241
|
+
|
|
2242
|
+
def get_table_dist_key(self):
|
|
2243
|
+
|
|
2244
|
+
# NOTE: this query should be re-written better. the problem is that it is
|
|
2245
|
+
# not possible to perform a cast on a table name with spaces...
|
|
2246
|
+
# Older Greenplum versions expose the distribution key as
|
|
2247
|
+
# gp_distribution_policy.attrnums, while GP 6.25+ renamed it to
|
|
2248
|
+
# gp_distribution_policy.distkey. Detect the column name first so the
|
|
2249
|
+
# transaction is never aborted by a failed catalog query.
|
|
2250
|
+
distkey_col_sql = "select attname from pg_catalog.pg_attribute " + \
|
|
2251
|
+
"where attrelid = 'pg_catalog.gp_distribution_policy'::regclass " + \
|
|
2252
|
+
"and attname in ('distkey', 'attrnums') " + \
|
|
2253
|
+
"order by attname desc limit 1; "
|
|
2254
|
+
distkey_cols = self.db.query(to_db_text(distkey_col_sql)).getresult()
|
|
2255
|
+
distkey_col = 'attrnums'
|
|
2256
|
+
if len(distkey_cols) > 0:
|
|
2257
|
+
distkey_col = to_text((distkey_cols[0])[0])
|
|
2258
|
+
if distkey_col not in ('distkey', 'attrnums'):
|
|
2259
|
+
distkey_col = 'attrnums'
|
|
2260
|
+
|
|
2261
|
+
sql = "select attname from pg_attribute a, gp_distribution_policy p , pg_class c, pg_namespace n "+\
|
|
2262
|
+
"where a.attrelid = c.oid and " + \
|
|
2263
|
+
"a.attrelid = p.localoid and " + \
|
|
2264
|
+
"a.attnum = any (p.%s) and " % distkey_col + \
|
|
2265
|
+
"c.relnamespace = n.oid and " + \
|
|
2266
|
+
"n.nspname = '%s' and c.relname = '%s'; " % (quote_unident(self.schema), quote_unident(self.table))
|
|
2267
|
+
|
|
2268
|
+
|
|
2269
|
+
resultList = self.db.query(to_db_text(sql)).getresult()
|
|
2270
|
+
attrs = []
|
|
2271
|
+
count = 0
|
|
2272
|
+
while count < len(resultList):
|
|
2273
|
+
attrs.append(to_text((resultList[count])[0]))
|
|
2274
|
+
count = count + 1
|
|
2275
|
+
|
|
2276
|
+
return attrs
|
|
2277
|
+
|
|
2278
|
+
def table_supports_update(self):
|
|
2279
|
+
"""Columns being updated cannot appear in the distribution key."""
|
|
2280
|
+
distKeyList = self.get_table_dist_key()
|
|
2281
|
+
distkey = set()
|
|
2282
|
+
for dk in distKeyList:
|
|
2283
|
+
distkey.add(quote_ident(dk))
|
|
2284
|
+
|
|
2285
|
+
self.distkey = distkey
|
|
2286
|
+
if len(distkey) != 0:
|
|
2287
|
+
# not randomly distributed - check that UPDATE_COLUMNS isn't part of the distribution key
|
|
2288
|
+
updateColumnList = self.getconfig('gpload:output:update_columns',
|
|
2289
|
+
list,
|
|
2290
|
+
returnOriginal=True)
|
|
2291
|
+
update_columns = convertListToDelimited(updateColumnList)
|
|
2292
|
+
update_columns = set(update_columns)
|
|
2293
|
+
a = distkey.intersection(update_columns)
|
|
2294
|
+
if len(a):
|
|
2295
|
+
self.control_file_error('update_columns cannot reference column(s) in distribution key (%s)' % ', '.join(list(distkey)))
|
|
2296
|
+
|
|
2297
|
+
def do_method_update(self):
|
|
2298
|
+
"""Load the data in and update an existing table based upon it"""
|
|
2299
|
+
|
|
2300
|
+
self.table_supports_update()
|
|
2301
|
+
self.create_staging_table()
|
|
2302
|
+
|
|
2303
|
+
self.create_external_table()
|
|
2304
|
+
self.do_insert(self.staging_table_name)
|
|
2305
|
+
# These rows are inserted temporarily for processing, so set inserted rows back to zero.
|
|
2306
|
+
self.rowsInserted = 0
|
|
2307
|
+
self.do_update(self.staging_table_name, 0)
|
|
2308
|
+
|
|
2309
|
+
def do_method_merge(self):
|
|
2310
|
+
"""insert data not already in the table, update remaining items"""
|
|
2311
|
+
|
|
2312
|
+
self.table_supports_update()
|
|
2313
|
+
self.create_staging_table()
|
|
2314
|
+
self.create_external_table()
|
|
2315
|
+
self.do_insert(self.staging_table_name)
|
|
2316
|
+
self.rowsInserted = 0 # MPP-13024. No rows inserted yet (only to temp table).
|
|
2317
|
+
self.do_update(self.staging_table_name, 0)
|
|
2318
|
+
|
|
2319
|
+
# insert new rows to the target table
|
|
2320
|
+
match = self.map_stuff('gpload:output:match_columns',lambda x,y:'into_table.%s=from_table.%s'%(x,y),0)
|
|
2321
|
+
matchColumns = self.getconfig('gpload:output:match_columns',list)
|
|
2322
|
+
|
|
2323
|
+
cols = list(filter(lambda a:a[2] != None, self.into_columns))
|
|
2324
|
+
sql = 'INSERT INTO %s ' % self.get_qualified_tablename()
|
|
2325
|
+
sql += '(%s) ' % ','.join(list(map(lambda a:a[0], cols)))
|
|
2326
|
+
sql += '(SELECT %s ' % ','.join(list(map(lambda a:'from_table.%s' % a[0], cols)))
|
|
2327
|
+
sql += 'FROM (SELECT *, row_number() OVER (PARTITION BY %s) AS gpload_row_number ' % ','.join(matchColumns)
|
|
2328
|
+
sql += 'FROM %s) AS from_table ' % self.staging_table_name
|
|
2329
|
+
sql += 'LEFT OUTER JOIN %s into_table ' % self.get_qualified_tablename()
|
|
2330
|
+
sql += 'ON %s '%' AND '.join(match)
|
|
2331
|
+
where = self.map_stuff('gpload:output:match_columns',lambda x,y:'into_table.%s IS NULL'%x,0)
|
|
2332
|
+
sql += 'WHERE %s ' % ' AND '.join(where)
|
|
2333
|
+
sql += 'AND gpload_row_number=1)'
|
|
2334
|
+
|
|
2335
|
+
self.log(self.LOG, sql)
|
|
2336
|
+
if not self.options.D:
|
|
2337
|
+
try:
|
|
2338
|
+
self.rowsInserted = self.db.query(to_db_text(sql))
|
|
2339
|
+
except Exception as e:
|
|
2340
|
+
# We need to be a bit careful about the error since it may contain non-unicode characters
|
|
2341
|
+
strE = str(e)
|
|
2342
|
+
strF = str(sql)
|
|
2343
|
+
self.log(self.ERROR, strE + ' encountered while running ' + strF)
|
|
2344
|
+
|
|
2345
|
+
|
|
2346
|
+
def do_truncate(self, tblname):
|
|
2347
|
+
self.log(self.LOG, "Truncate table %s" %(tblname))
|
|
2348
|
+
if not self.options.D:
|
|
2349
|
+
try:
|
|
2350
|
+
truncateSQLtext = "truncate %s" % tblname
|
|
2351
|
+
self.db.query(to_db_text(truncateSQLtext))
|
|
2352
|
+
except Exception as e:
|
|
2353
|
+
self.log(self.ERROR, 'could not execute truncate target %s: %s' % (tblname, str(e)))
|
|
2354
|
+
|
|
2355
|
+
def do_method(self):
|
|
2356
|
+
# Is the table to be truncated before the load?
|
|
2357
|
+
preload = self.getconfig('gpload:preload', list, default=None)
|
|
2358
|
+
method = self.getconfig('gpload:output:mode', str, 'insert').lower()
|
|
2359
|
+
self.log_errors = self.getconfig('gpload:input:log_errors', bool, False)
|
|
2360
|
+
truncate = False
|
|
2361
|
+
self.reuse_tables = False
|
|
2362
|
+
|
|
2363
|
+
if not self.options.no_auto_trans and not method=='insert':
|
|
2364
|
+
self.db.query("BEGIN")
|
|
2365
|
+
|
|
2366
|
+
if preload:
|
|
2367
|
+
truncate = self.getconfig('gpload:preload:truncate',bool,False)
|
|
2368
|
+
self.reuse_tables = self.getconfig('gpload:preload:reuse_tables',bool,False)
|
|
2369
|
+
if truncate == True:
|
|
2370
|
+
if method=='insert':
|
|
2371
|
+
self.do_truncate(self.schemaTable)
|
|
2372
|
+
else:
|
|
2373
|
+
self.log(self.ERROR, 'preload truncate operation should be used with insert ' +
|
|
2374
|
+
'operation only. used with %s' % method)
|
|
2375
|
+
|
|
2376
|
+
# sql pre or post processing?
|
|
2377
|
+
sql = self.getconfig('gpload:sql', list, default=None)
|
|
2378
|
+
before = None
|
|
2379
|
+
after = None
|
|
2380
|
+
if sql:
|
|
2381
|
+
before = self.getconfig('gpload:sql:before', str, default=None)
|
|
2382
|
+
after = self.getconfig('gpload:sql:after', str, default=None)
|
|
2383
|
+
if before:
|
|
2384
|
+
self.log(self.LOG, "Pre-SQL from user: %s" % before)
|
|
2385
|
+
if not self.options.D:
|
|
2386
|
+
try:
|
|
2387
|
+
self.db.query(to_db_text(before))
|
|
2388
|
+
except Exception as e:
|
|
2389
|
+
self.log(self.ERROR, 'could not execute SQL in sql:before "%s": %s' %
|
|
2390
|
+
(before, str(e)))
|
|
2391
|
+
|
|
2392
|
+
|
|
2393
|
+
if method=='insert':
|
|
2394
|
+
self.do_method_insert()
|
|
2395
|
+
elif method=='update':
|
|
2396
|
+
self.do_method_update()
|
|
2397
|
+
elif method=='merge':
|
|
2398
|
+
self.do_method_merge()
|
|
2399
|
+
else:
|
|
2400
|
+
self.control_file_error('unsupported method %s' % method)
|
|
2401
|
+
|
|
2402
|
+
# truncate the staging table to avoid dumping it's content - see MPP-15474
|
|
2403
|
+
if method=='merge' or method=='update':
|
|
2404
|
+
self.do_truncate(self.staging_table_name)
|
|
2405
|
+
|
|
2406
|
+
if after:
|
|
2407
|
+
self.log(self.LOG, "Post-SQL from user: %s" % after)
|
|
2408
|
+
if not self.options.D:
|
|
2409
|
+
try:
|
|
2410
|
+
self.db.query(to_db_text(after))
|
|
2411
|
+
except Exception as e:
|
|
2412
|
+
self.log(self.ERROR, 'could not execute SQL in sql:after "%s": %s' %
|
|
2413
|
+
(after, str(e)))
|
|
2414
|
+
|
|
2415
|
+
if not self.options.no_auto_trans and not method=='insert':
|
|
2416
|
+
self.db.query("COMMIT")
|
|
2417
|
+
|
|
2418
|
+
|
|
2419
|
+
def run2(self):
|
|
2420
|
+
self.log(self.DEBUG, 'config ' + str(self.config))
|
|
2421
|
+
start = time.time()
|
|
2422
|
+
self.read_config()
|
|
2423
|
+
self.setup_connection()
|
|
2424
|
+
self.read_table_metadata()
|
|
2425
|
+
self.read_columns()
|
|
2426
|
+
self.read_mapping()
|
|
2427
|
+
self.start_gpfdists()
|
|
2428
|
+
self.do_method()
|
|
2429
|
+
self.log(self.INFO, 'running time: %.2f seconds'%(time.time()-start))
|
|
2430
|
+
|
|
2431
|
+
def run(self):
|
|
2432
|
+
self.db = None
|
|
2433
|
+
self.rowsInserted = 0
|
|
2434
|
+
self.rowsUpdated = 0
|
|
2435
|
+
signal.signal(signal.SIGINT, handle_kill)
|
|
2436
|
+
signal.signal(signal.SIGTERM, handle_kill)
|
|
2437
|
+
# win32 doesn't do SIGQUIT
|
|
2438
|
+
if not platform.system() in ['Windows', 'Microsoft']:
|
|
2439
|
+
signal.signal(signal.SIGQUIT, handle_kill)
|
|
2440
|
+
signal.signal(signal.SIGHUP, signal.SIG_IGN)
|
|
2441
|
+
|
|
2442
|
+
try:
|
|
2443
|
+
try:
|
|
2444
|
+
self.run2()
|
|
2445
|
+
except Exception:
|
|
2446
|
+
traceback.print_exc(file=self.logfile)
|
|
2447
|
+
self.logfile.flush()
|
|
2448
|
+
self.exitValue = 2
|
|
2449
|
+
if (self.options.qv > self.INFO):
|
|
2450
|
+
traceback.print_exc()
|
|
2451
|
+
else:
|
|
2452
|
+
self.log(self.ERROR, "unexpected error -- backtrace " +
|
|
2453
|
+
"written to log file")
|
|
2454
|
+
finally:
|
|
2455
|
+
if self.cleanupSql:
|
|
2456
|
+
self.log(self.LOG, 'removing temporary data')
|
|
2457
|
+
self.setup_connection()
|
|
2458
|
+
for a in self.cleanupSql:
|
|
2459
|
+
try:
|
|
2460
|
+
self.log(self.DEBUG, a)
|
|
2461
|
+
self.db.query(to_db_text(a))
|
|
2462
|
+
except Exception:
|
|
2463
|
+
traceback.print_exc(file=self.logfile)
|
|
2464
|
+
self.logfile.flush()
|
|
2465
|
+
traceback.print_exc()
|
|
2466
|
+
if self.subprocesses:
|
|
2467
|
+
self.log(self.LOG, 'killing gpfdist')
|
|
2468
|
+
for a in self.subprocesses:
|
|
2469
|
+
try:
|
|
2470
|
+
if platform.system() in ['Windows', 'Microsoft']:
|
|
2471
|
+
# win32 API is better but hard for us
|
|
2472
|
+
# to install, so we use the crude method
|
|
2473
|
+
subprocess.Popen("taskkill /F /T /PID %i" % a.pid,
|
|
2474
|
+
shell=True, stdout=subprocess.PIPE,
|
|
2475
|
+
stderr=subprocess.PIPE)
|
|
2476
|
+
|
|
2477
|
+
else:
|
|
2478
|
+
os.kill(a.pid, signal.SIGTERM)
|
|
2479
|
+
except OSError:
|
|
2480
|
+
pass
|
|
2481
|
+
for t in self.threads:
|
|
2482
|
+
t.join()
|
|
2483
|
+
|
|
2484
|
+
self.log(self.INFO, 'rows Inserted = ' + str(self.rowsInserted))
|
|
2485
|
+
self.log(self.INFO, 'rows Updated = ' + str(self.rowsUpdated))
|
|
2486
|
+
self.log(self.INFO, 'data formatting errors = ' + str(NUM_WARN_ROWS))
|
|
2487
|
+
if self.exitValue==0:
|
|
2488
|
+
self.log(self.INFO, 'gpload succeeded')
|
|
2489
|
+
elif self.exitValue==1:
|
|
2490
|
+
self.log(self.INFO, 'gpload succeeded with warnings')
|
|
2491
|
+
else:
|
|
2492
|
+
self.log(self.INFO, 'gpload failed')
|
|
2493
|
+
|
|
2494
|
+
## MPP-19015 - Extra python thread shutdown time is needed on HP-UX
|
|
2495
|
+
if platform.uname()[0] == 'HP-UX':
|
|
2496
|
+
time.sleep(1)
|
|
2497
|
+
|
|
2498
|
+
|
|
2499
|
+
# ---------------------------------------------------------------------------
|
|
2500
|
+
# Programmatic interface
|
|
2501
|
+
# ---------------------------------------------------------------------------
|
|
2502
|
+
|
|
2503
|
+
class GploadError(RuntimeError):
|
|
2504
|
+
"""Raised by :meth:`GploadResult.raise_for_status` on a failed load."""
|
|
2505
|
+
|
|
2506
|
+
def __init__(self, result):
|
|
2507
|
+
super().__init__('gpload failed with exit code %d' % result.returncode)
|
|
2508
|
+
self.result = result
|
|
2509
|
+
|
|
2510
|
+
|
|
2511
|
+
class GploadResult:
|
|
2512
|
+
"""Outcome of one gpload run.
|
|
2513
|
+
|
|
2514
|
+
``returncode`` follows the gpload/load.sh convention: 0 - success,
|
|
2515
|
+
1 - loaded with rejected rows, 2 - failure, 124 - API timeout.
|
|
2516
|
+
"""
|
|
2517
|
+
|
|
2518
|
+
def __init__(self, returncode, stdout='', stderr='', log_file=None,
|
|
2519
|
+
command=None):
|
|
2520
|
+
self.returncode = returncode
|
|
2521
|
+
self.stdout = stdout
|
|
2522
|
+
self.stderr = stderr
|
|
2523
|
+
self.log_file = log_file
|
|
2524
|
+
self.command = list(command) if command else []
|
|
2525
|
+
|
|
2526
|
+
@property
|
|
2527
|
+
def ok(self):
|
|
2528
|
+
return self.returncode == 0
|
|
2529
|
+
|
|
2530
|
+
@property
|
|
2531
|
+
def has_warnings(self):
|
|
2532
|
+
return self.returncode == 1
|
|
2533
|
+
|
|
2534
|
+
@property
|
|
2535
|
+
def failed(self):
|
|
2536
|
+
return self.returncode >= 2
|
|
2537
|
+
|
|
2538
|
+
def raise_for_status(self):
|
|
2539
|
+
"""Raise :class:`GploadError` unless the load succeeded."""
|
|
2540
|
+
if self.returncode != 0:
|
|
2541
|
+
raise GploadError(self)
|
|
2542
|
+
return self
|
|
2543
|
+
|
|
2544
|
+
def __repr__(self):
|
|
2545
|
+
return 'GploadResult(returncode=%d, log_file=%r)' % (
|
|
2546
|
+
self.returncode, self.log_file)
|
|
2547
|
+
|
|
2548
|
+
|
|
2549
|
+
class Gpload:
|
|
2550
|
+
"""Programmatic interface to gpload.
|
|
2551
|
+
|
|
2552
|
+
The load runs in a child process: gpload calls ``sys.exit()`` on errors
|
|
2553
|
+
and replaces the process signal handlers, so calling it in-process would
|
|
2554
|
+
affect the host application. A load that exceeds ``timeout`` is killed
|
|
2555
|
+
together with its gpfdist children (exit code 124).
|
|
2556
|
+
|
|
2557
|
+
Example::
|
|
2558
|
+
|
|
2559
|
+
gpload = Gpload()
|
|
2560
|
+
result = gpload.load('/path/config.yaml', log='/var/log/gpload.log')
|
|
2561
|
+
if result.has_warnings:
|
|
2562
|
+
...
|
|
2563
|
+
result.raise_for_status()
|
|
2564
|
+
"""
|
|
2565
|
+
|
|
2566
|
+
def __init__(self, script=None, python=None, timeout=600):
|
|
2567
|
+
script = script or globals().get('__file__')
|
|
2568
|
+
if not script:
|
|
2569
|
+
raise ValueError('cannot determine gpload.py path; pass script=')
|
|
2570
|
+
self.script = os.path.abspath(script)
|
|
2571
|
+
self.python = python or sys.executable
|
|
2572
|
+
self.timeout = timeout
|
|
2573
|
+
|
|
2574
|
+
def build_command(self, config, log=None, host=None, port=None,
|
|
2575
|
+
user=None, database=None, dry_run=False, quiet=False,
|
|
2576
|
+
verbose=False, very_verbose=False, no_auto_trans=False,
|
|
2577
|
+
gpfdist_timeout=None, extra_args=None):
|
|
2578
|
+
"""Build the gpload command line without running it."""
|
|
2579
|
+
cmd = [self.python, self.script, '-f', os.fspath(config)]
|
|
2580
|
+
if host is not None:
|
|
2581
|
+
cmd += ['-h', str(host)]
|
|
2582
|
+
if port is not None:
|
|
2583
|
+
cmd += ['-p', str(port)]
|
|
2584
|
+
if user is not None:
|
|
2585
|
+
cmd += ['-U', str(user)]
|
|
2586
|
+
if database is not None:
|
|
2587
|
+
cmd += ['-d', str(database)]
|
|
2588
|
+
if log is not None:
|
|
2589
|
+
cmd += ['-l', os.fspath(log)]
|
|
2590
|
+
if dry_run:
|
|
2591
|
+
cmd.append('-D')
|
|
2592
|
+
if quiet:
|
|
2593
|
+
cmd.append('-q')
|
|
2594
|
+
if verbose:
|
|
2595
|
+
cmd.append('-v')
|
|
2596
|
+
if very_verbose:
|
|
2597
|
+
cmd.append('-V')
|
|
2598
|
+
if no_auto_trans:
|
|
2599
|
+
cmd.append('--no_auto_trans')
|
|
2600
|
+
if gpfdist_timeout is not None:
|
|
2601
|
+
cmd += ['--gpfdist_timeout', str(gpfdist_timeout)]
|
|
2602
|
+
if extra_args:
|
|
2603
|
+
cmd += [str(a) for a in extra_args]
|
|
2604
|
+
return cmd
|
|
2605
|
+
|
|
2606
|
+
@staticmethod
|
|
2607
|
+
def _kill(proc):
|
|
2608
|
+
"""Kill the child and its gpfdist children."""
|
|
2609
|
+
if os.name == 'posix':
|
|
2610
|
+
try:
|
|
2611
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
2612
|
+
except OSError:
|
|
2613
|
+
pass
|
|
2614
|
+
else:
|
|
2615
|
+
proc.kill()
|
|
2616
|
+
|
|
2617
|
+
def load(self, config, timeout=None, env=None, cwd=None, **options):
|
|
2618
|
+
"""Run gpload for ``config`` and return a :class:`GploadResult`.
|
|
2619
|
+
|
|
2620
|
+
``options`` are the keyword arguments of :meth:`build_command`.
|
|
2621
|
+
``env`` entries are merged into the inherited environment.
|
|
2622
|
+
"""
|
|
2623
|
+
cmd = self.build_command(config, **options)
|
|
2624
|
+
log_file = options.get('log')
|
|
2625
|
+
run_env = os.environ.copy()
|
|
2626
|
+
if env:
|
|
2627
|
+
run_env.update(env)
|
|
2628
|
+
run_timeout = self.timeout if timeout is None else timeout
|
|
2629
|
+
|
|
2630
|
+
popen_kwargs = {}
|
|
2631
|
+
if os.name == 'posix':
|
|
2632
|
+
# Own process group: on timeout gpload and its gpfdist children
|
|
2633
|
+
# are killed together.
|
|
2634
|
+
popen_kwargs['start_new_session'] = True
|
|
2635
|
+
|
|
2636
|
+
proc = subprocess.Popen(
|
|
2637
|
+
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
2638
|
+
text=True, encoding='utf-8', errors='replace',
|
|
2639
|
+
env=run_env, cwd=os.fspath(cwd) if cwd else None,
|
|
2640
|
+
**popen_kwargs)
|
|
2641
|
+
try:
|
|
2642
|
+
stdout, stderr = proc.communicate(timeout=run_timeout)
|
|
2643
|
+
except subprocess.TimeoutExpired:
|
|
2644
|
+
self._kill(proc)
|
|
2645
|
+
stdout, stderr = proc.communicate()
|
|
2646
|
+
return GploadResult(124, stdout or '', stderr or '', log_file, cmd)
|
|
2647
|
+
except BaseException:
|
|
2648
|
+
# Ctrl-C in the caller must not leave gpload/gpfdist running:
|
|
2649
|
+
# the child is in its own process group and does not get SIGINT.
|
|
2650
|
+
self._kill(proc)
|
|
2651
|
+
proc.communicate()
|
|
2652
|
+
raise
|
|
2653
|
+
|
|
2654
|
+
return GploadResult(proc.returncode, stdout or '', stderr or '',
|
|
2655
|
+
log_file, cmd)
|
|
2656
|
+
|
|
2657
|
+
|
|
2658
|
+
def run_gpload(config, **kwargs):
|
|
2659
|
+
"""Shortcut for ``Gpload().load(config, **kwargs)``."""
|
|
2660
|
+
return Gpload().load(config, **kwargs)
|
|
2661
|
+
|
|
2662
|
+
|
|
2663
|
+
class GploadTableResult:
|
|
2664
|
+
"""Outcome of loading one ``<data_dir>/<database>/<table>`` directory.
|
|
2665
|
+
|
|
2666
|
+
``returncode`` uses the load.sh mapping: 0 - success, 1 - loaded with
|
|
2667
|
+
rejected rows, 2 - failure (including a timeout).
|
|
2668
|
+
"""
|
|
2669
|
+
|
|
2670
|
+
def __init__(self, database, table, files=(), returncode=0,
|
|
2671
|
+
gpload_result=None, loaded_files=(), failed_files=(),
|
|
2672
|
+
error_file=None, error_line=None, message=None,
|
|
2673
|
+
config_path=None, config_text=None):
|
|
2674
|
+
self.database = database
|
|
2675
|
+
self.table = table
|
|
2676
|
+
self.files = [os.fspath(path) for path in files]
|
|
2677
|
+
self.returncode = returncode
|
|
2678
|
+
self.gpload_result = gpload_result
|
|
2679
|
+
self.loaded_files = [os.fspath(path) for path in loaded_files]
|
|
2680
|
+
self.failed_files = [os.fspath(path) for path in failed_files]
|
|
2681
|
+
self.error_file = error_file
|
|
2682
|
+
self.error_line = error_line
|
|
2683
|
+
self.message = message
|
|
2684
|
+
self.config_path = config_path
|
|
2685
|
+
self.config_text = config_text
|
|
2686
|
+
|
|
2687
|
+
@property
|
|
2688
|
+
def source_dir(self):
|
|
2689
|
+
if not self.files:
|
|
2690
|
+
return None
|
|
2691
|
+
return os.path.dirname(self.files[0])
|
|
2692
|
+
|
|
2693
|
+
@property
|
|
2694
|
+
def ok(self):
|
|
2695
|
+
return self.returncode == 0
|
|
2696
|
+
|
|
2697
|
+
@property
|
|
2698
|
+
def has_warnings(self):
|
|
2699
|
+
return self.returncode == 1
|
|
2700
|
+
|
|
2701
|
+
@property
|
|
2702
|
+
def failed(self):
|
|
2703
|
+
return self.returncode >= 2
|
|
2704
|
+
|
|
2705
|
+
def __repr__(self):
|
|
2706
|
+
return 'GploadTableResult(database=%r, table=%r, returncode=%d)' % (
|
|
2707
|
+
self.database, self.table, self.returncode)
|
|
2708
|
+
|
|
2709
|
+
|
|
2710
|
+
class GploadBatchResult:
|
|
2711
|
+
"""Aggregated outcome of :meth:`GploadBatch.load`."""
|
|
2712
|
+
|
|
2713
|
+
def __init__(self, tables=()):
|
|
2714
|
+
self.tables = list(tables)
|
|
2715
|
+
|
|
2716
|
+
@property
|
|
2717
|
+
def returncode(self):
|
|
2718
|
+
code = 0
|
|
2719
|
+
for table in self.tables:
|
|
2720
|
+
if table.returncode >= 2:
|
|
2721
|
+
return 2
|
|
2722
|
+
if table.returncode == 1:
|
|
2723
|
+
code = 1
|
|
2724
|
+
return code
|
|
2725
|
+
|
|
2726
|
+
@property
|
|
2727
|
+
def ok(self):
|
|
2728
|
+
return self.returncode == 0
|
|
2729
|
+
|
|
2730
|
+
@property
|
|
2731
|
+
def has_warnings(self):
|
|
2732
|
+
return self.returncode == 1
|
|
2733
|
+
|
|
2734
|
+
@property
|
|
2735
|
+
def failed(self):
|
|
2736
|
+
return self.returncode >= 2
|
|
2737
|
+
|
|
2738
|
+
@property
|
|
2739
|
+
def loaded_tables(self):
|
|
2740
|
+
return [table for table in self.tables if table.ok]
|
|
2741
|
+
|
|
2742
|
+
@property
|
|
2743
|
+
def warning_tables(self):
|
|
2744
|
+
return [table for table in self.tables if table.has_warnings]
|
|
2745
|
+
|
|
2746
|
+
@property
|
|
2747
|
+
def failed_tables(self):
|
|
2748
|
+
return [table for table in self.tables if table.failed]
|
|
2749
|
+
|
|
2750
|
+
def raise_for_status(self):
|
|
2751
|
+
"""Raise :class:`GploadError` unless every table loaded cleanly."""
|
|
2752
|
+
if self.returncode != 0:
|
|
2753
|
+
raise GploadError(self)
|
|
2754
|
+
return self
|
|
2755
|
+
|
|
2756
|
+
def __iter__(self):
|
|
2757
|
+
return iter(self.tables)
|
|
2758
|
+
|
|
2759
|
+
def __len__(self):
|
|
2760
|
+
return len(self.tables)
|
|
2761
|
+
|
|
2762
|
+
def __repr__(self):
|
|
2763
|
+
return 'GploadBatchResult(tables=%d, returncode=%d)' % (
|
|
2764
|
+
len(self.tables), self.returncode)
|
|
2765
|
+
|
|
2766
|
+
|
|
2767
|
+
def default_config_template(script=None):
|
|
2768
|
+
"""Return the default ``gpload.yaml.ori`` path.
|
|
2769
|
+
|
|
2770
|
+
The template is looked up next to ``gpload.py`` (source checkout) and in
|
|
2771
|
+
the ``share/gpload-py3`` directory used by an installed package.
|
|
2772
|
+
"""
|
|
2773
|
+
script = script or globals().get('__file__')
|
|
2774
|
+
candidates = []
|
|
2775
|
+
if script:
|
|
2776
|
+
candidates.append(os.path.join(
|
|
2777
|
+
os.path.dirname(os.path.abspath(script)), 'gpload.yaml.ori'))
|
|
2778
|
+
|
|
2779
|
+
try:
|
|
2780
|
+
candidates.append(os.path.join(
|
|
2781
|
+
sysconfig.get_path('data'), 'share', 'gpload-py3',
|
|
2782
|
+
'gpload.yaml.ori'))
|
|
2783
|
+
except (KeyError, TypeError):
|
|
2784
|
+
pass
|
|
2785
|
+
|
|
2786
|
+
try:
|
|
2787
|
+
candidates.append(os.path.join(
|
|
2788
|
+
site.USER_BASE, 'share', 'gpload-py3', 'gpload.yaml.ori'))
|
|
2789
|
+
except (AttributeError, TypeError):
|
|
2790
|
+
pass
|
|
2791
|
+
|
|
2792
|
+
for candidate in candidates:
|
|
2793
|
+
if os.path.isfile(candidate):
|
|
2794
|
+
return candidate
|
|
2795
|
+
return candidates[0] if candidates else 'gpload.yaml.ori'
|
|
2796
|
+
|
|
2797
|
+
|
|
2798
|
+
class GploadBatch:
|
|
2799
|
+
"""Batch loader for a ``<data_dir>/<database>/<table>/*.csv`` tree.
|
|
2800
|
+
|
|
2801
|
+
This is the programmatic equivalent of ``script/load.sh``: for every
|
|
2802
|
+
table directory it renders a control file from ``gpload.yaml.ori`` (using
|
|
2803
|
+
the CSV header for ``HEADER``/``COLUMNS``), loads all CSV files of that
|
|
2804
|
+
directory in one gpload run, removes successfully loaded files and moves
|
|
2805
|
+
the files of a failed load to ``failed/``.
|
|
2806
|
+
|
|
2807
|
+
Each table is loaded in a separate child process, so no database driver
|
|
2808
|
+
is needed in the calling process.
|
|
2809
|
+
|
|
2810
|
+
Example::
|
|
2811
|
+
|
|
2812
|
+
from gpload import GploadBatch
|
|
2813
|
+
|
|
2814
|
+
batch = GploadBatch(
|
|
2815
|
+
'/opt/gpstall/data', # <db>/<table>/*.csv
|
|
2816
|
+
log_dir='/var/log/gpstall', # default: <data_dir>/../log
|
|
2817
|
+
user='gpadmin',
|
|
2818
|
+
password='secret',
|
|
2819
|
+
host='127.0.0.1',
|
|
2820
|
+
port=5432,
|
|
2821
|
+
)
|
|
2822
|
+
result = batch.load()
|
|
2823
|
+
print(result.returncode, len(result.tables))
|
|
2824
|
+
"""
|
|
2825
|
+
|
|
2826
|
+
def __init__(self, data_dir, log_dir=None, config_template=None, *,
|
|
2827
|
+
user=None, password=None, host=None, port=None,
|
|
2828
|
+
gpfdist_host=None, port_range='[8081,8082]',
|
|
2829
|
+
error_limit=50000, script=None, python=None, timeout=600,
|
|
2830
|
+
file_pattern='*.csv', keep_temp_configs=False):
|
|
2831
|
+
self.data_dir = os.path.abspath(os.fspath(data_dir))
|
|
2832
|
+
if log_dir is None:
|
|
2833
|
+
log_dir = os.path.join(os.path.dirname(self.data_dir), 'log')
|
|
2834
|
+
self.log_dir = os.path.abspath(os.fspath(log_dir))
|
|
2835
|
+
|
|
2836
|
+
script = script or globals().get('__file__')
|
|
2837
|
+
if not script:
|
|
2838
|
+
raise ValueError('cannot determine gpload.py path; pass script=')
|
|
2839
|
+
self.script = os.path.abspath(script)
|
|
2840
|
+
self.python = python or sys.executable
|
|
2841
|
+
|
|
2842
|
+
if config_template is None:
|
|
2843
|
+
config_template = default_config_template(self.script)
|
|
2844
|
+
self.config_template = os.path.abspath(os.fspath(config_template))
|
|
2845
|
+
|
|
2846
|
+
self.user = user if user is not None else (
|
|
2847
|
+
os.environ.get('PGUSER') or getpass.getuser())
|
|
2848
|
+
self.password = password if password is not None else (
|
|
2849
|
+
os.environ.get('PGPASSWORD'))
|
|
2850
|
+
self.host = host if host is not None else (
|
|
2851
|
+
os.environ.get('PGHOST') or '127.0.0.1')
|
|
2852
|
+
self.port = port if port is not None else (
|
|
2853
|
+
os.environ.get('PGPORT') or 5432)
|
|
2854
|
+
self.gpfdist_host = gpfdist_host if gpfdist_host is not None else (
|
|
2855
|
+
self.host)
|
|
2856
|
+
self.port_range = port_range
|
|
2857
|
+
self.error_limit = error_limit
|
|
2858
|
+
|
|
2859
|
+
self.timeout = timeout
|
|
2860
|
+
self.file_pattern = file_pattern
|
|
2861
|
+
self.keep_temp_configs = keep_temp_configs
|
|
2862
|
+
|
|
2863
|
+
self.log_file = os.path.join(self.log_dir, 'gpload.log')
|
|
2864
|
+
self.latest_failed_file = os.path.join(
|
|
2865
|
+
self.log_dir, 'latest_failed_file')
|
|
2866
|
+
|
|
2867
|
+
self.loader = Gpload(
|
|
2868
|
+
script=self.script, python=self.python, timeout=timeout)
|
|
2869
|
+
|
|
2870
|
+
# -- discovery ---------------------------------------------------------
|
|
2871
|
+
|
|
2872
|
+
@staticmethod
|
|
2873
|
+
def _subdirectories(path):
|
|
2874
|
+
try:
|
|
2875
|
+
with os.scandir(path) as entries:
|
|
2876
|
+
entries = sorted(entries, key=lambda entry: entry.name)
|
|
2877
|
+
except OSError:
|
|
2878
|
+
return []
|
|
2879
|
+
|
|
2880
|
+
names = []
|
|
2881
|
+
for entry in entries:
|
|
2882
|
+
if entry.name.startswith('.'):
|
|
2883
|
+
continue
|
|
2884
|
+
try:
|
|
2885
|
+
if entry.is_dir():
|
|
2886
|
+
names.append(entry.name)
|
|
2887
|
+
except OSError:
|
|
2888
|
+
pass
|
|
2889
|
+
return names
|
|
2890
|
+
|
|
2891
|
+
def table_files(self, table_dir):
|
|
2892
|
+
"""Return the CSV files of one table directory, sorted."""
|
|
2893
|
+
files = []
|
|
2894
|
+
try:
|
|
2895
|
+
with os.scandir(table_dir) as entries:
|
|
2896
|
+
entries = list(entries)
|
|
2897
|
+
except OSError:
|
|
2898
|
+
return files
|
|
2899
|
+
|
|
2900
|
+
for entry in entries:
|
|
2901
|
+
try:
|
|
2902
|
+
is_file = entry.is_file()
|
|
2903
|
+
except OSError:
|
|
2904
|
+
is_file = False
|
|
2905
|
+
if is_file and fnmatch.fnmatch(entry.name, self.file_pattern):
|
|
2906
|
+
files.append(os.path.join(table_dir, entry.name))
|
|
2907
|
+
return sorted(files)
|
|
2908
|
+
|
|
2909
|
+
def scan(self):
|
|
2910
|
+
"""Return ``[(database, table, [csv_file, ...]), ...]``.
|
|
2911
|
+
|
|
2912
|
+
Only tables that contain at least one matching file are returned.
|
|
2913
|
+
"""
|
|
2914
|
+
if not os.path.isdir(self.data_dir):
|
|
2915
|
+
raise FileNotFoundError(
|
|
2916
|
+
'data directory not found: %s' % self.data_dir)
|
|
2917
|
+
|
|
2918
|
+
tables = []
|
|
2919
|
+
for database in self._subdirectories(self.data_dir):
|
|
2920
|
+
database_dir = os.path.join(self.data_dir, database)
|
|
2921
|
+
for table in self._subdirectories(database_dir):
|
|
2922
|
+
table_dir = os.path.join(database_dir, table)
|
|
2923
|
+
files = self.table_files(table_dir)
|
|
2924
|
+
if files:
|
|
2925
|
+
tables.append((database, table, files))
|
|
2926
|
+
return tables
|
|
2927
|
+
|
|
2928
|
+
# -- control file rendering -------------------------------------------
|
|
2929
|
+
|
|
2930
|
+
@staticmethod
|
|
2931
|
+
def _read_header(path):
|
|
2932
|
+
with open(path, 'r', encoding='utf-8-sig', newline='') as handle:
|
|
2933
|
+
try:
|
|
2934
|
+
row = next(csv.reader(handle))
|
|
2935
|
+
except (StopIteration, csv.Error):
|
|
2936
|
+
return []
|
|
2937
|
+
return [column.strip() for column in row if column.strip()]
|
|
2938
|
+
|
|
2939
|
+
def render_config(self, database, table, files):
|
|
2940
|
+
"""Render the control file template for one table."""
|
|
2941
|
+
files = list(files)
|
|
2942
|
+
if not files:
|
|
2943
|
+
raise ValueError('no CSV files for %s.%s' % (database, table))
|
|
2944
|
+
|
|
2945
|
+
with open(self.config_template, 'r', encoding='utf-8') as handle:
|
|
2946
|
+
template = handle.read()
|
|
2947
|
+
|
|
2948
|
+
columns = self._read_header(files[0])
|
|
2949
|
+
if not columns:
|
|
2950
|
+
raise ValueError(
|
|
2951
|
+
'empty CSV header in %s' % files[0])
|
|
2952
|
+
|
|
2953
|
+
table_dir = os.path.dirname(os.path.abspath(files[0]))
|
|
2954
|
+
file_block = '\n - %s' % os.path.join(
|
|
2955
|
+
table_dir, self.file_pattern)
|
|
2956
|
+
|
|
2957
|
+
header_block = '- HEADER: true\n - COLUMNS:'
|
|
2958
|
+
for column in columns:
|
|
2959
|
+
header_block += '\n - %s:' % column
|
|
2960
|
+
|
|
2961
|
+
values = {
|
|
2962
|
+
'DATABASE': database,
|
|
2963
|
+
'TABLE': table,
|
|
2964
|
+
'FILE': file_block,
|
|
2965
|
+
'HEADER': header_block,
|
|
2966
|
+
'GPUSER': self.user,
|
|
2967
|
+
'GPHOST': self.host,
|
|
2968
|
+
'GPPORT': self.port,
|
|
2969
|
+
'GPDHOST': self.gpfdist_host,
|
|
2970
|
+
'GPDPORT_RANGE': self.port_range,
|
|
2971
|
+
'ERROR_LIMIT': self.error_limit,
|
|
2972
|
+
}
|
|
2973
|
+
pattern = re.compile(r'\{(' + '|'.join(values) + r')\}')
|
|
2974
|
+
return pattern.sub(
|
|
2975
|
+
lambda match: str(values[match.group(1)]), template)
|
|
2976
|
+
|
|
2977
|
+
# -- loading -----------------------------------------------------------
|
|
2978
|
+
|
|
2979
|
+
@staticmethod
|
|
2980
|
+
def _map_returncode(returncode):
|
|
2981
|
+
if returncode == 0:
|
|
2982
|
+
return 0
|
|
2983
|
+
if returncode == 1:
|
|
2984
|
+
return 1
|
|
2985
|
+
return 2
|
|
2986
|
+
|
|
2987
|
+
def _environment(self):
|
|
2988
|
+
env = os.environ.copy()
|
|
2989
|
+
if self.password is not None:
|
|
2990
|
+
env['PGPASSWORD'] = self.password
|
|
2991
|
+
|
|
2992
|
+
# load.sh removes $GPHOME/* from LD_LIBRARY_PATH: the system psycopg2
|
|
2993
|
+
# is linked against the OS libpq and cannot load the older Greenplum
|
|
2994
|
+
# one. gpfdist is started by gpload.py in its own shell, which
|
|
2995
|
+
# sources the Greenplum environment again.
|
|
2996
|
+
gphome = env.get('GPHOME')
|
|
2997
|
+
ld_path = env.get('LD_LIBRARY_PATH')
|
|
2998
|
+
if gphome and ld_path:
|
|
2999
|
+
prefix = gphome.rstrip(os.sep) + os.sep
|
|
3000
|
+
kept = [path for path in ld_path.split(os.pathsep)
|
|
3001
|
+
if path and not path.startswith(prefix)]
|
|
3002
|
+
if kept:
|
|
3003
|
+
env['LD_LIBRARY_PATH'] = os.pathsep.join(kept)
|
|
3004
|
+
else:
|
|
3005
|
+
env.pop('LD_LIBRARY_PATH', None)
|
|
3006
|
+
return env
|
|
3007
|
+
|
|
3008
|
+
def _append_log_separator(self):
|
|
3009
|
+
with open(self.log_file, 'a', encoding='utf-8') as handle:
|
|
3010
|
+
handle.write('------------------------------\n')
|
|
3011
|
+
handle.flush()
|
|
3012
|
+
return handle.tell()
|
|
3013
|
+
|
|
3014
|
+
def _read_log_since(self, offset):
|
|
3015
|
+
try:
|
|
3016
|
+
with open(self.log_file, 'r', encoding='utf-8',
|
|
3017
|
+
errors='replace') as handle:
|
|
3018
|
+
handle.seek(offset)
|
|
3019
|
+
return handle.read()
|
|
3020
|
+
except OSError:
|
|
3021
|
+
return ''
|
|
3022
|
+
|
|
3023
|
+
@staticmethod
|
|
3024
|
+
def _parse_error_location(text):
|
|
3025
|
+
error_file = None
|
|
3026
|
+
error_line = None
|
|
3027
|
+
for line in (text or '').splitlines()[-50:]:
|
|
3028
|
+
if 'error file:' in line:
|
|
3029
|
+
error_file = line.split('error file:', 1)[1].strip()
|
|
3030
|
+
if 'error line:' in line:
|
|
3031
|
+
error_line = line.split('error line:', 1)[1].strip()
|
|
3032
|
+
return error_file or None, error_line or None
|
|
3033
|
+
|
|
3034
|
+
@staticmethod
|
|
3035
|
+
def _remove_loaded(files):
|
|
3036
|
+
removed = []
|
|
3037
|
+
errors = []
|
|
3038
|
+
for path in files:
|
|
3039
|
+
try:
|
|
3040
|
+
os.unlink(path)
|
|
3041
|
+
removed.append(path)
|
|
3042
|
+
except OSError as exc:
|
|
3043
|
+
errors.append('%s: %s' % (path, exc))
|
|
3044
|
+
message = None
|
|
3045
|
+
if errors:
|
|
3046
|
+
message = 'could not remove loaded files: ' + '; '.join(errors)
|
|
3047
|
+
return removed, message
|
|
3048
|
+
|
|
3049
|
+
@staticmethod
|
|
3050
|
+
def _move_to_failed(files, failed_dir):
|
|
3051
|
+
os.makedirs(failed_dir, exist_ok=True)
|
|
3052
|
+
moved = []
|
|
3053
|
+
for path in files:
|
|
3054
|
+
if not os.path.exists(path):
|
|
3055
|
+
continue
|
|
3056
|
+
destination = os.path.join(
|
|
3057
|
+
failed_dir, os.path.basename(path))
|
|
3058
|
+
try:
|
|
3059
|
+
os.replace(path, destination)
|
|
3060
|
+
moved.append(destination)
|
|
3061
|
+
except OSError:
|
|
3062
|
+
pass
|
|
3063
|
+
return moved
|
|
3064
|
+
|
|
3065
|
+
def _quarantine(self, table_dir, files, error_file, error_line):
|
|
3066
|
+
failed_dir = os.path.join(table_dir, 'failed')
|
|
3067
|
+
os.makedirs(failed_dir, exist_ok=True)
|
|
3068
|
+
|
|
3069
|
+
try:
|
|
3070
|
+
os.unlink(self.latest_failed_file)
|
|
3071
|
+
except OSError:
|
|
3072
|
+
pass
|
|
3073
|
+
|
|
3074
|
+
target = None
|
|
3075
|
+
if error_file:
|
|
3076
|
+
candidate = error_file if os.path.isabs(error_file) else \
|
|
3077
|
+
os.path.join(table_dir, error_file)
|
|
3078
|
+
candidate = os.path.abspath(candidate)
|
|
3079
|
+
# Only quarantine a file that really belongs to this table.
|
|
3080
|
+
if os.path.dirname(candidate) == os.path.abspath(table_dir):
|
|
3081
|
+
target = candidate
|
|
3082
|
+
|
|
3083
|
+
if error_line and target and os.path.isfile(target):
|
|
3084
|
+
destination = os.path.join(
|
|
3085
|
+
failed_dir,
|
|
3086
|
+
os.path.basename(target) + '_' + error_line)
|
|
3087
|
+
try:
|
|
3088
|
+
os.replace(target, destination)
|
|
3089
|
+
moved = [destination]
|
|
3090
|
+
entries = [destination]
|
|
3091
|
+
except OSError:
|
|
3092
|
+
moved = self._move_to_failed(files, failed_dir)
|
|
3093
|
+
entries = moved
|
|
3094
|
+
else:
|
|
3095
|
+
moved = self._move_to_failed(files, failed_dir)
|
|
3096
|
+
try:
|
|
3097
|
+
entries = sorted(
|
|
3098
|
+
os.path.join(failed_dir, name)
|
|
3099
|
+
for name in os.listdir(failed_dir))
|
|
3100
|
+
except OSError:
|
|
3101
|
+
entries = []
|
|
3102
|
+
|
|
3103
|
+
with open(self.latest_failed_file, 'w', encoding='utf-8') as handle:
|
|
3104
|
+
for entry in entries:
|
|
3105
|
+
handle.write(entry + '\n')
|
|
3106
|
+
handle.flush()
|
|
3107
|
+
return moved
|
|
3108
|
+
|
|
3109
|
+
def load_table(self, database, table, files=None):
|
|
3110
|
+
"""Load all CSV files of one ``<database>/<table>`` directory."""
|
|
3111
|
+
table_dir = os.path.join(self.data_dir, database, table)
|
|
3112
|
+
if files is None:
|
|
3113
|
+
files = self.table_files(table_dir)
|
|
3114
|
+
files = [os.path.abspath(os.fspath(path)) for path in files]
|
|
3115
|
+
if not files:
|
|
3116
|
+
return GploadTableResult(database, table)
|
|
3117
|
+
os.makedirs(self.log_dir, exist_ok=True)
|
|
3118
|
+
|
|
3119
|
+
try:
|
|
3120
|
+
config_text = self.render_config(database, table, files)
|
|
3121
|
+
except (OSError, UnicodeError, ValueError, csv.Error) as exc:
|
|
3122
|
+
return GploadTableResult(
|
|
3123
|
+
database, table, files=files, returncode=2,
|
|
3124
|
+
message='cannot render control file: %s' % exc)
|
|
3125
|
+
|
|
3126
|
+
descriptor, config_path = tempfile.mkstemp(
|
|
3127
|
+
prefix='gpload-batch-', suffix='.yaml')
|
|
3128
|
+
try:
|
|
3129
|
+
with os.fdopen(descriptor, 'w', encoding='utf-8') as handle:
|
|
3130
|
+
handle.write(config_text)
|
|
3131
|
+
|
|
3132
|
+
log_offset = self._append_log_separator()
|
|
3133
|
+
run = self.loader.load(
|
|
3134
|
+
config_path, log=self.log_file, env=self._environment(),
|
|
3135
|
+
timeout=self.timeout)
|
|
3136
|
+
output = self._read_log_since(log_offset) + run.stdout + \
|
|
3137
|
+
run.stderr
|
|
3138
|
+
finally:
|
|
3139
|
+
if not self.keep_temp_configs:
|
|
3140
|
+
try:
|
|
3141
|
+
os.unlink(config_path)
|
|
3142
|
+
except OSError:
|
|
3143
|
+
pass
|
|
3144
|
+
|
|
3145
|
+
returncode = self._map_returncode(run.returncode)
|
|
3146
|
+
if returncode == 0:
|
|
3147
|
+
loaded, message = self._remove_loaded(files)
|
|
3148
|
+
return GploadTableResult(
|
|
3149
|
+
database, table, files=files, returncode=0,
|
|
3150
|
+
gpload_result=run, loaded_files=loaded, message=message,
|
|
3151
|
+
config_path=config_path, config_text=config_text)
|
|
3152
|
+
|
|
3153
|
+
error_file, error_line = self._parse_error_location(output)
|
|
3154
|
+
failed = self._quarantine(
|
|
3155
|
+
table_dir, files, error_file, error_line)
|
|
3156
|
+
message = 'gpload exited with code %d' % run.returncode
|
|
3157
|
+
if run.returncode == 124:
|
|
3158
|
+
message = 'gpload timed out after %s seconds' % self.timeout
|
|
3159
|
+
return GploadTableResult(
|
|
3160
|
+
database, table, files=files, returncode=returncode,
|
|
3161
|
+
gpload_result=run, failed_files=failed, error_file=error_file,
|
|
3162
|
+
error_line=error_line, message=message,
|
|
3163
|
+
config_path=config_path, config_text=config_text)
|
|
3164
|
+
|
|
3165
|
+
def load_dir(self, on_table=None):
|
|
3166
|
+
"""Alias for :meth:`load`."""
|
|
3167
|
+
return self.load(on_table=on_table)
|
|
3168
|
+
|
|
3169
|
+
def load(self, on_table=None):
|
|
3170
|
+
"""Load every table directory and return a :class:`GploadBatchResult`.
|
|
3171
|
+
|
|
3172
|
+
``on_table`` is an optional callback invoked with each
|
|
3173
|
+
:class:`GploadTableResult` as soon as that table is processed.
|
|
3174
|
+
"""
|
|
3175
|
+
if not os.path.isdir(self.data_dir):
|
|
3176
|
+
raise FileNotFoundError(
|
|
3177
|
+
'data directory not found: %s' % self.data_dir)
|
|
3178
|
+
if not os.path.isfile(self.config_template):
|
|
3179
|
+
raise FileNotFoundError(
|
|
3180
|
+
'control file template not found: %s' % self.config_template)
|
|
3181
|
+
|
|
3182
|
+
os.makedirs(self.log_dir, exist_ok=True)
|
|
3183
|
+
|
|
3184
|
+
results = []
|
|
3185
|
+
for database, table, files in self.scan():
|
|
3186
|
+
result = self.load_table(database, table, files)
|
|
3187
|
+
results.append(result)
|
|
3188
|
+
if on_table is not None:
|
|
3189
|
+
on_table(result)
|
|
3190
|
+
return GploadBatchResult(results)
|
|
3191
|
+
|
|
3192
|
+
|
|
3193
|
+
def run_gpload_batch(data_dir, **kwargs):
|
|
3194
|
+
"""Shortcut for ``GploadBatch(data_dir, **kwargs).load()``."""
|
|
3195
|
+
return GploadBatch(data_dir, **kwargs).load()
|
|
3196
|
+
|
|
3197
|
+
|
|
3198
|
+
def main(argv=None):
|
|
3199
|
+
"""Command-line entry point used by the ``gpload-py3`` console script."""
|
|
3200
|
+
instance = gpload(sys.argv[1:] if argv is None else argv)
|
|
3201
|
+
instance.run()
|
|
3202
|
+
return instance.exitValue
|
|
3203
|
+
|
|
3204
|
+
|
|
3205
|
+
if __name__ == '__main__':
|
|
3206
|
+
sys.exit(main())
|