bigquery-fdw 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bigquery_fdw/__init__.py +0 -0
- bigquery_fdw/bqclient.py +114 -0
- bigquery_fdw/bqclient_test.py +96 -0
- bigquery_fdw/fdw.py +461 -0
- bigquery_fdw-2.2.0.dist-info/METADATA +153 -0
- bigquery_fdw-2.2.0.dist-info/RECORD +10 -0
- bigquery_fdw-2.2.0.dist-info/WHEEL +5 -0
- bigquery_fdw-2.2.0.dist-info/entry_points.txt +2 -0
- bigquery_fdw-2.2.0.dist-info/licenses/LICENSE +21 -0
- bigquery_fdw-2.2.0.dist-info/top_level.txt +1 -0
bigquery_fdw/__init__.py
ADDED
|
File without changes
|
bigquery_fdw/bqclient.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""
|
|
2
|
+
See:
|
|
3
|
+
https://cloud.google.com/bigquery/docs/reference/libraries
|
|
4
|
+
https://googlecloudplatform.github.io/google-cloud-python/stable/bigquery/usage.html?highlight=insert_data
|
|
5
|
+
https://github.com/GoogleCloudPlatform/python-docs-samples/tree/master/bigquery/cloud-client
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import datetime
|
|
9
|
+
|
|
10
|
+
from google.cloud import bigquery
|
|
11
|
+
from google import auth
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BqClient:
|
|
15
|
+
|
|
16
|
+
# Set vars
|
|
17
|
+
client = None
|
|
18
|
+
queryJob = None
|
|
19
|
+
location = None # Override dataset location
|
|
20
|
+
|
|
21
|
+
def setClient(self):
|
|
22
|
+
"""
|
|
23
|
+
Set BigQuery client with a Json key
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
self.client = bigquery.Client()
|
|
28
|
+
except auth.exceptions.DefaultCredentialsError as e:
|
|
29
|
+
raise RuntimeError(
|
|
30
|
+
'BigQuery client is not instantiated properly: ' + str(e))
|
|
31
|
+
|
|
32
|
+
def getClient(self):
|
|
33
|
+
"""
|
|
34
|
+
Returns `client`
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
return self.client
|
|
38
|
+
|
|
39
|
+
def runQuery(self, query, parameters=[], sqlDialect='standard'):
|
|
40
|
+
"""
|
|
41
|
+
Run BigQuery query
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
if self.client:
|
|
45
|
+
if parameters: # Parameterized query
|
|
46
|
+
# Prepare job configuration
|
|
47
|
+
job_config = bigquery.QueryJobConfig()
|
|
48
|
+
job_config.query_parameters = parameters
|
|
49
|
+
|
|
50
|
+
self.queryJob = self.client.query(
|
|
51
|
+
query, job_config=job_config, location=None)
|
|
52
|
+
else: # Non parameterized query
|
|
53
|
+
self.queryJob = self.client.query(query, location=None)
|
|
54
|
+
|
|
55
|
+
# Set SQL dialect
|
|
56
|
+
self.queryJob.UseLegacySQL = False
|
|
57
|
+
# if sqlDialect == 'legacy':
|
|
58
|
+
# self.queryJob.UseLegacySQL = True
|
|
59
|
+
# else:
|
|
60
|
+
# self.queryJob.UseLegacySQL = False
|
|
61
|
+
|
|
62
|
+
else:
|
|
63
|
+
raise RuntimeError(
|
|
64
|
+
'BigQuery client is not instantiated properly (from `runQuery`).')
|
|
65
|
+
|
|
66
|
+
def getQueryJob(self):
|
|
67
|
+
"""
|
|
68
|
+
Returns `queryJob`
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
return self.queryJob
|
|
72
|
+
|
|
73
|
+
def readResult(self):
|
|
74
|
+
"""
|
|
75
|
+
Returns query result
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
if self.queryJob:
|
|
79
|
+
result = self.queryJob.result()
|
|
80
|
+
return list(result)
|
|
81
|
+
else:
|
|
82
|
+
raise RuntimeError('No query is pending a result.')
|
|
83
|
+
|
|
84
|
+
def setParameter(self, type_, value):
|
|
85
|
+
"""
|
|
86
|
+
Prepare a parameter for a parameterized query
|
|
87
|
+
As documented by Google, only standard SQL syntax supports parameters in queries
|
|
88
|
+
|
|
89
|
+
See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/bigquery/cloud-client/query_params.py
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
return bigquery.ScalarQueryParameter(
|
|
93
|
+
# Set the name to None to use positional parameters (? symbol
|
|
94
|
+
# in the query). Note that you cannot mix named and positional
|
|
95
|
+
# parameters.
|
|
96
|
+
None, type_, self.varToString(value))
|
|
97
|
+
|
|
98
|
+
def varToString(self, var):
|
|
99
|
+
"""
|
|
100
|
+
Format a variable to a string
|
|
101
|
+
|
|
102
|
+
Examples:
|
|
103
|
+
`varToString(1234)` -> '1234'
|
|
104
|
+
`varToString('1234')` -> '1234'
|
|
105
|
+
`varToString(datetime.datetime.now())` -> '2017-10-03 11:47:54'
|
|
106
|
+
`varToString(datetime.datetime.now().date())` -> '2017-10-03'
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
if type(var) is datetime.date:
|
|
110
|
+
return var.strftime('%Y-%m-%d')
|
|
111
|
+
elif type(var) is datetime.datetime:
|
|
112
|
+
return var.strftime('%Y-%m-%d %H:%M:%S')
|
|
113
|
+
|
|
114
|
+
return str(var)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
|
|
5
|
+
from .bqclient import BqClient
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def run_test(query):
|
|
9
|
+
"""
|
|
10
|
+
BigQuery client test
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
# BqClient instance
|
|
14
|
+
bq = set_bq_instance()
|
|
15
|
+
|
|
16
|
+
# Authentication
|
|
17
|
+
set_client(bq)
|
|
18
|
+
print(" * BigQuery client instance:")
|
|
19
|
+
print(get_client(bq))
|
|
20
|
+
|
|
21
|
+
# Run query
|
|
22
|
+
run_query(bq, query)
|
|
23
|
+
print(" * Query instance:")
|
|
24
|
+
print(get_query_job(bq))
|
|
25
|
+
|
|
26
|
+
# Print the results.
|
|
27
|
+
print(" * Query results:")
|
|
28
|
+
for row in read_results(bq):
|
|
29
|
+
print(row)
|
|
30
|
+
|
|
31
|
+
return True
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def set_bq_instance():
|
|
35
|
+
"""
|
|
36
|
+
Create instance of BqClient
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
return BqClient()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def set_client(bq):
|
|
43
|
+
"""
|
|
44
|
+
Set BigQuery client
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
return bq.setClient()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def get_client(bq):
|
|
51
|
+
"""
|
|
52
|
+
Get BigQuery client
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
return bq.getClient()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def run_query(bq, query):
|
|
59
|
+
"""
|
|
60
|
+
Run query
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
return bq.runQuery(query)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def get_query_job(bq):
|
|
67
|
+
"""
|
|
68
|
+
Get query job
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
return bq.getQueryJob()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def read_results(bq):
|
|
75
|
+
"""
|
|
76
|
+
Read results
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
return [row for row in bq.readResult()]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def main():
|
|
83
|
+
|
|
84
|
+
# Parse arguments
|
|
85
|
+
parser = argparse.ArgumentParser()
|
|
86
|
+
parser.add_argument("-a", "--auth_type", type=str, help="Authentication type",
|
|
87
|
+
choices=['service_account'], default='service_account')
|
|
88
|
+
parser.add_argument("-q", "--query", type=str,
|
|
89
|
+
help="Query to run", required=True)
|
|
90
|
+
args = parser.parse_args()
|
|
91
|
+
|
|
92
|
+
run_test(query=args.query)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
if __name__ == '__main__':
|
|
96
|
+
main()
|
bigquery_fdw/fdw.py
ADDED
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
from collections import OrderedDict, namedtuple
|
|
2
|
+
|
|
3
|
+
from multicorn import ForeignDataWrapper
|
|
4
|
+
from multicorn.utils import log_to_postgres, ERROR, WARNING, INFO, DEBUG
|
|
5
|
+
|
|
6
|
+
from .bqclient import BqClient
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ConstantForeignDataWrapper(ForeignDataWrapper):
|
|
10
|
+
|
|
11
|
+
# Default vars
|
|
12
|
+
client = None # BqClient instance
|
|
13
|
+
partitionPseudoColumn = 'partition_date' # Name of the partition pseudo column
|
|
14
|
+
# Pseudo column to fetch `count(*)` when using the remote counting and grouping feature
|
|
15
|
+
countPseudoColumn = '_fdw_count'
|
|
16
|
+
castingRules = None # Dict of casting rules when using the `fdw_casting` option
|
|
17
|
+
|
|
18
|
+
def __init__(self, options, columns):
|
|
19
|
+
"""
|
|
20
|
+
Initialize instance, set class level vars
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
super(ConstantForeignDataWrapper, self).__init__(options, columns)
|
|
24
|
+
|
|
25
|
+
# Set options
|
|
26
|
+
self.setOptions(options)
|
|
27
|
+
|
|
28
|
+
# Set table columns
|
|
29
|
+
self.columns = columns
|
|
30
|
+
|
|
31
|
+
# Set data types and conversion rules mapping
|
|
32
|
+
self.setDatatypes()
|
|
33
|
+
self.setConversionRules()
|
|
34
|
+
|
|
35
|
+
def setOptions(self, options):
|
|
36
|
+
"""
|
|
37
|
+
Set table options at class level
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
# Set options at class scope
|
|
41
|
+
try:
|
|
42
|
+
self.dataset = options['fdw_dataset']
|
|
43
|
+
self.table = options['fdw_table']
|
|
44
|
+
self.convertToTz = options.get('fdw_convert_tz')
|
|
45
|
+
|
|
46
|
+
# Set verbose option
|
|
47
|
+
self.setOptionVerbose(options.get('fdw_verbose'))
|
|
48
|
+
|
|
49
|
+
# Set SQL dialect
|
|
50
|
+
self.setOptionSqlDialect(options.get('fdw_sql_dialect'))
|
|
51
|
+
|
|
52
|
+
# Set grouping option
|
|
53
|
+
self.setOptionGroupBy(options.get('fdw_group'))
|
|
54
|
+
|
|
55
|
+
# Set casting rules
|
|
56
|
+
self.setOptionCasting(options.get('fdw_casting'))
|
|
57
|
+
except KeyError:
|
|
58
|
+
log_to_postgres(
|
|
59
|
+
"You must specify these options when creating the FDW: fdw_dataset, fdw_table", ERROR)
|
|
60
|
+
|
|
61
|
+
def setDatatypes(self):
|
|
62
|
+
"""
|
|
63
|
+
Set data types mapping
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
# Create a named tuple
|
|
67
|
+
datatype = namedtuple('datatype', 'postgres bq_standard bq_legacy')
|
|
68
|
+
|
|
69
|
+
self.datatypes = [
|
|
70
|
+
datatype('text', 'STRING', 'STRING'),
|
|
71
|
+
# datatype('bytea', 'BYTES', 'BYTES'), # Not supported, need testing for support
|
|
72
|
+
datatype('bigint', 'INT64', 'INTEGER'),
|
|
73
|
+
datatype('double precision', 'FLOAT64', 'FLOAT'),
|
|
74
|
+
datatype('boolean', 'BOOL', 'BOOLEAN'),
|
|
75
|
+
datatype('timestamp without time zone', 'TIMESTAMP', 'TIMESTAMP'),
|
|
76
|
+
datatype('date', 'DATE', 'DATE'),
|
|
77
|
+
datatype('time without time zone', 'TIME', 'TIME'),
|
|
78
|
+
datatype('timestamp without time zone', 'DATETIME', 'DATETIME'),
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
def setConversionRules(self):
|
|
82
|
+
"""
|
|
83
|
+
Set list of allowed conversion rules
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
# Create a named tuple
|
|
87
|
+
conversionRule = namedtuple(
|
|
88
|
+
'conversionRule', 'bq_standard_from bq_standard_to')
|
|
89
|
+
|
|
90
|
+
self.conversionRules = [
|
|
91
|
+
conversionRule('INT64', ['BOOL', 'FLOAT64', 'INT64', 'STRING']),
|
|
92
|
+
conversionRule('FLOAT64', ['FLOAT64', 'INT64', 'STRING']),
|
|
93
|
+
conversionRule('BOOL', ['BOOL', 'INT64', 'STRING']),
|
|
94
|
+
conversionRule('STRING', ['BOOL', 'BYTES', 'DATE', 'DATETIME',
|
|
95
|
+
'FLOAT64', 'INT64', 'STRING', 'TIME', 'TIMESTAMP']),
|
|
96
|
+
conversionRule('BYTES', ['BYTES', 'STRING']),
|
|
97
|
+
conversionRule(
|
|
98
|
+
'DATE', ['DATE', 'DATETIME', 'STRING', 'TIMESTAMP']),
|
|
99
|
+
conversionRule(
|
|
100
|
+
'DATETIME', ['DATE', 'DATETIME', 'STRING', 'TIME', 'TIMESTAMP']),
|
|
101
|
+
conversionRule('TIME', ['STRING', 'TIME']),
|
|
102
|
+
conversionRule(
|
|
103
|
+
'TIMESTAMP', ['DATE', 'DATETIME', 'STRING', 'TIME', 'TIMESTAMP']),
|
|
104
|
+
conversionRule('ARRAY', ['ARRAY']),
|
|
105
|
+
conversionRule('STRUCT', ['STRUCT']),
|
|
106
|
+
]
|
|
107
|
+
|
|
108
|
+
def setOptionSqlDialect(self, standard_sql=None):
|
|
109
|
+
"""
|
|
110
|
+
Set a flag for the SQL dialect.
|
|
111
|
+
It can be `standard` or `legacy`. `standard` will be the default
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
self.dialect = 'standard'
|
|
115
|
+
|
|
116
|
+
if standard_sql == 'legacy':
|
|
117
|
+
self.dialect = 'legacy'
|
|
118
|
+
|
|
119
|
+
# Verbose log
|
|
120
|
+
if self.verbose:
|
|
121
|
+
log_to_postgres("Set SQL dialect to `" + self.dialect + "`", INFO)
|
|
122
|
+
|
|
123
|
+
def setOptionGroupBy(self, group):
|
|
124
|
+
"""
|
|
125
|
+
Set a flag `self.groupBy` as `True` if `group` contains the string 'true'
|
|
126
|
+
Otherwise, set it as `False`
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
if group == 'true':
|
|
130
|
+
self.groupBy = True
|
|
131
|
+
return
|
|
132
|
+
|
|
133
|
+
self.groupBy = False
|
|
134
|
+
|
|
135
|
+
def setOptionVerbose(self, verbose):
|
|
136
|
+
"""
|
|
137
|
+
Set a flag `self.verbose` as `True` if `verbose` contains the string 'true'
|
|
138
|
+
Otherwise, set it as `False`
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
if verbose == 'true':
|
|
142
|
+
self.verbose = True
|
|
143
|
+
return
|
|
144
|
+
|
|
145
|
+
self.verbose = False
|
|
146
|
+
|
|
147
|
+
def setOptionCasting(self, castingRules):
|
|
148
|
+
"""
|
|
149
|
+
Conversion rules are received as a string, for example: '{"key": "FLOAT64", "datetime": "DATE"}'
|
|
150
|
+
|
|
151
|
+
The string will be converted to a dict
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
if castingRules:
|
|
155
|
+
# Cast string as a dict
|
|
156
|
+
try:
|
|
157
|
+
import ast
|
|
158
|
+
self.castingRules = ast.literal_eval(castingRules)
|
|
159
|
+
except Exception as e:
|
|
160
|
+
log_to_postgres(
|
|
161
|
+
"fdw_casting conversion failed: `" + str(e) + "`", ERROR)
|
|
162
|
+
|
|
163
|
+
# For security reasons, ensure that the string was correctly casted as a dict
|
|
164
|
+
try:
|
|
165
|
+
if type(self.castingRules) is not dict:
|
|
166
|
+
raise ValueError('fdw_casting format is incorrect.')
|
|
167
|
+
except Exception as e:
|
|
168
|
+
log_to_postgres(
|
|
169
|
+
"fdw_casting conversion failed: `" + str(e) + "`", ERROR)
|
|
170
|
+
|
|
171
|
+
def getClient(self):
|
|
172
|
+
"""
|
|
173
|
+
Manage a pool of instances of BqClient
|
|
174
|
+
If the user uses different private keys in different tables, this method will return
|
|
175
|
+
the correct instance of BqClient class associated to the table private key
|
|
176
|
+
"""
|
|
177
|
+
|
|
178
|
+
# Returns an existing instance
|
|
179
|
+
if self.client:
|
|
180
|
+
return self.client
|
|
181
|
+
|
|
182
|
+
# Or create a new instance
|
|
183
|
+
return self.setClient()
|
|
184
|
+
|
|
185
|
+
def setClient(self):
|
|
186
|
+
"""
|
|
187
|
+
Attempt to connect to BigQuery client
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
try:
|
|
191
|
+
# Attempt connection
|
|
192
|
+
bq = BqClient()
|
|
193
|
+
bq.setClient()
|
|
194
|
+
|
|
195
|
+
# Verbose log
|
|
196
|
+
if self.verbose:
|
|
197
|
+
log_to_postgres(
|
|
198
|
+
"Connection to BigQuery client with BqClient instance ID " + str(id(bq)), INFO)
|
|
199
|
+
|
|
200
|
+
# Add to pool
|
|
201
|
+
self.client = bq
|
|
202
|
+
|
|
203
|
+
return bq
|
|
204
|
+
except RuntimeError:
|
|
205
|
+
log_to_postgres(
|
|
206
|
+
"Connection to BigQuery client failed", ERROR)
|
|
207
|
+
|
|
208
|
+
def execute(self, quals, columns):
|
|
209
|
+
"""
|
|
210
|
+
Executes a query
|
|
211
|
+
"""
|
|
212
|
+
|
|
213
|
+
# # Verbose log
|
|
214
|
+
# if self.verbose:
|
|
215
|
+
# log_to_postgres('Quals...', INFO)
|
|
216
|
+
# log_to_postgres(quals, INFO)
|
|
217
|
+
# log_to_postgres('Columns...', INFO)
|
|
218
|
+
# log_to_postgres(columns, INFO)
|
|
219
|
+
|
|
220
|
+
# Returns instance of BqClient
|
|
221
|
+
client = self.getClient()
|
|
222
|
+
|
|
223
|
+
# Prepare query
|
|
224
|
+
query, parameters = self.buildQuery(quals, columns)
|
|
225
|
+
|
|
226
|
+
# Run query
|
|
227
|
+
client.runQuery(query, parameters, self.dialect)
|
|
228
|
+
|
|
229
|
+
# Return query output
|
|
230
|
+
for row in client.readResult():
|
|
231
|
+
# Create an ordered dict with the column name and value
|
|
232
|
+
# Example: `OrderedDict([('column1', 'value1'), ('column2', value2)])`
|
|
233
|
+
line = OrderedDict()
|
|
234
|
+
for column in columns:
|
|
235
|
+
line[column] = row[column]
|
|
236
|
+
|
|
237
|
+
yield line
|
|
238
|
+
|
|
239
|
+
def buildQuery(self, quals, columns):
|
|
240
|
+
"""
|
|
241
|
+
Builds a BigQuery query
|
|
242
|
+
"""
|
|
243
|
+
|
|
244
|
+
# Set query var
|
|
245
|
+
query = ''
|
|
246
|
+
|
|
247
|
+
# Add SELECT clause
|
|
248
|
+
query += 'SELECT ' + self.buildColumnList(columns)
|
|
249
|
+
|
|
250
|
+
# Add FROM clause
|
|
251
|
+
query += " FROM `" + self.dataset + "." + self.table + "` "
|
|
252
|
+
|
|
253
|
+
# Add WHERE clause
|
|
254
|
+
clause, parameters = self.buildWhereClause(quals)
|
|
255
|
+
query += clause
|
|
256
|
+
|
|
257
|
+
# Add group by
|
|
258
|
+
if self.groupBy:
|
|
259
|
+
groupByColumns = self.buildColumnList(columns, 'GROUP_BY')
|
|
260
|
+
if groupByColumns:
|
|
261
|
+
query += ' GROUP BY ' + \
|
|
262
|
+
self.buildColumnList(columns, 'GROUP_BY')
|
|
263
|
+
|
|
264
|
+
# Verbose log
|
|
265
|
+
if self.verbose:
|
|
266
|
+
log_to_postgres("Prepared query: `" + query + "`", INFO)
|
|
267
|
+
|
|
268
|
+
return query, parameters
|
|
269
|
+
|
|
270
|
+
def buildColumnList(self, columns, usage='SELECT'):
|
|
271
|
+
"""
|
|
272
|
+
Build the SELECT clause of the SQL query
|
|
273
|
+
"""
|
|
274
|
+
|
|
275
|
+
clause = ''
|
|
276
|
+
|
|
277
|
+
# Disable aliases for Group By
|
|
278
|
+
useAliases = True
|
|
279
|
+
if usage == 'GROUP_BY':
|
|
280
|
+
useAliases = False
|
|
281
|
+
|
|
282
|
+
if columns: # If we have columns
|
|
283
|
+
for column in columns:
|
|
284
|
+
if column == self.countPseudoColumn: # Pseudo column to count grouped rows
|
|
285
|
+
if usage == 'SELECT':
|
|
286
|
+
clause += "count(*) " + \
|
|
287
|
+
self.addColumnAlias(column, useAliases) + ", "
|
|
288
|
+
elif column == self.partitionPseudoColumn: # Partition pseudo column
|
|
289
|
+
clause += "_PARTITIONTIME " + \
|
|
290
|
+
self.addColumnAlias(column, useAliases) + ", "
|
|
291
|
+
else: # Any other column
|
|
292
|
+
# Get column data type
|
|
293
|
+
dataType = self.getBigQueryDatatype(column)
|
|
294
|
+
|
|
295
|
+
# Save column original name
|
|
296
|
+
columnOriginalName = column
|
|
297
|
+
|
|
298
|
+
# If the data type is a date or a timestamp
|
|
299
|
+
if dataType in ['DATE', 'TIMESTAMP']:
|
|
300
|
+
column = self.setTimeZone(column, dataType)
|
|
301
|
+
|
|
302
|
+
# Data type casting
|
|
303
|
+
column = self.castColumn(
|
|
304
|
+
column, columnOriginalName, dataType)
|
|
305
|
+
|
|
306
|
+
clause += column + " " + \
|
|
307
|
+
self.addColumnAlias(
|
|
308
|
+
columnOriginalName, useAliases) + ", "
|
|
309
|
+
|
|
310
|
+
# Remove final `, `
|
|
311
|
+
clause = clause.strip(', ')
|
|
312
|
+
elif usage == 'SELECT': # Otherwise fetch all
|
|
313
|
+
clause += "*"
|
|
314
|
+
|
|
315
|
+
return clause
|
|
316
|
+
|
|
317
|
+
def setTimeZone(self, column, dataType):
|
|
318
|
+
"""
|
|
319
|
+
If the option `fdw_convert_tz` is used, convert the time zone automatically from UTC to the desired time zone
|
|
320
|
+
"""
|
|
321
|
+
|
|
322
|
+
# Option is set
|
|
323
|
+
if self.convertToTz:
|
|
324
|
+
if dataType == 'DATE': # BigQuery column type is `DATE`
|
|
325
|
+
return 'DATE(' + column + ', "' + self.convertToTz + '") '
|
|
326
|
+
else: # BigQuery column type is `TIMESTAMP`
|
|
327
|
+
return 'DATETIME(' + column + ', "' + self.convertToTz + '") '
|
|
328
|
+
|
|
329
|
+
# Option is not set
|
|
330
|
+
return column
|
|
331
|
+
|
|
332
|
+
def castColumn(self, column, columnOriginalName, dataType):
|
|
333
|
+
"""
|
|
334
|
+
If the option `fdw_casting` is used, this method will attempt to cast the column to the new type
|
|
335
|
+
"""
|
|
336
|
+
|
|
337
|
+
if self.castingRules and columnOriginalName in self.castingRules: # If we have casting rule for this column
|
|
338
|
+
# Get desired casting
|
|
339
|
+
castTo = self.castingRules[columnOriginalName]
|
|
340
|
+
|
|
341
|
+
# Find if we have a matching rule
|
|
342
|
+
rule = [
|
|
343
|
+
conversionRule for conversionRule in self.conversionRules if conversionRule.bq_standard_from == dataType.upper()]
|
|
344
|
+
|
|
345
|
+
if rule:
|
|
346
|
+
# Check if casting from the original data type to the new one is supported
|
|
347
|
+
if castTo.upper() in rule[0].bq_standard_to:
|
|
348
|
+
return 'CAST(' + column + ' as ' + castTo.upper() + ')'
|
|
349
|
+
else:
|
|
350
|
+
log_to_postgres("Casting from the data type `" + dataType.upper(
|
|
351
|
+
) + "` to the data type `" + castTo.upper() + "` is not permitted.", ERROR)
|
|
352
|
+
else:
|
|
353
|
+
log_to_postgres(
|
|
354
|
+
"Casting from the data type `" + dataType.upper() + "` is not permitted.", ERROR)
|
|
355
|
+
|
|
356
|
+
# Option is not set
|
|
357
|
+
return column
|
|
358
|
+
|
|
359
|
+
def addColumnAlias(self, alias, useAliases=True):
|
|
360
|
+
"""
|
|
361
|
+
Returns a string "as `alias`" if `useAliases` is `True`
|
|
362
|
+
"""
|
|
363
|
+
|
|
364
|
+
if useAliases:
|
|
365
|
+
return " as " + alias
|
|
366
|
+
|
|
367
|
+
return ''
|
|
368
|
+
|
|
369
|
+
def buildWhereClause(self, quals):
|
|
370
|
+
"""
|
|
371
|
+
Build the WHERE clause of the SQL query
|
|
372
|
+
"""
|
|
373
|
+
|
|
374
|
+
clause = ''
|
|
375
|
+
parameters = []
|
|
376
|
+
|
|
377
|
+
# Add WHERE clause
|
|
378
|
+
# `quals` example: `[Qual('test', '=', 'test 2'), Qual('test', '~~', '3')]`
|
|
379
|
+
if quals:
|
|
380
|
+
clause += "WHERE "
|
|
381
|
+
for qual in quals:
|
|
382
|
+
if qual.field_name == self.partitionPseudoColumn:
|
|
383
|
+
clause += "_PARTITIONTIME " + \
|
|
384
|
+
str(self.getOperator(qual.operator)) + " ?"
|
|
385
|
+
|
|
386
|
+
# Format date as a timestamp
|
|
387
|
+
value = qual.value.strftime("%Y-%m-%d 00:00:00")
|
|
388
|
+
|
|
389
|
+
# Force data type to `TIMESTAMP`
|
|
390
|
+
parameters.append(self.setParameter(
|
|
391
|
+
qual.field_name, 'TIMESTAMP', value))
|
|
392
|
+
else:
|
|
393
|
+
clause += str(qual.field_name) + " " + \
|
|
394
|
+
str(self.getOperator(qual.operator)) + " ?"
|
|
395
|
+
parameters.append(self.setParameter(
|
|
396
|
+
qual.field_name, self.getBigQueryDatatype(qual.field_name), qual.value))
|
|
397
|
+
|
|
398
|
+
# Add ` AND `
|
|
399
|
+
clause += " AND "
|
|
400
|
+
|
|
401
|
+
# Remove final ` AND `
|
|
402
|
+
clause = clause.strip(' AND ')
|
|
403
|
+
|
|
404
|
+
return (clause, parameters)
|
|
405
|
+
|
|
406
|
+
def getOperator(self, operator):
|
|
407
|
+
"""
|
|
408
|
+
Validate operator
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
# List of BigQuery operators supported
|
|
412
|
+
# Exhaustive list: https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators#operators
|
|
413
|
+
# Non listed operators may or may not work
|
|
414
|
+
operators = ['=', '<', '>', '<=', '>=', '!=', '<>', 'LIKE', 'NOT LIKE']
|
|
415
|
+
|
|
416
|
+
# Mapping between multicorn operators and BigQuery operators
|
|
417
|
+
mapping = {}
|
|
418
|
+
mapping['~~'] = 'LIKE'
|
|
419
|
+
mapping['!~~'] = 'NOT LIKE'
|
|
420
|
+
|
|
421
|
+
if operator in operators: # Operator is natively supported
|
|
422
|
+
return operator
|
|
423
|
+
elif operator in mapping: # Multicorn operator has a BigQuery equivalent
|
|
424
|
+
return mapping[operator]
|
|
425
|
+
else: # Operator is not supported
|
|
426
|
+
log_to_postgres(
|
|
427
|
+
"Operator `" + operator + "` is not currently supported", ERROR)
|
|
428
|
+
|
|
429
|
+
def getBigQueryDatatype(self, column, dialect='standard'):
|
|
430
|
+
"""
|
|
431
|
+
Returns the BigQuery standard SQL data type of a PostgreSQL column
|
|
432
|
+
|
|
433
|
+
Example: `column1` has the PostgreSQL type `bigint` which is called `INT64` in BigQuery standard SQL
|
|
434
|
+
"""
|
|
435
|
+
|
|
436
|
+
# Get PostgreSQL column type
|
|
437
|
+
# Example: `timestamp without time zone`
|
|
438
|
+
pgDatatype = self.columns[column].base_type_name
|
|
439
|
+
|
|
440
|
+
for datatype in self.datatypes: # For each known data types
|
|
441
|
+
if datatype.postgres == pgDatatype: # If the PostgreSQL data type matches the known data type
|
|
442
|
+
# Returns equivalent BigQuery data type
|
|
443
|
+
if dialect == 'legacy':
|
|
444
|
+
return datatype.bq_legacy
|
|
445
|
+
else:
|
|
446
|
+
return datatype.bq_standard
|
|
447
|
+
|
|
448
|
+
# Return a default data type in an attempt to save the day
|
|
449
|
+
return 'STRING'
|
|
450
|
+
|
|
451
|
+
def setParameter(self, column, type_, value):
|
|
452
|
+
"""
|
|
453
|
+
Set a parameter in BigQuery client
|
|
454
|
+
"""
|
|
455
|
+
|
|
456
|
+
# Verbose log
|
|
457
|
+
if self.verbose:
|
|
458
|
+
log_to_postgres(
|
|
459
|
+
"Add query parameter `" + self.client.varToString(value) + "` for column `" + column + "` with the type `" + type_ + "`", INFO)
|
|
460
|
+
|
|
461
|
+
return self.client.setParameter(type_, value)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bigquery-fdw
|
|
3
|
+
Version: 2.2.0
|
|
4
|
+
Summary: BigQuery Foreign Data Wrapper for PostgreSQL
|
|
5
|
+
Home-page: https://github.com/gabfl/bigquery_fdw
|
|
6
|
+
Author: Gabriel Bordeaux
|
|
7
|
+
Author-email: pypi@gab.lc
|
|
8
|
+
License: MIT
|
|
9
|
+
Classifier: Topic :: Database
|
|
10
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: MacOS
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Natural Language :: English
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python
|
|
17
|
+
Classifier: Development Status :: 4 - Beta
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: google-cloud-bigquery==3.47.0
|
|
22
|
+
Requires-Dist: google-auth==2.62.2
|
|
23
|
+
Requires-Dist: google-auth-oauthlib==1.5.0
|
|
24
|
+
Dynamic: author
|
|
25
|
+
Dynamic: author-email
|
|
26
|
+
Dynamic: classifier
|
|
27
|
+
Dynamic: description
|
|
28
|
+
Dynamic: description-content-type
|
|
29
|
+
Dynamic: home-page
|
|
30
|
+
Dynamic: license
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
Dynamic: requires-dist
|
|
33
|
+
Dynamic: requires-python
|
|
34
|
+
Dynamic: summary
|
|
35
|
+
|
|
36
|
+
# bigquery_fdw: BigQuery Foreign Data Wrapper for PostgreSQL
|
|
37
|
+
|
|
38
|
+
[](https://pypi.org/project/bigquery-fdw/)
|
|
39
|
+
[](https://github.com/gabfl/bigquery_fdw/actions)
|
|
40
|
+
[](https://codecov.io/gh/gabfl/bigquery_fdw)
|
|
41
|
+
[](https://raw.githubusercontent.com/gabfl/bigquery_fdw/main/LICENSE)
|
|
42
|
+
|
|
43
|
+
bigquery_fdw is a BigQuery foreign data wrapper for PostgreSQL using [Multicorn](https://github.com/Segfault-Inc/Multicorn).
|
|
44
|
+
|
|
45
|
+
It allows to write queries in PostgreSQL SQL syntax using a foreign table. It supports most of BigQuery's [data types](docs/data_types.md) and [operators](docs/operators.md).
|
|
46
|
+
|
|
47
|
+
## Features and limitations
|
|
48
|
+
|
|
49
|
+
- Table partitioning is supported. [You can use partitions in your SQL queries](docs/table_partitioning.md).
|
|
50
|
+
- Queries are parameterized when sent to BigQuery
|
|
51
|
+
- BigQuery's standard SQL support (legacy SQL is not supported)
|
|
52
|
+
- Authentication works with a "[Service Account](docs/service_account.md)" Json private key
|
|
53
|
+
|
|
54
|
+
[Read more](docs/README.md).
|
|
55
|
+
|
|
56
|
+
## Requirements
|
|
57
|
+
|
|
58
|
+
- PostgreSQL 14 to 18
|
|
59
|
+
- Python >= 3.10
|
|
60
|
+
|
|
61
|
+
## Get started
|
|
62
|
+
|
|
63
|
+
### Using docker
|
|
64
|
+
|
|
65
|
+
See [getting started with Docker](docs/docker.md)
|
|
66
|
+
|
|
67
|
+
### Installation on Debian/Ubuntu
|
|
68
|
+
|
|
69
|
+
#### Dependencies required to install bigquery_fdw:
|
|
70
|
+
|
|
71
|
+
You need to install the following dependencies:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Install required packages
|
|
75
|
+
apt update
|
|
76
|
+
apt install -y postgresql-server-dev-18 python3-setuptools python3-dev make gcc git
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
PostgreSQL versions 14 to 18 are tested in CI.
|
|
80
|
+
|
|
81
|
+
#### Installation
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
# Install Multicorn
|
|
85
|
+
# multicorn2 is a fork of Segfault-Inc/Multicorn that adds support for recent PostgreSQL versions.
|
|
86
|
+
# The original pgsql-io/multicorn2 repository was removed in October 2026; this copy is pinned to its v3.2 release.
|
|
87
|
+
# Alternatively, up to PostgreSQL 12, you can use gabfl/Multicorn that adds better support for Python3.
|
|
88
|
+
# You may also choose to build against the original project instead.
|
|
89
|
+
git clone https://github.com/ShaheedHaque/multicorn2.git Multicorn && cd Multicorn
|
|
90
|
+
git checkout 4e0feaa8c8a4e7db80ff442d71fb7786d5519e29
|
|
91
|
+
make && make install
|
|
92
|
+
|
|
93
|
+
# Install bigquery_fdw
|
|
94
|
+
pip3 install bigquery-fdw
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Major dependencies installed automatically during the installation process:
|
|
98
|
+
|
|
99
|
+
- [Google Cloud BigQuery](https://pypi.org/project/google-cloud-bigquery/)
|
|
100
|
+
- [Multicorn](https://github.com/ShaheedHaque/multicorn2)
|
|
101
|
+
|
|
102
|
+
## Authentication
|
|
103
|
+
|
|
104
|
+
bigquery_fdw relies on [Google Cloud API's default authentication](https://cloud.google.com/docs/authentication/getting-started#linux-or-macos).
|
|
105
|
+
|
|
106
|
+
Your need to have an environment variable `GOOGLE_APPLICATION_CREDENTIALS` that has to be accessible by bigquery_fdw. Setting environment variables varies depending on OS but for Ubuntu or Debian, the preferred way is to edit `/etc/postgresql/[version]/main/environment` and add:
|
|
107
|
+
```
|
|
108
|
+
GOOGLE_APPLICATION_CREDENTIALS = '/path/to/key.json'
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Restarting PostgreSQL is required for the environment variable to be loaded.
|
|
112
|
+
|
|
113
|
+
## Usage
|
|
114
|
+
|
|
115
|
+
We recommend testing the [BigQuery client connectivity](docs/test_client.md) before trying to use the FDW.
|
|
116
|
+
|
|
117
|
+
With `psql`:
|
|
118
|
+
|
|
119
|
+
```sql
|
|
120
|
+
CREATE EXTENSION multicorn;
|
|
121
|
+
|
|
122
|
+
CREATE SERVER bigquery_srv FOREIGN DATA WRAPPER multicorn
|
|
123
|
+
OPTIONS (
|
|
124
|
+
wrapper 'bigquery_fdw.fdw.ConstantForeignDataWrapper'
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
CREATE FOREIGN TABLE my_bigquery_table (
|
|
128
|
+
column1 text,
|
|
129
|
+
column2 bigint
|
|
130
|
+
) SERVER bigquery_srv
|
|
131
|
+
OPTIONS (
|
|
132
|
+
fdw_dataset 'my_dataset',
|
|
133
|
+
fdw_table 'my_table'
|
|
134
|
+
);
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Options
|
|
138
|
+
|
|
139
|
+
List of options implemented in `CREATE FOREIGN TABLE` syntax:
|
|
140
|
+
|
|
141
|
+
| Option | Default | Description |
|
|
142
|
+
|-----|----|----|
|
|
143
|
+
| `fdw_dataset` | - | BigQuery dataset name |
|
|
144
|
+
| `fdw_table` | - | BigQuery table name |
|
|
145
|
+
| `fdw_convert_tz` | - | Convert BigQuery time zone for dates and timestamps to selected time zone. Example: `'US/Eastern'`. |
|
|
146
|
+
| `fdw_group` | `'false'` | See [Remote grouping and counting](docs/remote_grouping.md). |
|
|
147
|
+
| `fdw_casting` | - | See [Casting](docs/casting.md). |
|
|
148
|
+
| `fdw_verbose` | `'false'` | Set to `'true'` to output debug information in PostrgeSQL's logs |
|
|
149
|
+
| `fdw_sql_dialect` | `'standard'` | BigQuery SQL dialect. Currently only `standard` is supported. |
|
|
150
|
+
|
|
151
|
+
## More documentation
|
|
152
|
+
|
|
153
|
+
See [bigquery_fdw documentation](docs/README.md).
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
bigquery_fdw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
bigquery_fdw/bqclient.py,sha256=KUGLS39JA2aWtXlqtsrCjvxxSTXdgLkisNzXGlfjzJA,3440
|
|
3
|
+
bigquery_fdw/bqclient_test.py,sha256=v7HUbo4p-kPvQY0sJYXVEzW83tjSbGvDkNYTWogYSes,1556
|
|
4
|
+
bigquery_fdw/fdw.py,sha256=KRZ1SJsQ0no-IdjliMn915ygm8tManoXWSzwy98q8G0,16042
|
|
5
|
+
bigquery_fdw-2.2.0.dist-info/licenses/LICENSE,sha256=WqS7GgVJk_T1GouVCLKKsR1q83EOah9Hpb72mq0pQ64,1073
|
|
6
|
+
bigquery_fdw-2.2.0.dist-info/METADATA,sha256=f6ekcTeYu3Cj3n74Rgndpnm_Dyh1qSm33Pmzm-PbR2U,5470
|
|
7
|
+
bigquery_fdw-2.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
8
|
+
bigquery_fdw-2.2.0.dist-info/entry_points.txt,sha256=xJLSDIqPEAwwv2Lahm16mzJfDse44QQYhhC7-WKB4-g,67
|
|
9
|
+
bigquery_fdw-2.2.0.dist-info/top_level.txt,sha256=R9SGcYIac_hb4IO_zPBpaqodCaVGPLsdIxtO2Zfnxks,13
|
|
10
|
+
bigquery_fdw-2.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2017 Gabriel Bordeaux
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
bigquery_fdw
|