bigquery-fdw 2.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -0,0 +1,114 @@
1
+ """
2
+ See:
3
+ https://cloud.google.com/bigquery/docs/reference/libraries
4
+ https://googlecloudplatform.github.io/google-cloud-python/stable/bigquery/usage.html?highlight=insert_data
5
+ https://github.com/GoogleCloudPlatform/python-docs-samples/tree/master/bigquery/cloud-client
6
+ """
7
+
8
+ import datetime
9
+
10
+ from google.cloud import bigquery
11
+ from google import auth
12
+
13
+
14
+ class BqClient:
15
+
16
+ # Set vars
17
+ client = None
18
+ queryJob = None
19
+ location = None # Override dataset location
20
+
21
+ def setClient(self):
22
+ """
23
+ Set BigQuery client with a Json key
24
+ """
25
+
26
+ try:
27
+ self.client = bigquery.Client()
28
+ except auth.exceptions.DefaultCredentialsError as e:
29
+ raise RuntimeError(
30
+ 'BigQuery client is not instantiated properly: ' + str(e))
31
+
32
+ def getClient(self):
33
+ """
34
+ Returns `client`
35
+ """
36
+
37
+ return self.client
38
+
39
+ def runQuery(self, query, parameters=[], sqlDialect='standard'):
40
+ """
41
+ Run BigQuery query
42
+ """
43
+
44
+ if self.client:
45
+ if parameters: # Parameterized query
46
+ # Prepare job configuration
47
+ job_config = bigquery.QueryJobConfig()
48
+ job_config.query_parameters = parameters
49
+
50
+ self.queryJob = self.client.query(
51
+ query, job_config=job_config, location=None)
52
+ else: # Non parameterized query
53
+ self.queryJob = self.client.query(query, location=None)
54
+
55
+ # Set SQL dialect
56
+ self.queryJob.UseLegacySQL = False
57
+ # if sqlDialect == 'legacy':
58
+ # self.queryJob.UseLegacySQL = True
59
+ # else:
60
+ # self.queryJob.UseLegacySQL = False
61
+
62
+ else:
63
+ raise RuntimeError(
64
+ 'BigQuery client is not instantiated properly (from `runQuery`).')
65
+
66
+ def getQueryJob(self):
67
+ """
68
+ Returns `queryJob`
69
+ """
70
+
71
+ return self.queryJob
72
+
73
+ def readResult(self):
74
+ """
75
+ Returns query result
76
+ """
77
+
78
+ if self.queryJob:
79
+ result = self.queryJob.result()
80
+ return list(result)
81
+ else:
82
+ raise RuntimeError('No query is pending a result.')
83
+
84
+ def setParameter(self, type_, value):
85
+ """
86
+ Prepare a parameter for a parameterized query
87
+ As documented by Google, only standard SQL syntax supports parameters in queries
88
+
89
+ See https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/bigquery/cloud-client/query_params.py
90
+ """
91
+
92
+ return bigquery.ScalarQueryParameter(
93
+ # Set the name to None to use positional parameters (? symbol
94
+ # in the query). Note that you cannot mix named and positional
95
+ # parameters.
96
+ None, type_, self.varToString(value))
97
+
98
+ def varToString(self, var):
99
+ """
100
+ Format a variable to a string
101
+
102
+ Examples:
103
+ `varToString(1234)` -> '1234'
104
+ `varToString('1234')` -> '1234'
105
+ `varToString(datetime.datetime.now())` -> '2017-10-03 11:47:54'
106
+ `varToString(datetime.datetime.now().date())` -> '2017-10-03'
107
+ """
108
+
109
+ if type(var) is datetime.date:
110
+ return var.strftime('%Y-%m-%d')
111
+ elif type(var) is datetime.datetime:
112
+ return var.strftime('%Y-%m-%d %H:%M:%S')
113
+
114
+ return str(var)
@@ -0,0 +1,96 @@
1
+ #!/usr/bin/python3
2
+
3
+ import argparse
4
+
5
+ from .bqclient import BqClient
6
+
7
+
8
+ def run_test(query):
9
+ """
10
+ BigQuery client test
11
+ """
12
+
13
+ # BqClient instance
14
+ bq = set_bq_instance()
15
+
16
+ # Authentication
17
+ set_client(bq)
18
+ print(" * BigQuery client instance:")
19
+ print(get_client(bq))
20
+
21
+ # Run query
22
+ run_query(bq, query)
23
+ print(" * Query instance:")
24
+ print(get_query_job(bq))
25
+
26
+ # Print the results.
27
+ print(" * Query results:")
28
+ for row in read_results(bq):
29
+ print(row)
30
+
31
+ return True
32
+
33
+
34
+ def set_bq_instance():
35
+ """
36
+ Create instance of BqClient
37
+ """
38
+
39
+ return BqClient()
40
+
41
+
42
+ def set_client(bq):
43
+ """
44
+ Set BigQuery client
45
+ """
46
+
47
+ return bq.setClient()
48
+
49
+
50
+ def get_client(bq):
51
+ """
52
+ Get BigQuery client
53
+ """
54
+
55
+ return bq.getClient()
56
+
57
+
58
+ def run_query(bq, query):
59
+ """
60
+ Run query
61
+ """
62
+
63
+ return bq.runQuery(query)
64
+
65
+
66
+ def get_query_job(bq):
67
+ """
68
+ Get query job
69
+ """
70
+
71
+ return bq.getQueryJob()
72
+
73
+
74
+ def read_results(bq):
75
+ """
76
+ Read results
77
+ """
78
+
79
+ return [row for row in bq.readResult()]
80
+
81
+
82
+ def main():
83
+
84
+ # Parse arguments
85
+ parser = argparse.ArgumentParser()
86
+ parser.add_argument("-a", "--auth_type", type=str, help="Authentication type",
87
+ choices=['service_account'], default='service_account')
88
+ parser.add_argument("-q", "--query", type=str,
89
+ help="Query to run", required=True)
90
+ args = parser.parse_args()
91
+
92
+ run_test(query=args.query)
93
+
94
+
95
+ if __name__ == '__main__':
96
+ main()
bigquery_fdw/fdw.py ADDED
@@ -0,0 +1,461 @@
1
+ from collections import OrderedDict, namedtuple
2
+
3
+ from multicorn import ForeignDataWrapper
4
+ from multicorn.utils import log_to_postgres, ERROR, WARNING, INFO, DEBUG
5
+
6
+ from .bqclient import BqClient
7
+
8
+
9
+ class ConstantForeignDataWrapper(ForeignDataWrapper):
10
+
11
+ # Default vars
12
+ client = None # BqClient instance
13
+ partitionPseudoColumn = 'partition_date' # Name of the partition pseudo column
14
+ # Pseudo column to fetch `count(*)` when using the remote counting and grouping feature
15
+ countPseudoColumn = '_fdw_count'
16
+ castingRules = None # Dict of casting rules when using the `fdw_casting` option
17
+
18
+ def __init__(self, options, columns):
19
+ """
20
+ Initialize instance, set class level vars
21
+ """
22
+
23
+ super(ConstantForeignDataWrapper, self).__init__(options, columns)
24
+
25
+ # Set options
26
+ self.setOptions(options)
27
+
28
+ # Set table columns
29
+ self.columns = columns
30
+
31
+ # Set data types and conversion rules mapping
32
+ self.setDatatypes()
33
+ self.setConversionRules()
34
+
35
+ def setOptions(self, options):
36
+ """
37
+ Set table options at class level
38
+ """
39
+
40
+ # Set options at class scope
41
+ try:
42
+ self.dataset = options['fdw_dataset']
43
+ self.table = options['fdw_table']
44
+ self.convertToTz = options.get('fdw_convert_tz')
45
+
46
+ # Set verbose option
47
+ self.setOptionVerbose(options.get('fdw_verbose'))
48
+
49
+ # Set SQL dialect
50
+ self.setOptionSqlDialect(options.get('fdw_sql_dialect'))
51
+
52
+ # Set grouping option
53
+ self.setOptionGroupBy(options.get('fdw_group'))
54
+
55
+ # Set casting rules
56
+ self.setOptionCasting(options.get('fdw_casting'))
57
+ except KeyError:
58
+ log_to_postgres(
59
+ "You must specify these options when creating the FDW: fdw_dataset, fdw_table", ERROR)
60
+
61
+ def setDatatypes(self):
62
+ """
63
+ Set data types mapping
64
+ """
65
+
66
+ # Create a named tuple
67
+ datatype = namedtuple('datatype', 'postgres bq_standard bq_legacy')
68
+
69
+ self.datatypes = [
70
+ datatype('text', 'STRING', 'STRING'),
71
+ # datatype('bytea', 'BYTES', 'BYTES'), # Not supported, need testing for support
72
+ datatype('bigint', 'INT64', 'INTEGER'),
73
+ datatype('double precision', 'FLOAT64', 'FLOAT'),
74
+ datatype('boolean', 'BOOL', 'BOOLEAN'),
75
+ datatype('timestamp without time zone', 'TIMESTAMP', 'TIMESTAMP'),
76
+ datatype('date', 'DATE', 'DATE'),
77
+ datatype('time without time zone', 'TIME', 'TIME'),
78
+ datatype('timestamp without time zone', 'DATETIME', 'DATETIME'),
79
+ ]
80
+
81
+ def setConversionRules(self):
82
+ """
83
+ Set list of allowed conversion rules
84
+ """
85
+
86
+ # Create a named tuple
87
+ conversionRule = namedtuple(
88
+ 'conversionRule', 'bq_standard_from bq_standard_to')
89
+
90
+ self.conversionRules = [
91
+ conversionRule('INT64', ['BOOL', 'FLOAT64', 'INT64', 'STRING']),
92
+ conversionRule('FLOAT64', ['FLOAT64', 'INT64', 'STRING']),
93
+ conversionRule('BOOL', ['BOOL', 'INT64', 'STRING']),
94
+ conversionRule('STRING', ['BOOL', 'BYTES', 'DATE', 'DATETIME',
95
+ 'FLOAT64', 'INT64', 'STRING', 'TIME', 'TIMESTAMP']),
96
+ conversionRule('BYTES', ['BYTES', 'STRING']),
97
+ conversionRule(
98
+ 'DATE', ['DATE', 'DATETIME', 'STRING', 'TIMESTAMP']),
99
+ conversionRule(
100
+ 'DATETIME', ['DATE', 'DATETIME', 'STRING', 'TIME', 'TIMESTAMP']),
101
+ conversionRule('TIME', ['STRING', 'TIME']),
102
+ conversionRule(
103
+ 'TIMESTAMP', ['DATE', 'DATETIME', 'STRING', 'TIME', 'TIMESTAMP']),
104
+ conversionRule('ARRAY', ['ARRAY']),
105
+ conversionRule('STRUCT', ['STRUCT']),
106
+ ]
107
+
108
+ def setOptionSqlDialect(self, standard_sql=None):
109
+ """
110
+ Set a flag for the SQL dialect.
111
+ It can be `standard` or `legacy`. `standard` will be the default
112
+ """
113
+
114
+ self.dialect = 'standard'
115
+
116
+ if standard_sql == 'legacy':
117
+ self.dialect = 'legacy'
118
+
119
+ # Verbose log
120
+ if self.verbose:
121
+ log_to_postgres("Set SQL dialect to `" + self.dialect + "`", INFO)
122
+
123
+ def setOptionGroupBy(self, group):
124
+ """
125
+ Set a flag `self.groupBy` as `True` if `group` contains the string 'true'
126
+ Otherwise, set it as `False`
127
+ """
128
+
129
+ if group == 'true':
130
+ self.groupBy = True
131
+ return
132
+
133
+ self.groupBy = False
134
+
135
+ def setOptionVerbose(self, verbose):
136
+ """
137
+ Set a flag `self.verbose` as `True` if `verbose` contains the string 'true'
138
+ Otherwise, set it as `False`
139
+ """
140
+
141
+ if verbose == 'true':
142
+ self.verbose = True
143
+ return
144
+
145
+ self.verbose = False
146
+
147
+ def setOptionCasting(self, castingRules):
148
+ """
149
+ Conversion rules are received as a string, for example: '{"key": "FLOAT64", "datetime": "DATE"}'
150
+
151
+ The string will be converted to a dict
152
+ """
153
+
154
+ if castingRules:
155
+ # Cast string as a dict
156
+ try:
157
+ import ast
158
+ self.castingRules = ast.literal_eval(castingRules)
159
+ except Exception as e:
160
+ log_to_postgres(
161
+ "fdw_casting conversion failed: `" + str(e) + "`", ERROR)
162
+
163
+ # For security reasons, ensure that the string was correctly casted as a dict
164
+ try:
165
+ if type(self.castingRules) is not dict:
166
+ raise ValueError('fdw_casting format is incorrect.')
167
+ except Exception as e:
168
+ log_to_postgres(
169
+ "fdw_casting conversion failed: `" + str(e) + "`", ERROR)
170
+
171
+ def getClient(self):
172
+ """
173
+ Manage a pool of instances of BqClient
174
+ If the user uses different private keys in different tables, this method will return
175
+ the correct instance of BqClient class associated to the table private key
176
+ """
177
+
178
+ # Returns an existing instance
179
+ if self.client:
180
+ return self.client
181
+
182
+ # Or create a new instance
183
+ return self.setClient()
184
+
185
+ def setClient(self):
186
+ """
187
+ Attempt to connect to BigQuery client
188
+ """
189
+
190
+ try:
191
+ # Attempt connection
192
+ bq = BqClient()
193
+ bq.setClient()
194
+
195
+ # Verbose log
196
+ if self.verbose:
197
+ log_to_postgres(
198
+ "Connection to BigQuery client with BqClient instance ID " + str(id(bq)), INFO)
199
+
200
+ # Add to pool
201
+ self.client = bq
202
+
203
+ return bq
204
+ except RuntimeError:
205
+ log_to_postgres(
206
+ "Connection to BigQuery client failed", ERROR)
207
+
208
+ def execute(self, quals, columns):
209
+ """
210
+ Executes a query
211
+ """
212
+
213
+ # # Verbose log
214
+ # if self.verbose:
215
+ # log_to_postgres('Quals...', INFO)
216
+ # log_to_postgres(quals, INFO)
217
+ # log_to_postgres('Columns...', INFO)
218
+ # log_to_postgres(columns, INFO)
219
+
220
+ # Returns instance of BqClient
221
+ client = self.getClient()
222
+
223
+ # Prepare query
224
+ query, parameters = self.buildQuery(quals, columns)
225
+
226
+ # Run query
227
+ client.runQuery(query, parameters, self.dialect)
228
+
229
+ # Return query output
230
+ for row in client.readResult():
231
+ # Create an ordered dict with the column name and value
232
+ # Example: `OrderedDict([('column1', 'value1'), ('column2', value2)])`
233
+ line = OrderedDict()
234
+ for column in columns:
235
+ line[column] = row[column]
236
+
237
+ yield line
238
+
239
+ def buildQuery(self, quals, columns):
240
+ """
241
+ Builds a BigQuery query
242
+ """
243
+
244
+ # Set query var
245
+ query = ''
246
+
247
+ # Add SELECT clause
248
+ query += 'SELECT ' + self.buildColumnList(columns)
249
+
250
+ # Add FROM clause
251
+ query += " FROM `" + self.dataset + "." + self.table + "` "
252
+
253
+ # Add WHERE clause
254
+ clause, parameters = self.buildWhereClause(quals)
255
+ query += clause
256
+
257
+ # Add group by
258
+ if self.groupBy:
259
+ groupByColumns = self.buildColumnList(columns, 'GROUP_BY')
260
+ if groupByColumns:
261
+ query += ' GROUP BY ' + \
262
+ self.buildColumnList(columns, 'GROUP_BY')
263
+
264
+ # Verbose log
265
+ if self.verbose:
266
+ log_to_postgres("Prepared query: `" + query + "`", INFO)
267
+
268
+ return query, parameters
269
+
270
+ def buildColumnList(self, columns, usage='SELECT'):
271
+ """
272
+ Build the SELECT clause of the SQL query
273
+ """
274
+
275
+ clause = ''
276
+
277
+ # Disable aliases for Group By
278
+ useAliases = True
279
+ if usage == 'GROUP_BY':
280
+ useAliases = False
281
+
282
+ if columns: # If we have columns
283
+ for column in columns:
284
+ if column == self.countPseudoColumn: # Pseudo column to count grouped rows
285
+ if usage == 'SELECT':
286
+ clause += "count(*) " + \
287
+ self.addColumnAlias(column, useAliases) + ", "
288
+ elif column == self.partitionPseudoColumn: # Partition pseudo column
289
+ clause += "_PARTITIONTIME " + \
290
+ self.addColumnAlias(column, useAliases) + ", "
291
+ else: # Any other column
292
+ # Get column data type
293
+ dataType = self.getBigQueryDatatype(column)
294
+
295
+ # Save column original name
296
+ columnOriginalName = column
297
+
298
+ # If the data type is a date or a timestamp
299
+ if dataType in ['DATE', 'TIMESTAMP']:
300
+ column = self.setTimeZone(column, dataType)
301
+
302
+ # Data type casting
303
+ column = self.castColumn(
304
+ column, columnOriginalName, dataType)
305
+
306
+ clause += column + " " + \
307
+ self.addColumnAlias(
308
+ columnOriginalName, useAliases) + ", "
309
+
310
+ # Remove final `, `
311
+ clause = clause.strip(', ')
312
+ elif usage == 'SELECT': # Otherwise fetch all
313
+ clause += "*"
314
+
315
+ return clause
316
+
317
+ def setTimeZone(self, column, dataType):
318
+ """
319
+ If the option `fdw_convert_tz` is used, convert the time zone automatically from UTC to the desired time zone
320
+ """
321
+
322
+ # Option is set
323
+ if self.convertToTz:
324
+ if dataType == 'DATE': # BigQuery column type is `DATE`
325
+ return 'DATE(' + column + ', "' + self.convertToTz + '") '
326
+ else: # BigQuery column type is `TIMESTAMP`
327
+ return 'DATETIME(' + column + ', "' + self.convertToTz + '") '
328
+
329
+ # Option is not set
330
+ return column
331
+
332
+ def castColumn(self, column, columnOriginalName, dataType):
333
+ """
334
+ If the option `fdw_casting` is used, this method will attempt to cast the column to the new type
335
+ """
336
+
337
+ if self.castingRules and columnOriginalName in self.castingRules: # If we have casting rule for this column
338
+ # Get desired casting
339
+ castTo = self.castingRules[columnOriginalName]
340
+
341
+ # Find if we have a matching rule
342
+ rule = [
343
+ conversionRule for conversionRule in self.conversionRules if conversionRule.bq_standard_from == dataType.upper()]
344
+
345
+ if rule:
346
+ # Check if casting from the original data type to the new one is supported
347
+ if castTo.upper() in rule[0].bq_standard_to:
348
+ return 'CAST(' + column + ' as ' + castTo.upper() + ')'
349
+ else:
350
+ log_to_postgres("Casting from the data type `" + dataType.upper(
351
+ ) + "` to the data type `" + castTo.upper() + "` is not permitted.", ERROR)
352
+ else:
353
+ log_to_postgres(
354
+ "Casting from the data type `" + dataType.upper() + "` is not permitted.", ERROR)
355
+
356
+ # Option is not set
357
+ return column
358
+
359
+ def addColumnAlias(self, alias, useAliases=True):
360
+ """
361
+ Returns a string "as `alias`" if `useAliases` is `True`
362
+ """
363
+
364
+ if useAliases:
365
+ return " as " + alias
366
+
367
+ return ''
368
+
369
+ def buildWhereClause(self, quals):
370
+ """
371
+ Build the WHERE clause of the SQL query
372
+ """
373
+
374
+ clause = ''
375
+ parameters = []
376
+
377
+ # Add WHERE clause
378
+ # `quals` example: `[Qual('test', '=', 'test 2'), Qual('test', '~~', '3')]`
379
+ if quals:
380
+ clause += "WHERE "
381
+ for qual in quals:
382
+ if qual.field_name == self.partitionPseudoColumn:
383
+ clause += "_PARTITIONTIME " + \
384
+ str(self.getOperator(qual.operator)) + " ?"
385
+
386
+ # Format date as a timestamp
387
+ value = qual.value.strftime("%Y-%m-%d 00:00:00")
388
+
389
+ # Force data type to `TIMESTAMP`
390
+ parameters.append(self.setParameter(
391
+ qual.field_name, 'TIMESTAMP', value))
392
+ else:
393
+ clause += str(qual.field_name) + " " + \
394
+ str(self.getOperator(qual.operator)) + " ?"
395
+ parameters.append(self.setParameter(
396
+ qual.field_name, self.getBigQueryDatatype(qual.field_name), qual.value))
397
+
398
+ # Add ` AND `
399
+ clause += " AND "
400
+
401
+ # Remove final ` AND `
402
+ clause = clause.strip(' AND ')
403
+
404
+ return (clause, parameters)
405
+
406
+ def getOperator(self, operator):
407
+ """
408
+ Validate operator
409
+ """
410
+
411
+ # List of BigQuery operators supported
412
+ # Exhaustive list: https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators#operators
413
+ # Non listed operators may or may not work
414
+ operators = ['=', '<', '>', '<=', '>=', '!=', '<>', 'LIKE', 'NOT LIKE']
415
+
416
+ # Mapping between multicorn operators and BigQuery operators
417
+ mapping = {}
418
+ mapping['~~'] = 'LIKE'
419
+ mapping['!~~'] = 'NOT LIKE'
420
+
421
+ if operator in operators: # Operator is natively supported
422
+ return operator
423
+ elif operator in mapping: # Multicorn operator has a BigQuery equivalent
424
+ return mapping[operator]
425
+ else: # Operator is not supported
426
+ log_to_postgres(
427
+ "Operator `" + operator + "` is not currently supported", ERROR)
428
+
429
+ def getBigQueryDatatype(self, column, dialect='standard'):
430
+ """
431
+ Returns the BigQuery standard SQL data type of a PostgreSQL column
432
+
433
+ Example: `column1` has the PostgreSQL type `bigint` which is called `INT64` in BigQuery standard SQL
434
+ """
435
+
436
+ # Get PostgreSQL column type
437
+ # Example: `timestamp without time zone`
438
+ pgDatatype = self.columns[column].base_type_name
439
+
440
+ for datatype in self.datatypes: # For each known data types
441
+ if datatype.postgres == pgDatatype: # If the PostgreSQL data type matches the known data type
442
+ # Returns equivalent BigQuery data type
443
+ if dialect == 'legacy':
444
+ return datatype.bq_legacy
445
+ else:
446
+ return datatype.bq_standard
447
+
448
+ # Return a default data type in an attempt to save the day
449
+ return 'STRING'
450
+
451
+ def setParameter(self, column, type_, value):
452
+ """
453
+ Set a parameter in BigQuery client
454
+ """
455
+
456
+ # Verbose log
457
+ if self.verbose:
458
+ log_to_postgres(
459
+ "Add query parameter `" + self.client.varToString(value) + "` for column `" + column + "` with the type `" + type_ + "`", INFO)
460
+
461
+ return self.client.setParameter(type_, value)
@@ -0,0 +1,153 @@
1
+ Metadata-Version: 2.4
2
+ Name: bigquery-fdw
3
+ Version: 2.2.0
4
+ Summary: BigQuery Foreign Data Wrapper for PostgreSQL
5
+ Home-page: https://github.com/gabfl/bigquery_fdw
6
+ Author: Gabriel Bordeaux
7
+ Author-email: pypi@gab.lc
8
+ License: MIT
9
+ Classifier: Topic :: Database
10
+ Classifier: Topic :: Database :: Database Engines/Servers
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: MacOS
13
+ Classifier: Operating System :: POSIX :: Linux
14
+ Classifier: Natural Language :: English
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python
17
+ Classifier: Development Status :: 4 - Beta
18
+ Requires-Python: >=3.10
19
+ Description-Content-Type: text/markdown
20
+ License-File: LICENSE
21
+ Requires-Dist: google-cloud-bigquery==3.47.0
22
+ Requires-Dist: google-auth==2.62.2
23
+ Requires-Dist: google-auth-oauthlib==1.5.0
24
+ Dynamic: author
25
+ Dynamic: author-email
26
+ Dynamic: classifier
27
+ Dynamic: description
28
+ Dynamic: description-content-type
29
+ Dynamic: home-page
30
+ Dynamic: license
31
+ Dynamic: license-file
32
+ Dynamic: requires-dist
33
+ Dynamic: requires-python
34
+ Dynamic: summary
35
+
36
+ # bigquery_fdw: BigQuery Foreign Data Wrapper for PostgreSQL
37
+
38
+ [![Pypi](https://img.shields.io/pypi/v/bigquery-fdw.svg)](https://pypi.org/project/bigquery-fdw/)
39
+ [![Build Status](https://github.com/gabfl/bigquery_fdw/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/gabfl/bigquery_fdw/actions)
40
+ [![codecov](https://codecov.io/gh/gabfl/bigquery_fdw/branch/main/graph/badge.svg)](https://codecov.io/gh/gabfl/bigquery_fdw)
41
+ [![MIT licensed](https://img.shields.io/badge/license-MIT-green.svg)](https://raw.githubusercontent.com/gabfl/bigquery_fdw/main/LICENSE)
42
+
43
+ bigquery_fdw is a BigQuery foreign data wrapper for PostgreSQL using [Multicorn](https://github.com/Segfault-Inc/Multicorn).
44
+
45
+ It allows to write queries in PostgreSQL SQL syntax using a foreign table. It supports most of BigQuery's [data types](docs/data_types.md) and [operators](docs/operators.md).
46
+
47
+ ## Features and limitations
48
+
49
+ - Table partitioning is supported. [You can use partitions in your SQL queries](docs/table_partitioning.md).
50
+ - Queries are parameterized when sent to BigQuery
51
+ - BigQuery's standard SQL support (legacy SQL is not supported)
52
+ - Authentication works with a "[Service Account](docs/service_account.md)" Json private key
53
+
54
+ [Read more](docs/README.md).
55
+
56
+ ## Requirements
57
+
58
+ - PostgreSQL 14 to 18
59
+ - Python >= 3.10
60
+
61
+ ## Get started
62
+
63
+ ### Using docker
64
+
65
+ See [getting started with Docker](docs/docker.md)
66
+
67
+ ### Installation on Debian/Ubuntu
68
+
69
+ #### Dependencies required to install bigquery_fdw:
70
+
71
+ You need to install the following dependencies:
72
+
73
+ ```bash
74
+ # Install required packages
75
+ apt update
76
+ apt install -y postgresql-server-dev-18 python3-setuptools python3-dev make gcc git
77
+ ```
78
+
79
+ PostgreSQL versions 14 to 18 are tested in CI.
80
+
81
+ #### Installation
82
+
83
+ ```bash
84
+ # Install Multicorn
85
+ # multicorn2 is a fork of Segfault-Inc/Multicorn that adds support for recent PostgreSQL versions.
86
+ # The original pgsql-io/multicorn2 repository was removed in October 2026; this copy is pinned to its v3.2 release.
87
+ # Alternatively, up to PostgreSQL 12, you can use gabfl/Multicorn that adds better support for Python3.
88
+ # You may also choose to build against the original project instead.
89
+ git clone https://github.com/ShaheedHaque/multicorn2.git Multicorn && cd Multicorn
90
+ git checkout 4e0feaa8c8a4e7db80ff442d71fb7786d5519e29
91
+ make && make install
92
+
93
+ # Install bigquery_fdw
94
+ pip3 install bigquery-fdw
95
+ ```
96
+
97
+ Major dependencies installed automatically during the installation process:
98
+
99
+ - [Google Cloud BigQuery](https://pypi.org/project/google-cloud-bigquery/)
100
+ - [Multicorn](https://github.com/ShaheedHaque/multicorn2)
101
+
102
+ ## Authentication
103
+
104
+ bigquery_fdw relies on [Google Cloud API's default authentication](https://cloud.google.com/docs/authentication/getting-started#linux-or-macos).
105
+
106
+ Your need to have an environment variable `GOOGLE_APPLICATION_CREDENTIALS` that has to be accessible by bigquery_fdw. Setting environment variables varies depending on OS but for Ubuntu or Debian, the preferred way is to edit `/etc/postgresql/[version]/main/environment` and add:
107
+ ```
108
+ GOOGLE_APPLICATION_CREDENTIALS = '/path/to/key.json'
109
+ ```
110
+
111
+ Restarting PostgreSQL is required for the environment variable to be loaded.
112
+
113
+ ## Usage
114
+
115
+ We recommend testing the [BigQuery client connectivity](docs/test_client.md) before trying to use the FDW.
116
+
117
+ With `psql`:
118
+
119
+ ```sql
120
+ CREATE EXTENSION multicorn;
121
+
122
+ CREATE SERVER bigquery_srv FOREIGN DATA WRAPPER multicorn
123
+ OPTIONS (
124
+ wrapper 'bigquery_fdw.fdw.ConstantForeignDataWrapper'
125
+ );
126
+
127
+ CREATE FOREIGN TABLE my_bigquery_table (
128
+ column1 text,
129
+ column2 bigint
130
+ ) SERVER bigquery_srv
131
+ OPTIONS (
132
+ fdw_dataset 'my_dataset',
133
+ fdw_table 'my_table'
134
+ );
135
+ ```
136
+
137
+ ## Options
138
+
139
+ List of options implemented in `CREATE FOREIGN TABLE` syntax:
140
+
141
+ | Option | Default | Description |
142
+ |-----|----|----|
143
+ | `fdw_dataset` | - | BigQuery dataset name |
144
+ | `fdw_table` | - | BigQuery table name |
145
+ | `fdw_convert_tz` | - | Convert BigQuery time zone for dates and timestamps to selected time zone. Example: `'US/Eastern'`. |
146
+ | `fdw_group` | `'false'` | See [Remote grouping and counting](docs/remote_grouping.md). |
147
+ | `fdw_casting` | - | See [Casting](docs/casting.md). |
148
+ | `fdw_verbose` | `'false'` | Set to `'true'` to output debug information in PostrgeSQL's logs |
149
+ | `fdw_sql_dialect` | `'standard'` | BigQuery SQL dialect. Currently only `standard` is supported. |
150
+
151
+ ## More documentation
152
+
153
+ See [bigquery_fdw documentation](docs/README.md).
@@ -0,0 +1,10 @@
1
+ bigquery_fdw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ bigquery_fdw/bqclient.py,sha256=KUGLS39JA2aWtXlqtsrCjvxxSTXdgLkisNzXGlfjzJA,3440
3
+ bigquery_fdw/bqclient_test.py,sha256=v7HUbo4p-kPvQY0sJYXVEzW83tjSbGvDkNYTWogYSes,1556
4
+ bigquery_fdw/fdw.py,sha256=KRZ1SJsQ0no-IdjliMn915ygm8tManoXWSzwy98q8G0,16042
5
+ bigquery_fdw-2.2.0.dist-info/licenses/LICENSE,sha256=WqS7GgVJk_T1GouVCLKKsR1q83EOah9Hpb72mq0pQ64,1073
6
+ bigquery_fdw-2.2.0.dist-info/METADATA,sha256=f6ekcTeYu3Cj3n74Rgndpnm_Dyh1qSm33Pmzm-PbR2U,5470
7
+ bigquery_fdw-2.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
8
+ bigquery_fdw-2.2.0.dist-info/entry_points.txt,sha256=xJLSDIqPEAwwv2Lahm16mzJfDse44QQYhhC7-WKB4-g,67
9
+ bigquery_fdw-2.2.0.dist-info/top_level.txt,sha256=R9SGcYIac_hb4IO_zPBpaqodCaVGPLsdIxtO2Zfnxks,13
10
+ bigquery_fdw-2.2.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ bq_client_test = bigquery_fdw.bqclient_test:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2017 Gabriel Bordeaux
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1 @@
1
+ bigquery_fdw