bqseine 0.0.1__tar.gz

bqseine-0.0.1/.gitignore

@@ -0,0 +1,207 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

bqseine-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bqseine-0.0.1/PKG-INFO

@@ -0,0 +1,65 @@
+Metadata-Version: 2.4
+Name: bqseine
+Version: 0.0.1
+Summary: ETL for BigQuery
+Project-URL: Homepage, https://github.com/shaafiee/seine
+Project-URL: Issues, https://github.com/shaafiee/siene/issues
+Author-email: Shaafiee <shaafiee@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# BQSeine
+## Python dict to BigQuery data loader
+Seine is a data loader that pushes data in a dictionary to BigQuery in relational normalized form.
+
+## Usage
+```
+from bqseine import sync
+sourceData = [
+	{
+		'item': 'Juice',
+		'price': 20.0,
+		'stock': [
+			{
+				'batch': '2025-01-20',
+				'qty': 300
+			},
+			{
+				'batch': '2025-02-02',
+				'qty': 50
+			}
+		]
+	},
+	{
+		'item': 'Burger',
+		'price': 30.0,
+		'stock': [
+			{
+				'batch': '2025-02-10',
+				'qty': 200
+			}
+		]
+	}
+]
+sync('someGoogleProject', sourceData, 'catalog', 'US')
+### The arguments above are:
+### sync(<Google project name>, <dict>, <main table name>, <BigQuery region>)*
+```
+
+The above example will generate the following tables in BigQuery:
+### catalog
+| seine_id | seine_parent_id | item | price | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 0 | 'Juice' | 20.0 | now() |
+| 2 | 0 | 'Burger' | 30.0 | now() |
+
+### catalog_stock
+| seine_id | seine_parent_id | batch | qty | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 1 | '2025-01-20' | 300 | now() |
+| 2 | 1 | '2025-02-02' | 50 | now() |
+| 3 | 2 | '2025-02-10' | 200 | now() |

bqseine-0.0.1/README.md

@@ -0,0 +1,51 @@
+# BQSeine
+## Python dict to BigQuery data loader
+Seine is a data loader that pushes data in a dictionary to BigQuery in relational normalized form.
+
+## Usage
+```
+from bqseine import sync
+sourceData = [
+	{
+		'item': 'Juice',
+		'price': 20.0,
+		'stock': [
+			{
+				'batch': '2025-01-20',
+				'qty': 300
+			},
+			{
+				'batch': '2025-02-02',
+				'qty': 50
+			}
+		]
+	},
+	{
+		'item': 'Burger',
+		'price': 30.0,
+		'stock': [
+			{
+				'batch': '2025-02-10',
+				'qty': 200
+			}
+		]
+	}
+]
+sync('someGoogleProject', sourceData, 'catalog', 'US')
+### The arguments above are:
+### sync(<Google project name>, <dict>, <main table name>, <BigQuery region>)*
+```
+
+The above example will generate the following tables in BigQuery:
+### catalog
+| seine_id | seine_parent_id | item | price | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 0 | 'Juice' | 20.0 | now() |
+| 2 | 0 | 'Burger' | 30.0 | now() |
+
+### catalog_stock
+| seine_id | seine_parent_id | batch | qty | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 1 | '2025-01-20' | 300 | now() |
+| 2 | 1 | '2025-02-02' | 50 | now() |
+| 3 | 2 | '2025-02-10' | 200 | now() |

bqseine-0.0.1/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling >= 1.26"]
+build-backend = "hatchling.build"
+
+[project]
+name = "bqseine"
+version = "0.0.1"
+authors = [
+  { name="Shaafiee", email="shaafiee@gmail.com" },
+]
+description = "ETL for BigQuery"
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "MIT"
+license-files = ["LICEN[CS]E*"]
+
+[project.urls]
+Homepage = "https://github.com/shaafiee/seine"
+Issues = "https://github.com/shaafiee/siene/issues"

bqseine-0.0.1/src/bqseine/polyp.py

@@ -0,0 +1,381 @@
+from gcp_secrets.secrets import *
+from google.cloud import bigquery
+from google.cloud.exceptions import NotFound
+from datetime import date, datetime
+from db_lib import *
+import json
+
+
+lastSeineId = {}
+tableSchema = {}
+tableCurrentSchema = {}
+tableCurrentSchemaType = {}
+tableReset = {}
+#client  = bigquery.Client()
+
+#    * EXCEPT(is_generated, generation_expression, is_stored, is_updatable)
+tableColumnsQuery = """
+SELECT 
+    column_name 
+  FROM 
+    `__dataset__`.INFORMATION_SCHEMA.COLUMNS 
+  WHERE 
+    table_name = '__table__';
+"""
+
+
+def resolveType(value):
+	isDatetime = False
+	tempDatetime = None
+	try:
+		timeDatetime = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
+		value = timeDatetime
+		isDatetime = True
+	except:
+		pass
+
+	if isinstance(value, bool):
+		return "BOOL"
+	elif isinstance(value, int):
+		return "INT64"
+	elif isinstance(value, float):
+		return "FLOAT64"
+	elif type(value) == datetime:
+		return "DATETIME"
+	else:
+		return "STRING"
+
+
+def testValue(value, tableKey, fieldKey):
+	global tableCurrentSchemaType
+	try:
+		timeDatetime = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
+		return timeDatetime.strftime('%Y-%m-%dT%H:%M:%S')
+	except:
+		pass
+
+	fieldType = None
+	if fieldKey in tableCurrentSchemaType[tableKey].keys():
+		fieldType = tableCurrentSchemaType[tableKey][fieldKey]
+	else:
+		fieldType = resolveType(value)
+		tableCurrentSchemaType[tableKey][fieldKey] = fieldType
+
+	if fieldType == "BOOL":
+		if isinstance(value, bool):
+			return value
+		else:
+			return False
+	elif fieldType == "INT64":
+		if isinstance(value, int):
+			return value
+		else:
+			return 0
+	elif fieldType == "FLOAT64":
+		if isinstance(value, float):
+			return value
+		else:
+			return 0
+	elif fieldType == "STRING":
+		if isinstance(value, str):
+			return value
+		else:
+			return ""
+	else:
+		if isinstance(value, str):
+			return value
+		else:
+			return json.dumps(value)
+
+	return ""
+
+
+def incrementId(curKey):
+	global lastSeineId
+	if curKey in lastSeineId.keys():
+		lastSeineId[curKey] += 1
+	else:
+		lastSeineId[curKey] = 1
+
+
+def sync(myGoogleProject, blob, curKey, bqRegion = 'US', firstReset = False, idField = None):
+	global lastSeineId
+	global tableSchema
+	global tableReset
+	global tableCurrentSchema
+	global tableCurrentSchemaType
+
+	if len(curKey) < 1:
+		raise ExceptionType("A default current key (second arg) should be provided")
+
+	stack = []
+	if isinstance(blob, list):
+		for part in blob:
+			stack.insert(0, (curKey, part, curKey, 0))
+	else:
+		stack = [(curKey, blob, curKey, 0)]
+		
+	tableChecked = False
+
+	client = None
+	try:
+		client = bigquery.Client(project=myGoogleProject)
+	except:
+		raise ExceptionType(f"Could not connect to {myGoogleProject}")
+
+	if firstReset:
+		lastSeineId = {}
+		tableSchema = {}
+		tableReset = {}
+		tableCurrentSchema = {}
+		tableCurrentSchemaType = {}
+
+	keyNotInSchema = {}
+	dataToLoad = {}
+	seineDataset = False
+	dataset = ""
+	datasetName = "seine_" + curKey
+	try:
+		seineDataset = client.get_dataset(myGoogleProject + f".{datasetName}")
+	except NotFound:
+		seineDataset = bigquery.Dataset(myGoogleProject + f".{datasetName}")
+		seineDataset.location = bqRegion
+		seineDataset = client.create_dataset(seineDataset, timeout=30)
+		print("Created dataset {}".format(seineDataset.dataset_id))
+	
+	valuesArray = []
+	currentDepth = 0
+	while stack:
+		curKey, curDict, lastKey, parentId = stack.pop()
+		currentDepth += 1
+		fieldTypes = {}
+		fields = []
+		fieldsJson = {}
+		values = []
+		fieldType = {}
+		valuePlaceholders = {}
+		counter = 0
+
+		#seineDataset = client.get_dataset(myGoogleProject + f".{datasetName}")
+		print("-------------------------")
+		print(curDict)
+		noUpdateNeeded = False
+
+		if curKey not in lastSeineId.keys():
+			lastSeineId[curKey] = 1
+
+		if parentId == 0 and curKey == idField and isinstance(curDict, int):
+			try:
+				curTableName = myGoogleProject + f".{datasetName}." + curKey
+				queryJob = client.query(f"select {idField} from {curTableName} where {idField} = {curDict}")
+				returned = queryJob.result()
+				foundId = False
+				for row in returned:
+					foundId = True
+				if foundId:
+					print(f"ID exists {idField} = {curDict}")
+					print(returned)
+					continue
+			except:
+				print("================== QUERY FAILED =======================")
+				pass
+
+		#if curKey not in keyNotInSchema.keys():
+		keyNotInSchema[curKey] = []
+		if curKey not in tableCurrentSchema.keys():
+			try:
+				curTableName = myGoogleProject + f".{datasetName}." + curKey
+				queryJob = client.query(f"select max(seine_id) as max_id from {curTableName}")
+				returned = queryJob.result()
+				print(returned)
+				for row in returned:
+					if row.max_id is not None and isinstance(row.max_id, int):
+						lastSeineId[curKey] = int(row.max_id) + 1
+			except:
+				pass
+			tableCurrentSchema[curKey] = []
+			tableCurrentSchemaType[curKey] = {}
+		
+		if isinstance(curDict, list):
+			if not isinstance(curDict[0], dict):
+				fields.append(curKey)
+				fieldTypes[curKey] = resolveType(json.dumps(curDict))
+				valuePlaceholders[curKey] = testValue(json.dumps(curDict), curKey, curKey)
+				if curKey not in tableCurrentSchema[curKey]:
+					tableCurrentSchema[curKey].append(curKey)
+					keyNotInSchema[curKey].append(curKey)
+					#tableCurrentSchemaType[curKey][curKey] = resolveType(json.dumps(curDict))
+			else:
+				noUpdateNeeded = True
+				for tempDict in curDict:
+					stack.insert(0, (curKey, tempDict, lastKey, lastSeineId[lastKey]))
+				continue
+
+		elif isinstance(curDict, dict):
+			for key, value in curDict.items():
+				if isinstance(value, list):
+					if len(value) > 0:
+						if not isinstance(value[0], dict) and key not in ["edges"]:
+							fields.append(key)
+							fieldTypes[key] = resolveType(json.dumps(value))
+							valuePlaceholders[key] = testValue(json.dumps(value), curKey, key)
+							if key not in tableCurrentSchema[curKey]:
+								tableCurrentSchema[curKey].append(key)
+								keyNotInSchema[curKey].append(key)
+								#tableCurrentSchemaType[curKey][key] = resolveType(json.dumps(value))
+						elif key in ["edges"]:
+							noUpdateNeeded = True
+							for part in value:
+								stack.insert(0, (curKey, part, lastKey,  lastSeineId[lastKey]))
+						else:
+							for part in value:
+								stack.insert(0, (curKey + "_" + key, part, curKey,  lastSeineId[lastKey]))
+							#fields.append(key)
+							#fieldTypes[key] = resolveType(json.dumps(value))
+							#valuePlaceholders[key] = testValue(json.dumps(value), curKey, key)
+							#if key not in tableCurrentSchema[curKey]:
+							#	tableCurrentSchema[curKey].append(key)
+							#	keyNotInSchema[curKey].append(key)
+					else:
+						fields.append(key)
+						fieldTypes[key] = resolveType(json.dumps(value))
+						valuePlaceholders[key] = testValue(json.dumps(value), curKey, key)
+						if key not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(key)
+							keyNotInSchema[curKey].append(key)
+							#tableCurrentSchemaType[curKey][key] = resolveType(json.dumps(value))
+				elif isinstance(value, dict):
+					if key in ["node"]:
+						noUpdateNeeded = True
+						stack.insert(0, (curKey, value, lastKey,  lastSeineId[lastKey]))
+					else:
+						stack.insert(0, (curKey + "_" + key, value, curKey,  lastSeineId[lastKey]))
+						fields.append(key)
+						fieldTypes[key] = resolveType(lastSeineId[curKey])
+						valuePlaceholders[key] = testValue(lastSeineId[curKey], curKey, key)
+						if key not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(key)
+							keyNotInSchema[curKey].append(key)
+							#tableCurrentSchemaType[curKey][key] = resolveType(json.dumps(value))
+				else:
+					# Add schema
+					if key not in fields:
+						fields.append(key)
+						fieldTypes[key] = resolveType(value)
+						valuePlaceholders[key] = testValue(value, curKey, key)
+						if key not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(key)
+							keyNotInSchema[curKey].append(key)
+							#tableCurrentSchemaType[curKey][key] = resolveType(value)
+
+		elif isinstance(curDict, str) or isinstance(curDict, int) or isinstance(curDict, float):
+			fields.append(curKey)
+			fieldTypes[curKey] = resolveType(curDict)
+			valuePlaceholders[curKey] = testValue(curDict, curKey, key)
+			if curKey not in tableCurrentSchema[curKey]:
+				tableCurrentSchema[curKey].append(curKey)
+				keyNotInSchema[curKey].append(curKey)
+				#tableCurrentSchemaType[curKey][curKey] = resolveType(curDict)
+
+		else:
+			fields.append(curKey)
+			fieldTypes[curKey] = resolveType(curDict)
+			valuePlaceholders[curKey] = testValue(curDict, curKey, key)
+			if curKey not in tableCurrentSchema[curKey]:
+				tableCurrentSchema[curKey].append(curKey)
+				keyNotInSchema[curKey].append(curKey)
+				#tableCurrentSchemaType[curKey][curKey] = resolveType(curDict)
+
+		#if len(fields) < 1 and parentId is None:
+		#	continue
+
+		if noUpdateNeeded:
+			continue
+
+		if len(keyNotInSchema[curKey]) > 0 or curKey not in tableReset.keys():
+			tableReset[curKey] = True
+			curTableName = myGoogleProject + f".{datasetName}." + curKey
+			curTable = False
+			tableSchema[curKey] = []
+			tableSchema[curKey].append(bigquery.SchemaField("seine_id", "INT64"))
+			tableSchema[curKey].append(bigquery.SchemaField("seine_parent_id", "INT64"))
+			tableSchema[curKey].append(bigquery.SchemaField("injected", "DATETIME"))
+			tableCurrentSchema[curKey].append("seine_id");
+			tableCurrentSchema[curKey].append("seine_parent_id");
+			tableCurrentSchema[curKey].append("injected");
+			try:
+				curTable = client.get_table(curTableName)
+				#colQuery = tableColumnsQuery.replace("__dataset__", seineDataset).replace("__table__", key)
+				#queryJob = client.query(colQuery)
+				#returned = queryJob.result()
+				existingSchema = curTable.schema
+				tableSchema[curKey] = existingSchema
+				existingColumns = []
+				jobCconfig = bigquery.QueryJobConfig(
+					destination=curTableName,
+					schema_update_options=[bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION],
+					write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
+				)
+				for schemaElement in existingSchema: 
+					existingColumns.append(schemaElement.name)
+				schemaAdjusted = False
+				for fidx, field in enumerate(fields):
+					if field not in existingColumns:
+						if field not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(field);
+						tableSchema[curKey].append(bigquery.SchemaField(field, fieldTypes[field]))
+						existingSchema.append(bigquery.SchemaField(field, fieldTypes[field]))
+						if not schemaAdjusted:
+							schemaAdjusted = True
+				if schemaAdjusted:
+					curTable.schema = existingSchema
+					try:
+						curTable = client.update_table(curTable, ["schema"])
+					except:
+						pass
+
+				#if returned.total_rows > 0:
+				#	counter = 0
+				#	for row in returned:
+				#		print(row)
+				#		counter += 1
+				#		if counter < 2:
+				#			continue
+				#		if row.max_id + 1 > lastSeineId[curKey]:
+				#			lastSeineId[curKey] = int(row.max_id) + 1
+				#			print(f"{curKey}" + str(lastSeineId[curKey]))
+			except NotFound:
+				for fidx, field in enumerate(fields):
+					tableSchema[curKey].append(bigquery.SchemaField(field, fieldTypes[field]))
+					if field not in tableCurrentSchema[curKey]:
+						tableCurrentSchema[curKey].append(field);
+				curTable = bigquery.Table(curTableName, schema=tableSchema[curKey])
+				curTable = client.create_table(curTable)
+				lastSeineId[curKey] = 1
+			keyNotInSchema[curKey] = []
+
+		if curKey not in dataToLoad.keys():
+			dataToLoad[curKey] = []
+		tempRow = {
+			"seine_id": lastSeineId[curKey],
+			"seine_parent_id": parentId,
+			"injected": datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
+		}
+		for idx, field in enumerate(fields):
+			tempRow[field] = valuePlaceholders[field]
+		print(tempRow)
+		dataToLoad[curKey].append(tempRow)
+		lastSeineId[curKey] += 1
+
+	conn, cur = dbConnect()
+	for tableName in dataToLoad.keys():
+		curTable = client.get_table(myGoogleProject + f".{datasetName}." + tableName)
+		errors = client.insert_rows_json(
+			curTable, dataToLoad[tableName], row_ids=[None] * len(dataToLoad[tableName])
+		)
+		if errors == []:
+			print("Loaded " + str(len(dataToLoad[tableName])) + " rows into " + tableName)
+		else:
+			print("FAILED: loading " + str(len(dataToLoad[tableName])) + " rows into " + tableName)
+			print(errors)
+