bqseine 0.1.0__tar.gz

bqseine-0.1.0/.gitignore

@@ -0,0 +1,207 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+#poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+#pdm.lock
+#pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+#pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Abstra
+# Abstra is an AI-powered process automation framework.
+# Ignore directories containing user credentials, local state, and settings.
+# Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#  Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#  that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#  and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#  you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor
+#  Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/

bqseine-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

bqseine-0.1.0/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.4
+Name: bqseine
+Version: 0.1.0
+Summary: ETL for BigQuery
+Project-URL: Homepage, https://github.com/shaafiee/seine
+Project-URL: Issues, https://github.com/shaafiee/siene/issues
+Author-email: Shaafiee <shaafiee@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+
+# BQSeine
+## Python dict to BigQuery data loader
+Seine is a data loader that pushes data in a dictionary to BigQuery in relational normalized form. Seine also has functionality to use Gemini to get data insights from Bigquery.
+
+## Usage
+
+### AI insights
+```
+from bqseine.agent import chat
+response = chat(question, context)
+```
+
+In making the  call to chat(<question>, <context>), the <context> argument explains to the LLM how the tables in Bigquery and their fields refer to each other and what they mean.
+
+### Data loading
+```
+from bqseine.polyp import sync
+sourceData = [
+	{
+		'item': 'Juice',
+		'price': 20.0,
+		'stock': [
+			{
+				'batch': '2025-01-20',
+				'qty': 300
+			},
+			{
+				'batch': '2025-02-02',
+				'qty': 50
+			}
+		]
+	},
+	{
+		'item': 'Burger',
+		'price': 30.0,
+		'stock': [
+			{
+				'batch': '2025-02-10',
+				'qty': 200
+			}
+		]
+	}
+]
+sync('someGoogleProject', sourceData, 'catalog', 'US')
+### The arguments above are:
+### sync(<Google project name>, <dict>, <main table name>, <BigQuery region>)*
+```
+
+The above example will generate the following tables in BigQuery:
+#### catalog
+| seine_id | seine_parent_id | item | price | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 0 | 'Juice' | 20.0 | now() |
+| 2 | 0 | 'Burger' | 30.0 | now() |
+
+#### catalog_stock
+| seine_id | seine_parent_id | batch | qty | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 1 | '2025-01-20' | 300 | now() |
+| 2 | 1 | '2025-02-02' | 50 | now() |
+| 3 | 2 | '2025-02-10' | 200 | now() |

bqseine-0.1.0/README.md

@@ -0,0 +1,61 @@
+# BQSeine
+## Python dict to BigQuery data loader
+Seine is a data loader that pushes data in a dictionary to BigQuery in relational normalized form. Seine also has functionality to use Gemini to get data insights from Bigquery.
+
+## Usage
+
+### AI insights
+```
+from bqseine.agent import chat
+response = chat(question, context)
+```
+
+In making the  call to chat(<question>, <context>), the <context> argument explains to the LLM how the tables in Bigquery and their fields refer to each other and what they mean.
+
+### Data loading
+```
+from bqseine.polyp import sync
+sourceData = [
+	{
+		'item': 'Juice',
+		'price': 20.0,
+		'stock': [
+			{
+				'batch': '2025-01-20',
+				'qty': 300
+			},
+			{
+				'batch': '2025-02-02',
+				'qty': 50
+			}
+		]
+	},
+	{
+		'item': 'Burger',
+		'price': 30.0,
+		'stock': [
+			{
+				'batch': '2025-02-10',
+				'qty': 200
+			}
+		]
+	}
+]
+sync('someGoogleProject', sourceData, 'catalog', 'US')
+### The arguments above are:
+### sync(<Google project name>, <dict>, <main table name>, <BigQuery region>)*
+```
+
+The above example will generate the following tables in BigQuery:
+#### catalog
+| seine_id | seine_parent_id | item | price | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 0 | 'Juice' | 20.0 | now() |
+| 2 | 0 | 'Burger' | 30.0 | now() |
+
+#### catalog_stock
+| seine_id | seine_parent_id | batch | qty | injected |
+| --- | --- | --- | --- | --- |
+| 1 | 1 | '2025-01-20' | 300 | now() |
+| 2 | 1 | '2025-02-02' | 50 | now() |
+| 3 | 2 | '2025-02-10' | 200 | now() |

bqseine-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[build-system]
+requires = ["hatchling >= 1.26"]
+build-backend = "hatchling.build"
+
+[project]
+name = "bqseine"
+version = "0.1.0"
+authors = [
+  { name="Shaafiee", email="shaafiee@gmail.com" },
+]
+description = "ETL for BigQuery"
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+license = "MIT"
+license-files = ["LICEN[CS]E*"]
+
+[project.urls]
+Homepage = "https://github.com/shaafiee/seine"
+Issues = "https://github.com/shaafiee/siene/issues"

bqseine-0.1.0/src/bqseine/agent.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import os
+import json
+from typing import Dict, Any, List, Optional
+
+from google import genai
+from google.genai import types as gtypes
+from google.cloud import bigquery
+
+from context import sales, stocr, traffic
+
+from decimal import Decimal
+from datetime import date, datetime
+
+# --- BigQuery setup (ADC: env, WI, or SA on the host) ---
+bq = bigquery.Client()
+
+# --- Gemini (AI Studio) ---
+# Uses GOOGLE_API_KEY if set, otherwise falls back to application-default creds when supported.
+gclient = genai.Client()
+
+# ---------- Tool implementations ----------
+
+def json_safe(obj):
+	"""Recursively convert BigQuery / Python types to JSON-serializable types."""
+	if isinstance(obj, Decimal):
+		# choose float or str depending on how precise you need
+		return float(obj)
+	if isinstance(obj, (datetime, date)):
+		return obj.isoformat()
+	if isinstance(obj, dict):
+		return {k: json_safe(v) for k, v in obj.items()}
+	if isinstance(obj, (list, tuple)):
+		return [json_safe(v) for v in obj]
+	# BigQuery Row and similar should already be converted before calling this
+	return obj
+
+def list_datasets(project_id: Optional[str] = None) -> List[str]:
+	"""List accessible BigQuery datasets in a project (defaults to client's project)."""
+	proj = project_id or bq.project
+	return [d.dataset_id for d in bq.list_datasets(project=proj)]
+
+def list_tables(dataset: str, project_id: Optional[str] = None) -> List[str]:
+	"""List tables in a dataset."""
+	proj = project_id or bq.project
+	return [t.table_id for t in bq.list_tables(f"{proj}.{dataset}")]
+
+def get_table_schema(table: str, dataset: str, project_id: Optional[str] = None) -> List[Dict[str, Any]]:
+	"""Return table schema (name, type, mode) as JSON."""
+	proj = project_id or bq.project
+	t = bq.get_table(f"{proj}.{dataset}.{table}")
+	return [{"name": f.name, "type": f.field_type, "mode": f.mode} for f in t.schema]
+
+def run_query(sql: str, params: Optional[Dict[str, Any]] = None, dry_run: bool = False) -> Dict[str, Any]:
+	lowered = sql.strip().lower()
+	if lowered.startswith(("insert", "update", "delete", "merge", "create", "drop", "alter")):
+		return {
+			"error": "Refusing to run DDL/DML in this agent. SELECT-only allowed.",
+			"sql": sql,
+		}
+
+	qparams = []
+	if params:
+		for k, v in params.items():
+			if isinstance(v, bool):
+				typ = "BOOL"
+			elif isinstance(v, int):
+				typ = "INT64"
+			elif isinstance(v, float):
+				typ = "FLOAT64"
+			else:
+				typ = "STRING"
+			qparams.append(bigquery.ScalarQueryParameter(k, typ, v))
+
+	job_config = bigquery.QueryJobConfig(
+		dry_run=dry_run,
+		use_query_cache=True,
+	)
+	if qparams:
+		job_config.query_parameters = qparams
+
+	try:
+		job = bq.query(sql, job_config=job_config)
+
+		if dry_run:
+			return json_safe({
+				"rows": [],
+				"total_rows": 0,
+				"slot_ms": job.slot_millis,
+				"job_id": job.job_id,
+				"cache_hit": getattr(job, "cache_hit", None),
+				"sql": sql,
+			})
+
+		raw_rows = [dict(r) for r in job.result()]
+		return json_safe({
+			"rows": raw_rows,
+			"total_rows": len(raw_rows),
+			"slot_ms": job.slot_millis,
+			"job_id": job.job_id,
+			"cache_hit": getattr(job, "cache_hit", None),
+			"sql": sql,
+		})
+
+	except Exception as e:
+		return json_safe({
+			"error": str(e),
+			"sql": sql,
+		})
+
+# ---------- Tool declarations for Gemini ----------
+
+bq_tools = gtypes.Tool(function_declarations=[
+	{
+		"name": "list_datasets",
+		"description": "List accessible BigQuery datasets for a project.",
+		"parameters": {
+			"type": "object",
+			"properties": {
+	"project_id": {"type": "string", "description": "GCP project ID"},
+			}
+		},
+	},
+	{
+		"name": "list_tables",
+		"description": "List tables in a dataset.",
+		"parameters": {
+			"type": "object",
+			"properties": {
+	"dataset": {"type": "string"},
+	"project_id": {"type": "string"},
+			},
+			"required": ["dataset"]
+		},
+	},
+	{
+		"name": "get_table_schema",
+		"description": "Get schema for a table as JSON.",
+		"parameters": {
+			"type": "object",
+			"properties": {
+	"table": {"type": "string"},
+	"dataset": {"type": "string"},
+	"project_id": {"type": "string"},
+			},
+			"required": ["table", "dataset"]
+		},
+	},
+	{
+		"name": "run_query",
+		"description": "Run a SELECT-only BigQuery query with optional named parameters.",
+		"parameters": {
+			"type": "object",
+			"properties": {
+	"sql": {"type": "string"},
+	"params": {"type": "object"},
+	"dry_run": {"type": "boolean"}
+			},
+			"required": ["sql"]
+		},
+	},
+])
+
+# Optional: force the model to use tools when needed (or try AUTO first).
+tool_config_any = gtypes.ToolConfig(
+	function_calling_config=gtypes.FunctionCallingConfig(mode="AUTO")
+)
+gen_config = gtypes.GenerateContentConfig(
+	tools=[bq_tools],
+	tool_config=tool_config_any,
+	temperature=0.1,
+)
+
+SYSTEM_PROMPT = """You are a data analyst assistant for BigQuery.
+- Prefer SELECT-only SQL.
+- When missing a column/table name, use list_datasets/list_tables/get_table_schema.
+- For final answers, provide a brief natural-language summary, include the SQL you ran in a fenced code block, and include the output from the SQL, styled as an HTML <table>, in a fenced code block.
+"""
+def dispatch_tool(name: str, args: Dict[str, Any]) -> Dict[str, Any]:
+	try:
+		if name == "list_datasets":
+			return json_safe({"datasets": list_datasets(**args)})
+		if name == "list_tables":
+			return json_safe({"tables": list_tables(**args)})
+		if name == "get_table_schema":
+			return json_safe({"schema": get_table_schema(**args)})
+		if name == "run_query":
+			return json_safe(run_query(**args))
+		return {"error": f"Unknown tool: {name}"}
+	except Exception as e:
+		return {"error": str(e)}
+
+def chat(user_data: list[str],
+			  history: list[gtypes.Content] | None = None,
+			  model: str = "gemini-2.5-pro") -> gtypes.GenerateContentResponse:
+	user_prompt = user_data[0]
+	additional_instructions = user_data[1]
+	if history is None:
+		history = []
+
+	if not history:
+		history.append(
+			gtypes.Content(
+				role="user",
+				parts=[gtypes.Part(text=f"{SYSTEM_PROMPT}\n{additional_instructions}")]
+			)
+		)
+
+	# Add the new user message
+	history.append(
+		gtypes.Content(
+			role="user",
+			parts=[gtypes.Part(text=user_prompt)]
+		)
+	)
+
+	# First call: model may respond with text + function_call parts
+	resp = gclient.models.generate_content(
+		model=model,
+		contents=history,
+		config=gen_config,
+	)
+
+	# Collect any tool calls from parts
+	tool_calls = []
+	for part in resp.candidates[0].content.parts:
+		if part.function_call:
+			tool_calls.append(part.function_call)
+
+	if not tool_calls:
+		# No tools requested → just return; caller can use resp.text safely
+		return resp
+
+	# Execute each requested tool
+	tool_response_contents: list[gtypes.Content] = []
+	for fc in tool_calls:
+		result = dispatch_tool(fc.name, dict(fc.args))
+		tool_response_contents.append(
+			gtypes.Content(
+				role="tool",
+				parts=[
+					gtypes.Part(
+						function_response=gtypes.FunctionResponse(
+							name=fc.name,
+							response=result,
+						)
+					)
+				],
+			)
+		)
+
+	# Extend history with the model's function_call turn + tool responses
+	history.append(resp.candidates[0].content)
+	history.extend(tool_response_contents)
+
+	# Second call: model now sees tool outputs and should produce final answer text
+	final_resp = gclient.models.generate_content(
+		model=model,
+		contents=history,
+		config=gen_config,
+	)
+
+	return final_resp
+
+if __name__ == "__main__":
+	#"Generate a table of all the fabrics for which a slipcover was sold in October of 2025, with first column showing the fabric name, second the volume sold and third the total earnings in USD."
+	#"Generate weekly STOCR 28 the second quarter of 2025."
+	# Example usage
+	"""
+	question = (
+		"Find the top 5 product categories by revenue in 2025 Q3 from "
+		"`myproj.analytics.sales` and show revenue and order_count. "
+		"If you don't know columns, inspect the schema first."
+	)
+	"""
+	question = (
+		"Generate a table of all the fabrics for which a slipcover was sold in October of 2025, with first column showing the fabric name, second the volume sold and third the total earnings in USD."
+	)
+	answer = chat_once([question, f"{sales}\n{stocr}\n{traffic}"])
+	print(answer.text)

bqseine-0.1.0/src/bqseine/polyp.py

@@ -0,0 +1,381 @@
+from gcp_secrets.secrets import *
+from google.cloud import bigquery
+from google.cloud.exceptions import NotFound
+from datetime import date, datetime
+from db_lib import *
+import json
+
+
+lastSeineId = {}
+tableSchema = {}
+tableCurrentSchema = {}
+tableCurrentSchemaType = {}
+tableReset = {}
+#client  = bigquery.Client()
+
+#    * EXCEPT(is_generated, generation_expression, is_stored, is_updatable)
+tableColumnsQuery = """
+SELECT 
+    column_name 
+  FROM 
+    `__dataset__`.INFORMATION_SCHEMA.COLUMNS 
+  WHERE 
+    table_name = '__table__';
+"""
+
+
+def resolveType(value):
+	isDatetime = False
+	tempDatetime = None
+	try:
+		timeDatetime = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
+		value = timeDatetime
+		isDatetime = True
+	except:
+		pass
+
+	if isinstance(value, bool):
+		return "BOOL"
+	elif isinstance(value, int):
+		return "INT64"
+	elif isinstance(value, float):
+		return "FLOAT64"
+	elif type(value) == datetime:
+		return "DATETIME"
+	else:
+		return "STRING"
+
+
+def testValue(value, tableKey, fieldKey):
+	global tableCurrentSchemaType
+	try:
+		timeDatetime = datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
+		return timeDatetime.strftime('%Y-%m-%dT%H:%M:%S')
+	except:
+		pass
+
+	fieldType = None
+	if fieldKey in tableCurrentSchemaType[tableKey].keys():
+		fieldType = tableCurrentSchemaType[tableKey][fieldKey]
+	else:
+		fieldType = resolveType(value)
+		tableCurrentSchemaType[tableKey][fieldKey] = fieldType
+
+	if fieldType == "BOOL":
+		if isinstance(value, bool):
+			return value
+		else:
+			return False
+	elif fieldType == "INT64":
+		if isinstance(value, int):
+			return value
+		else:
+			return 0
+	elif fieldType == "FLOAT64":
+		if isinstance(value, float):
+			return value
+		else:
+			return 0
+	elif fieldType == "STRING":
+		if isinstance(value, str):
+			return value
+		else:
+			return ""
+	else:
+		if isinstance(value, str):
+			return value
+		else:
+			return json.dumps(value)
+
+	return ""
+
+
+def incrementId(curKey):
+	global lastSeineId
+	if curKey in lastSeineId.keys():
+		lastSeineId[curKey] += 1
+	else:
+		lastSeineId[curKey] = 1
+
+
+def sync(myGoogleProject, blob, curKey, bqRegion = 'US', firstReset = False, idField = None):
+	global lastSeineId
+	global tableSchema
+	global tableReset
+	global tableCurrentSchema
+	global tableCurrentSchemaType
+
+	if len(curKey) < 1:
+		raise ExceptionType("A default current key (second arg) should be provided")
+
+	stack = []
+	if isinstance(blob, list):
+		for part in blob:
+			stack.insert(0, (curKey, part, curKey, 0))
+	else:
+		stack = [(curKey, blob, curKey, 0)]
+		
+	tableChecked = False
+
+	client = None
+	try:
+		client = bigquery.Client(project=myGoogleProject)
+	except:
+		raise ExceptionType(f"Could not connect to {myGoogleProject}")
+
+	if firstReset:
+		lastSeineId = {}
+		tableSchema = {}
+		tableReset = {}
+		tableCurrentSchema = {}
+		tableCurrentSchemaType = {}
+
+	keyNotInSchema = {}
+	dataToLoad = {}
+	seineDataset = False
+	dataset = ""
+	datasetName = "seine_" + curKey
+	try:
+		seineDataset = client.get_dataset(myGoogleProject + f".{datasetName}")
+	except NotFound:
+		seineDataset = bigquery.Dataset(myGoogleProject + f".{datasetName}")
+		seineDataset.location = bqRegion
+		seineDataset = client.create_dataset(seineDataset, timeout=30)
+		print("Created dataset {}".format(seineDataset.dataset_id))
+	
+	valuesArray = []
+	currentDepth = 0
+	while stack:
+		curKey, curDict, lastKey, parentId = stack.pop()
+		currentDepth += 1
+		fieldTypes = {}
+		fields = []
+		fieldsJson = {}
+		values = []
+		fieldType = {}
+		valuePlaceholders = {}
+		counter = 0
+
+		#seineDataset = client.get_dataset(myGoogleProject + f".{datasetName}")
+		print("-------------------------")
+		print(curDict)
+		noUpdateNeeded = False
+
+		if curKey not in lastSeineId.keys():
+			lastSeineId[curKey] = 1
+
+		if parentId == 0 and curKey == idField and isinstance(curDict, int):
+			try:
+				curTableName = myGoogleProject + f".{datasetName}." + curKey
+				queryJob = client.query(f"select {idField} from {curTableName} where {idField} = {curDict}")
+				returned = queryJob.result()
+				foundId = False
+				for row in returned:
+					foundId = True
+				if foundId:
+					print(f"ID exists {idField} = {curDict}")
+					print(returned)
+					continue
+			except:
+				print("================== QUERY FAILED =======================")
+				pass
+
+		#if curKey not in keyNotInSchema.keys():
+		keyNotInSchema[curKey] = []
+		if curKey not in tableCurrentSchema.keys():
+			try:
+				curTableName = myGoogleProject + f".{datasetName}." + curKey
+				queryJob = client.query(f"select max(seine_id) as max_id from {curTableName}")
+				returned = queryJob.result()
+				print(returned)
+				for row in returned:
+					if row.max_id is not None and isinstance(row.max_id, int):
+						lastSeineId[curKey] = int(row.max_id) + 1
+			except:
+				pass
+			tableCurrentSchema[curKey] = []
+			tableCurrentSchemaType[curKey] = {}
+		
+		if isinstance(curDict, list):
+			if not isinstance(curDict[0], dict):
+				fields.append(curKey)
+				fieldTypes[curKey] = resolveType(json.dumps(curDict))
+				valuePlaceholders[curKey] = testValue(json.dumps(curDict), curKey, curKey)
+				if curKey not in tableCurrentSchema[curKey]:
+					tableCurrentSchema[curKey].append(curKey)
+					keyNotInSchema[curKey].append(curKey)
+					#tableCurrentSchemaType[curKey][curKey] = resolveType(json.dumps(curDict))
+			else:
+				noUpdateNeeded = True
+				for tempDict in curDict:
+					stack.insert(0, (curKey, tempDict, lastKey, lastSeineId[lastKey]))
+				continue
+
+		elif isinstance(curDict, dict):
+			for key, value in curDict.items():
+				if isinstance(value, list):
+					if len(value) > 0:
+						if not isinstance(value[0], dict) and key not in ["edges"]:
+							fields.append(key)
+							fieldTypes[key] = resolveType(json.dumps(value))
+							valuePlaceholders[key] = testValue(json.dumps(value), curKey, key)
+							if key not in tableCurrentSchema[curKey]:
+								tableCurrentSchema[curKey].append(key)
+								keyNotInSchema[curKey].append(key)
+								#tableCurrentSchemaType[curKey][key] = resolveType(json.dumps(value))
+						elif key in ["edges"]:
+							noUpdateNeeded = True
+							for part in value:
+								stack.insert(0, (curKey, part, lastKey,  lastSeineId[lastKey]))
+						else:
+							for part in value:
+								stack.insert(0, (curKey + "_" + key, part, curKey,  lastSeineId[lastKey]))
+							#fields.append(key)
+							#fieldTypes[key] = resolveType(json.dumps(value))
+							#valuePlaceholders[key] = testValue(json.dumps(value), curKey, key)
+							#if key not in tableCurrentSchema[curKey]:
+							#	tableCurrentSchema[curKey].append(key)
+							#	keyNotInSchema[curKey].append(key)
+					else:
+						fields.append(key)
+						fieldTypes[key] = resolveType(json.dumps(value))
+						valuePlaceholders[key] = testValue(json.dumps(value), curKey, key)
+						if key not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(key)
+							keyNotInSchema[curKey].append(key)
+							#tableCurrentSchemaType[curKey][key] = resolveType(json.dumps(value))
+				elif isinstance(value, dict):
+					if key in ["node"]:
+						noUpdateNeeded = True
+						stack.insert(0, (curKey, value, lastKey,  lastSeineId[lastKey]))
+					else:
+						stack.insert(0, (curKey + "_" + key, value, curKey,  lastSeineId[lastKey]))
+						fields.append(key)
+						fieldTypes[key] = resolveType(lastSeineId[curKey])
+						valuePlaceholders[key] = testValue(lastSeineId[curKey], curKey, key)
+						if key not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(key)
+							keyNotInSchema[curKey].append(key)
+							#tableCurrentSchemaType[curKey][key] = resolveType(json.dumps(value))
+				else:
+					# Add schema
+					if key not in fields:
+						fields.append(key)
+						fieldTypes[key] = resolveType(value)
+						valuePlaceholders[key] = testValue(value, curKey, key)
+						if key not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(key)
+							keyNotInSchema[curKey].append(key)
+							#tableCurrentSchemaType[curKey][key] = resolveType(value)
+
+		elif isinstance(curDict, str) or isinstance(curDict, int) or isinstance(curDict, float):
+			fields.append(curKey)
+			fieldTypes[curKey] = resolveType(curDict)
+			valuePlaceholders[curKey] = testValue(curDict, curKey, key)
+			if curKey not in tableCurrentSchema[curKey]:
+				tableCurrentSchema[curKey].append(curKey)
+				keyNotInSchema[curKey].append(curKey)
+				#tableCurrentSchemaType[curKey][curKey] = resolveType(curDict)
+
+		else:
+			fields.append(curKey)
+			fieldTypes[curKey] = resolveType(curDict)
+			valuePlaceholders[curKey] = testValue(curDict, curKey, key)
+			if curKey not in tableCurrentSchema[curKey]:
+				tableCurrentSchema[curKey].append(curKey)
+				keyNotInSchema[curKey].append(curKey)
+				#tableCurrentSchemaType[curKey][curKey] = resolveType(curDict)
+
+		#if len(fields) < 1 and parentId is None:
+		#	continue
+
+		if noUpdateNeeded:
+			continue
+
+		if len(keyNotInSchema[curKey]) > 0 or curKey not in tableReset.keys():
+			tableReset[curKey] = True
+			curTableName = myGoogleProject + f".{datasetName}." + curKey
+			curTable = False
+			tableSchema[curKey] = []
+			tableSchema[curKey].append(bigquery.SchemaField("seine_id", "INT64"))
+			tableSchema[curKey].append(bigquery.SchemaField("seine_parent_id", "INT64"))
+			tableSchema[curKey].append(bigquery.SchemaField("injected", "DATETIME"))
+			tableCurrentSchema[curKey].append("seine_id");
+			tableCurrentSchema[curKey].append("seine_parent_id");
+			tableCurrentSchema[curKey].append("injected");
+			try:
+				curTable = client.get_table(curTableName)
+				#colQuery = tableColumnsQuery.replace("__dataset__", seineDataset).replace("__table__", key)
+				#queryJob = client.query(colQuery)
+				#returned = queryJob.result()
+				existingSchema = curTable.schema
+				tableSchema[curKey] = existingSchema
+				existingColumns = []
+				jobCconfig = bigquery.QueryJobConfig(
+					destination=curTableName,
+					schema_update_options=[bigquery.SchemaUpdateOption.ALLOW_FIELD_ADDITION],
+					write_disposition=bigquery.WriteDisposition.WRITE_APPEND,
+				)
+				for schemaElement in existingSchema: 
+					existingColumns.append(schemaElement.name)
+				schemaAdjusted = False
+				for fidx, field in enumerate(fields):
+					if field not in existingColumns:
+						if field not in tableCurrentSchema[curKey]:
+							tableCurrentSchema[curKey].append(field);
+						tableSchema[curKey].append(bigquery.SchemaField(field, fieldTypes[field]))
+						existingSchema.append(bigquery.SchemaField(field, fieldTypes[field]))
+						if not schemaAdjusted:
+							schemaAdjusted = True
+				if schemaAdjusted:
+					curTable.schema = existingSchema
+					try:
+						curTable = client.update_table(curTable, ["schema"])
+					except:
+						pass
+
+				#if returned.total_rows > 0:
+				#	counter = 0
+				#	for row in returned:
+				#		print(row)
+				#		counter += 1
+				#		if counter < 2:
+				#			continue
+				#		if row.max_id + 1 > lastSeineId[curKey]:
+				#			lastSeineId[curKey] = int(row.max_id) + 1
+				#			print(f"{curKey}" + str(lastSeineId[curKey]))
+			except NotFound:
+				for fidx, field in enumerate(fields):
+					tableSchema[curKey].append(bigquery.SchemaField(field, fieldTypes[field]))
+					if field not in tableCurrentSchema[curKey]:
+						tableCurrentSchema[curKey].append(field);
+				curTable = bigquery.Table(curTableName, schema=tableSchema[curKey])
+				curTable = client.create_table(curTable)
+				lastSeineId[curKey] = 1
+			keyNotInSchema[curKey] = []
+
+		if curKey not in dataToLoad.keys():
+			dataToLoad[curKey] = []
+		tempRow = {
+			"seine_id": lastSeineId[curKey],
+			"seine_parent_id": parentId,
+			"injected": datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
+		}
+		for idx, field in enumerate(fields):
+			tempRow[field] = valuePlaceholders[field]
+		print(tempRow)
+		dataToLoad[curKey].append(tempRow)
+		lastSeineId[curKey] += 1
+
+	conn, cur = dbConnect()
+	for tableName in dataToLoad.keys():
+		curTable = client.get_table(myGoogleProject + f".{datasetName}." + tableName)
+		errors = client.insert_rows_json(
+			curTable, dataToLoad[tableName], row_ids=[None] * len(dataToLoad[tableName])
+		)
+		if errors == []:
+			print("Loaded " + str(len(dataToLoad[tableName])) + " rows into " + tableName)
+		else:
+			print("FAILED: loading " + str(len(dataToLoad[tableName])) + " rows into " + tableName)
+			print(errors)
+