python-datastore-sqlalchemy 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python_datastore_sqlalchemy-0.0.1/LICENSE +9 -0
- python_datastore_sqlalchemy-0.0.1/PKG-INFO +65 -0
- python_datastore_sqlalchemy-0.0.1/README.md +28 -0
- python_datastore_sqlalchemy-0.0.1/pyproject.toml +21 -0
- python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/PKG-INFO +65 -0
- python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/SOURCES.txt +25 -0
- python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/dependency_links.txt +1 -0
- python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/entry_points.txt +2 -0
- python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/not-zip-safe +1 -0
- python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/requires.txt +1 -0
- python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/top_level.txt +1 -0
- python_datastore_sqlalchemy-0.0.1/setup.cfg +4 -0
- python_datastore_sqlalchemy-0.0.1/setup.py +70 -0
- python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/__init__.py +27 -0
- python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/_helpers.py +135 -0
- python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/_types.py +147 -0
- python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/base.py +291 -0
- python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/datastore_dbapi.py +2322 -0
- python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/parse_url.py +287 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_datastore.py +272 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_gql.py +1202 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_integration_queries.py +402 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_orm.py +79 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_unit_base.py +226 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_unit_dbapi.py +1416 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_unit_helpers.py +124 -0
- python_datastore_sqlalchemy-0.0.1/tests/test_unit_types.py +203 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright 2025 The sqlalchemy-datastore Authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
6
|
+
|
|
7
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: python-datastore-sqlalchemy
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: SQLAlchemy dialect for google cloud datastore
|
|
5
|
+
Home-page: https://github.com/splasky/python-datastore-sqlalchemy
|
|
6
|
+
Author: HY Chang(splasky)
|
|
7
|
+
Author-email: hychang.1997.tw@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Documentation, https://github.com/splasky/python-datastore-sqlalchemy/wiki
|
|
10
|
+
Project-URL: Source, https://github.com/splasky/python-datastore-sqlalchemy
|
|
11
|
+
Project-URL: Tracker, https://github.com/splasky/python-datastore-sqlalchemy/issues
|
|
12
|
+
Keywords: SQLAlchemy GCP Datastore
|
|
13
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
18
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
19
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: sqlalchemy<3.0.0,>=1.4.16
|
|
24
|
+
Dynamic: author
|
|
25
|
+
Dynamic: author-email
|
|
26
|
+
Dynamic: classifier
|
|
27
|
+
Dynamic: description
|
|
28
|
+
Dynamic: description-content-type
|
|
29
|
+
Dynamic: home-page
|
|
30
|
+
Dynamic: keywords
|
|
31
|
+
Dynamic: license
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
Dynamic: project-url
|
|
34
|
+
Dynamic: requires-dist
|
|
35
|
+
Dynamic: requires-python
|
|
36
|
+
Dynamic: summary
|
|
37
|
+
|
|
38
|
+
SQLAlchemy dialect for google cloud datastore(firestore mode)
|
|
39
|
+
========================
|
|
40
|
+
How to install
|
|
41
|
+
```
|
|
42
|
+
python3 setup.py install
|
|
43
|
+
```
|
|
44
|
+
How to use
|
|
45
|
+
```python
|
|
46
|
+
from sqlalchemy import *
|
|
47
|
+
from sqlalchemy.engine import create_engine
|
|
48
|
+
from sqlalchemy.schema import *
|
|
49
|
+
engine = create_engine('datastore://test-api-1', credentials='path/to/credentials.json')
|
|
50
|
+
conn = engine.connect()
|
|
51
|
+
result = conn.execute("SELECT * from test_table)
|
|
52
|
+
print(result.fetchall())
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Preview
|
|
56
|
+
<img src="assets/pie.png">
|
|
57
|
+
|
|
58
|
+
> [!WARNING]
|
|
59
|
+
> Please note: Not all GQL and SQL syntax has been fully tested. Should you encounter any bugs, please post the relevant query and open an issue on GitHub.
|
|
60
|
+
|
|
61
|
+
## How to contribute
|
|
62
|
+
Feel free to open issues and pull requests on GitHub.
|
|
63
|
+
|
|
64
|
+
## Development Notes
|
|
65
|
+
- [Develop a SQLAlchemy and it's dialects](https://hackmd.io/lsBW5GCVR82SORyWZ1cssA?view)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
SQLAlchemy dialect for google cloud datastore(firestore mode)
|
|
2
|
+
========================
|
|
3
|
+
How to install
|
|
4
|
+
```
|
|
5
|
+
python3 setup.py install
|
|
6
|
+
```
|
|
7
|
+
How to use
|
|
8
|
+
```python
|
|
9
|
+
from sqlalchemy import *
|
|
10
|
+
from sqlalchemy.engine import create_engine
|
|
11
|
+
from sqlalchemy.schema import *
|
|
12
|
+
engine = create_engine('datastore://test-api-1', credentials='path/to/credentials.json')
|
|
13
|
+
conn = engine.connect()
|
|
14
|
+
result = conn.execute("SELECT * from test_table)
|
|
15
|
+
print(result.fetchall())
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Preview
|
|
19
|
+
<img src="assets/pie.png">
|
|
20
|
+
|
|
21
|
+
> [!WARNING]
|
|
22
|
+
> Please note: Not all GQL and SQL syntax has been fully tested. Should you encounter any bugs, please post the relevant query and open an issue on GitHub.
|
|
23
|
+
|
|
24
|
+
## How to contribute
|
|
25
|
+
Feel free to open issues and pull requests on GitHub.
|
|
26
|
+
|
|
27
|
+
## Development Notes
|
|
28
|
+
- [Develop a SQLAlchemy and it's dialects](https://hackmd.io/lsBW5GCVR82SORyWZ1cssA?view)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
[tool.ruff]
|
|
2
|
+
line-length = 120
|
|
3
|
+
|
|
4
|
+
[tool.ruff.lint]
|
|
5
|
+
select = ["E", "F", "W", "I", "N", "B", "C", "G", "S"]
|
|
6
|
+
ignore = [
|
|
7
|
+
"E501", # Line too long - allow longer lines for SQL strings in tests
|
|
8
|
+
"S608", # SQL injection - intentional for test cases
|
|
9
|
+
"B007", # Loop variable not used - common in test assertions
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
[tool.ruff.lint.per-file-ignores]
|
|
13
|
+
"tests/*" = ["S608", "E501", "S101", "B007", "S603"]
|
|
14
|
+
|
|
15
|
+
[tool.mypy]
|
|
16
|
+
ignore_missing_imports = true
|
|
17
|
+
disable_error_code = ["var-annotated", "import-untyped"]
|
|
18
|
+
|
|
19
|
+
[[tool.mypy.overrides]]
|
|
20
|
+
module = ["tests.*", "sqlalchemy_datastore.*"]
|
|
21
|
+
ignore_errors = true
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: python-datastore-sqlalchemy
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: SQLAlchemy dialect for google cloud datastore
|
|
5
|
+
Home-page: https://github.com/splasky/python-datastore-sqlalchemy
|
|
6
|
+
Author: HY Chang(splasky)
|
|
7
|
+
Author-email: hychang.1997.tw@gmail.com
|
|
8
|
+
License: MIT
|
|
9
|
+
Project-URL: Documentation, https://github.com/splasky/python-datastore-sqlalchemy/wiki
|
|
10
|
+
Project-URL: Source, https://github.com/splasky/python-datastore-sqlalchemy
|
|
11
|
+
Project-URL: Tracker, https://github.com/splasky/python-datastore-sqlalchemy/issues
|
|
12
|
+
Keywords: SQLAlchemy GCP Datastore
|
|
13
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Programming Language :: Python
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
18
|
+
Classifier: Topic :: Database :: Front-Ends
|
|
19
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: sqlalchemy<3.0.0,>=1.4.16
|
|
24
|
+
Dynamic: author
|
|
25
|
+
Dynamic: author-email
|
|
26
|
+
Dynamic: classifier
|
|
27
|
+
Dynamic: description
|
|
28
|
+
Dynamic: description-content-type
|
|
29
|
+
Dynamic: home-page
|
|
30
|
+
Dynamic: keywords
|
|
31
|
+
Dynamic: license
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
Dynamic: project-url
|
|
34
|
+
Dynamic: requires-dist
|
|
35
|
+
Dynamic: requires-python
|
|
36
|
+
Dynamic: summary
|
|
37
|
+
|
|
38
|
+
SQLAlchemy dialect for google cloud datastore(firestore mode)
|
|
39
|
+
========================
|
|
40
|
+
How to install
|
|
41
|
+
```
|
|
42
|
+
python3 setup.py install
|
|
43
|
+
```
|
|
44
|
+
How to use
|
|
45
|
+
```python
|
|
46
|
+
from sqlalchemy import *
|
|
47
|
+
from sqlalchemy.engine import create_engine
|
|
48
|
+
from sqlalchemy.schema import *
|
|
49
|
+
engine = create_engine('datastore://test-api-1', credentials='path/to/credentials.json')
|
|
50
|
+
conn = engine.connect()
|
|
51
|
+
result = conn.execute("SELECT * from test_table)
|
|
52
|
+
print(result.fetchall())
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Preview
|
|
56
|
+
<img src="assets/pie.png">
|
|
57
|
+
|
|
58
|
+
> [!WARNING]
|
|
59
|
+
> Please note: Not all GQL and SQL syntax has been fully tested. Should you encounter any bugs, please post the relevant query and open an issue on GitHub.
|
|
60
|
+
|
|
61
|
+
## How to contribute
|
|
62
|
+
Feel free to open issues and pull requests on GitHub.
|
|
63
|
+
|
|
64
|
+
## Development Notes
|
|
65
|
+
- [Develop a SQLAlchemy and it's dialects](https://hackmd.io/lsBW5GCVR82SORyWZ1cssA?view)
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
setup.py
|
|
5
|
+
python_datastore_sqlalchemy.egg-info/PKG-INFO
|
|
6
|
+
python_datastore_sqlalchemy.egg-info/SOURCES.txt
|
|
7
|
+
python_datastore_sqlalchemy.egg-info/dependency_links.txt
|
|
8
|
+
python_datastore_sqlalchemy.egg-info/entry_points.txt
|
|
9
|
+
python_datastore_sqlalchemy.egg-info/not-zip-safe
|
|
10
|
+
python_datastore_sqlalchemy.egg-info/requires.txt
|
|
11
|
+
python_datastore_sqlalchemy.egg-info/top_level.txt
|
|
12
|
+
sqlalchemy_datastore/__init__.py
|
|
13
|
+
sqlalchemy_datastore/_helpers.py
|
|
14
|
+
sqlalchemy_datastore/_types.py
|
|
15
|
+
sqlalchemy_datastore/base.py
|
|
16
|
+
sqlalchemy_datastore/datastore_dbapi.py
|
|
17
|
+
sqlalchemy_datastore/parse_url.py
|
|
18
|
+
tests/test_datastore.py
|
|
19
|
+
tests/test_gql.py
|
|
20
|
+
tests/test_integration_queries.py
|
|
21
|
+
tests/test_orm.py
|
|
22
|
+
tests/test_unit_base.py
|
|
23
|
+
tests/test_unit_dbapi.py
|
|
24
|
+
tests/test_unit_helpers.py
|
|
25
|
+
tests/test_unit_types.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sqlalchemy<3.0.0,>=1.4.16
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
sqlalchemy_datastore
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Copyright (c) 2025 hychang <hychang.1997.tw@gmail.com>
|
|
2
|
+
#
|
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
4
|
+
# this software and associated documentation files (the "Software"), to deal in
|
|
5
|
+
# the Software without restriction, including without limitation the rights to
|
|
6
|
+
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
7
|
+
# the Software, and to permit persons to whom the Software is furnished to do so,
|
|
8
|
+
# subject to the following conditions:
|
|
9
|
+
#
|
|
10
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
# copies or substantial portions of the Software.
|
|
12
|
+
#
|
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
15
|
+
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
16
|
+
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
17
|
+
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
18
|
+
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
19
|
+
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
|
|
23
|
+
from setuptools import setup, find_packages
|
|
24
|
+
|
|
25
|
+
v = open(
|
|
26
|
+
os.path.join(os.path.dirname(__file__), "sqlalchemy_datastore", "__init__.py")
|
|
27
|
+
)
|
|
28
|
+
version_match = re.compile(r'.*__version__ = "(.*?)"', re.S).match(v.read())
|
|
29
|
+
if not version_match:
|
|
30
|
+
raise RuntimeError("Unable to find version string in __init__.py.")
|
|
31
|
+
VERSION = version_match.group(1)
|
|
32
|
+
v.close()
|
|
33
|
+
|
|
34
|
+
readme = os.path.join(os.path.dirname(__file__), "README.md")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
setup(
|
|
38
|
+
name="python-datastore-sqlalchemy",
|
|
39
|
+
version=VERSION,
|
|
40
|
+
description="SQLAlchemy dialect for google cloud datastore",
|
|
41
|
+
long_description=open(readme).read(),
|
|
42
|
+
long_description_content_type="text/markdown",
|
|
43
|
+
url="https://github.com/splasky/python-datastore-sqlalchemy",
|
|
44
|
+
author="HY Chang(splasky)",
|
|
45
|
+
author_email="hychang.1997.tw@gmail.com",
|
|
46
|
+
license="MIT",
|
|
47
|
+
classifiers=[
|
|
48
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
49
|
+
"Intended Audience :: Developers",
|
|
50
|
+
"Programming Language :: Python",
|
|
51
|
+
"Programming Language :: Python :: 3",
|
|
52
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
53
|
+
"Topic :: Database :: Front-Ends",
|
|
54
|
+
"Operating System :: POSIX :: Linux",
|
|
55
|
+
],
|
|
56
|
+
keywords="SQLAlchemy GCP Datastore",
|
|
57
|
+
python_requires=">=3.9",
|
|
58
|
+
project_urls={
|
|
59
|
+
"Documentation": "https://github.com/splasky/python-datastore-sqlalchemy/wiki",
|
|
60
|
+
"Source": "https://github.com/splasky/python-datastore-sqlalchemy",
|
|
61
|
+
"Tracker": "https://github.com/splasky/python-datastore-sqlalchemy/issues",
|
|
62
|
+
},
|
|
63
|
+
packages=find_packages(include=["sqlalchemy_datastore"]),
|
|
64
|
+
include_package_data=True,
|
|
65
|
+
install_requires=["sqlalchemy>=1.4.16,<3.0.0",],
|
|
66
|
+
zip_safe=False,
|
|
67
|
+
entry_points={
|
|
68
|
+
"sqlalchemy.dialects": ["datastore = sqlalchemy_datastore:CloudDatastoreDialect"]
|
|
69
|
+
},
|
|
70
|
+
)
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Copyright (c) 2025 hychang <hychang.1997.tw@gmail.com>
|
|
2
|
+
#
|
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
4
|
+
# this software and associated documentation files (the "Software"), to deal in
|
|
5
|
+
# the Software without restriction, including without limitation the rights to
|
|
6
|
+
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
7
|
+
# the Software, and to permit persons to whom the Software is furnished to do so,
|
|
8
|
+
# subject to the following conditions:
|
|
9
|
+
#
|
|
10
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
# copies or substantial portions of the Software.
|
|
12
|
+
#
|
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
15
|
+
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
16
|
+
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
17
|
+
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
18
|
+
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
19
|
+
|
|
20
|
+
from .base import CloudDatastoreDialect
|
|
21
|
+
|
|
22
|
+
__version__ = "0.0.1"
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"__version__",
|
|
26
|
+
"CloudDatastoreDialect",
|
|
27
|
+
]
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# Copyright (c) 2017 The sqlalchemy-bigquery Authors
|
|
2
|
+
#
|
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
4
|
+
# this software and associated documentation files (the "Software"), to deal in
|
|
5
|
+
# the Software without restriction, including without limitation the rights to
|
|
6
|
+
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
7
|
+
# the Software, and to permit persons to whom the Software is furnished to do so,
|
|
8
|
+
# subject to the following conditions:
|
|
9
|
+
#
|
|
10
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
# copies or substantial portions of the Software.
|
|
12
|
+
#
|
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
15
|
+
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
16
|
+
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
17
|
+
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
18
|
+
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
19
|
+
|
|
20
|
+
import base64
|
|
21
|
+
import functools
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
from typing import Optional, Tuple
|
|
26
|
+
|
|
27
|
+
import google.auth
|
|
28
|
+
import sqlalchemy
|
|
29
|
+
from google.api_core import client_info
|
|
30
|
+
from google.auth.credentials import Credentials
|
|
31
|
+
from google.cloud import datastore
|
|
32
|
+
from google.oauth2 import service_account
|
|
33
|
+
|
|
34
|
+
USER_AGENT_TEMPLATE = "sqlalchemy/{}"
|
|
35
|
+
SCOPES = (
|
|
36
|
+
"https://www.googleapis.com/auth/datastore",
|
|
37
|
+
"https://www.googleapis.com/auth/cloud-platform",
|
|
38
|
+
"https://www.googleapis.com/auth/drive",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def google_client_info(
|
|
43
|
+
user_agent: Optional[str] = None,
|
|
44
|
+
) -> client_info.ClientInfo:
|
|
45
|
+
"""
|
|
46
|
+
Return a client_info object, with an optional user agent
|
|
47
|
+
string. If user_agent is None, use a default value.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
if user_agent is None:
|
|
51
|
+
user_agent = USER_AGENT_TEMPLATE.format(sqlalchemy.__version__)
|
|
52
|
+
return client_info.ClientInfo(user_agent=user_agent)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def create_datastore_client(
|
|
56
|
+
credentials_info: Optional[dict] = None,
|
|
57
|
+
credentials_path: Optional[str] = None,
|
|
58
|
+
credentials_base64: Optional[str] = None,
|
|
59
|
+
project_id: Optional[str] = None,
|
|
60
|
+
user_agent: Optional[client_info.ClientInfo] = None,
|
|
61
|
+
database: Optional[str] = None
|
|
62
|
+
) -> Tuple[datastore.Client, Optional[Credentials]]:
|
|
63
|
+
"""Construct a BigQuery client object.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
credentials_info Optional[dict]:
|
|
67
|
+
credentials_path Optional[str]:
|
|
68
|
+
credentials_base64 Optional[str]:
|
|
69
|
+
location (Optional[str]):
|
|
70
|
+
Default location for jobs / datasets / tables.
|
|
71
|
+
project_id (Optional[str]):
|
|
72
|
+
Project ID for the project which the client acts on behalf of.
|
|
73
|
+
user_agent (Optional[google.api_core.client_info.ClientInfo]):
|
|
74
|
+
The client info used to send a user-agent string along with API
|
|
75
|
+
requests. If ``None``, then default info will be used. Generally,
|
|
76
|
+
you only need to set this if you're developing your own library
|
|
77
|
+
or partner tool.
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
default_project = None
|
|
81
|
+
database = database if database != "(default)" else None
|
|
82
|
+
if os.getenv("DATASTORE_EMULATOR_HOST") is not None:
|
|
83
|
+
client = datastore.Client(project=project_id)
|
|
84
|
+
return client, None
|
|
85
|
+
|
|
86
|
+
if credentials_base64:
|
|
87
|
+
credentials_info = json.loads(base64.b64decode(credentials_base64))
|
|
88
|
+
|
|
89
|
+
if credentials_path:
|
|
90
|
+
credentials = service_account.Credentials.from_service_account_file(
|
|
91
|
+
credentials_path
|
|
92
|
+
)
|
|
93
|
+
credentials = credentials.with_scopes(SCOPES)
|
|
94
|
+
default_project = credentials.project_id
|
|
95
|
+
elif credentials_info:
|
|
96
|
+
credentials = service_account.Credentials.from_service_account_info(
|
|
97
|
+
credentials_info
|
|
98
|
+
)
|
|
99
|
+
credentials = credentials.with_scopes(SCOPES)
|
|
100
|
+
default_project = credentials.project_id
|
|
101
|
+
else:
|
|
102
|
+
credentials, default_project = google.auth.default(scopes=SCOPES)
|
|
103
|
+
|
|
104
|
+
if project_id is None:
|
|
105
|
+
project_id = default_project
|
|
106
|
+
|
|
107
|
+
info = google_client_info(user_agent=user_agent.to_user_agent() if user_agent is not None else None)
|
|
108
|
+
|
|
109
|
+
return datastore.Client(
|
|
110
|
+
client_info=info,
|
|
111
|
+
project=project_id,
|
|
112
|
+
credentials=credentials,
|
|
113
|
+
database=database
|
|
114
|
+
), credentials
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def substitute_re_method(r, flags=0, repl=None):
|
|
118
|
+
if repl is None:
|
|
119
|
+
return lambda f: substitute_re_method(r, flags, f)
|
|
120
|
+
|
|
121
|
+
r = re.compile(r, flags)
|
|
122
|
+
|
|
123
|
+
@functools.wraps(repl)
|
|
124
|
+
def sub(self, s, *args, **kw):
|
|
125
|
+
def repl_(m):
|
|
126
|
+
return repl(self, m, *args, **kw)
|
|
127
|
+
|
|
128
|
+
return r.sub(repl_, s)
|
|
129
|
+
|
|
130
|
+
return sub
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def substitute_string_re_method(r, *, repl, flags=0):
|
|
134
|
+
r = re.compile(r, flags)
|
|
135
|
+
return lambda self, s: r.sub(repl, s)
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# Copyright (c) 2017 The sqlalchemy-bigquery Authors
|
|
2
|
+
#
|
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
4
|
+
# this software and associated documentation files (the "Software"), to deal in
|
|
5
|
+
# the Software without restriction, including without limitation the rights to
|
|
6
|
+
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
7
|
+
# the Software, and to permit persons to whom the Software is furnished to do so,
|
|
8
|
+
# subject to the following conditions:
|
|
9
|
+
#
|
|
10
|
+
# The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
# copies or substantial portions of the Software.
|
|
12
|
+
#
|
|
13
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
15
|
+
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
16
|
+
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
17
|
+
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
18
|
+
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
19
|
+
|
|
20
|
+
import sqlalchemy.types
|
|
21
|
+
import sqlalchemy.util
|
|
22
|
+
from google.cloud.bigquery.schema import SchemaField
|
|
23
|
+
|
|
24
|
+
_type_map = {
|
|
25
|
+
"ARRAY": sqlalchemy.types.ARRAY,
|
|
26
|
+
"BIGNUMERIC": sqlalchemy.types.Numeric,
|
|
27
|
+
"BOOLEAN": sqlalchemy.types.Boolean,
|
|
28
|
+
"BOOL": sqlalchemy.types.Boolean,
|
|
29
|
+
"BYTES": sqlalchemy.types.BINARY,
|
|
30
|
+
"DATETIME": sqlalchemy.types.DATETIME,
|
|
31
|
+
"DATE": sqlalchemy.types.DATE,
|
|
32
|
+
"FLOAT64": sqlalchemy.types.Float,
|
|
33
|
+
"FLOAT": sqlalchemy.types.Float,
|
|
34
|
+
"INT64": sqlalchemy.types.Integer,
|
|
35
|
+
"INTEGER": sqlalchemy.types.Integer,
|
|
36
|
+
"NUMERIC": sqlalchemy.types.Numeric,
|
|
37
|
+
"RECORD": sqlalchemy.types.JSON,
|
|
38
|
+
"STRING": sqlalchemy.types.String,
|
|
39
|
+
"STRUCT": sqlalchemy.types.JSON,
|
|
40
|
+
"TIMESTAMP": sqlalchemy.types.TIMESTAMP,
|
|
41
|
+
"TIME": sqlalchemy.types.TIME,
|
|
42
|
+
"GEOGRAPHY": sqlalchemy.types.JSON,
|
|
43
|
+
"NULL": sqlalchemy.types.NullType,
|
|
44
|
+
"KEY": sqlalchemy.types.JSON
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
# By convention, dialect-provided types are spelled with all upper case.
|
|
48
|
+
ARRAY = _type_map["ARRAY"]
|
|
49
|
+
BIGNUMERIC = _type_map["NUMERIC"]
|
|
50
|
+
BOOLEAN = _type_map["BOOLEAN"]
|
|
51
|
+
BOOL = _type_map["BOOL"]
|
|
52
|
+
BYTES = _type_map["BYTES"]
|
|
53
|
+
DATETIME = _type_map["DATETIME"]
|
|
54
|
+
DATE = _type_map["DATE"]
|
|
55
|
+
FLOAT64 = _type_map["FLOAT64"]
|
|
56
|
+
FLOAT = _type_map["FLOAT"]
|
|
57
|
+
INT64 = _type_map["INT64"]
|
|
58
|
+
INTEGER = _type_map["INTEGER"]
|
|
59
|
+
NUMERIC = _type_map["NUMERIC"]
|
|
60
|
+
RECORD = _type_map["RECORD"]
|
|
61
|
+
STRING = _type_map["STRING"]
|
|
62
|
+
TIMESTAMP = _type_map["TIMESTAMP"]
|
|
63
|
+
TIME = _type_map["TIME"]
|
|
64
|
+
GEOPOINT = _type_map["GEOGRAPHY"]
|
|
65
|
+
STRUCT_FIELD_TYPES = _type_map["STRUCT"]
|
|
66
|
+
RECORD_FIELD_TYPES = _type_map["RECORD"]
|
|
67
|
+
NULL_TYPE = _type_map["NULL"]
|
|
68
|
+
KEY_TYPE = _type_map["KEY"]
|
|
69
|
+
|
|
70
|
+
_property_type = {
|
|
71
|
+
"Date/Time": DATETIME(),
|
|
72
|
+
"GeoPt": GEOPOINT(),
|
|
73
|
+
"Integer": INTEGER(),
|
|
74
|
+
"Key": KEY_TYPE(),
|
|
75
|
+
"Blob": BYTES(),
|
|
76
|
+
"Boolean": BOOLEAN(),
|
|
77
|
+
"EmbeddedEntity": STRUCT_FIELD_TYPES(),
|
|
78
|
+
"Float": FLOAT(),
|
|
79
|
+
"NULL": NULL_TYPE(),
|
|
80
|
+
"String": STRING(),
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _get_transitive_schema_fields(fields):
|
|
85
|
+
"""
|
|
86
|
+
Recurse into record type and return all the nested field names.
|
|
87
|
+
As contributed by @sumedhsakdeo on issue #17
|
|
88
|
+
"""
|
|
89
|
+
results = []
|
|
90
|
+
for field in fields:
|
|
91
|
+
results += [field]
|
|
92
|
+
if field.field_type in STRUCT_FIELD_TYPES and field.mode != "REPEATED":
|
|
93
|
+
sub_fields = [
|
|
94
|
+
SchemaField.from_api_repr(
|
|
95
|
+
dict(f.to_api_repr(), name=f"{field.name}.{f.name}")
|
|
96
|
+
)
|
|
97
|
+
for f in field.fields
|
|
98
|
+
]
|
|
99
|
+
results += _get_transitive_schema_fields(sub_fields)
|
|
100
|
+
return results
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _get_sqla_column_type(field):
|
|
104
|
+
try:
|
|
105
|
+
coltype = _type_map[field.field_type]
|
|
106
|
+
except KeyError:
|
|
107
|
+
sqlalchemy.util.warn(
|
|
108
|
+
"Did not recognize type '%s' of column '%s'"
|
|
109
|
+
% (field.field_type, field.name)
|
|
110
|
+
)
|
|
111
|
+
coltype = sqlalchemy.types.NullType
|
|
112
|
+
else:
|
|
113
|
+
if field.field_type.endswith("NUMERIC"):
|
|
114
|
+
coltype = coltype(precision=field.precision, scale=field.scale)
|
|
115
|
+
elif field.field_type == "STRING" or field.field_type == "BYTES":
|
|
116
|
+
coltype = coltype(field.max_length)
|
|
117
|
+
elif field.field_type == "RECORD" or field.field_type == "STRUCT":
|
|
118
|
+
coltype = dict(
|
|
119
|
+
*(
|
|
120
|
+
(subfield.name, _get_sqla_column_type(subfield))
|
|
121
|
+
for subfield in field.fields
|
|
122
|
+
)
|
|
123
|
+
)
|
|
124
|
+
else:
|
|
125
|
+
coltype = coltype()
|
|
126
|
+
|
|
127
|
+
if field.mode == "REPEATED":
|
|
128
|
+
coltype = ARRAY(coltype)
|
|
129
|
+
|
|
130
|
+
return coltype
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def get_columns(bq_schema):
|
|
134
|
+
fields = _get_transitive_schema_fields(bq_schema)
|
|
135
|
+
return [
|
|
136
|
+
{
|
|
137
|
+
"name": field.name,
|
|
138
|
+
"type": _get_sqla_column_type(field),
|
|
139
|
+
"nullable": field.mode == "NULLABLE" or field.mode == "REPEATED",
|
|
140
|
+
"comment": field.description,
|
|
141
|
+
"default": None,
|
|
142
|
+
"precision": field.precision,
|
|
143
|
+
"scale": field.scale,
|
|
144
|
+
"max_length": field.max_length,
|
|
145
|
+
}
|
|
146
|
+
for field in fields
|
|
147
|
+
]
|