python-datastore-sqlalchemy 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. python_datastore_sqlalchemy-0.0.1/LICENSE +9 -0
  2. python_datastore_sqlalchemy-0.0.1/PKG-INFO +65 -0
  3. python_datastore_sqlalchemy-0.0.1/README.md +28 -0
  4. python_datastore_sqlalchemy-0.0.1/pyproject.toml +21 -0
  5. python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/PKG-INFO +65 -0
  6. python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/SOURCES.txt +25 -0
  7. python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/dependency_links.txt +1 -0
  8. python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/entry_points.txt +2 -0
  9. python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/not-zip-safe +1 -0
  10. python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/requires.txt +1 -0
  11. python_datastore_sqlalchemy-0.0.1/python_datastore_sqlalchemy.egg-info/top_level.txt +1 -0
  12. python_datastore_sqlalchemy-0.0.1/setup.cfg +4 -0
  13. python_datastore_sqlalchemy-0.0.1/setup.py +70 -0
  14. python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/__init__.py +27 -0
  15. python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/_helpers.py +135 -0
  16. python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/_types.py +147 -0
  17. python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/base.py +291 -0
  18. python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/datastore_dbapi.py +2322 -0
  19. python_datastore_sqlalchemy-0.0.1/sqlalchemy_datastore/parse_url.py +287 -0
  20. python_datastore_sqlalchemy-0.0.1/tests/test_datastore.py +272 -0
  21. python_datastore_sqlalchemy-0.0.1/tests/test_gql.py +1202 -0
  22. python_datastore_sqlalchemy-0.0.1/tests/test_integration_queries.py +402 -0
  23. python_datastore_sqlalchemy-0.0.1/tests/test_orm.py +79 -0
  24. python_datastore_sqlalchemy-0.0.1/tests/test_unit_base.py +226 -0
  25. python_datastore_sqlalchemy-0.0.1/tests/test_unit_dbapi.py +1416 -0
  26. python_datastore_sqlalchemy-0.0.1/tests/test_unit_helpers.py +124 -0
  27. python_datastore_sqlalchemy-0.0.1/tests/test_unit_types.py +203 -0
@@ -0,0 +1,9 @@
1
+ MIT License
2
+
3
+ Copyright 2025 The sqlalchemy-datastore Authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
6
+
7
+ The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
8
+
9
+ THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,65 @@
1
+ Metadata-Version: 2.4
2
+ Name: python-datastore-sqlalchemy
3
+ Version: 0.0.1
4
+ Summary: SQLAlchemy dialect for google cloud datastore
5
+ Home-page: https://github.com/splasky/python-datastore-sqlalchemy
6
+ Author: HY Chang(splasky)
7
+ Author-email: hychang.1997.tw@gmail.com
8
+ License: MIT
9
+ Project-URL: Documentation, https://github.com/splasky/python-datastore-sqlalchemy/wiki
10
+ Project-URL: Source, https://github.com/splasky/python-datastore-sqlalchemy
11
+ Project-URL: Tracker, https://github.com/splasky/python-datastore-sqlalchemy/issues
12
+ Keywords: SQLAlchemy GCP Datastore
13
+ Classifier: Development Status :: 2 - Pre-Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Programming Language :: Python
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: Implementation :: CPython
18
+ Classifier: Topic :: Database :: Front-Ends
19
+ Classifier: Operating System :: POSIX :: Linux
20
+ Requires-Python: >=3.9
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: sqlalchemy<3.0.0,>=1.4.16
24
+ Dynamic: author
25
+ Dynamic: author-email
26
+ Dynamic: classifier
27
+ Dynamic: description
28
+ Dynamic: description-content-type
29
+ Dynamic: home-page
30
+ Dynamic: keywords
31
+ Dynamic: license
32
+ Dynamic: license-file
33
+ Dynamic: project-url
34
+ Dynamic: requires-dist
35
+ Dynamic: requires-python
36
+ Dynamic: summary
37
+
38
+ SQLAlchemy dialect for google cloud datastore(firestore mode)
39
+ ========================
40
+ How to install
41
+ ```
42
+ python3 setup.py install
43
+ ```
44
+ How to use
45
+ ```python
46
+ from sqlalchemy import *
47
+ from sqlalchemy.engine import create_engine
48
+ from sqlalchemy.schema import *
49
+ engine = create_engine('datastore://test-api-1', credentials='path/to/credentials.json')
50
+ conn = engine.connect()
51
+ result = conn.execute("SELECT * from test_table)
52
+ print(result.fetchall())
53
+ ```
54
+
55
+ ## Preview
56
+ <img src="assets/pie.png">
57
+
58
+ > [!WARNING]
59
+ > Please note: Not all GQL and SQL syntax has been fully tested. Should you encounter any bugs, please post the relevant query and open an issue on GitHub.
60
+
61
+ ## How to contribute
62
+ Feel free to open issues and pull requests on GitHub.
63
+
64
+ ## Development Notes
65
+ - [Develop a SQLAlchemy and it's dialects](https://hackmd.io/lsBW5GCVR82SORyWZ1cssA?view)
@@ -0,0 +1,28 @@
1
+ SQLAlchemy dialect for google cloud datastore(firestore mode)
2
+ ========================
3
+ How to install
4
+ ```
5
+ python3 setup.py install
6
+ ```
7
+ How to use
8
+ ```python
9
+ from sqlalchemy import *
10
+ from sqlalchemy.engine import create_engine
11
+ from sqlalchemy.schema import *
12
+ engine = create_engine('datastore://test-api-1', credentials='path/to/credentials.json')
13
+ conn = engine.connect()
14
+ result = conn.execute("SELECT * from test_table)
15
+ print(result.fetchall())
16
+ ```
17
+
18
+ ## Preview
19
+ <img src="assets/pie.png">
20
+
21
+ > [!WARNING]
22
+ > Please note: Not all GQL and SQL syntax has been fully tested. Should you encounter any bugs, please post the relevant query and open an issue on GitHub.
23
+
24
+ ## How to contribute
25
+ Feel free to open issues and pull requests on GitHub.
26
+
27
+ ## Development Notes
28
+ - [Develop a SQLAlchemy and it's dialects](https://hackmd.io/lsBW5GCVR82SORyWZ1cssA?view)
@@ -0,0 +1,21 @@
1
+ [tool.ruff]
2
+ line-length = 120
3
+
4
+ [tool.ruff.lint]
5
+ select = ["E", "F", "W", "I", "N", "B", "C", "G", "S"]
6
+ ignore = [
7
+ "E501", # Line too long - allow longer lines for SQL strings in tests
8
+ "S608", # SQL injection - intentional for test cases
9
+ "B007", # Loop variable not used - common in test assertions
10
+ ]
11
+
12
+ [tool.ruff.lint.per-file-ignores]
13
+ "tests/*" = ["S608", "E501", "S101", "B007", "S603"]
14
+
15
+ [tool.mypy]
16
+ ignore_missing_imports = true
17
+ disable_error_code = ["var-annotated", "import-untyped"]
18
+
19
+ [[tool.mypy.overrides]]
20
+ module = ["tests.*", "sqlalchemy_datastore.*"]
21
+ ignore_errors = true
@@ -0,0 +1,65 @@
1
+ Metadata-Version: 2.4
2
+ Name: python-datastore-sqlalchemy
3
+ Version: 0.0.1
4
+ Summary: SQLAlchemy dialect for google cloud datastore
5
+ Home-page: https://github.com/splasky/python-datastore-sqlalchemy
6
+ Author: HY Chang(splasky)
7
+ Author-email: hychang.1997.tw@gmail.com
8
+ License: MIT
9
+ Project-URL: Documentation, https://github.com/splasky/python-datastore-sqlalchemy/wiki
10
+ Project-URL: Source, https://github.com/splasky/python-datastore-sqlalchemy
11
+ Project-URL: Tracker, https://github.com/splasky/python-datastore-sqlalchemy/issues
12
+ Keywords: SQLAlchemy GCP Datastore
13
+ Classifier: Development Status :: 2 - Pre-Alpha
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Programming Language :: Python
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: Implementation :: CPython
18
+ Classifier: Topic :: Database :: Front-Ends
19
+ Classifier: Operating System :: POSIX :: Linux
20
+ Requires-Python: >=3.9
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: sqlalchemy<3.0.0,>=1.4.16
24
+ Dynamic: author
25
+ Dynamic: author-email
26
+ Dynamic: classifier
27
+ Dynamic: description
28
+ Dynamic: description-content-type
29
+ Dynamic: home-page
30
+ Dynamic: keywords
31
+ Dynamic: license
32
+ Dynamic: license-file
33
+ Dynamic: project-url
34
+ Dynamic: requires-dist
35
+ Dynamic: requires-python
36
+ Dynamic: summary
37
+
38
+ SQLAlchemy dialect for google cloud datastore(firestore mode)
39
+ ========================
40
+ How to install
41
+ ```
42
+ python3 setup.py install
43
+ ```
44
+ How to use
45
+ ```python
46
+ from sqlalchemy import *
47
+ from sqlalchemy.engine import create_engine
48
+ from sqlalchemy.schema import *
49
+ engine = create_engine('datastore://test-api-1', credentials='path/to/credentials.json')
50
+ conn = engine.connect()
51
+ result = conn.execute("SELECT * from test_table)
52
+ print(result.fetchall())
53
+ ```
54
+
55
+ ## Preview
56
+ <img src="assets/pie.png">
57
+
58
+ > [!WARNING]
59
+ > Please note: Not all GQL and SQL syntax has been fully tested. Should you encounter any bugs, please post the relevant query and open an issue on GitHub.
60
+
61
+ ## How to contribute
62
+ Feel free to open issues and pull requests on GitHub.
63
+
64
+ ## Development Notes
65
+ - [Develop a SQLAlchemy and it's dialects](https://hackmd.io/lsBW5GCVR82SORyWZ1cssA?view)
@@ -0,0 +1,25 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ setup.py
5
+ python_datastore_sqlalchemy.egg-info/PKG-INFO
6
+ python_datastore_sqlalchemy.egg-info/SOURCES.txt
7
+ python_datastore_sqlalchemy.egg-info/dependency_links.txt
8
+ python_datastore_sqlalchemy.egg-info/entry_points.txt
9
+ python_datastore_sqlalchemy.egg-info/not-zip-safe
10
+ python_datastore_sqlalchemy.egg-info/requires.txt
11
+ python_datastore_sqlalchemy.egg-info/top_level.txt
12
+ sqlalchemy_datastore/__init__.py
13
+ sqlalchemy_datastore/_helpers.py
14
+ sqlalchemy_datastore/_types.py
15
+ sqlalchemy_datastore/base.py
16
+ sqlalchemy_datastore/datastore_dbapi.py
17
+ sqlalchemy_datastore/parse_url.py
18
+ tests/test_datastore.py
19
+ tests/test_gql.py
20
+ tests/test_integration_queries.py
21
+ tests/test_orm.py
22
+ tests/test_unit_base.py
23
+ tests/test_unit_dbapi.py
24
+ tests/test_unit_helpers.py
25
+ tests/test_unit_types.py
@@ -0,0 +1,2 @@
1
+ [sqlalchemy.dialects]
2
+ datastore = sqlalchemy_datastore:CloudDatastoreDialect
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,70 @@
1
+ # Copyright (c) 2025 hychang <hychang.1997.tw@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
4
+ # this software and associated documentation files (the "Software"), to deal in
5
+ # the Software without restriction, including without limitation the rights to
6
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
7
+ # the Software, and to permit persons to whom the Software is furnished to do so,
8
+ # subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in all
11
+ # copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
15
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
16
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
17
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
18
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19
+
20
+ import os
21
+ import re
22
+
23
+ from setuptools import setup, find_packages
24
+
25
+ v = open(
26
+ os.path.join(os.path.dirname(__file__), "sqlalchemy_datastore", "__init__.py")
27
+ )
28
+ version_match = re.compile(r'.*__version__ = "(.*?)"', re.S).match(v.read())
29
+ if not version_match:
30
+ raise RuntimeError("Unable to find version string in __init__.py.")
31
+ VERSION = version_match.group(1)
32
+ v.close()
33
+
34
+ readme = os.path.join(os.path.dirname(__file__), "README.md")
35
+
36
+
37
+ setup(
38
+ name="python-datastore-sqlalchemy",
39
+ version=VERSION,
40
+ description="SQLAlchemy dialect for google cloud datastore",
41
+ long_description=open(readme).read(),
42
+ long_description_content_type="text/markdown",
43
+ url="https://github.com/splasky/python-datastore-sqlalchemy",
44
+ author="HY Chang(splasky)",
45
+ author_email="hychang.1997.tw@gmail.com",
46
+ license="MIT",
47
+ classifiers=[
48
+ "Development Status :: 2 - Pre-Alpha",
49
+ "Intended Audience :: Developers",
50
+ "Programming Language :: Python",
51
+ "Programming Language :: Python :: 3",
52
+ "Programming Language :: Python :: Implementation :: CPython",
53
+ "Topic :: Database :: Front-Ends",
54
+ "Operating System :: POSIX :: Linux",
55
+ ],
56
+ keywords="SQLAlchemy GCP Datastore",
57
+ python_requires=">=3.9",
58
+ project_urls={
59
+ "Documentation": "https://github.com/splasky/python-datastore-sqlalchemy/wiki",
60
+ "Source": "https://github.com/splasky/python-datastore-sqlalchemy",
61
+ "Tracker": "https://github.com/splasky/python-datastore-sqlalchemy/issues",
62
+ },
63
+ packages=find_packages(include=["sqlalchemy_datastore"]),
64
+ include_package_data=True,
65
+ install_requires=["sqlalchemy>=1.4.16,<3.0.0",],
66
+ zip_safe=False,
67
+ entry_points={
68
+ "sqlalchemy.dialects": ["datastore = sqlalchemy_datastore:CloudDatastoreDialect"]
69
+ },
70
+ )
@@ -0,0 +1,27 @@
1
+ # Copyright (c) 2025 hychang <hychang.1997.tw@gmail.com>
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
4
+ # this software and associated documentation files (the "Software"), to deal in
5
+ # the Software without restriction, including without limitation the rights to
6
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
7
+ # the Software, and to permit persons to whom the Software is furnished to do so,
8
+ # subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in all
11
+ # copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
15
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
16
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
17
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
18
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19
+
20
+ from .base import CloudDatastoreDialect
21
+
22
+ __version__ = "0.0.1"
23
+
24
+ __all__ = [
25
+ "__version__",
26
+ "CloudDatastoreDialect",
27
+ ]
@@ -0,0 +1,135 @@
1
+ # Copyright (c) 2017 The sqlalchemy-bigquery Authors
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
4
+ # this software and associated documentation files (the "Software"), to deal in
5
+ # the Software without restriction, including without limitation the rights to
6
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
7
+ # the Software, and to permit persons to whom the Software is furnished to do so,
8
+ # subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in all
11
+ # copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
15
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
16
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
17
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
18
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19
+
20
+ import base64
21
+ import functools
22
+ import json
23
+ import os
24
+ import re
25
+ from typing import Optional, Tuple
26
+
27
+ import google.auth
28
+ import sqlalchemy
29
+ from google.api_core import client_info
30
+ from google.auth.credentials import Credentials
31
+ from google.cloud import datastore
32
+ from google.oauth2 import service_account
33
+
34
+ USER_AGENT_TEMPLATE = "sqlalchemy/{}"
35
+ SCOPES = (
36
+ "https://www.googleapis.com/auth/datastore",
37
+ "https://www.googleapis.com/auth/cloud-platform",
38
+ "https://www.googleapis.com/auth/drive",
39
+ )
40
+
41
+
42
+ def google_client_info(
43
+ user_agent: Optional[str] = None,
44
+ ) -> client_info.ClientInfo:
45
+ """
46
+ Return a client_info object, with an optional user agent
47
+ string. If user_agent is None, use a default value.
48
+ """
49
+
50
+ if user_agent is None:
51
+ user_agent = USER_AGENT_TEMPLATE.format(sqlalchemy.__version__)
52
+ return client_info.ClientInfo(user_agent=user_agent)
53
+
54
+
55
+ def create_datastore_client(
56
+ credentials_info: Optional[dict] = None,
57
+ credentials_path: Optional[str] = None,
58
+ credentials_base64: Optional[str] = None,
59
+ project_id: Optional[str] = None,
60
+ user_agent: Optional[client_info.ClientInfo] = None,
61
+ database: Optional[str] = None
62
+ ) -> Tuple[datastore.Client, Optional[Credentials]]:
63
+ """Construct a BigQuery client object.
64
+
65
+ Args:
66
+ credentials_info Optional[dict]:
67
+ credentials_path Optional[str]:
68
+ credentials_base64 Optional[str]:
69
+ location (Optional[str]):
70
+ Default location for jobs / datasets / tables.
71
+ project_id (Optional[str]):
72
+ Project ID for the project which the client acts on behalf of.
73
+ user_agent (Optional[google.api_core.client_info.ClientInfo]):
74
+ The client info used to send a user-agent string along with API
75
+ requests. If ``None``, then default info will be used. Generally,
76
+ you only need to set this if you're developing your own library
77
+ or partner tool.
78
+ """
79
+
80
+ default_project = None
81
+ database = database if database != "(default)" else None
82
+ if os.getenv("DATASTORE_EMULATOR_HOST") is not None:
83
+ client = datastore.Client(project=project_id)
84
+ return client, None
85
+
86
+ if credentials_base64:
87
+ credentials_info = json.loads(base64.b64decode(credentials_base64))
88
+
89
+ if credentials_path:
90
+ credentials = service_account.Credentials.from_service_account_file(
91
+ credentials_path
92
+ )
93
+ credentials = credentials.with_scopes(SCOPES)
94
+ default_project = credentials.project_id
95
+ elif credentials_info:
96
+ credentials = service_account.Credentials.from_service_account_info(
97
+ credentials_info
98
+ )
99
+ credentials = credentials.with_scopes(SCOPES)
100
+ default_project = credentials.project_id
101
+ else:
102
+ credentials, default_project = google.auth.default(scopes=SCOPES)
103
+
104
+ if project_id is None:
105
+ project_id = default_project
106
+
107
+ info = google_client_info(user_agent=user_agent.to_user_agent() if user_agent is not None else None)
108
+
109
+ return datastore.Client(
110
+ client_info=info,
111
+ project=project_id,
112
+ credentials=credentials,
113
+ database=database
114
+ ), credentials
115
+
116
+
117
+ def substitute_re_method(r, flags=0, repl=None):
118
+ if repl is None:
119
+ return lambda f: substitute_re_method(r, flags, f)
120
+
121
+ r = re.compile(r, flags)
122
+
123
+ @functools.wraps(repl)
124
+ def sub(self, s, *args, **kw):
125
+ def repl_(m):
126
+ return repl(self, m, *args, **kw)
127
+
128
+ return r.sub(repl_, s)
129
+
130
+ return sub
131
+
132
+
133
+ def substitute_string_re_method(r, *, repl, flags=0):
134
+ r = re.compile(r, flags)
135
+ return lambda self, s: r.sub(repl, s)
@@ -0,0 +1,147 @@
1
+ # Copyright (c) 2017 The sqlalchemy-bigquery Authors
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a copy of
4
+ # this software and associated documentation files (the "Software"), to deal in
5
+ # the Software without restriction, including without limitation the rights to
6
+ # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
7
+ # the Software, and to permit persons to whom the Software is furnished to do so,
8
+ # subject to the following conditions:
9
+ #
10
+ # The above copyright notice and this permission notice shall be included in all
11
+ # copies or substantial portions of the Software.
12
+ #
13
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
15
+ # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
16
+ # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
17
+ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
18
+ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19
+
20
+ import sqlalchemy.types
21
+ import sqlalchemy.util
22
+ from google.cloud.bigquery.schema import SchemaField
23
+
24
+ _type_map = {
25
+ "ARRAY": sqlalchemy.types.ARRAY,
26
+ "BIGNUMERIC": sqlalchemy.types.Numeric,
27
+ "BOOLEAN": sqlalchemy.types.Boolean,
28
+ "BOOL": sqlalchemy.types.Boolean,
29
+ "BYTES": sqlalchemy.types.BINARY,
30
+ "DATETIME": sqlalchemy.types.DATETIME,
31
+ "DATE": sqlalchemy.types.DATE,
32
+ "FLOAT64": sqlalchemy.types.Float,
33
+ "FLOAT": sqlalchemy.types.Float,
34
+ "INT64": sqlalchemy.types.Integer,
35
+ "INTEGER": sqlalchemy.types.Integer,
36
+ "NUMERIC": sqlalchemy.types.Numeric,
37
+ "RECORD": sqlalchemy.types.JSON,
38
+ "STRING": sqlalchemy.types.String,
39
+ "STRUCT": sqlalchemy.types.JSON,
40
+ "TIMESTAMP": sqlalchemy.types.TIMESTAMP,
41
+ "TIME": sqlalchemy.types.TIME,
42
+ "GEOGRAPHY": sqlalchemy.types.JSON,
43
+ "NULL": sqlalchemy.types.NullType,
44
+ "KEY": sqlalchemy.types.JSON
45
+ }
46
+
47
+ # By convention, dialect-provided types are spelled with all upper case.
48
+ ARRAY = _type_map["ARRAY"]
49
+ BIGNUMERIC = _type_map["NUMERIC"]
50
+ BOOLEAN = _type_map["BOOLEAN"]
51
+ BOOL = _type_map["BOOL"]
52
+ BYTES = _type_map["BYTES"]
53
+ DATETIME = _type_map["DATETIME"]
54
+ DATE = _type_map["DATE"]
55
+ FLOAT64 = _type_map["FLOAT64"]
56
+ FLOAT = _type_map["FLOAT"]
57
+ INT64 = _type_map["INT64"]
58
+ INTEGER = _type_map["INTEGER"]
59
+ NUMERIC = _type_map["NUMERIC"]
60
+ RECORD = _type_map["RECORD"]
61
+ STRING = _type_map["STRING"]
62
+ TIMESTAMP = _type_map["TIMESTAMP"]
63
+ TIME = _type_map["TIME"]
64
+ GEOPOINT = _type_map["GEOGRAPHY"]
65
+ STRUCT_FIELD_TYPES = _type_map["STRUCT"]
66
+ RECORD_FIELD_TYPES = _type_map["RECORD"]
67
+ NULL_TYPE = _type_map["NULL"]
68
+ KEY_TYPE = _type_map["KEY"]
69
+
70
+ _property_type = {
71
+ "Date/Time": DATETIME(),
72
+ "GeoPt": GEOPOINT(),
73
+ "Integer": INTEGER(),
74
+ "Key": KEY_TYPE(),
75
+ "Blob": BYTES(),
76
+ "Boolean": BOOLEAN(),
77
+ "EmbeddedEntity": STRUCT_FIELD_TYPES(),
78
+ "Float": FLOAT(),
79
+ "NULL": NULL_TYPE(),
80
+ "String": STRING(),
81
+ }
82
+
83
+
84
+ def _get_transitive_schema_fields(fields):
85
+ """
86
+ Recurse into record type and return all the nested field names.
87
+ As contributed by @sumedhsakdeo on issue #17
88
+ """
89
+ results = []
90
+ for field in fields:
91
+ results += [field]
92
+ if field.field_type in STRUCT_FIELD_TYPES and field.mode != "REPEATED":
93
+ sub_fields = [
94
+ SchemaField.from_api_repr(
95
+ dict(f.to_api_repr(), name=f"{field.name}.{f.name}")
96
+ )
97
+ for f in field.fields
98
+ ]
99
+ results += _get_transitive_schema_fields(sub_fields)
100
+ return results
101
+
102
+
103
+ def _get_sqla_column_type(field):
104
+ try:
105
+ coltype = _type_map[field.field_type]
106
+ except KeyError:
107
+ sqlalchemy.util.warn(
108
+ "Did not recognize type '%s' of column '%s'"
109
+ % (field.field_type, field.name)
110
+ )
111
+ coltype = sqlalchemy.types.NullType
112
+ else:
113
+ if field.field_type.endswith("NUMERIC"):
114
+ coltype = coltype(precision=field.precision, scale=field.scale)
115
+ elif field.field_type == "STRING" or field.field_type == "BYTES":
116
+ coltype = coltype(field.max_length)
117
+ elif field.field_type == "RECORD" or field.field_type == "STRUCT":
118
+ coltype = dict(
119
+ *(
120
+ (subfield.name, _get_sqla_column_type(subfield))
121
+ for subfield in field.fields
122
+ )
123
+ )
124
+ else:
125
+ coltype = coltype()
126
+
127
+ if field.mode == "REPEATED":
128
+ coltype = ARRAY(coltype)
129
+
130
+ return coltype
131
+
132
+
133
+ def get_columns(bq_schema):
134
+ fields = _get_transitive_schema_fields(bq_schema)
135
+ return [
136
+ {
137
+ "name": field.name,
138
+ "type": _get_sqla_column_type(field),
139
+ "nullable": field.mode == "NULLABLE" or field.mode == "REPEATED",
140
+ "comment": field.description,
141
+ "default": None,
142
+ "precision": field.precision,
143
+ "scale": field.scale,
144
+ "max_length": field.max_length,
145
+ }
146
+ for field in fields
147
+ ]