databricks-sqlalchemy 1.0.0__tar.gz → 1.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. databricks_sqlalchemy-1.0.1/CHANGELOG.md +5 -0
  2. {databricks_sqlalchemy-1.0.0 → databricks_sqlalchemy-1.0.1}/PKG-INFO +39 -60
  3. {databricks_sqlalchemy-1.0.0 → databricks_sqlalchemy-1.0.1}/README.md +37 -58
  4. {databricks_sqlalchemy-1.0.0 → databricks_sqlalchemy-1.0.1}/pyproject.toml +3 -10
  5. databricks_sqlalchemy-1.0.1/src/databricks/sqlalchemy/__init__.py +1 -0
  6. databricks_sqlalchemy-1.0.1/src/databricks/sqlalchemy/dialect/__init__.py +340 -0
  7. databricks_sqlalchemy-1.0.1/src/databricks/sqlalchemy/dialect/base.py +17 -0
  8. databricks_sqlalchemy-1.0.1/src/databricks/sqlalchemy/dialect/compiler.py +38 -0
  9. databricks_sqlalchemy-1.0.0/CHANGELOG.md +0 -274
  10. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/__init__.py +0 -4
  11. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/_ddl.py +0 -100
  12. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/_parse.py +0 -385
  13. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/_types.py +0 -323
  14. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/base.py +0 -436
  15. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/dependency_test/test_dependency.py +0 -22
  16. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/py.typed +0 -0
  17. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/pytest.ini +0 -4
  18. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/requirements.py +0 -249
  19. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/setup.cfg +0 -4
  20. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/_extra.py +0 -70
  21. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/_future.py +0 -331
  22. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/_regression.py +0 -311
  23. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/_unsupported.py +0 -450
  24. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/conftest.py +0 -13
  25. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/overrides/_componentreflectiontest.py +0 -189
  26. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/overrides/_ctetest.py +0 -33
  27. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test/test_suite.py +0 -13
  28. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test_local/__init__.py +0 -5
  29. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test_local/conftest.py +0 -44
  30. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test_local/e2e/MOCK_DATA.xlsx +0 -0
  31. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test_local/e2e/test_basic.py +0 -543
  32. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test_local/test_ddl.py +0 -96
  33. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test_local/test_parsing.py +0 -160
  34. databricks_sqlalchemy-1.0.0/src/databricks/sqlalchemy/test_local/test_types.py +0 -161
  35. {databricks_sqlalchemy-1.0.0 → databricks_sqlalchemy-1.0.1}/LICENSE +0 -0
@@ -0,0 +1,5 @@
1
+ # Release History
2
+
3
+ # 1.0.1
4
+
5
+ - This is databricks-sqlalchemy plugin based on sqlalchemy v1 and has all the databricks-sql-python v2.9.6 needed sqlalchemy features
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: databricks-sqlalchemy
3
- Version: 1.0.0
3
+ Version: 1.0.1
4
4
  Summary: Databricks SQLAlchemy plugin for Python
5
5
  License: Apache-2.0
6
6
  Author: Databricks
@@ -14,17 +14,14 @@ Classifier: Programming Language :: Python :: 3.10
14
14
  Classifier: Programming Language :: Python :: 3.11
15
15
  Classifier: Programming Language :: Python :: 3.12
16
16
  Requires-Dist: databricks_sql_connector_core (>=4.0.0)
17
- Requires-Dist: sqlalchemy (>=2.0.21)
17
+ Requires-Dist: sqlalchemy (>=1.3.24,<2.0.0)
18
18
  Project-URL: Bug Tracker, https://github.com/databricks/databricks-sqlalchemy/issues
19
19
  Project-URL: Homepage, https://github.com/databricks/databricks-sqlalchemy
20
20
  Description-Content-Type: text/markdown
21
21
 
22
- ## Databricks dialect for SQLALchemy 2.0
22
+ ## Databricks dialect for SQLALchemy 1.0
23
23
 
24
- The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. A working example demonstrating usage can be found in `examples/sqlalchemy.py`.
25
-
26
- ## Usage with SQLAlchemy <= 2.0
27
- A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`.
24
+ The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. A working example demonstrating usage can be found in `example.py`.
28
25
 
29
26
 
30
27
  ## Installation
@@ -32,7 +29,7 @@ A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4
32
29
  To install the dialect and its dependencies:
33
30
 
34
31
  ```shell
35
- pip install databricks-sqlalchemy
32
+ pip install databricks-sqlalchemy~=1.0
36
33
  ```
37
34
 
38
35
  If you also plan to use `alembic` you can alternatively run:
@@ -65,41 +62,45 @@ access_token = os.getenv("DATABRICKS_TOKEN")
65
62
  catalog = os.getenv("DATABRICKS_CATALOG")
66
63
  schema = os.getenv("DATABRICKS_SCHEMA")
67
64
 
68
- engine = create_engine(
69
- f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}"
70
- )
65
+ if sqlalchemy.__version__.startswith("1.3"):
66
+ # SQLAlchemy 1.3.x fails to parse the http_path, catalog, and schema from our connection string
67
+ # Pass these in as connect_args instead
68
+
69
+ conn_string = f"databricks://token:{access_token}@{host}"
70
+ connect_args = dict(catalog=catalog, schema=schema, http_path=http_path)
71
+ all_connect_args = {**extra_connect_args, **connect_args}
72
+ engine = create_engine(conn_string, connect_args=all_connect_args)
73
+ else:
74
+ engine = create_engine(
75
+ f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}",
76
+ connect_args=extra_connect_args,
77
+ )
78
+
71
79
  ```
72
80
 
73
81
  ## Types
74
82
 
75
- The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks.
83
+ The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/13/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/13/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks.
76
84
 
77
85
  |SQLAlchemy Type|Databricks SQL Type|
78
86
  |-|-|
79
- [`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html)
80
- [`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)|
81
- [`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html)
82
- [`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html)
83
- [`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)|
84
- [`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html)
85
- [`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)|
86
- [`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html)
87
- [`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html)
88
- [`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)|
89
- [`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)|
90
- [`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html)
91
- [`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
92
- [`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
93
- [`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
94
- [`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
95
- [`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
96
- [`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)
97
-
98
- In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks:
99
-
100
- - [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html)
101
- - [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html)
102
- - [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)
87
+ [`BigInteger`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html)
88
+ [`LargeBinary`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)|
89
+ [`Boolean`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html)
90
+ [`Date`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html)
91
+ [`DateTime`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)|
92
+ [`Enum`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)|
93
+ [`Float`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html)
94
+ [`Integer`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html)
95
+ [`Numeric`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)|
96
+ [`PickleType`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)|
97
+ [`SmallInteger`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html)
98
+ [`String`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
99
+ [`Text`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
100
+ [`Time`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
101
+ [`Unicode`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
102
+ [`UnicodeText`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
103
+ [`Uuid`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)
103
104
 
104
105
 
105
106
  ### `LargeBinary()` and `PickleType()`
@@ -112,24 +113,6 @@ Support for `CHECK` constraints is not implemented in this dialect. Support is p
112
113
 
113
114
  SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported.
114
115
 
115
- ### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()`
116
-
117
- Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models.
118
-
119
- The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`.
120
-
121
- If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead.
122
-
123
- _Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._
124
-
125
- ```python
126
- from sqlalchemy import DateTime
127
- from databricks.sqlalchemy import TIMESTAMP
128
-
129
- class SomeModel(Base):
130
- some_date_without_timezone = DateTime()
131
- some_date_with_timezone = TIMESTAMP()
132
- ```
133
116
 
134
117
  ### `String()`, `Text()`, `Unicode()`, and `UnicodeText()`
135
118
 
@@ -154,7 +137,7 @@ class SomeModel(Base):
154
137
 
155
138
  Identity and generated value support is currently limited in this dialect.
156
139
 
157
- When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead.
140
+ When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/13/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/13/core/defaults.html#identity-ddl) instead.
158
141
 
159
142
  Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type.
160
143
 
@@ -168,10 +151,6 @@ class SomeModel(Base):
168
151
 
169
152
  When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned).
170
153
 
171
- ## Parameters
172
-
173
- `databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`.
174
-
175
154
  ## Usage with pandas
176
155
 
177
156
  Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks.
@@ -202,7 +181,7 @@ with engine.connect() as conn:
202
181
  df.to_sql('squares',conn)
203
182
  ```
204
183
 
205
- ## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys)
184
+ ## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/13/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/13/core/constraints.html#defining-foreign-keys)
206
185
 
207
186
  Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column.
208
187
 
@@ -1,9 +1,6 @@
1
- ## Databricks dialect for SQLALchemy 2.0
1
+ ## Databricks dialect for SQLALchemy 1.0
2
2
 
3
- The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. A working example demonstrating usage can be found in `examples/sqlalchemy.py`.
4
-
5
- ## Usage with SQLAlchemy <= 2.0
6
- A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`.
3
+ The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. A working example demonstrating usage can be found in `example.py`.
7
4
 
8
5
 
9
6
  ## Installation
@@ -11,7 +8,7 @@ A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4
11
8
  To install the dialect and its dependencies:
12
9
 
13
10
  ```shell
14
- pip install databricks-sqlalchemy
11
+ pip install databricks-sqlalchemy~=1.0
15
12
  ```
16
13
 
17
14
  If you also plan to use `alembic` you can alternatively run:
@@ -44,41 +41,45 @@ access_token = os.getenv("DATABRICKS_TOKEN")
44
41
  catalog = os.getenv("DATABRICKS_CATALOG")
45
42
  schema = os.getenv("DATABRICKS_SCHEMA")
46
43
 
47
- engine = create_engine(
48
- f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}"
49
- )
44
+ if sqlalchemy.__version__.startswith("1.3"):
45
+ # SQLAlchemy 1.3.x fails to parse the http_path, catalog, and schema from our connection string
46
+ # Pass these in as connect_args instead
47
+
48
+ conn_string = f"databricks://token:{access_token}@{host}"
49
+ connect_args = dict(catalog=catalog, schema=schema, http_path=http_path)
50
+ all_connect_args = {**extra_connect_args, **connect_args}
51
+ engine = create_engine(conn_string, connect_args=all_connect_args)
52
+ else:
53
+ engine = create_engine(
54
+ f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}",
55
+ connect_args=extra_connect_args,
56
+ )
57
+
50
58
  ```
51
59
 
52
60
  ## Types
53
61
 
54
- The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks.
62
+ The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/13/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/13/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks.
55
63
 
56
64
  |SQLAlchemy Type|Databricks SQL Type|
57
65
  |-|-|
58
- [`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html)
59
- [`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)|
60
- [`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html)
61
- [`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html)
62
- [`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)|
63
- [`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html)
64
- [`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)|
65
- [`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html)
66
- [`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html)
67
- [`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)|
68
- [`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)|
69
- [`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html)
70
- [`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
71
- [`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
72
- [`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
73
- [`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
74
- [`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
75
- [`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)
76
-
77
- In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks:
78
-
79
- - [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html)
80
- - [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html)
81
- - [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)
66
+ [`BigInteger`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html)
67
+ [`LargeBinary`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)|
68
+ [`Boolean`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html)
69
+ [`Date`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html)
70
+ [`DateTime`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)|
71
+ [`Enum`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)|
72
+ [`Float`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html)
73
+ [`Integer`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html)
74
+ [`Numeric`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)|
75
+ [`PickleType`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)|
76
+ [`SmallInteger`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html)
77
+ [`String`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
78
+ [`Text`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
79
+ [`Time`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
80
+ [`Unicode`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
81
+ [`UnicodeText`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)|
82
+ [`Uuid`](https://docs.sqlalchemy.org/en/13/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)
82
83
 
83
84
 
84
85
  ### `LargeBinary()` and `PickleType()`
@@ -91,24 +92,6 @@ Support for `CHECK` constraints is not implemented in this dialect. Support is p
91
92
 
92
93
  SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported.
93
94
 
94
- ### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()`
95
-
96
- Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models.
97
-
98
- The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`.
99
-
100
- If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead.
101
-
102
- _Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._
103
-
104
- ```python
105
- from sqlalchemy import DateTime
106
- from databricks.sqlalchemy import TIMESTAMP
107
-
108
- class SomeModel(Base):
109
- some_date_without_timezone = DateTime()
110
- some_date_with_timezone = TIMESTAMP()
111
- ```
112
95
 
113
96
  ### `String()`, `Text()`, `Unicode()`, and `UnicodeText()`
114
97
 
@@ -133,7 +116,7 @@ class SomeModel(Base):
133
116
 
134
117
  Identity and generated value support is currently limited in this dialect.
135
118
 
136
- When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead.
119
+ When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/13/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/13/core/defaults.html#identity-ddl) instead.
137
120
 
138
121
  Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type.
139
122
 
@@ -147,10 +130,6 @@ class SomeModel(Base):
147
130
 
148
131
  When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned).
149
132
 
150
- ## Parameters
151
-
152
- `databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`.
153
-
154
133
  ## Usage with pandas
155
134
 
156
135
  Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks.
@@ -181,7 +160,7 @@ with engine.connect() as conn:
181
160
  df.to_sql('squares',conn)
182
161
  ```
183
162
 
184
- ## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys)
163
+ ## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/13/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/13/core/constraints.html#defining-foreign-keys)
185
164
 
186
165
  Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column.
187
166
 
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "databricks-sqlalchemy"
3
- version = "1.0.0"
3
+ version = "1.0.1"
4
4
  description = "Databricks SQLAlchemy plugin for Python"
5
5
  authors = ["Databricks <databricks-sql-connector-maintainers@databricks.com>"]
6
6
  license = "Apache-2.0"
@@ -11,7 +11,7 @@ include = ["CHANGELOG.md"]
11
11
  [tool.poetry.dependencies]
12
12
  python = "^3.8.0"
13
13
  databricks_sql_connector_core = { version = ">=4.0.0"}
14
- sqlalchemy = { version = ">=2.0.21" }
14
+ sqlalchemy = { version = "^1.3.24" }
15
15
 
16
16
  [tool.poetry.dev-dependencies]
17
17
  pytest = "^7.1.2"
@@ -33,11 +33,4 @@ build-backend = "poetry.core.masonry.api"
33
33
 
34
34
  [tool.black]
35
35
  exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/'
36
- #
37
- [tool.pytest.ini_options]
38
- markers = {"reviewed" = "Test case has been reviewed by Databricks"}
39
- minversion = "6.0"
40
- log_cli = "false"
41
- log_cli_level = "INFO"
42
- testpaths = ["tests", "src/databricks/sqlalchemy/test_local"]
43
- env_files = ["test.env"]
36
+
@@ -0,0 +1 @@
1
+ from databricks.sqlalchemy.dialect import DatabricksDialect