crate 2.3.0__tar.gz → 2.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {crate-2.3.0 → crate-2.4.0}/CHANGES.rst +8 -0
- {crate-2.3.0 → crate-2.4.0}/PKG-INFO +3 -3
- {crate-2.3.0 → crate-2.4.0}/docs/by-example/http.rst +1 -1
- {crate-2.3.0 → crate-2.4.0}/pyproject.toml +4 -4
- crate-2.4.0/src/crate/client/_codec.py +79 -0
- crate-2.4.0/src/crate/client/_options.py +120 -0
- crate-2.4.0/src/crate/client/_pool.py +120 -0
- crate-2.4.0/src/crate/client/_request.py +131 -0
- crate-2.4.0/src/crate/client/_response.py +164 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/http.py +85 -354
- {crate-2.3.0 → crate-2.4.0}/tests/client/test_http.py +4 -1
- crate-2.4.0/tests/client/test_options.py +149 -0
- crate-2.4.0/tests/client/test_pool.py +148 -0
- crate-2.4.0/tests/client/test_request.py +177 -0
- crate-2.4.0/tests/client/test_response.py +217 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/test_serialization.py +1 -1
- {crate-2.3.0 → crate-2.4.0}/.gitignore +0 -0
- {crate-2.3.0 → crate-2.4.0}/CONTRIBUTING.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/DEVELOP.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/LICENSE +0 -0
- {crate-2.3.0 → crate-2.4.0}/NOTICE +0 -0
- {crate-2.3.0 → crate-2.4.0}/README.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/.gitignore +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/Makefile +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/_extra/robots.txt +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/blobs.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/build.json +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/by-example/blob.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/by-example/client.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/by-example/connection.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/by-example/cursor.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/by-example/https.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/by-example/index.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/conf.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/connect.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/data-types.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/docutils.conf +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/getting-started.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/index-all.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/index.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/other-options.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/query.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/docs/requirements.txt +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/__init__.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/_pep440.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/blob.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/connection.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/converter.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/cursor.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/client/exceptions.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/testing/__init__.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/testing/layer.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/src/crate/testing/util.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/__init__.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/import/test_a.json +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/mappings/locations.sql +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/pki/cacert_invalid.pem +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/pki/cacert_valid.pem +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/pki/client_invalid.pem +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/pki/client_valid.pem +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/pki/readme.rst +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/pki/server_valid.pem +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/assets/settings/test_a.json +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/__init__.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/settings.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/test_blob.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/test_connection.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/test_cursor.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/test_exceptions.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/client/test_utils.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/conftest.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/test_docs.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/testing/__init__.py +0 -0
- {crate-2.3.0 → crate-2.4.0}/tests/testing/test_layer.py +0 -0
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
Changes for crate
|
|
3
3
|
=================
|
|
4
4
|
|
|
5
|
+
2026/10/30 2.4.0
|
|
6
|
+
================
|
|
7
|
+
- Breaking change: Raised the minimum ``urllib3`` version to 2.8.0. This
|
|
8
|
+
fixes CVE-2026-97687, CVE-2026-97688, and CVE-2026-97689. Support for
|
|
9
|
+
urllib3 1.x has been removed.
|
|
10
|
+
- Raised the minimum ``orjson`` version to 3.11.6, that fixes
|
|
11
|
+
CVE-2025-67221: ``orjson.dumps`` crashing on deeply nested input.
|
|
12
|
+
|
|
5
13
|
2026/09/17 2.3.0
|
|
6
14
|
================
|
|
7
15
|
- Added ``DefaultTypeConverter`` support that decodes ``DataType.UUID``
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: crate
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.4.0
|
|
4
4
|
Summary: CrateDB Python Client
|
|
5
5
|
Author-email: "Crate.io" <office@crate.io>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -24,8 +24,8 @@ Classifier: Topic :: Internet :: WWW/HTTP
|
|
|
24
24
|
Classifier: Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator
|
|
25
25
|
Classifier: Topic :: System :: Networking
|
|
26
26
|
Requires-Python: >=3.10
|
|
27
|
-
Requires-Dist: orjson
|
|
28
|
-
Requires-Dist: urllib3
|
|
27
|
+
Requires-Dist: orjson>=3.11.6
|
|
28
|
+
Requires-Dist: urllib3>=2.8.0
|
|
29
29
|
Requires-Dist: verlib2
|
|
30
30
|
Description-Content-Type: text/x-rst
|
|
31
31
|
|
|
@@ -220,7 +220,7 @@ When connecting to non-CrateDB servers, the HttpClient will raise a ConnectionEr
|
|
|
220
220
|
>>> http_client.server_infos(http_client._get_server())
|
|
221
221
|
Traceback (most recent call last):
|
|
222
222
|
...
|
|
223
|
-
crate.client.exceptions.ProgrammingError: Invalid server response of content-type 'text/html':
|
|
223
|
+
crate.client.exceptions.ProgrammingError: Invalid server response of content-type 'text/html...':
|
|
224
224
|
...
|
|
225
225
|
>>> http_client.close()
|
|
226
226
|
|
|
@@ -49,14 +49,14 @@ classifiers = [
|
|
|
49
49
|
"Topic :: System :: Networking",
|
|
50
50
|
]
|
|
51
51
|
dependencies = [
|
|
52
|
-
"orjson",
|
|
53
|
-
"urllib3",
|
|
52
|
+
"orjson>=3.11.6",
|
|
53
|
+
"urllib3>=2.8.0",
|
|
54
54
|
"verlib2",
|
|
55
55
|
]
|
|
56
56
|
|
|
57
57
|
[dependency-groups]
|
|
58
58
|
dev = [
|
|
59
|
-
"certifi",
|
|
59
|
+
"certifi>=2026.10.23",
|
|
60
60
|
"coverage",
|
|
61
61
|
"mypy<2.4",
|
|
62
62
|
"pytest<10",
|
|
@@ -174,7 +174,7 @@ lint.per-file-ignores."tests/*" = [
|
|
|
174
174
|
"S310", # False positive; it's a https url
|
|
175
175
|
"S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
|
|
176
176
|
]
|
|
177
|
-
lint.per-file-ignores."src/crate/client/{connection.py,http.py}" = [
|
|
177
|
+
lint.per-file-ignores."src/crate/client/{connection.py,http.py,_pool.py,_response.py}" = [
|
|
178
178
|
"A004", # Import `ConnectionError` is shadowing a Python builtin
|
|
179
179
|
"A005", # Import `ConnectionError` is shadowing a Python builtin
|
|
180
180
|
]
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# -*- coding: utf-8; -*-
|
|
2
|
+
#
|
|
3
|
+
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
|
|
4
|
+
# license agreements. See the NOTICE file distributed with this work for
|
|
5
|
+
# additional information regarding copyright ownership. Crate licenses
|
|
6
|
+
# this file to you under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License. You may
|
|
8
|
+
# obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
14
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
15
|
+
# License for the specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
#
|
|
18
|
+
# However, if you have executed another commercial license agreement
|
|
19
|
+
# with Crate these terms will supersede the license and you may use the
|
|
20
|
+
# software solely pursuant to the terms of the relevant commercial agreement.
|
|
21
|
+
|
|
22
|
+
import calendar
|
|
23
|
+
import datetime as dt
|
|
24
|
+
import typing as t
|
|
25
|
+
from decimal import Decimal
|
|
26
|
+
|
|
27
|
+
import orjson
|
|
28
|
+
|
|
29
|
+
epoch_aware = dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc)
|
|
30
|
+
epoch_naive = dt.datetime(1970, 1, 1)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def json_encoder(obj: t.Any) -> t.Union[int, str]:
|
|
34
|
+
"""
|
|
35
|
+
Encoder function for orjson, with additional type support.
|
|
36
|
+
|
|
37
|
+
- Python's `Decimal` type will be serialized to `str`.
|
|
38
|
+
- Python's `dt.datetime` and `dt.date` types will be
|
|
39
|
+
serialized to `int` after converting to milliseconds
|
|
40
|
+
since epoch.
|
|
41
|
+
- Python's `dt.time` will be serialized to `str`, following
|
|
42
|
+
the ISO format.
|
|
43
|
+
|
|
44
|
+
https://github.com/ijl/orjson#default
|
|
45
|
+
https://cratedb.com/docs/crate/reference/en/latest/general/ddl/data-types.html#type-timestamp
|
|
46
|
+
"""
|
|
47
|
+
if isinstance(obj, Decimal):
|
|
48
|
+
return str(obj)
|
|
49
|
+
if isinstance(obj, dt.datetime):
|
|
50
|
+
if obj.tzinfo is not None:
|
|
51
|
+
delta = obj - epoch_aware
|
|
52
|
+
else:
|
|
53
|
+
delta = obj - epoch_naive
|
|
54
|
+
return int(
|
|
55
|
+
delta.microseconds / 1000.0
|
|
56
|
+
+ (delta.seconds + delta.days * 24 * 3600) * 1000.0
|
|
57
|
+
)
|
|
58
|
+
if isinstance(obj, dt.time):
|
|
59
|
+
return obj.isoformat()
|
|
60
|
+
if isinstance(obj, dt.date):
|
|
61
|
+
return calendar.timegm(obj.timetuple()) * 1000
|
|
62
|
+
raise TypeError
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def json_dumps(obj: t.Any) -> bytes:
|
|
66
|
+
"""
|
|
67
|
+
Serialize to JSON format, using `orjson`, with additional type support.
|
|
68
|
+
|
|
69
|
+
https://github.com/ijl/orjson
|
|
70
|
+
"""
|
|
71
|
+
return orjson.dumps(
|
|
72
|
+
obj,
|
|
73
|
+
default=json_encoder,
|
|
74
|
+
option=(
|
|
75
|
+
orjson.OPT_PASSTHROUGH_DATETIME
|
|
76
|
+
| orjson.OPT_NON_STR_KEYS
|
|
77
|
+
| orjson.OPT_SERIALIZE_NUMPY
|
|
78
|
+
),
|
|
79
|
+
)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# -*- coding: utf-8; -*-
|
|
2
|
+
#
|
|
3
|
+
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
|
|
4
|
+
# license agreements. See the NOTICE file distributed with this work for
|
|
5
|
+
# additional information regarding copyright ownership. Crate licenses
|
|
6
|
+
# this file to you under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License. You may
|
|
8
|
+
# obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
14
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
15
|
+
# License for the specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
#
|
|
18
|
+
# However, if you have executed another commercial license agreement
|
|
19
|
+
# with Crate these terms will supersede the license and you may use the
|
|
20
|
+
# software solely pursuant to the terms of the relevant commercial agreement.
|
|
21
|
+
|
|
22
|
+
import re
|
|
23
|
+
import socket
|
|
24
|
+
import typing as t
|
|
25
|
+
from urllib.parse import urlparse
|
|
26
|
+
|
|
27
|
+
SQL_PATH = "/_sql?types=true"
|
|
28
|
+
|
|
29
|
+
_HTTP_PAT = re.compile("https?://.+", re.I)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def server_url(server: str) -> str:
|
|
33
|
+
"""
|
|
34
|
+
Normalizes a given server string to an url
|
|
35
|
+
|
|
36
|
+
>>> print(server_url('a'))
|
|
37
|
+
http://a
|
|
38
|
+
>>> print(server_url('a/path'))
|
|
39
|
+
http://a/path
|
|
40
|
+
>>> print(server_url('a:9345'))
|
|
41
|
+
http://a:9345
|
|
42
|
+
>>> print(server_url('a:9345/path'))
|
|
43
|
+
http://a:9345/path
|
|
44
|
+
>>> print(server_url('https://a:9345'))
|
|
45
|
+
https://a:9345
|
|
46
|
+
>>> print(server_url('https://a:9345/path'))
|
|
47
|
+
https://a:9345/path
|
|
48
|
+
>>> print(server_url('https://a'))
|
|
49
|
+
https://a
|
|
50
|
+
>>> print(server_url('https://a/path'))
|
|
51
|
+
https://a/path
|
|
52
|
+
>>> print(server_url('demo.crate.io'))
|
|
53
|
+
http://demo.crate.io
|
|
54
|
+
"""
|
|
55
|
+
if not _HTTP_PAT.match(server):
|
|
56
|
+
server = "http://%s" % server
|
|
57
|
+
parsed = urlparse(server)
|
|
58
|
+
return parsed.geturl()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def to_server_list(servers: t.Union[str, t.Iterable[str]]) -> t.List[str]:
|
|
62
|
+
if isinstance(servers, str):
|
|
63
|
+
servers = servers.split()
|
|
64
|
+
return [server_url(s) for s in servers]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def credentials_from_url(
|
|
68
|
+
url: str, username: t.Optional[str], password: t.Optional[str]
|
|
69
|
+
) -> t.Tuple[t.Optional[str], t.Optional[str]]:
|
|
70
|
+
"""
|
|
71
|
+
Return the user name and password from the userinfo of `url`, falling
|
|
72
|
+
back to the given ones for the parts the URL does not contain.
|
|
73
|
+
"""
|
|
74
|
+
parsed = urlparse(url)
|
|
75
|
+
if parsed.username is not None:
|
|
76
|
+
username = parsed.username
|
|
77
|
+
if parsed.password is not None:
|
|
78
|
+
password = parsed.password
|
|
79
|
+
return username, password
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def validate_compress(compress: t.Any) -> t.Union[int, bool]:
|
|
83
|
+
if not isinstance(compress, (bool, int)):
|
|
84
|
+
raise TypeError(
|
|
85
|
+
f"compress must be bool or int, got {type(compress).__name__!r}"
|
|
86
|
+
)
|
|
87
|
+
return compress
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def sql_path(error_trace: bool, path: str = SQL_PATH) -> str:
|
|
91
|
+
if error_trace:
|
|
92
|
+
path += "&error_trace=true"
|
|
93
|
+
return path
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def keepalive_socket_options(
|
|
97
|
+
keepalive: bool = True,
|
|
98
|
+
tcp_keepidle: t.Optional[int] = None,
|
|
99
|
+
tcp_keepintvl: t.Optional[int] = None,
|
|
100
|
+
tcp_keepcnt: t.Optional[int] = None,
|
|
101
|
+
) -> t.List[t.Tuple[int, int, int]]:
|
|
102
|
+
"""
|
|
103
|
+
Return the `setsockopt` arguments for TCP keepalive, or an empty list
|
|
104
|
+
when `keepalive` is off.
|
|
105
|
+
"""
|
|
106
|
+
if not keepalive:
|
|
107
|
+
return []
|
|
108
|
+
|
|
109
|
+
# always use TCP keepalive
|
|
110
|
+
opts = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
|
|
111
|
+
|
|
112
|
+
# hasattr check because some options depend on system capabilities
|
|
113
|
+
# see https://docs.python.org/3/library/socket.html#socket.SOMAXCONN
|
|
114
|
+
if hasattr(socket, "TCP_KEEPIDLE") and tcp_keepidle is not None:
|
|
115
|
+
opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, tcp_keepidle))
|
|
116
|
+
if hasattr(socket, "TCP_KEEPINTVL") and tcp_keepintvl is not None:
|
|
117
|
+
opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, tcp_keepintvl))
|
|
118
|
+
if hasattr(socket, "TCP_KEEPCNT") and tcp_keepcnt is not None:
|
|
119
|
+
opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, tcp_keepcnt))
|
|
120
|
+
return opts
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# -*- coding: utf-8; -*-
|
|
2
|
+
#
|
|
3
|
+
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
|
|
4
|
+
# license agreements. See the NOTICE file distributed with this work for
|
|
5
|
+
# additional information regarding copyright ownership. Crate licenses
|
|
6
|
+
# this file to you under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License. You may
|
|
8
|
+
# obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
14
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
15
|
+
# License for the specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
#
|
|
18
|
+
# However, if you have executed another commercial license agreement
|
|
19
|
+
# with Crate these terms will supersede the license and you may use the
|
|
20
|
+
# software solely pursuant to the terms of the relevant commercial agreement.
|
|
21
|
+
|
|
22
|
+
import heapq
|
|
23
|
+
import logging
|
|
24
|
+
import threading
|
|
25
|
+
import time
|
|
26
|
+
import typing as t
|
|
27
|
+
|
|
28
|
+
from crate.client.exceptions import ConnectionError
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class _ServerPool:
|
|
32
|
+
"""
|
|
33
|
+
Round-robin server selection. A dropped server is taken out of rotation
|
|
34
|
+
for `retry_interval` seconds.
|
|
35
|
+
|
|
36
|
+
Subclasses set `_logger` to log server changes on their own logger.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
retry_interval = 30
|
|
40
|
+
"""Retry interval for failed servers in seconds."""
|
|
41
|
+
|
|
42
|
+
_logger = logging.getLogger(__name__)
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
servers: t.List[str],
|
|
47
|
+
clock: t.Callable[[], float] = time.time,
|
|
48
|
+
) -> None:
|
|
49
|
+
self._active_servers = servers
|
|
50
|
+
self._inactive_servers: t.List[t.Tuple[float, str, str]] = []
|
|
51
|
+
self._lock = threading.RLock()
|
|
52
|
+
self._clock = clock
|
|
53
|
+
|
|
54
|
+
def _get_server(self) -> str:
|
|
55
|
+
"""
|
|
56
|
+
Get server to use for request.
|
|
57
|
+
Also process inactive server list, re-add them after given interval.
|
|
58
|
+
"""
|
|
59
|
+
with self._lock:
|
|
60
|
+
inactive_server_count = len(self._inactive_servers)
|
|
61
|
+
for _ in range(inactive_server_count):
|
|
62
|
+
try:
|
|
63
|
+
ts, server, message = heapq.heappop(self._inactive_servers)
|
|
64
|
+
except IndexError:
|
|
65
|
+
pass
|
|
66
|
+
else:
|
|
67
|
+
if (ts + self.retry_interval) > self._clock():
|
|
68
|
+
# Not yet, put it back
|
|
69
|
+
heapq.heappush(
|
|
70
|
+
self._inactive_servers, (ts, server, message)
|
|
71
|
+
)
|
|
72
|
+
else:
|
|
73
|
+
self._active_servers.append(server)
|
|
74
|
+
self._logger.warning(
|
|
75
|
+
"Restored server %s into active pool", server
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# if none is old enough, use oldest
|
|
79
|
+
if not self._active_servers:
|
|
80
|
+
ts, server, message = heapq.heappop(self._inactive_servers)
|
|
81
|
+
self._active_servers.append(server)
|
|
82
|
+
self._logger.info("Restored server %s into active pool", server)
|
|
83
|
+
|
|
84
|
+
server = self._active_servers[0]
|
|
85
|
+
self._roundrobin()
|
|
86
|
+
|
|
87
|
+
return server
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def active_servers(self) -> t.List[str]:
|
|
91
|
+
"""get the active servers for this client"""
|
|
92
|
+
with self._lock:
|
|
93
|
+
return list(self._active_servers)
|
|
94
|
+
|
|
95
|
+
def _drop_server(self, server: str, message: str) -> None:
|
|
96
|
+
"""
|
|
97
|
+
Drop server from active list and adds it to the inactive ones.
|
|
98
|
+
"""
|
|
99
|
+
try:
|
|
100
|
+
self._active_servers.remove(server)
|
|
101
|
+
except ValueError:
|
|
102
|
+
pass
|
|
103
|
+
else:
|
|
104
|
+
heapq.heappush(
|
|
105
|
+
self._inactive_servers, (self._clock(), server, message)
|
|
106
|
+
)
|
|
107
|
+
self._logger.warning("Removed server %s from active pool", server)
|
|
108
|
+
|
|
109
|
+
# if this is the last server raise exception, otherwise try next
|
|
110
|
+
if not self._active_servers:
|
|
111
|
+
raise ConnectionError(
|
|
112
|
+
("No more Servers available, exception from last server: %s")
|
|
113
|
+
% message
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
def _roundrobin(self) -> None:
|
|
117
|
+
"""
|
|
118
|
+
Very simple round-robin implementation
|
|
119
|
+
"""
|
|
120
|
+
self._active_servers.append(self._active_servers.pop(0))
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
# -*- coding: utf-8; -*-
|
|
2
|
+
#
|
|
3
|
+
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
|
|
4
|
+
# license agreements. See the NOTICE file distributed with this work for
|
|
5
|
+
# additional information regarding copyright ownership. Crate licenses
|
|
6
|
+
# this file to you under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License. You may
|
|
8
|
+
# obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
14
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
15
|
+
# License for the specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
#
|
|
18
|
+
# However, if you have executed another commercial license agreement
|
|
19
|
+
# with Crate these terms will supersede the license and you may use the
|
|
20
|
+
# software solely pursuant to the terms of the relevant commercial agreement.
|
|
21
|
+
|
|
22
|
+
import gzip
|
|
23
|
+
import io
|
|
24
|
+
import os
|
|
25
|
+
import typing as t
|
|
26
|
+
from base64 import b64encode
|
|
27
|
+
|
|
28
|
+
from crate.client._codec import json_dumps
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def super_len(o: t.Any) -> t.Optional[int]:
|
|
32
|
+
if hasattr(o, "__len__"):
|
|
33
|
+
return len(o)
|
|
34
|
+
if hasattr(o, "len"):
|
|
35
|
+
return o.len
|
|
36
|
+
if hasattr(o, "fileno"):
|
|
37
|
+
try:
|
|
38
|
+
fileno = o.fileno()
|
|
39
|
+
except io.UnsupportedOperation:
|
|
40
|
+
pass
|
|
41
|
+
else:
|
|
42
|
+
return os.fstat(fileno).st_size
|
|
43
|
+
if hasattr(o, "getvalue"):
|
|
44
|
+
# e.g. BytesIO, cStringIO.StringI
|
|
45
|
+
return len(o.getvalue())
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def build_headers(
|
|
50
|
+
headers: t.Optional[t.Dict[str, str]],
|
|
51
|
+
data: t.Any,
|
|
52
|
+
username: t.Optional[str] = None,
|
|
53
|
+
password: t.Optional[str] = None,
|
|
54
|
+
schema: t.Optional[str] = None,
|
|
55
|
+
jwt_token: t.Optional[str] = None,
|
|
56
|
+
) -> t.Dict[str, str]:
|
|
57
|
+
"""
|
|
58
|
+
Add the length, authentication, schema and content type headers.
|
|
59
|
+
|
|
60
|
+
A given `headers` dict is updated in place and returned.
|
|
61
|
+
"""
|
|
62
|
+
if headers is None:
|
|
63
|
+
headers = {}
|
|
64
|
+
if "Content-Length" not in headers:
|
|
65
|
+
length = super_len(data)
|
|
66
|
+
if length is not None:
|
|
67
|
+
headers["Content-Length"] = str(length)
|
|
68
|
+
|
|
69
|
+
# Sanity checks.
|
|
70
|
+
if jwt_token is not None and username is not None:
|
|
71
|
+
raise ValueError(
|
|
72
|
+
"Either JWT tokens are accepted, or user credentials, but not both"
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
# Authentication token
|
|
76
|
+
if jwt_token is not None and "Authorization" not in headers:
|
|
77
|
+
headers["Authorization"] = "Bearer %s" % jwt_token
|
|
78
|
+
|
|
79
|
+
# Authentication credentials
|
|
80
|
+
if username is not None:
|
|
81
|
+
if "Authorization" not in headers and username is not None:
|
|
82
|
+
credentials = username + ":"
|
|
83
|
+
if password is not None:
|
|
84
|
+
credentials += password
|
|
85
|
+
headers["Authorization"] = "Basic %s" % b64encode(
|
|
86
|
+
credentials.encode("utf-8")
|
|
87
|
+
).decode("utf-8")
|
|
88
|
+
# For backwards compatibility with Crate <= 2.2
|
|
89
|
+
if "X-User" not in headers:
|
|
90
|
+
headers["X-User"] = username
|
|
91
|
+
|
|
92
|
+
if schema is not None:
|
|
93
|
+
headers["Default-Schema"] = schema
|
|
94
|
+
headers["Accept"] = "application/json"
|
|
95
|
+
headers["Content-Type"] = "application/json"
|
|
96
|
+
return headers
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def compress_body(
|
|
100
|
+
data: bytes, compress: t.Union[int, bool]
|
|
101
|
+
) -> t.Tuple[bytes, bool]:
|
|
102
|
+
"""
|
|
103
|
+
Return the request body to send and whether it is gzip-compressed.
|
|
104
|
+
|
|
105
|
+
`compress` is either a flag, or the body size in bytes from which on
|
|
106
|
+
to compress.
|
|
107
|
+
"""
|
|
108
|
+
compress_enabled = compress is True or (
|
|
109
|
+
not isinstance(compress, bool) and len(data) >= compress
|
|
110
|
+
)
|
|
111
|
+
if compress_enabled:
|
|
112
|
+
return gzip.compress(data, compresslevel=6), True
|
|
113
|
+
return data, False
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def create_sql_payload(
|
|
117
|
+
stmt: str,
|
|
118
|
+
args: t.Optional[t.Union[t.List, t.Tuple, t.Dict]],
|
|
119
|
+
bulk_args: t.Optional[t.List],
|
|
120
|
+
) -> bytes:
|
|
121
|
+
if not isinstance(stmt, str):
|
|
122
|
+
raise ValueError("stmt is not a string")
|
|
123
|
+
if args and bulk_args:
|
|
124
|
+
raise ValueError("Cannot provide both: args and bulk_args")
|
|
125
|
+
|
|
126
|
+
data: t.Dict[str, t.Any] = {"stmt": stmt}
|
|
127
|
+
if args:
|
|
128
|
+
data["args"] = args
|
|
129
|
+
if bulk_args:
|
|
130
|
+
data["bulk_args"] = bulk_args
|
|
131
|
+
return json_dumps(data)
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# -*- coding: utf-8; -*-
|
|
2
|
+
#
|
|
3
|
+
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
|
|
4
|
+
# license agreements. See the NOTICE file distributed with this work for
|
|
5
|
+
# additional information regarding copyright ownership. Crate licenses
|
|
6
|
+
# this file to you under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
# you may not use this file except in compliance with the License. You may
|
|
8
|
+
# obtain a copy of the License at
|
|
9
|
+
#
|
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
#
|
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
14
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
15
|
+
# License for the specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
#
|
|
18
|
+
# However, if you have executed another commercial license agreement
|
|
19
|
+
# with Crate these terms will supersede the license and you may use the
|
|
20
|
+
# software solely pursuant to the terms of the relevant commercial agreement.
|
|
21
|
+
|
|
22
|
+
import typing as t
|
|
23
|
+
from urllib.parse import urlparse
|
|
24
|
+
|
|
25
|
+
import orjson
|
|
26
|
+
|
|
27
|
+
from crate.client.exceptions import (
|
|
28
|
+
BlobLocationNotFoundException,
|
|
29
|
+
ConnectionError,
|
|
30
|
+
IntegrityError,
|
|
31
|
+
ProgrammingError,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
SRV_UNAVAILABLE_STATUSES = {502, 503, 504, 509}
|
|
35
|
+
PRESERVE_ACTIVE_SERVER_EXCEPTIONS = {ConnectionResetError, BrokenPipeError}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def raise_for_status(
|
|
39
|
+
status: int,
|
|
40
|
+
reason: t.Optional[str],
|
|
41
|
+
headers: t.Mapping[str, str],
|
|
42
|
+
body: bytes,
|
|
43
|
+
) -> None:
|
|
44
|
+
"""
|
|
45
|
+
Raise `IntegrityError` exceptions for `DuplicateKeyException` errors.
|
|
46
|
+
"""
|
|
47
|
+
try:
|
|
48
|
+
return _raise_for_status_real(status, reason, headers, body)
|
|
49
|
+
except ProgrammingError as ex:
|
|
50
|
+
if "DuplicateKeyException" in ex.message:
|
|
51
|
+
raise IntegrityError(ex.message, error_trace=ex.error_trace) from ex
|
|
52
|
+
raise
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _raise_for_status_real(
|
|
56
|
+
status: int,
|
|
57
|
+
reason: t.Optional[str],
|
|
58
|
+
headers: t.Mapping[str, str],
|
|
59
|
+
body: bytes,
|
|
60
|
+
) -> None:
|
|
61
|
+
"""make sure that only crate.exceptions are raised that are defined in
|
|
62
|
+
the DB-API specification"""
|
|
63
|
+
message = ""
|
|
64
|
+
if 400 <= status < 500:
|
|
65
|
+
message = "%s Client Error: %s" % (status, reason)
|
|
66
|
+
elif 500 <= status < 600:
|
|
67
|
+
message = "%s Server Error: %s" % (status, reason)
|
|
68
|
+
else:
|
|
69
|
+
return
|
|
70
|
+
if status == 503:
|
|
71
|
+
raise ConnectionError(message)
|
|
72
|
+
if headers.get("content-type", "").startswith("application/json"):
|
|
73
|
+
data = orjson.loads(body)
|
|
74
|
+
error = data.get("error", {})
|
|
75
|
+
error_trace = data.get("error_trace", None)
|
|
76
|
+
if "results" in data:
|
|
77
|
+
errors = [
|
|
78
|
+
res["error_message"]
|
|
79
|
+
for res in data["results"]
|
|
80
|
+
if res.get("error_message")
|
|
81
|
+
]
|
|
82
|
+
if errors:
|
|
83
|
+
raise ProgrammingError("\n".join(errors))
|
|
84
|
+
if isinstance(error, dict):
|
|
85
|
+
raise ProgrammingError(
|
|
86
|
+
error.get("message", ""), error_trace=error_trace
|
|
87
|
+
)
|
|
88
|
+
raise ProgrammingError(error, error_trace=error_trace)
|
|
89
|
+
raise ProgrammingError(message)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def parse_server_infos(
|
|
93
|
+
server: str, content: t.Dict[str, t.Any]
|
|
94
|
+
) -> t.Tuple[str, t.Optional[str], str]:
|
|
95
|
+
node_name = content.get("name")
|
|
96
|
+
node_version = content.get("version", {}).get("number", "0.0.0")
|
|
97
|
+
return server, node_name, node_version
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def blob_put_result(
|
|
101
|
+
status: int,
|
|
102
|
+
reason: t.Optional[str],
|
|
103
|
+
headers: t.Mapping[str, str],
|
|
104
|
+
body: bytes,
|
|
105
|
+
table: str,
|
|
106
|
+
digest: str,
|
|
107
|
+
) -> bool:
|
|
108
|
+
if status == 201:
|
|
109
|
+
# blob created
|
|
110
|
+
return True
|
|
111
|
+
if status == 409:
|
|
112
|
+
# blob exists
|
|
113
|
+
return False
|
|
114
|
+
if status in (400, 404):
|
|
115
|
+
raise BlobLocationNotFoundException(table, digest)
|
|
116
|
+
raise_for_status(status, reason, headers, body)
|
|
117
|
+
return False
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def blob_del_result(
|
|
121
|
+
status: int,
|
|
122
|
+
reason: t.Optional[str],
|
|
123
|
+
headers: t.Mapping[str, str],
|
|
124
|
+
body: bytes,
|
|
125
|
+
) -> bool:
|
|
126
|
+
if status == 204:
|
|
127
|
+
return True
|
|
128
|
+
if status == 404:
|
|
129
|
+
return False
|
|
130
|
+
raise_for_status(status, reason, headers, body)
|
|
131
|
+
return False
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def blob_exists_result(
|
|
135
|
+
status: int,
|
|
136
|
+
reason: t.Optional[str],
|
|
137
|
+
headers: t.Mapping[str, str],
|
|
138
|
+
body: bytes,
|
|
139
|
+
) -> bool:
|
|
140
|
+
if status == 200:
|
|
141
|
+
return True
|
|
142
|
+
elif status == 404:
|
|
143
|
+
return False
|
|
144
|
+
raise_for_status(status, reason, headers, body)
|
|
145
|
+
return False
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def redirect_target(
|
|
149
|
+
status: int, location: t.Union[str, None, t.Literal[False]]
|
|
150
|
+
) -> t.Optional[str]:
|
|
151
|
+
"""
|
|
152
|
+
Return the server to follow a redirect to, as ``scheme://netloc``.
|
|
153
|
+
"""
|
|
154
|
+
if location and 300 <= status <= 308:
|
|
155
|
+
redirect_url = urlparse(location)
|
|
156
|
+
return f"{redirect_url.scheme}://{redirect_url.netloc}"
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def should_preserve_server(ex: BaseException) -> bool:
|
|
161
|
+
return any(
|
|
162
|
+
t in [type(arg) for arg in ex.args]
|
|
163
|
+
for t in PRESERVE_ACTIVE_SERVER_EXCEPTIONS
|
|
164
|
+
)
|